diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..c77ac7a07ebe7c333af2fb59c925c6c4fb4ef811 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +evaluation-pipeline/assets/babylm.png filter=lfs diff=lfs merge=lfs -text +evaluation-pipeline/docs/img/fewshot_example_gpt3.png filter=lfs diff=lfs merge=lfs -text +evaluation-pipeline/sample_predictions.json filter=lfs diff=lfs merge=lfs -text diff --git a/evaluation-pipeline/.coveragerc b/evaluation-pipeline/.coveragerc new file mode 100644 index 0000000000000000000000000000000000000000..a15db19f10959c9b203a25417dc188319db45d86 --- /dev/null +++ b/evaluation-pipeline/.coveragerc @@ -0,0 +1,21 @@ +[run] +omit = + # Requires manual data download + lm_eval/tasks/jigsaw_unintended_bias.py + +[report] +exclude_lines = + # Skip any pass lines such as may be used for @abstractmethod + pass + + # Have to re-enable the standard pragma + pragma: no cover + + # Don't complain about missing debug-only code: + def __repr__ + if self\.debug + + # Don't complain if tests don't hit defensive assertion code: + raise AssertionError + raise NotImplementedError + return NotImplemented diff --git a/evaluation-pipeline/.flake8 b/evaluation-pipeline/.flake8 new file mode 100644 index 0000000000000000000000000000000000000000..73f6455d132003fce0034f41d72eeb901b68f039 --- /dev/null +++ b/evaluation-pipeline/.flake8 @@ -0,0 +1,5 @@ +[flake8] +ignore = E203, E266, E501, W503, F403, F401, C901 +max-line-length = 127 +max-complexity = 10 +select = B,C,E,F,W,T4,B9 diff --git a/evaluation-pipeline/.gitignore b/evaluation-pipeline/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..32e929fd3f1972c10fef0d1e742602c19567e1fb --- /dev/null +++ b/evaluation-pipeline/.gitignore @@ -0,0 +1,56 @@ +# Distribution / Packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +*.pyc +*.log +*.db +MANIFEST + +# Directories +data/ +outputs/ +lm_cache/ + +# Editors +.idea +.vscode +.ipynb_checkpoints + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Unit test / coverage reports +.coverage +.coverage.* +.cache + +# local testing files +test/ + +# baseline models +baselines/ + +# outputs from running scripts +.out +.err diff --git a/evaluation-pipeline/.gitmodules b/evaluation-pipeline/.gitmodules new file mode 100644 index 0000000000000000000000000000000000000000..ff05f05e1e0270c913a040a0b8f31a68c181222c --- /dev/null +++ b/evaluation-pipeline/.gitmodules @@ -0,0 +1,3 @@ +[submodule "lm_eval/datasets/biomedical"] + path = lm_eval/datasets/biomedical + url = https://github.com/bigscience-workshop/biomedical diff --git a/evaluation-pipeline/.pre-commit-config.yaml b/evaluation-pipeline/.pre-commit-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3812ceb399172688e72498b8b5377463c1acd200 --- /dev/null +++ b/evaluation-pipeline/.pre-commit-config.yaml @@ -0,0 +1,44 @@ +# Ignore test linting to avoid conflicting changes to version stability. +exclude: ^tests/testdata/ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.1.0 + hooks: + - id: check-added-large-files + - id: check-ast + - id: check-byte-order-marker + - id: check-case-conflict + - id: check-json + - id: check-merge-conflict + - id: check-symlinks + - id: check-yaml + - id: destroyed-symlinks + - id: detect-private-key + - id: end-of-file-fixer + - id: no-commit-to-branch + - id: requirements-txt-fixer + - id: trailing-whitespace + - id: fix-byte-order-marker + exclude: docs/CNAME + - id: fix-encoding-pragma + args: [--remove] + - id: mixed-line-ending + args: [--fix=lf] + - repo: https://gitlab.com/pycqa/flake8 + rev: 3.7.9 + hooks: + - id: flake8 + - repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black + language_version: python3.8 + - repo: https://github.com/codespell-project/codespell + rev: v2.1.0 + hooks: + - id: codespell + exclude: > + (?x)^( + .*\.json|ignore.txt + )$ + args: [--check-filenames, --check-hidden, --ignore-words=ignore.txt] diff --git a/evaluation-pipeline/CODEOWNERS b/evaluation-pipeline/CODEOWNERS new file mode 100644 index 0000000000000000000000000000000000000000..d90f881cd70dc051b6fd33739ee69ad3c1376b55 --- /dev/null +++ b/evaluation-pipeline/CODEOWNERS @@ -0,0 +1 @@ +* @jon-tow @leogao2 @StellaAthena diff --git a/evaluation-pipeline/LICENSE.md b/evaluation-pipeline/LICENSE.md new file mode 100644 index 0000000000000000000000000000000000000000..12e6063183935e876e232db276568baf4954b492 --- /dev/null +++ b/evaluation-pipeline/LICENSE.md @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 EleutherAI + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/evaluation-pipeline/README.md b/evaluation-pipeline/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9dae60779aa7efed802f2e4e970c98677291e145 --- /dev/null +++ b/evaluation-pipeline/README.md @@ -0,0 +1,237 @@ +# BabyLM Evaluation Pipeline +![BabyLM Challenge](assets/babylm.png) + +## Overview + +This code provides the backend for the BabyLM Challenge's evaluation pipeline. + +We provide support for zero-shot evaluations on BLiMP, as well as scripts for fine-tuning HuggingFace-based models on GLUE and MSGS tasks. + +We also provide a [Colab demo](https://colab.research.google.com/drive/1HX2D3wztO81tKcqCeV_ecRcEUseBVuTc?usp=sharing) of the evaluation pipeline as a demonstration of how to use the code. + +If you have questions about or suggestions for this code, please open an issue and consider [joining our Slack](https://join.slack.com/t/babylmchallenge/shared_invite/zt-1s8el4mro-qvVO447l3POBZcUNvMWQcg). We also welcome pull requests! + +## Installation + +To install dependencies, run this: + +```bash +git clone https://github.com/babylm/evaluation-pipeline +cd evaluation-pipeline +pip install -e ".[dev]" +pip install torch==1.11.0+cu113 torchvision==0.12.0+cu113 torchaudio==0.11.0 --extra-index-url https://download.pytorch.org/whl/cu113 +``` + +If your GPU is compatible with CUDA 10, replace all instances of `cu113` with `cu102`. + +### Data +We provide versions of BLiMP, GLUE, and MSGS which have been filtered according to the vocabulary of the `strict-small` dataset. We filter for examples where each word has appeared in our training set at least twice. + +Unzip the dataset into the root directory of this repository: `unzip filter_data.zip`. + +## Usage +### Zero-shot Evaluation +To evaluate a model on zero-shot tasks like BLiMP and the held-out BLiMP supplement tasks: + +```bash +python babylm_eval.py 'path/to/model_and_tokenizer' 'model_type' +``` + +Where `model_type` is one of "encoder", "decoder" or "encoder-decoder". + +### Fine-tuning +To fine-tune and evaluate a model on tasks that require fine-tuning, like the (Super)GLUE tasks or held-out MSGS tasks: + +```bash +./finetune_all_tasks.sh 'path/to/model_and_tokenizer' +``` + +#### Hyperparameters +This script contains hyperparameter defaults that should work for a variety of model sizes, architectures, and tasks. You may adjust these hyperparameters as you wish, though we ask that you submit the best hyperparmeter settings in a README file if you don't use the defaults. + +Here are the defaults that we use: +| Hyperparameter | Value | +| -------------- | ----- | +| Initial learning rate | 5e-5 | +| Batch size | 64 | +| Maximum epochs | 10 | +| Evaluate every (steps) | 200 | +| Patience | 10 | +| Random seed | 12 | + +## Uploading Results +We provide a shell script that will collect your results into a single file: + +```bash +./collect_results.py path/to/model_and_tokenizer +``` + +This will output a file called `all_predictions.json` in the root folder of this repository. We will ask you to upload this file to a submission portal. + +We will also ask you to share a link where we can download your model and tokenizer. + +### Format of Predictions +If you wish to submit your results and you are not using the `collect_results.py` script, please ensure that your predictions file conforms to the submission format (example provided here as `sample_predictions.json`). This is a file consisting of line-separated JSON objects, where each line corresponds to a single subtask. + +For each line, the JSON object includes a `task` field ("blimp", "glue", "supplement", or "msgs"), a `sub_task` field (the specific task, like "cola" or "anaphor_agreement"), and a `predictions` field, which is a list of JSON objects containing example IDs and predictions for those examples. Here is an example: + +``` +{"task": "glue", "sub_task": "mnli", "predictions": [{"id": "mnli_0", "pred": 0}, {"id": "mnli_1": "pred": 1}, ..., {"id": "mnli_6561", "pred": 1}]} +``` + +### Age-of-acquisition prediction Evaluation +This evaluation is based on Portelance, Duan, Lupyan and Frank 2023 (see citation below). + +If you want to run it, run the zero-shot evaluation script with the "--run_aoa" flag: + +```bash +python babylm_eval.py 'path/to/model_and_tokenizer' 'model_type' --run_aoa +``` + +Note, the evaluation requires access to forward pass labels from your tokenizer. It currently expects the tokenizer to either produce them under the key "labels" if the model type is a "decoder" where labels represent the shifted "input_ids", or if no labels are provided, it will set the "labels" to be equal to the "input_ids" (this is done automatically for "encoder" and "encoder-decoder" type models. In the event that your labels are not equal to the input_ids, please make sure your tokenizer contains them under the key "labels". + +Once it runs, it will produce two json files in a folder called "aoa_prediction" in the model directory provided. One of the files contains the estimated average surprisal of words for the model in child directed utterances taken from CHILDES. The other contains the results of the evaluation. Models are evaluated using leave-one-out cross validation. The results are Mean Absolute Deviation (MAD) scores in months between the actual average age-of-acquisition (AoA) of these words by American English speaking children and the predicted AoA based on the models average surprisal scores (the closer the MAD scores are to zero, the better). MAD scores are provided over all the words, over nouns, over predicates, and over function words. Previous work has found that models tend to do better at predicting the AoA of predicates and function words over nouns. + +The better the fit, the better a model's predictions and the actual AoA of words in kids (the smaller the MAD scores), the more the order in which models learn words resembles the order in which children tend to learn words. + +Note that, while we do not require you to run this evaluation or submit your score for our evaluation, we highly encourage you to compute this metric and discuss it in your paper! + +## Baselines +We provide a series of baseline models that we train on our strict or strict-small dataset. These are [hosted on HuggingFace](https://huggingface.co/babylm). + +We simply take the hyperparameters used to pre-train the original versions of these models, and train them on our strict or strict-small datasets. While we do reduce the context length and, in some cases, the batch size, these are otherwise minimally modified. + +Here are baseline scores. These are all accuracies, unless otherwise noted by (F1), where we use macro-F1. Random chance accuracy on all BLiMP tasks is 50. + +**Strict-small Track** + +*BLiMP* +| Model | Anaphor Agr. | Agr. Structure | Binding | Control/Raising | D-N Agr. | Ellipsis | Filler-Gap | Irregular Forms | Island Effects | NPI Licensing | Quantifiers | S-V Agr. | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| OPT-125m | 63.8 | 70.6 | 67.1 | 66.5 | 78.5 | 62 | 63.8 | 67.5 | 48.6 | 46.7 | 59.6 | 56.9 | +| RoBERTa-base | 81.5 | 67.1 | 67.3 | 67.9 | 90.8 | 76.4 | 63.5 | 87.4 | 39.9 | 55.9 | 70.5 | 65.4 | +| T5-base | 68.9 | 63.8 | 60.4 | 60.9 | 72.2 | 34.4 | 48.2 | 77.6 | 45.6 | 47.8 | 61.2 | 65.0 | + +*BLiMP Supplement* +| Model | Hypernym | QA Congruence (easy) | QA Congruence (tricky) | Subj.-Aux. Inversion | Turn Taking | +| --- | --- | --- | --- | --- | --- | +| OPT-125m | 50.0 | 54.7 | 31.5 | 80.3 | 57.1 | +| RoBERTa-base | 49.4 | 31.3 | 32.1 | 71.7 | 53.2 | +| T5-base | 48.0 | 40.6 | 21.2 | 64.9 | 45.0 | + +*(Super)GLUE* +| Model | CoLA | SST-2 | MRPC (F1) | QQP (F1) | MNLI | MNLI-mm | QNLI | RTE | BoolQ | MultiRC | WSC | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| *Majority label* | *69.5* | *50.2* | *82* | *53.1* | *35.7* | *35.7* | *35.4* | *53.1* | *50.5* | *59.9* | *53.2* | *61.4* | +| OPT-125m | 64.6 | 81.9 | 72.5 | 60.4 | 57.6 | 60.0 | 61.5 | 60.0 | 63.3 | 55.2 | 60.2 | +| RoBERTa-base | 70.8 | 87.0 | 79.2 | 73.7 | 73.2 | 74.0 | 77.0 | 61.6 | 66.3 | 61.4 | 61.4 | +| T5-base | 61.2 | 78.1 | 80.5 | 66.2 | 48.0 | 50.3 | 62.0 | 49.4 | 66.0 | 47.1 | 61.4 | + +*MSGS* +| Model | CR (Control) | LC (Control) | MV (Control) | RP (Control) | SC (Control) | CR_LC | CR_RTP | MV_LC | MV_RTP | SC_LC | SC_RP | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| OPT-125m | 86.4 | 86.1 | 99.8 | 100.0 | 94.3 | 66.5 | 67.0 | 66.5 | 67.6 | 80.2 | 67.5 | +| RoBERTa-base | 84.1 | 100.0 | 99.4 | 93.5 | 96.4 | 67.7 | 68.6 | 66.7 | 68.6 | 84.2 | 65.7 | +| T5-base | 78.4 | 100.0 | 72.7 | 95.5 | 94.4 | 66.7 | 69.7 | 66.6 | 66.9 | 73.6 | 67.8 | + +*Age-of-acquisition Prediction* +(Mean absolute deviation in months across LOO cross-validation folds) +| Model | Overall (591 words) | Nouns (322) | Predicates (167) | Function words (102) | +| --- | --- | --- | --- | --- | +| OPT-125m | 2.03 | 1.98 | 1.81 | 2.57 | +| RoBERTa-base | 2.06 | 1.99 | 1.85 | 2.65 | +| T5-base | 2.04 | 1.97 | 1.82 | 2.64 | + +------------- + +**Strict Track** + +*BLiMP* +| Model | Anaphor Agr. | Agr. Structure | Binding | Control/Raising | D-N Agr. | Ellipsis | Filler-Gap | Irregular Forms | Island Effects | NPI Licensing | Quantifiers | S-V Agr. | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| OPT-125m | 94.9 | 73.8 | 73.8 | 72.2 | 93.1 | 80.5 | 73.6 | 80.8 | 57.8 | 51.6 | 74.5 | 77.3 | +| RoBERTa-base | 89.5 | 71.3 | 71 | 67.1 | 93.1 | 83.8 | 68.0 | 89.6 | 54.5 | 66.3 | 70.3 | 76.2 | +| T5-base | 66.7 | 61.2 | 59.4 | 59.8 | 53.8 | 49.1 | 70.0 | 75.5 | 43.6 | 45.6 | 34.2 | 53.2 | + +*BLiMP Supplement* +| Model | Hypernym | QA Congruence (easy) | QA Congruence (tricky) | Subj.-Aux. Inversion | Turn Taking | +| --- | --- | --- | --- | --- | --- | +| OPT-125m | 46.3 | 76.5 | 47.9 | 85.3 | 82.9 | +| RoBERTa-base | 50.8 | 34.4 | 34.5 | 45.6 | 46.8 | +| T5-base | 51.1 | 45.3 | 25.5 | 69.2 | 48.9 | + +*(Super)GLUE* +| Model | CoLA | SST-2 | MRPC (F1) | QQP (F1) | MNLI | MNLI-mm | QNLI | RTE | BoolQ | MultiRC | WSC | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| *Majority label* | *69.5* | *50.2* | *82* | *53.1* | *35.7* | *35.7* | *35.4* | *53.1* | *50.5* | *59.9* | *53.2* | *61.4* | +| OPT-125m | 73.7 | 86.6 | 82.1 | 77.8 | 70.1 | 71.9 | 80.1 | 67.7 | 66.0 | 61.1 | 59.0 | +| RoBERTa-base | 75.9 | 88.6 | 80.5 | 78.5 | 68.7 | 78.0 | 82.3 | 51.5 | 59.9 | 61.3 | 61.4 | +| T5-base | 76.3 | 88.0 | 85.9 | 79.7 | 71.5 | 74.0 | 83.1 | 60.6 | 69.0 | 62.4 | 60.2 | + +*MSGS* +| Model | CR (Control) | LC (Control) | MV (Control) | RP (Control) | SC (Control) | CR_LC | CR_RTP | MV_LC | MV_RTP | SC_LC | SC_RP | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| OPT-125m | 97.2 | 82.6 | 100.0 | 99.8 | 88.1 | 75.3 | 67.1 | 66.3 | 66.8 | 84.8 | 62.0 | +| RoBERTa-base | 93.0 | 100.0 | 100.0 | 100.0 | 89.0 | 68.3 | 66.8 | 66.6 | 80.2 | 67.4 | 67.4 | +| T5-base | 95.1 | 100.0 | 100.0 | 99.8 | 88.7 | 76.7 | 69.4 | 67.0 | 67.7 | 72.7 | 68.0 | + +----------------------- + +These are naïve baselines that are meant to provide a starting point for investigation. We look forward to seeing how you will improve upon these! + +## Citation +If you use the datasets or code from this repository, please cite the BabyLM Call for Papers: + +``` +@article{warstadt2023papers, + title = {Call for Papers -- The BabyLM Challenge: Sample-efficient pretraining on a developmentally plausible corpus}, + author = {Warstadt, Alex and + Choshen, Leshem and + Mueller, Aaron and + Williams, Adina and + Wilcox, Ethan and + Zhuang, Chengxu}, + year = {2023}, + journal = {Computing Research Repository}, + volume = {arXiv:2301.11796} +} +``` + +Please also cite the lm-eval-harness paper: +``` +@software{eval-harness, + author = {Gao, Leo and + Tow, Jonathan and + Biderman, Stella and + Black, Sid and + DiPofi, Anthony and + Foster, Charles and + Golding, Laurence and + Hsu, Jeffrey and + McDonell, Kyle and + Muennighoff, Niklas and + Phang, Jason and + Reynolds, Laria and + Tang, Eric and + Thite, Anish and + Wang, Ben and + Wang, Kevin and + Zou, Andy}, + title = {A framework for few-shot language model evaluation}, + month = sep, + year = 2021, + publisher = {Zenodo}, + version = {v0.0.1}, + doi = {10.5281/zenodo.5371628}, + url = {https://doi.org/10.5281/zenodo.5371628} +} +``` + +Please cite the following if you choose to include the Age-of-acquisition prediction evaluation: +``` +@manuscript{portelance2023predicting, + author = {Portelance, Eva and Duan, Yuguang and Frank, Michael C. and Lupyan, Gary}, + title = {Predicting age of acquisition for children’s early vocabulary in five languages using language model surprisal}, + year = {2023}, + url = {https://github.com/evaportelance/multilingual-aoa-prediction} + } diff --git a/evaluation-pipeline/aoa_data/sent_words.json b/evaluation-pipeline/aoa_data/sent_words.json new file mode 100644 index 0000000000000000000000000000000000000000..05dd1aee0a6b6de2604896323ffac1291a016cd5 --- /dev/null +++ b/evaluation-pipeline/aoa_data/sent_words.json @@ -0,0 +1 @@ +[{"sent": "and then Fraser got it .", "words": ["then", "it", "and"]}, {"sent": "and theres another toe .", "words": ["another", "toe", "and"]}, {"sent": "put the shirt on and then xxx .", "words": ["the", "put", "shirt", "and", "on", "then"]}, {"sent": "you gonna get that outof your mouth and tell me what would you like ?", "words": ["what", "your", "mouth", "like", "you", "get", "and", "would", "me", "that"]}, {"sent": "and down .", "words": ["down", "and"]}, {"sent": "and draw tug and tow .", "words": ["draw", "and"]}, {"sent": "have a look in the cupboard and see what weve got ?", "words": ["the", "what", "and", "have", "in", "look", "see", "a"]}, {"sent": "is that nice and pretty now ?", "words": ["is", "nice", "and", "pretty", "that"]}, {"sent": "Fraser do you wanna go upstairs and watch it ?", "words": ["do", "watch", "go", "and", "you", "it", "wanna"]}, {"sent": "and Peters in there .", "words": ["there", "in", "and"]}, {"sent": "and Lunas favorite color is blue is it ?", "words": ["it", "blue", "is", "and"]}, {"sent": "its the one that youve taken Nanny and Grandad to .", "words": ["the", "to", "that", "and"]}, {"sent": "and you can have some blankets .", "words": ["can", "you", "have", "and", "some"]}, {"sent": "and what color is your hair ?", "words": ["what", "your", "is", "and", "hair"]}, {"sent": "and what is the rabbit doing ?", "words": ["the", "what", "is", "and"]}, {"sent": "okay open the door and then I can come out .", "words": ["the", "can", "open", "and", "out", "then", "door"]}, {"sent": "a little one and a big one .", "words": ["big", "and", "little", "a"]}, {"sent": "and toys ?", "words": ["and"]}, {"sent": "theres a tub there and theres one here .", "words": ["here", "there", "and", "a"]}, {"sent": "if your if your shorts had chicken pox theyd have dots all over them and spots wouldnt they ?", "words": ["all", "shorts", "your", "if", "them", "and", "have", "they", "chicken", "over"]}, {"sent": "and who else ?", "words": ["who", "and"]}, {"sent": "and then well put a little toothpaste on it ?", "words": ["put", "little", "and", "it", "on", "then", "a"]}, {"sent": "and what did the chipmunks hafta do ?", "words": ["the", "what", "do", "and", "did"]}, {"sent": "and why is it your stuff ?", "words": ["your", "is", "and", "it", "why"]}, {"sent": "Roman and Sophie yes and Roman and Sophie caught the balloons too .", "words": ["the", "too", "and"]}, {"sent": "and um .", "words": ["and"]}, {"sent": "with the pots and the buckets and the spade and the watering can .", "words": ["the", "can", "with", "and"]}, {"sent": "and then did you come back home and have your tea ?", "words": ["your", "back", "you", "and", "have", "did", "then", "home"]}, {"sent": "and then were gonna cook it .", "words": ["and", "cook", "it", "then", "were"]}, {"sent": "and when theyre in that walkie and doing that and all .", "words": ["all", "when", "and", "in", "that"]}, {"sent": "and a cockerel says cockadoodledoo .", "words": ["a", "and"]}, {"sent": "lets hope he doesnt drop that ice cream onto his uh new blue tshirt and then shes got yellow trousers on .", "words": ["ice cream", "blue", "his", "and", "on", "he", "drop", "ice", "yellow", "then", "new", "that"]}, {"sent": "and all that so when they get close to the filling station itll be like time to fill up .", "words": ["the", "all", "up", "when", "so", "to", "be", "like", "and", "get", "they", "close", "that"]}, {"sent": "here take it away and give her the medicine .", "words": ["the", "medicine", "her", "and", "here", "it", "away", "give", "take"]}, {"sent": "and whats this ?", "words": ["this", "and"]}, {"sent": "there the mommys swimming and shes on the edge of the pool .", "words": ["the", "of", "pool", "and", "there", "on"]}, {"sent": "youve seen that program though with AuntyMabel and Pippin havent you ?", "words": ["you", "that", "with", "and"]}, {"sent": "coke and mixed up chips all in the same bottle ?", "words": ["up", "all", "the", "and", "coke", "in", "same", "bottle"]}, {"sent": "it has got envelopes and books and pencils and pens and crayons and books and birthday cards .", "words": ["it", "and"]}, {"sent": "it was bleeding down your legs and ?", "words": ["your", "was", "and", "it", "down"]}, {"sent": "just then there was a flash of light and a rumbling sound in the sky .", "words": ["the", "sky", "of", "was", "light", "and", "there", "in", "then", "a"]}, {"sent": "and why why is that funny ?", "words": ["that", "is", "why", "and"]}, {"sent": "you pay thirty nine cents for the goldfish and about ten dollars for everything else .", "words": ["the", "for", "and", "you", "about"]}, {"sent": "and sometimes mum sends Pingu to buy a fish .", "words": ["to", "and", "fish", "buy", "a"]}, {"sent": "and which ones do you which one do you think is her what do you think was her favorite book today ?", "words": ["do", "think", "what", "is", "was", "her", "you", "and", "book", "which"]}, {"sent": "no youre attached forever and ever and ever .", "words": ["and"]}, {"sent": "pizza has mushrooms on it and it looks like peppers and pepperoni .", "words": ["like", "pizza", "and", "it", "on"]}, {"sent": "you and Spencer .", "words": ["and", "you"]}, {"sent": "and Id hafta thaw them out in the oven .", "words": ["the", "them", "and", "in", "out", "oven"]}, {"sent": "shall we find your little PostmanPat and .", "words": ["your", "we", "and", "little", "find"]}, {"sent": "and thats a green pepper .", "words": ["green", "a", "and"]}, {"sent": "and I saw it today and I said gee I would love to read that one .", "words": ["to", "and", "it", "read", "love", "would", "that"]}, {"sent": "and put an eyeball and a mouth .", "words": ["put", "mouth", "and", "an", "a"]}, {"sent": "and that one .", "words": ["that", "and"]}, {"sent": "and then little cat b cleaned off the tv .", "words": ["off", "the", "little", "and", "cat", "tv", "then"]}, {"sent": "not this Friday which ya cant switch and not next Friday where .", "words": ["where", "and", "this", "which", "not"]}, {"sent": "and well call off on the way and see Daddy and pick the child seat up .", "words": ["off", "the", "up", "pick", "and", "on", "see"]}, {"sent": "hes going to the toilet and hes taking his pants off ?", "words": ["the", "off", "to", "and", "pants", "his"]}, {"sent": "lets wait lets wait lets five minutes and then well go up .", "words": ["up", "wait", "go", "and", "then"]}, {"sent": "and now the dragon was looking at the presents and what happened after he looked at the presents ?", "words": ["the", "what", "was", "and", "he", "at"]}, {"sent": "and MrsWww and MissWww see we get better each time .", "words": ["better", "each", "we", "and", "get", "see"]}, {"sent": "and daddy pinched most of it just to put that thing on .", "words": ["put", "of", "to", "and", "it", "on", "that"]}, {"sent": "and she laid down in the big bed and it was too hard .", "words": ["the", "was", "bed", "and", "in", "it", "hard", "too", "down", "big", "she"]}, {"sent": "and wheres the train gonna go to ?", "words": ["the", "go", "train", "to", "and"]}, {"sent": "were going to sing a song about tummy and mummie ?", "words": ["to", "and", "sing", "about", "tummy", "were", "a"]}, {"sent": "two scarfs and a belt .", "words": ["belt", "a", "and"]}, {"sent": "and .", "words": ["and"]}, {"sent": "and you cant take them .", "words": ["them", "take", "you", "and"]}, {"sent": "and I said no .", "words": ["and"]}, {"sent": "and then youll find out .", "words": ["out", "then", "find", "and"]}, {"sent": "and no see Gordon and no see Henry .", "words": ["see", "and"]}, {"sent": "and what is what are they can you tell me what is going on in the house ?", "words": ["can", "what", "the", "is", "and", "you", "they", "in", "are", "house", "on", "me"]}, {"sent": "lets go and play in the other room .", "words": ["the", "go", "play", "and", "other", "in", "room"]}, {"sent": "well youve got buckets and spades and forks upstairs havent you ?", "words": ["you", "and"]}, {"sent": "and Ill give you one .", "words": ["give", "you", "and"]}, {"sent": "and on saturdays or sundays I bring one of them out here .", "words": ["of", "them", "bring", "and", "here", "out", "on"]}, {"sent": "and Ive just found it .", "words": ["it", "and"]}, {"sent": "and the ambulance goes there .", "words": ["the", "there", "and"]}, {"sent": "and then on the paper to dry it .", "words": ["the", "to", "paper", "dry", "and", "it", "on", "then"]}, {"sent": "get right under the table and you can pick it up .", "words": ["the", "can", "up", "table", "pick", "and", "get", "under", "you", "it"]}, {"sent": "during the mid summer when the arctic nights are as light as day snowy steps out of the nest and spreads his wings .", "words": ["the", "light", "of", "and", "are", "out", "his", "when"]}, {"sent": "is that Robby and Paul ?", "words": ["that", "and", "is"]}, {"sent": "and a little bit of a broken light switch .", "words": ["of", "light", "little", "broken", "and", "a"]}, {"sent": "and then you dont see the car .", "words": ["the", "car", "you", "and", "dont", "see", "then"]}, {"sent": "put your feet down and well rinse them off .", "words": ["off", "put", "your", "them", "and", "down"]}, {"sent": "and squash .", "words": ["and"]}, {"sent": "you were standing and he took the picture .", "words": ["the", "picture", "and", "you", "he", "were"]}, {"sent": "and AnnMarie is sick .", "words": ["sick", "is", "and"]}, {"sent": "and the digger can see if it can dig its way out .", "words": ["the", "can", "if", "and", "it", "out", "see"]}, {"sent": "thats the seat and the steering wheel .", "words": ["the", "and"]}, {"sent": "and he just was sitting watching LiveAndKicking and all xxx .", "words": ["all", "was", "he", "and"]}, {"sent": "get your cups and saucers Becky .", "words": ["get", "your", "and"]}, {"sent": "and theres his mouth .", "words": ["his", "mouth", "and"]}, {"sent": "and I bet the yellow triangle fits in there .", "words": ["the", "and", "there", "in", "yellow"]}, {"sent": "Baloo and .", "words": ["and"]}, {"sent": "fishy or something and um .", "words": ["and"]}, {"sent": "cut me and core me but please dont ignore me .", "words": ["cut", "and", "dont", "me", "but"]}, {"sent": "checking out their faces and their features and stuff so .", "words": ["so", "their", "out", "and"]}, {"sent": "and Ill do it later .", "words": ["do", "it", "and"]}, {"sent": "there and there .", "words": ["there", "and"]}, {"sent": "what is she saying now to that person ?", "words": ["what", "is", "to", "that", "she"]}, {"sent": "what sort of dinosaur do you think it is ?", "words": ["what", "do", "think", "is", "of", "you", "it"]}, {"sent": "what a funny breakfast .", "words": ["what", "a"]}, {"sent": "what do you have on the seesaw there ?", "words": ["the", "what", "do", "you", "have", "there", "on"]}, {"sent": "what does it say ?", "words": ["what", "does", "say", "it"]}, {"sent": "what Abe ?", "words": ["what"]}, {"sent": "what was what am I thinking of ?", "words": ["was", "what", "of", "am"]}, {"sent": "what well do .", "words": ["what", "do"]}, {"sent": "what ?", "words": ["what"]}, {"sent": "and what shape does Po have on his head ?", "words": ["what", "does", "head", "and", "have", "his", "on"]}, {"sent": "whats what ?", "words": ["what"]}, {"sent": "youd like what ?", "words": ["what", "like"]}, {"sent": "what do you think it is ?", "words": ["what", "do", "think", "is", "you", "it"]}, {"sent": "what is that ?", "words": ["what", "that", "is"]}, {"sent": "what is that kittycat eating ?", "words": ["what", "that", "is"]}, {"sent": "look what I have here .", "words": ["have", "what", "here", "look"]}, {"sent": "what do you do to babies ?", "words": ["to", "what", "do", "you"]}, {"sent": "what ?", "words": ["what"]}, {"sent": "mm mm mm tok tok tok do you know what her ?", "words": ["do", "what", "her", "you"]}, {"sent": "but what do I hafta do ?", "words": ["what", "do", "but"]}, {"sent": "what are those ?", "words": ["what", "are", "those"]}, {"sent": "do you know what that is ?", "words": ["do", "what", "is", "you", "that"]}, {"sent": "what if we uh .", "words": ["what", "if", "we"]}, {"sent": "what are those ?", "words": ["what", "are", "those"]}, {"sent": "what else is xxx besides the water ?", "words": ["the", "what", "water", "is"]}, {"sent": "did I tell you what we hafta do first .", "words": ["what", "do", "we", "first", "you", "did"]}, {"sent": "what did you see ?", "words": ["see", "what", "did", "you"]}, {"sent": "so what other presents did you get for your birthday by the way ?", "words": ["the", "what", "for", "your", "so", "you", "other", "get", "by", "did"]}, {"sent": "what what kind of a noise ?", "words": ["of", "what", "a"]}, {"sent": "what do you brush what do you put on your teeth to brush them ?", "words": ["what", "do", "put", "your", "brush", "to", "them", "you", "on"]}, {"sent": "what do you put in it ?", "words": ["what", "do", "put", "you", "in", "it"]}, {"sent": "you dont have what ?", "words": ["have", "dont", "what", "you"]}, {"sent": "what are you doing to me ?", "words": ["what", "to", "you", "are", "me"]}, {"sent": "what else an ostrich .", "words": ["what", "an"]}, {"sent": "oh well what kind of broom what do you want the broom to ?", "words": ["the", "what", "do", "of", "to", "you", "broom"]}, {"sent": "what do you want water for ?", "words": ["what", "do", "for", "you", "water"]}, {"sent": "what am I xxx ?", "words": ["what", "am"]}, {"sent": "because thats what we do .", "words": ["what", "do", "because", "we"]}, {"sent": "what can you see ?", "words": ["can", "what", "see", "you"]}, {"sent": "what about this ones gonna eat this ones gonna have two dinners because its got two mouths .", "words": ["what", "eat", "have", "this", "about", "because"]}, {"sent": "what should we do ?", "words": ["what", "do", "we"]}, {"sent": "what happened ?", "words": ["what"]}, {"sent": "what did she um what did she like playing with here ?", "words": ["what", "with", "like", "here", "did", "she"]}, {"sent": "what do you wanna see ?", "words": ["what", "do", "you", "wanna", "see"]}, {"sent": "this is vegetable soup what are those ?", "words": ["what", "those", "is", "this", "are", "soup"]}, {"sent": "and what are you going to play ?", "words": ["what", "to", "play", "you", "and", "are"]}, {"sent": "and what was the other little boy called .", "words": ["the", "what", "was", "other", "and", "little"]}, {"sent": "what were they doing ?", "words": ["what", "were", "they"]}, {"sent": "what are you gonna do when its full ?", "words": ["what", "do", "full", "you", "are", "when"]}, {"sent": "what ?", "words": ["what"]}, {"sent": "what shall we what shall we buy Daddy ?", "words": ["what", "buy", "we"]}, {"sent": "what about the money ?", "words": ["the", "what", "about", "money"]}, {"sent": "what a klutzy Tarzan .", "words": ["what", "a"]}, {"sent": "xxx what are you crazy ?", "words": ["what", "are", "you"]}, {"sent": "what else do we have here ?", "words": ["what", "do", "we", "have", "here"]}, {"sent": "what what is that boy mad at ?", "words": ["what", "is", "mad", "at", "that"]}, {"sent": "but what Im thinking of is about three days ago .", "words": ["what", "is", "of", "about", "but"]}, {"sent": "what ?", "words": ["what"]}, {"sent": "hey what is that ?", "words": ["what", "that", "is"]}, {"sent": "tell you what .", "words": ["what", "you"]}, {"sent": "but what did the fairy godmother tell her ?", "words": ["the", "what", "did", "her", "but"]}, {"sent": "um what kind of cookie .", "words": ["of", "what", "cookie"]}, {"sent": "put what ?", "words": ["what", "put"]}, {"sent": "what a surprise .", "words": ["what", "a"]}, {"sent": "what are you doing ?", "words": ["what", "are", "you"]}, {"sent": "what ?", "words": ["what"]}, {"sent": "Melissa what are you doing ?", "words": ["what", "are", "you"]}, {"sent": "what is it ?", "words": ["what", "it", "is"]}, {"sent": "what does yellow mean ?", "words": ["what", "does", "yellow"]}, {"sent": "what ?", "words": ["what"]}, {"sent": "we can see what the pictures are of .", "words": ["can", "what", "the", "we", "of", "are", "see"]}, {"sent": "oh this is what whats this ?", "words": ["this", "what", "is"]}, {"sent": "what well do is Ill Ill finish my breakfast .", "words": ["what", "do", "my", "is", "finish"]}, {"sent": "well you tell me what youre buying .", "words": ["me", "what", "you"]}, {"sent": "what kind of face is that ?", "words": ["what", "face", "is", "of", "that"]}, {"sent": "what kind of boats do you see in Maine ?", "words": ["what", "do", "of", "you", "in", "see"]}, {"sent": "what do you put rocks in it for ?", "words": ["what", "do", "put", "for", "you", "in", "it"]}, {"sent": "what do you think these are ?", "words": ["what", "do", "think", "these", "you", "are"]}, {"sent": "what do you have ?", "words": ["have", "what", "do", "you"]}, {"sent": "what are they doing ?", "words": ["what", "are", "they"]}, {"sent": "mommys what ?", "words": ["what"]}, {"sent": "what are those he is putting on ?", "words": ["what", "those", "is", "are", "he", "on"]}, {"sent": "what is this sposta be ?", "words": ["what", "be", "this", "is"]}, {"sent": "tell mama what this is ?", "words": ["what", "this", "is"]}, {"sent": "what about this one ?", "words": ["what", "about", "this"]}, {"sent": "what is that ?", "words": ["what", "that", "is"]}, {"sent": "what ?", "words": ["what"]}, {"sent": "I cant tell what youre saying .", "words": ["what"]}, {"sent": "what does he do ?", "words": ["what", "do", "does", "he"]}, {"sent": "what color do you have now ?", "words": ["have", "what", "do", "you"]}, {"sent": "what is that ?", "words": ["what", "that", "is"]}, {"sent": "there what did I find ?", "words": ["there", "what", "did", "find"]}, {"sent": "there now hes got to get some more bricks hasnt he so what do I hafta do with the crane ?", "words": ["the", "what", "do", "to", "so", "with", "get", "there", "more", "he", "some"]}, {"sent": "what colors that ?", "words": ["what", "that"]}, {"sent": "what is this ?", "words": ["what", "this", "is"]}, {"sent": "guess what weve gotta get .", "words": ["get", "what"]}, {"sent": "what do you what what do you think .", "words": ["what", "do", "think", "you"]}, {"sent": "and what do frogs do ?", "words": ["what", "do", "and"]}, {"sent": "you know what that man was called ?", "words": ["was", "what", "that", "you"]}, {"sent": "what color shirt is this ?", "words": ["what", "this", "is", "shirt"]}, {"sent": "and did you lick your icecream ?", "words": ["your", "you", "and", "did", "lick"]}, {"sent": "and why did he suggest going to Stockport at lunchtime ?", "words": ["to", "and", "he", "did", "at", "why"]}, {"sent": "where did you go yesterday ?", "words": ["where", "go", "did", "you"]}, {"sent": "he did .", "words": ["he", "did"]}, {"sent": "especially when you did that ribbit thing I think she liked .", "words": ["think", "when", "you", "did", "that", "she"]}, {"sent": "what did you do ?", "words": ["what", "do", "did", "you"]}, {"sent": "where did it fly off to ?", "words": ["where", "off", "to", "it", "did"]}, {"sent": "did you lot sing Winniethepooh ?", "words": ["sing", "did", "you"]}, {"sent": "what did we just have for dinner ?", "words": ["what", "for", "we", "have", "did"]}, {"sent": "did you get that toy yesterday too ?", "words": ["you", "get", "toy", "did", "too", "that"]}, {"sent": "yeah did you .", "words": ["did", "you"]}, {"sent": "did you go to Gwwws party and you had your face painted ?", "words": ["your", "face", "go", "to", "party", "you", "and", "did"]}, {"sent": "you tell me what you did at the park first .", "words": ["the", "what", "first", "you", "park", "did", "at", "me"]}, {"sent": "where did he go ?", "words": ["where", "he", "did", "go"]}, {"sent": "you did .", "words": ["did", "you"]}, {"sent": "what did you get for your birthday ?", "words": ["what", "for", "your", "you", "get", "did"]}, {"sent": "I did .", "words": ["did"]}, {"sent": "you did lick me .", "words": ["me", "lick", "did", "you"]}, {"sent": "how did we do it ?", "words": ["do", "how", "we", "it", "did"]}, {"sent": "did you hit your head or anything ?", "words": ["your", "head", "you", "did", "hit"]}, {"sent": "did you toot .", "words": ["did", "you"]}, {"sent": "did he remember me ?", "words": ["me", "he", "did"]}, {"sent": "did you call Di ?", "words": ["did", "you"]}, {"sent": "did he ?", "words": ["he", "did"]}, {"sent": "and what did you do then ?", "words": ["what", "do", "you", "and", "did", "then"]}, {"sent": "no Mommy did .", "words": ["did"]}, {"sent": "what did you do at Janets ?", "words": ["what", "do", "you", "did", "at"]}, {"sent": "an what did did you say in your letter ?", "words": ["what", "your", "say", "you", "an", "in", "did"]}, {"sent": "yeah but you did it didnt you .", "words": ["it", "did", "but", "you"]}, {"sent": "what did you have for supper at Roman and Sophies house ?", "words": ["what", "for", "you", "have", "and", "house", "did", "at"]}, {"sent": "we already did that one .", "words": ["that", "did", "we"]}, {"sent": "did he like the carrot ?", "words": ["the", "he", "did", "like"]}, {"sent": "when did you get them ?", "words": ["them", "you", "get", "did", "when"]}, {"sent": "you did a nice little pussy cat at school or nursery once .", "words": ["school", "nice", "little", "you", "cat", "did", "at", "a"]}, {"sent": "which dog did we see last week at Grandmas ?", "words": ["we", "last", "did", "at", "see", "which", "dog"]}, {"sent": "oh did it scare you .", "words": ["it", "did", "you"]}, {"sent": "what did you get ?", "words": ["get", "what", "did", "you"]}, {"sent": "Mich Im just thinking because Micah did this same study not the the longitudinal part but the play .", "words": ["the", "play", "this", "same", "did", "but", "not", "because"]}, {"sent": "what did you say Summer ?", "words": ["what", "say", "did", "you"]}, {"sent": "did I take did I turn the curling tongs off ?", "words": ["the", "take", "did", "off"]}, {"sent": "they did let him out eventually .", "words": ["they", "him", "did", "out"]}, {"sent": "did he try any of the books today ?", "words": ["the", "of", "any", "he", "did", "try"]}, {"sent": "did he really ?", "words": ["he", "did"]}, {"sent": "you only did one side .", "words": ["did", "you"]}, {"sent": "what did she call .", "words": ["what", "did", "she"]}, {"sent": "did you like that ?", "words": ["that", "did", "like", "you"]}, {"sent": "thats the birds new house did you know that ?", "words": ["the", "you", "house", "did", "new", "that"]}, {"sent": "she did a nice job didnt she ?", "words": ["did", "nice", "she", "a"]}, {"sent": "it went past in the road did it ?", "words": ["the", "in", "it", "did"]}, {"sent": "yes you did spill the coffee .", "words": ["the", "you", "coffee", "spill", "did"]}, {"sent": "Naima when we went today did you not know that I was gonna come back ?", "words": ["back", "we", "was", "you", "that", "did", "when", "not"]}, {"sent": "and this time he did not wanna let go of the spoon it seemed like .", "words": ["the", "go", "of", "like", "and", "this", "it", "he", "did", "wanna", "spoon", "not"]}, {"sent": "um we did the zoo and school .", "words": ["the", "zoo", "we", "school", "and", "did"]}, {"sent": "where did your granny and granpa take you ?", "words": ["where", "your", "and", "you", "did", "take"]}, {"sent": "well they did say that you were going to Tescos .", "words": ["to", "say", "you", "they", "did", "that", "were"]}, {"sent": "yes you did make a food mess and Mommy and Daddy are gonna hafta clean it up and youre gonna hafta help us .", "words": ["clean", "up", "help", "you", "and", "us", "food", "are", "it", "did", "make", "a"]}, {"sent": "why did you push that over ?", "words": ["push", "you", "did", "over", "that", "why"]}, {"sent": "what what what did they do on Teletubbies ?", "words": ["what", "do", "they", "did", "on"]}, {"sent": "did you see this ?", "words": ["see", "did", "this", "you"]}, {"sent": "did it landed on your face .", "words": ["your", "face", "it", "did", "on"]}, {"sent": "what did you do this morning when you got up ?", "words": ["up", "what", "do", "you", "this", "did", "when"]}, {"sent": "what else did you like ?", "words": ["what", "did", "like", "you"]}, {"sent": "tell me this did your granny go away on her holidays ?", "words": ["your", "go", "this", "away", "did", "on", "me", "her"]}, {"sent": "when did you get this ?", "words": ["you", "get", "this", "did", "when"]}, {"sent": "did we go to New York last week ?", "words": ["go", "we", "to", "last", "did"]}, {"sent": "and did Daddy do that one ?", "words": ["do", "that", "did", "and"]}, {"sent": "oh what did Daddy give you ?", "words": ["what", "give", "did", "you"]}, {"sent": "how come that did that ?", "words": ["how", "that", "did"]}, {"sent": "yeah I dont think Daddy did a good enough job here Joseph .", "words": ["think", "here", "dont", "did", "good", "a"]}, {"sent": "what did Ellie and you did ?", "words": ["what", "did", "you", "and"]}, {"sent": "why did you ?", "words": ["did", "why", "you"]}, {"sent": "oh did we see some of that when we went shopping ?", "words": ["when", "we", "of", "did", "see", "that", "some"]}, {"sent": "yeah oh what did you see a new picture oh another new picture .", "words": ["what", "picture", "you", "another", "did", "see", "new", "a"]}, {"sent": "you did it .", "words": ["it", "did", "you"]}, {"sent": "did the other have salt in it ?", "words": ["the", "other", "have", "salt", "in", "it", "did"]}, {"sent": "did you hurt yourself ?", "words": ["hurt", "yourself", "did", "you"]}, {"sent": "what presents did you get ?", "words": ["get", "what", "did", "you"]}, {"sent": "did you take did you take yourself back in ?", "words": ["yourself", "back", "you", "in", "did", "take"]}, {"sent": "did you tell me you went to the circus two times ?", "words": ["the", "to", "you", "did", "me"]}, {"sent": "but what did she give you ?", "words": ["what", "you", "did", "give", "she", "but"]}, {"sent": "jumped over the gate sure did .", "words": ["the", "over", "did"]}, {"sent": "did a Mommy take a splinter outof your hand a few days ago ?", "words": ["your", "hand", "did", "take", "a"]}, {"sent": "did ya have some ice cream ?", "words": ["ice cream", "did ya", "have", "did", "ice", "some"]}, {"sent": "I did .", "words": ["did"]}, {"sent": "did you enjoy that ?", "words": ["that", "did", "you"]}, {"sent": "we got so excited did we forget to say thank you ?", "words": ["we", "to", "so", "say", "you", "did"]}, {"sent": "did you go in the playground today ?", "words": ["the", "go", "you", "in", "did"]}, {"sent": "did it work ?", "words": ["it", "work", "did"]}, {"sent": "you had some chips but who did we go to Macdonalds with ?", "words": ["we", "go", "to", "with", "you", "did", "who", "some", "but"]}, {"sent": "did you ?", "words": ["did", "you"]}, {"sent": "and what did you say ?", "words": ["what", "say", "you", "and", "did"]}, {"sent": "what did you do when you went home ?", "words": ["what", "do", "you", "did", "when", "home"]}, {"sent": "did you have alotof fun ?", "words": ["have", "did", "you"]}, {"sent": "did you eat supper at Spencers house ?", "words": ["you", "eat", "house", "did", "at"]}, {"sent": "yeah where did you put the train ?", "words": ["where", "the", "put", "train", "you", "did"]}, {"sent": "he didnt wanna did he ?", "words": ["he", "did", "wanna"]}, {"sent": "did she say .", "words": ["say", "did", "she"]}, {"sent": "you did that in school today .", "words": ["school", "you", "in", "did", "that"]}, {"sent": "what did you find ?", "words": ["what", "did", "find", "you"]}, {"sent": "did you hear the pop then ?", "words": ["the", "pop", "hear", "you", "did", "then"]}, {"sent": "yeah but you didnt take it you didnt get him off of here see ?", "words": ["off", "of", "you", "get", "here", "him", "it", "see", "take", "but"]}, {"sent": "you make the awfulest things when somebody tells you to smile .", "words": ["the", "when", "to", "you", "make", "smile"]}, {"sent": "what are you reading ?", "words": ["what", "are", "you"]}, {"sent": "how many people well you count them and see ?", "words": ["how", "them", "and", "you", "see"]}, {"sent": "and you go and do the tea .", "words": ["the", "do", "go", "you", "and"]}, {"sent": "you guys are taking these pairs too fast .", "words": ["these", "fast", "you", "are", "too"]}, {"sent": "and you drop a penny in a certain place .", "words": ["you", "and", "penny", "in", "drop", "a"]}, {"sent": "there are some words in this story youre gonna hafta know before you even read the story so lets give you the vocabulary you need okay ?", "words": ["the", "give", "need", "so", "you", "there", "in", "are", "this", "read", "some", "story"]}, {"sent": "you know hes doing there .", "words": ["there", "you"]}, {"sent": "is that you ?", "words": ["that", "you", "is"]}, {"sent": "what did you do when you went home ?", "words": ["what", "do", "you", "did", "when", "home"]}, {"sent": "because Ill end up putting you in the boot of the car .", "words": ["up", "the", "of", "car", "you", "in", "because"]}, {"sent": "you standing the crayons upright ?", "words": ["the", "you"]}, {"sent": "you wanna sing that ?", "words": ["sing", "that", "wanna", "you"]}, {"sent": "do you know what that one is ?", "words": ["do", "what", "is", "you", "that"]}, {"sent": "there you are .", "words": ["there", "are", "you"]}, {"sent": "what do you think ?", "words": ["what", "do", "think", "you"]}, {"sent": "why you hitting ?", "words": ["why", "you"]}, {"sent": "you ready ?", "words": ["you"]}, {"sent": "what did you like best ?", "words": ["what", "did", "like", "you"]}, {"sent": "yeah one of them that you said .", "words": ["of", "that", "them", "you"]}, {"sent": "thank you .", "words": ["you"]}, {"sent": "listen mommy is gonna make you a juice .", "words": ["listen", "is", "you", "juice", "make", "a"]}, {"sent": "do you see Katie ?", "words": ["see", "do", "you"]}, {"sent": "would you rather be a viking than a cowboy ?", "words": ["be", "a", "you", "would"]}, {"sent": "I I dont have anymore food left for you .", "words": ["for", "you", "have", "dont", "food"]}, {"sent": "this is how you drink this way .", "words": ["how", "drink", "is", "you", "this"]}, {"sent": "how did you eat it ?", "words": ["how", "you", "eat", "it", "did"]}, {"sent": "who do you think those are coming ?", "words": ["do", "think", "those", "you", "are", "who"]}, {"sent": "what color you tell Mama .", "words": ["what", "you"]}, {"sent": "what are you getting ?", "words": ["what", "are", "you"]}, {"sent": "you gonna try ?", "words": ["try", "you"]}, {"sent": "do you want some paper ?", "words": ["do", "paper", "some", "you"]}, {"sent": "be like you know what .", "words": ["be", "what", "like", "you"]}, {"sent": "what do you have ?", "words": ["have", "what", "do", "you"]}, {"sent": "would you be able to get it out ?", "words": ["to", "be", "you", "get", "it", "out", "would"]}, {"sent": "do you wanna take the man outof it ?", "words": ["the", "do", "you", "it", "wanna", "take"]}, {"sent": "you wanna ride Cee again ?", "words": ["ride", "wanna", "you"]}, {"sent": "you were going on about your post lorry ?", "words": ["your", "you", "about", "on", "were"]}, {"sent": "yeh yyy to you too .", "words": ["to", "too", "you"]}, {"sent": "there you are .", "words": ["there", "are", "you"]}, {"sent": "do you know what the noise is ?", "words": ["the", "do", "what", "is", "you"]}, {"sent": "which piece do you wanna start with ?", "words": ["do", "with", "you", "wanna", "which"]}, {"sent": "do you not remember there was a big picture of TinkyWinky ?", "words": ["do", "of", "was", "picture", "you", "there", "big", "not", "a"]}, {"sent": "would you put it in the sink ?", "words": ["the", "put", "you", "in", "it", "sink", "would"]}, {"sent": "what do you think ?", "words": ["what", "do", "think", "you"]}, {"sent": "oh you can use it for all sorts .", "words": ["can", "all", "for", "you", "it"]}, {"sent": "whos taking you to nursery today Nathan ?", "words": ["to", "you"]}, {"sent": "you need some more ?", "words": ["some", "more", "need", "you"]}, {"sent": "he threw the egg at you and washed it ?", "words": ["the", "egg", "you", "and", "it", "he", "at"]}, {"sent": "you wanna count ?", "words": ["wanna", "you"]}, {"sent": "do you think theyll bring you some tomorrow ?", "words": ["do", "think", "bring", "you", "some"]}, {"sent": "och you are you are .", "words": ["are", "you"]}, {"sent": "is there any numbers that you see that you know ?", "words": ["is", "any", "you", "there", "see", "that"]}, {"sent": "are you giving them some more juice ?", "words": ["them", "you", "more", "are", "some", "juice"]}, {"sent": "ahhah what are you going to draw now ?", "words": ["what", "to", "draw", "you", "are"]}, {"sent": "you hafta roll the dice and when you roll this you get one of these parts to go on your bug .", "words": ["the", "these", "bug", "your", "go", "of", "to", "and", "you", "this", "get", "on", "when"]}, {"sent": "thank you very much .", "words": ["much", "you"]}, {"sent": "you want me to make a banana ?", "words": ["to", "you", "banana", "me", "make", "a"]}, {"sent": "do you eat pie with hamburgers ?", "words": ["eat", "do", "with", "you"]}, {"sent": "it doesnt matter what you have as long as you enjoy them .", "words": ["what", "them", "long", "you", "have", "it"]}, {"sent": "can you look ?", "words": ["can", "look", "you"]}, {"sent": "you like play in the dirt too ?", "words": ["the", "play", "like", "you", "in", "too"]}, {"sent": "you will ?", "words": ["will", "you"]}, {"sent": "well I guess you need the wooden sides on first I think .", "words": ["the", "think", "need", "first", "you", "on"]}, {"sent": "whatre you going to do ?", "words": ["to", "do", "you"]}, {"sent": "are you feeling poorly as well ?", "words": ["are", "you"]}, {"sent": "can you sing that for me Thomas ?", "words": ["can", "for", "you", "sing", "me", "that"]}, {"sent": "heres you wanna put these in the bag for King Soopers .", "words": ["the", "these", "put", "for", "you", "in", "wanna"]}, {"sent": "dont you know ?", "words": ["dont", "you"]}, {"sent": "you want something else to eat or to drink ?", "words": ["eat", "to", "drink", "you"]}, {"sent": "can you hold it ?", "words": ["can", "it", "hold", "you"]}, {"sent": "when you when its hot you hafta blow it right ?", "words": ["you", "hot", "it", "when", "blow"]}, {"sent": "I didnt get all of that but Im sure it made perfect sense to you .", "words": ["all", "of", "to", "you", "get", "it", "that", "but"]}, {"sent": "Ill give it to you Perri .", "words": ["to", "give", "it", "you"]}, {"sent": "you hafta look at the colors .", "words": ["the", "look", "at", "you"]}, {"sent": "here you go .", "words": ["here", "go", "you"]}, {"sent": "did you notice that ?", "words": ["that", "did", "you"]}, {"sent": "then you can carry your portable carwash with you wherever you go .", "words": ["can", "your", "go", "with", "you", "carry", "then"]}, {"sent": "what did you do ?", "words": ["what", "do", "did", "you"]}, {"sent": "and you werent very happy were you ?", "words": ["happy", "were", "you", "and"]}, {"sent": "can you open it ?", "words": ["can", "it", "open", "you"]}, {"sent": "can you read that ?", "words": ["can", "that", "read", "you"]}, {"sent": "we hadta chuck it out because you didnt play with it under your bed so we hadta chuck it out .", "words": ["your", "we", "so", "play", "with", "bed", "you", "under", "it", "out", "because"]}, {"sent": "you were really .", "words": ["were", "you"]}, {"sent": "have you still got some more cheese left ?", "words": ["you", "have", "more", "some", "cheese"]}, {"sent": "do you wanna .", "words": ["do", "wanna", "you"]}, {"sent": "do you like going on the train ?", "words": ["the", "do", "train", "like", "you", "on"]}, {"sent": "if you wanted to take one out how do you do it ?", "words": ["do", "how", "if", "to", "you", "it", "out", "take"]}, {"sent": "you hafta stay still and you hafta smile and you hafta ask him to smile .", "words": ["to", "and", "you", "him", "stay", "smile"]}, {"sent": "are you gonna read ?", "words": ["are", "read", "you"]}, {"sent": "well how come youre chewing them and you willnt chew other things .", "words": ["how", "them", "other", "you", "and"]}, {"sent": "peekaboo I see you .", "words": ["see", "you"]}, {"sent": "okay you can pretend then .", "words": ["can", "pretend", "then", "you"]}, {"sent": "what day do you go to nursery ?", "words": ["what", "do", "go", "to", "you"]}, {"sent": "are you gonna look for him ?", "words": ["for", "you", "him", "are", "look"]}, {"sent": "when I need some help Ill shout for you .", "words": ["help", "for", "need", "you", "some", "when"]}, {"sent": "what pictures do you want ?", "words": ["what", "do", "you"]}, {"sent": "then can you see how his hands look .", "words": ["can", "how", "you", "look", "his", "see", "then"]}, {"sent": "doesnt scare you huh ?", "words": ["you"]}, {"sent": "lets have a look at her a minute .", "words": ["have", "at", "look", "her", "a"]}, {"sent": "what do you think we should have for breakfast today ?", "words": ["what", "do", "think", "for", "we", "you", "have"]}, {"sent": "so they could all have .", "words": ["all", "so", "have", "they", "could"]}, {"sent": "you can have some orange if you like .", "words": ["can", "if", "like", "you", "have", "orange", "some"]}, {"sent": "I have no idea if theyll have xxx for today it might .", "words": ["have", "it", "for", "if"]}, {"sent": "have you found the little bear in this picture ?", "words": ["the", "picture", "bear", "have", "little", "you", "in", "this"]}, {"sent": "I think Keisha you have your your left foot in .", "words": ["think", "your", "you", "have", "foot", "in"]}, {"sent": "I have I have .", "words": ["have"]}, {"sent": "oh theres another car oh I see let me see that one I see this one has a seat and this one doesnt have one .", "words": ["see", "car", "and", "another", "this", "have", "me", "that", "a"]}, {"sent": "Lara seems to have got her jelly shoes on .", "words": ["jelly", "to", "have", "on", "her"]}, {"sent": "I like all of these so what have we been doing since we saw Erin yesterday ?", "words": ["all", "these", "what", "we", "of", "so", "like", "have"]}, {"sent": "but you havent got any shoes on yet have you ?", "words": ["any", "you", "have", "on", "but"]}, {"sent": "how do you have a worm bath so that doctor Weist will know ?", "words": ["do", "how", "so", "you", "have", "will", "that", "a"]}, {"sent": "she doesnt have a nice dress to wear .", "words": ["to", "nice", "have", "dress", "she", "a"]}, {"sent": "have a look .", "words": ["have", "look", "a"]}, {"sent": "what have you got to say ?", "words": ["what", "to", "say", "you", "have"]}, {"sent": "and do you regularly have um soup xxx ?", "words": ["do", "you", "have", "and", "soup"]}, {"sent": "and they have faces painted apple .", "words": ["have", "they", "apple", "and"]}, {"sent": "and she can drink wine so I think well have some wine .", "words": ["can", "think", "drink", "so", "and", "have", "some", "she"]}, {"sent": "Imean I have .", "words": ["have"]}, {"sent": "can he have some pretzels ?", "words": ["can", "some", "have", "he"]}, {"sent": "we neednt have got you a bed .", "words": ["we", "bed", "you", "have", "a"]}, {"sent": "you couldve let her have the rabbit .", "words": ["have", "the", "her", "you"]}, {"sent": "let me have a look .", "words": ["have", "me", "look", "a"]}, {"sent": "what did you have .", "words": ["have", "what", "did", "you"]}, {"sent": "oh you havent got many out Aran have you ?", "words": ["have", "out", "you"]}, {"sent": "I will still have me going all day .", "words": ["will", "me", "have", "all"]}, {"sent": "they have that little thing there ?", "words": ["little", "have", "they", "there", "that"]}, {"sent": "have they allgone ?", "words": ["have", "they"]}, {"sent": "Alex come and have some lunch .", "words": ["have", "some", "and"]}, {"sent": "okay you sure have been working on it for a long time .", "words": ["for", "long", "you", "have", "it", "on", "a"]}, {"sent": "we can play with all toys that we dont have at home .", "words": ["can", "all", "we", "play", "with", "have", "dont", "at", "that", "home"]}, {"sent": "have they been caught eating their tea ?", "words": ["have", "they", "their"]}, {"sent": "Ive got to have the corn ?", "words": ["have", "to", "corn", "the"]}, {"sent": "well have a look .", "words": ["have", "look", "a"]}, {"sent": "go upstairs then and have a weewee .", "words": ["go", "and", "have", "then", "a"]}, {"sent": "lets have a look .", "words": ["have", "look", "a"]}, {"sent": "have you asked Polly how her leg is ?", "words": ["how", "is", "you", "have", "leg", "her"]}, {"sent": "because I didnt have the receiver on .", "words": ["have", "the", "on", "because"]}, {"sent": "switch it off because were going to have some milk now .", "words": ["off", "to", "have", "it", "milk", "some", "were", "because"]}, {"sent": "you do have look at all of those colors oh my gosh .", "words": ["do", "all", "my", "those", "of", "you", "have", "at", "look"]}, {"sent": "what did you have ?", "words": ["have", "what", "did", "you"]}, {"sent": "we have alot of these w thats why I was kinda surprised she went to the babies today first maybe she didnt notice the animals right away but .", "words": ["the", "these", "we", "of", "was", "to", "first", "have", "away", "but", "she", "why"]}, {"sent": "have a look .", "words": ["have", "look", "a"]}, {"sent": "would you have you ever seen SesameStreet ?", "words": ["have", "you", "would"]}, {"sent": "oh is Mummy going to have any dinner then ?", "words": ["is", "to", "any", "have", "then"]}, {"sent": "have you heard Miss X is in hospital ?", "words": ["have", "in", "is", "you"]}, {"sent": "do you have a duckie ?", "words": ["have", "do", "a", "you"]}, {"sent": "would you like to bring the fruit over here in the digger and well have a look at it .", "words": ["the", "look", "to", "bring", "like", "you", "here", "and", "in", "have", "at", "it", "would", "over", "a"]}, {"sent": "lets have a look and see .", "words": ["and", "have", "look", "see", "a"]}, {"sent": "what would have happened if that butterfly would have sat on your nose ?", "words": ["butterfly", "what", "your", "if", "have", "would", "on", "that", "nose"]}, {"sent": "have they ?", "words": ["have", "they"]}, {"sent": "lets have a look at these fish .", "words": ["these", "fish", "have", "at", "look", "a"]}, {"sent": "yes they have blonde hair too uh huh .", "words": ["have", "they", "hair", "too"]}, {"sent": "how would you like to have me as a friend ?", "words": ["how", "to", "like", "you", "have", "would", "me", "a"]}, {"sent": "they no have they can have a ride .", "words": ["can", "have", "they", "ride", "a"]}, {"sent": "oil spills have many causes .", "words": ["have"]}, {"sent": "oh well you should have told him to .", "words": ["have", "to", "him", "you"]}, {"sent": "have you finished your milk ?", "words": ["have", "milk", "your", "you"]}, {"sent": "pussy cats have green eyes .", "words": ["have", "green"]}, {"sent": "have you been riding your bicycle lately ?", "words": ["have", "your", "bicycle", "you"]}, {"sent": "have you got a purple one ?", "words": ["have", "a", "you"]}, {"sent": "a tree and you have the tree .", "words": ["the", "tree", "you", "have", "and", "a"]}, {"sent": "where have I been ?", "words": ["where", "have"]}, {"sent": "another bed downstairs aswell you should have a bedroom downstairs as well .", "words": ["bedroom", "bed", "you", "another", "have", "a"]}, {"sent": "have you got any tablets ?", "words": ["have", "any", "you"]}, {"sent": "xxx cant have broccoli ?", "words": ["have"]}, {"sent": "have you had a busy day darling ?", "words": ["have", "a", "you"]}, {"sent": "xxx we have that at home .", "words": ["we", "have", "at", "that", "home"]}, {"sent": "did you have some toys out ?", "words": ["you", "have", "did", "out", "some"]}, {"sent": "okay can I have the pen ?", "words": ["can", "the", "have", "pen"]}, {"sent": "and have another biscuit .", "words": ["have", "another", "and"]}, {"sent": "lets have a look .", "words": ["have", "look", "a"]}, {"sent": "what else do we have ?", "words": ["have", "what", "do", "we"]}, {"sent": "I have a look round and I come to your till .", "words": ["your", "to", "and", "have", "look", "a"]}, {"sent": "have a good day .", "words": ["have", "good", "a"]}, {"sent": "well you cant have them .", "words": ["have", "them", "you"]}, {"sent": "giants have long everything .", "words": ["have", "long"]}, {"sent": "I think they didnt have enough room to put it the right way .", "words": ["the", "think", "put", "to", "have", "they", "room", "it"]}, {"sent": "you have a lot .", "words": ["have", "a lot", "a", "you"]}, {"sent": "both those ladys have been shopping .", "words": ["have", "those"]}, {"sent": "have you .", "words": ["have", "you"]}, {"sent": "you have some like this huh ?", "words": ["like", "you", "have", "this", "some"]}, {"sent": "we dont have as big a variety and ours are a little bit bigger .", "words": ["we", "and", "have", "dont", "little", "are", "big", "a"]}, {"sent": "oh lets have a look at her then .", "words": ["have", "at", "look", "then", "her", "a"]}, {"sent": "are we going to have the bones of the monkey ?", "words": ["the", "we", "of", "to", "monkey", "have", "are"]}, {"sent": "uh mommy do you have a seven .", "words": ["have", "do", "a", "you"]}, {"sent": "motorcycle chicken have you seen Naimas balls ?", "words": ["have", "motorcycle", "you", "chicken"]}, {"sent": "no you cant have no more .", "words": ["have", "more", "you"]}, {"sent": "we useta have one of those .", "words": ["have", "of", "those", "we"]}, {"sent": "well have AuntyMable and Pippin been talking about flowers with a nice smell ?", "words": ["nice", "with", "and", "have", "about", "a"]}, {"sent": "they have little bunks .", "words": ["have", "they", "little"]}, {"sent": "can I have some ice cream when you all have some ?", "words": ["can", "all", "ice cream", "you", "have", "ice", "some", "when"]}, {"sent": "you have enough room want Mommy to push you over ?", "words": ["push", "to", "you", "have", "room", "over"]}, {"sent": "there should be a mirror there so you can see your face once the doors have sprung open .", "words": ["can", "the", "your", "face", "open", "be", "so", "you", "have", "there", "see", "a"]}, {"sent": "you have that ?", "words": ["have", "that", "you"]}, {"sent": "dont have it going into her .", "words": ["into", "have", "dont", "it", "her"]}, {"sent": "have you noticed that ?", "words": ["have", "that", "you"]}, {"sent": "do you have everything you need asked Grandma .", "words": ["have", "do", "need", "you"]}, {"sent": "where shall we have your oven ?", "words": ["where", "your", "we", "have", "oven"]}, {"sent": "oh it doesnt go straight as your ball at home .", "words": ["your", "go", "ball", "it", "at", "home"]}, {"sent": "yes we do read at home a lot .", "words": ["do", "we", "a lot", "read", "at", "a", "home"]}, {"sent": "oh look at that pony hes walking .", "words": ["pony", "at", "that", "look"]}, {"sent": "are you not looking at me ?", "words": ["you", "are", "at", "me", "not"]}, {"sent": "I mean at school ?", "words": ["school", "at"]}, {"sent": "so Ill put her here very carefully with scotch tape and then we can fix it at work .", "words": ["can", "put", "work", "we", "so", "fix", "with", "tape", "and", "here", "it", "at", "then", "her"]}, {"sent": "oh yes look at the kitties .", "words": ["the", "at", "look"]}, {"sent": "ah look at those toy whoops .", "words": ["toy", "at", "those", "look"]}, {"sent": "if we have a look at this Thursday .", "words": ["if", "we", "have", "this", "at", "look", "a"]}, {"sent": "look at the clouds .", "words": ["the", "at", "look"]}, {"sent": "I worked at the weavers trade .", "words": ["the", "at"]}, {"sent": "look at my little .", "words": ["my", "at", "little", "look"]}, {"sent": "who has her room at the very top floor in our house ?", "words": ["the", "our", "in", "room", "house", "at", "who", "her"]}, {"sent": "whats at Blackpool for the train ?", "words": ["the", "train", "for", "at"]}, {"sent": "did you have fun at Ewwws birthday party ?", "words": ["party", "have", "you", "did", "at"]}, {"sent": "what happened at your birthdayparty Fraser ?", "words": ["what", "your", "at"]}, {"sent": "hm look at this stuff .", "words": ["this", "at", "look"]}, {"sent": "did you see a lion at the zoo ?", "words": ["the", "zoo", "you", "did", "at", "see", "lion", "a"]}, {"sent": "you wanna stop at Jojos circus .", "words": ["at", "stop", "wanna", "you"]}, {"sent": "wow ohmygoodness in colors look at your dinosaur sponges .", "words": ["in", "at", "your", "look"]}, {"sent": "youre doing a very good job of sitting at your place .", "words": ["your", "of", "good", "at", "a"]}, {"sent": "hes looking at you .", "words": ["you", "at"]}, {"sent": "Daddys at work .", "words": ["work", "at"]}, {"sent": "well actually we can have an apple at lunch and you can cut it up .", "words": ["can", "up", "cut", "we", "and", "have", "you", "an", "it", "at", "apple"]}, {"sent": "look at its so many .", "words": ["so", "at", "look"]}, {"sent": "oh think of all the stuff you did at home today .", "words": ["the", "all", "think", "of", "you", "did", "at", "home"]}, {"sent": "but listen sweetheart were rushing weve got plenty of time at the moment .", "words": ["the", "listen", "of", "at", "were", "but"]}, {"sent": "well wheres the daddy at Kalie ?", "words": ["the", "at"]}, {"sent": "willnt eat much of anything at the minute will you ?", "words": ["the", "of", "you", "eat", "at", "will", "much"]}, {"sent": "I dont think you need any cream on it at the moment .", "words": ["the", "think", "need", "any", "you", "dont", "it", "at", "on"]}, {"sent": "well if you put Mummys packet in a bowl on the table at a party everybody at the party would say look at those funny hula hoops .", "words": ["bowl", "the", "put", "look", "those", "if", "table", "say", "would", "you", "party", "in", "at", "on", "a"]}, {"sent": "did he stop at your house and bring you candy canes ?", "words": ["your", "stop", "bring", "and", "you", "he", "did", "house", "at", "candy"]}, {"sent": "oh look at that .", "words": ["at", "that", "look"]}, {"sent": "oh look at that child get in here .", "words": ["look", "get", "here", "in", "at", "that"]}, {"sent": "she is at the dog park what is she doing at the dog park ?", "words": ["the", "what", "is", "she", "park", "at", "dog"]}, {"sent": "we have this one at home or at my parents house actually .", "words": ["my", "we", "have", "this", "house", "at", "home"]}, {"sent": "oh where does one get candy at halloween ?", "words": ["where", "does", "get", "at", "candy"]}, {"sent": "at the school ?", "words": ["the", "school", "at"]}, {"sent": "now look at this .", "words": ["this", "at", "look"]}, {"sent": "oh have you been an attendant at a fire ?", "words": ["you", "have", "an", "at", "a"]}, {"sent": "hey do you hear the washing machine do you wanna go look at the clothes in the washing machine ?", "words": ["the", "do", "go", "hear", "you", "washing machine", "in", "at", "wanna", "look"]}, {"sent": "dont look at me like that .", "words": ["look", "like", "dont", "at", "me", "that"]}, {"sent": "look at theres the last one .", "words": ["the", "last", "at", "look"]}, {"sent": "oh look at that .", "words": ["at", "that", "look"]}, {"sent": "at least the second ones was oranges .", "words": ["the", "was", "at"]}, {"sent": "oh look at that face .", "words": ["at", "that", "face", "look"]}, {"sent": "look at th look at them there toys .", "words": ["there", "at", "them", "look"]}, {"sent": "look at my thumb Daddy .", "words": ["my", "at", "look"]}, {"sent": "what did you have at breakfast in the hotel ?", "words": ["the", "what", "you", "have", "in", "did", "at"]}, {"sent": "can you look at the book with mommy ?", "words": ["can", "the", "with", "you", "at", "look", "book"]}, {"sent": "you usually go to look at the trains dont you ?", "words": ["the", "go", "to", "you", "dont", "at", "look"]}, {"sent": "look at all of the rest of the toys .", "words": ["the", "all", "look", "of", "at"]}, {"sent": "the wind gets lonely at night with no one to play with .", "words": ["the", "to", "play", "with", "at", "wind"]}, {"sent": "youre laughing at the sound .", "words": ["the", "at"]}, {"sent": "I forgot the ponies at home .", "words": ["the", "home", "at"]}, {"sent": "you know tomorrow were gonna go to um see Jwww at university .", "words": ["go", "to", "you", "at", "see", "were"]}, {"sent": "did you have a good time yesterday at Daisy and Toms ?", "words": ["you", "have", "and", "did", "good", "at", "a"]}, {"sent": "no wonder you are tired at night .", "words": ["are", "at", "tired", "you"]}, {"sent": "Jeffs at home hes with Daddy .", "words": ["home", "with", "at"]}, {"sent": "but its a big one at home .", "words": ["a", "at", "big", "but", "home"]}, {"sent": "did you eat anything at Tinyteds ?", "words": ["eat", "at", "did", "you"]}, {"sent": "you had some nice food at the birthday party the other day didnt you ?", "words": ["the", "nice", "you", "party", "other", "food", "at", "some"]}, {"sent": "at the bottom of her hair ?", "words": ["the", "bottom", "of", "at", "hair", "her"]}, {"sent": "oh look at that rain Fraser .", "words": ["at", "that", "rain", "look"]}, {"sent": "dont throw them at anybody .", "words": ["dont", "them", "throw", "at"]}, {"sent": "look at the colors .", "words": ["the", "at", "look"]}, {"sent": "look at those teddies .", "words": ["at", "those", "look"]}, {"sent": "you wanna look at this .", "words": ["you", "this", "at", "wanna", "look"]}, {"sent": "ah look at this dog .", "words": ["this", "at", "dog", "look"]}, {"sent": "at your youre all ready married ?", "words": ["all", "your", "at"]}, {"sent": "look at the turtle .", "words": ["the", "turtle", "at", "look"]}, {"sent": "look at the wee cute one .", "words": ["the", "cute", "at", "look"]}, {"sent": "thats what you would have done if you had been around at the time Im sure .", "words": ["the", "what", "around", "if", "you", "have", "at", "would"]}, {"sent": "how come youll drink it at playgroup but you dont drink it in the day at home .", "words": ["the", "how", "drink", "you", "dont", "in", "it", "at", "but", "home"]}, {"sent": "he saw some clowns at the circus throw pies .", "words": ["the", "throw", "he", "at", "some"]}, {"sent": "her foots stuck at the moment .", "words": ["the", "stuck", "her", "at"]}, {"sent": "do you remember that package that you saw at the stationary store that had Mastersoftheuniverse on it ?", "words": ["the", "do", "store", "you", "it", "at", "on", "that"]}, {"sent": "because we got engaged at Easter and married in July which wasnt all that long .", "words": ["all", "we", "long", "and", "in", "that", "at", "which", "because"]}, {"sent": "theyve got a flat tyre at the front .", "words": ["the", "at", "a"]}, {"sent": "hes at the station look with all the people waiting at the station for the train .", "words": ["the", "all", "for", "train", "with", "at", "look"]}, {"sent": "you wanna look at a book again ?", "words": ["you", "at", "wanna", "look", "book", "a"]}, {"sent": "no theres no bear at the door is there ?", "words": ["the", "is", "bear", "there", "at", "door"]}, {"sent": "when its nap time at school what happens ?", "words": ["school", "what", "when", "at"]}, {"sent": "there was a fire at Pippins house wasnt there ?", "words": ["was", "there", "house", "at", "a"]}, {"sent": "because I think sometimes at this age they havent quite developed the idea of sharing or otherness yet have they and you can often get sort of xxx .", "words": ["the", "can", "think", "of", "and", "have", "this", "they", "you", "get", "at", "because"]}, {"sent": "look at this .", "words": ["this", "at", "look"]}, {"sent": "look at me do you hear me ?", "words": ["do", "look", "you", "hear", "at", "me"]}, {"sent": "look at that .", "words": ["at", "that", "look"]}, {"sent": "I think I should have stayed at home .", "words": ["have", "home", "think", "at"]}, {"sent": "is that why she appeared so quickly at the door ?", "words": ["the", "is", "so", "at", "that", "she", "why", "door"]}, {"sent": "look at those stars they are very shiny .", "words": ["look", "those", "they", "are", "at"]}, {"sent": "what did you have for lunch today at Rods house ?", "words": ["what", "for", "you", "have", "house", "did", "at"]}, {"sent": "oh look at this .", "words": ["this", "at", "look"]}, {"sent": "no not at all .", "words": ["all", "not", "at"]}, {"sent": "we saw killerwhales at Seaworld in Florida as well didnt we ?", "words": ["in", "at", "we"]}, {"sent": "well she seems very susceptible to big girl at the table position .", "words": ["the", "table", "to", "at", "big", "she"]}, {"sent": "look at this one .", "words": ["this", "at", "look"]}, {"sent": "at the .", "words": ["the", "at"]}, {"sent": "youre not looking at these instructions properly .", "words": ["these", "not", "at"]}, {"sent": "its quite strange it being so open at the front isnt it ?", "words": ["the", "open", "so", "it", "at"]}, {"sent": "put her in the chimney .", "words": ["the", "in", "put", "her"]}, {"sent": "gonna throw the ball to Daddy .", "words": ["the", "to", "ball", "throw"]}, {"sent": "do you think we could change the front to the back and the back to the front ?", "words": ["the", "do", "think", "back", "we", "to", "you", "and", "could"]}, {"sent": "Awww Winnie the Pooh .", "words": ["the"]}, {"sent": "heres the baby .", "words": ["the"]}, {"sent": "oh well did you put away the tomato sauce ?", "words": ["the", "put", "sauce", "you", "away", "did"]}, {"sent": "I felt like saying well I dont put a bolt to the back of his head .", "words": ["the", "put", "back", "of", "to", "head", "like", "dont", "his", "a"]}, {"sent": "the carpet square with the the building site on it .", "words": ["the", "on", "it", "with"]}, {"sent": "we took a very a really big ferry we were on the boat for about an hour and half that was quite an adventure .", "words": ["the", "for", "we", "was", "boat", "and", "big", "an", "that", "about", "on", "were", "a"]}, {"sent": "the brakes on ?", "words": ["the", "on"]}, {"sent": "ah PBBears favorite time in the bear in the .", "words": ["the", "in", "bear"]}, {"sent": "xxx the horse .", "words": ["the", "horse"]}, {"sent": "do you know how the wipers go ?", "words": ["the", "do", "how", "go", "you"]}, {"sent": "and theres the z .", "words": ["the", "and"]}, {"sent": "well open the curtains .", "words": ["open", "the"]}, {"sent": "where is the sugarbowl ?", "words": ["where", "the", "is"]}, {"sent": "so if you can make up a story then Ill have a story to go with the pictures .", "words": ["can", "up", "the", "if", "go", "so", "to", "then", "with", "you", "have", "story", "make", "a"]}, {"sent": "um this is funny Thomas just twirl this round at the light switch in the dolphin .", "words": ["the", "is", "light", "this", "in", "at"]}, {"sent": "and the kitty .", "words": ["the", "kitty", "and"]}, {"sent": "leave the light alone .", "words": ["the", "light"]}, {"sent": "look see the tree .", "words": ["the", "see", "tree", "look"]}, {"sent": "wheres the horsie ?", "words": ["the"]}, {"sent": "laughing all the way .", "words": ["the", "all"]}, {"sent": "turn the baby this way .", "words": ["the", "this"]}, {"sent": "the cow jumped over the moon .", "words": ["the", "over", "moon", "cow"]}, {"sent": "the ones which go to sleep .", "words": ["the", "go", "sleep", "to", "which"]}, {"sent": "wheres the clown ?", "words": ["the"]}, {"sent": "like the Disneychannel .", "words": ["the", "like"]}, {"sent": "oh weve got to do the bottom one first .", "words": ["the", "do", "bottom", "to", "first"]}, {"sent": "heres here go the dominoes you ready ?", "words": ["here", "the", "you", "go"]}, {"sent": "you can always put people behind the house .", "words": ["can", "the", "put", "behind", "you", "house"]}, {"sent": "I turned the taperecorder on sothat it could hear what youre saying .", "words": ["the", "what", "hear", "it", "on", "could"]}, {"sent": "oh oh theyre going in the car .", "words": ["the", "car", "in"]}, {"sent": "cat asks who is in the flower pot ?", "words": ["the", "is", "flower", "cat", "in", "who"]}, {"sent": "solution to the problems .", "words": ["the", "to"]}, {"sent": "there is the airplane .", "words": ["the", "there", "airplane", "is"]}, {"sent": "no Henry not on the carpet .", "words": ["on", "the", "not"]}, {"sent": "youre very into bananas at the moment arent you ?", "words": ["the", "into", "you", "at"]}, {"sent": "we saw the Somerfields lorrys on the motorway didnt we ?", "words": ["the", "on", "we"]}, {"sent": "I think you better go in the living room .", "words": ["the", "better", "think", "go", "living room", "you", "in", "room"]}, {"sent": "the holly berries are for the dicky birds .", "words": ["the", "are", "for"]}, {"sent": "dont think thats sposta be in the shopping is it ?", "words": ["the", "think", "is", "be", "dont", "in", "it"]}, {"sent": "put the v back in the bag .", "words": ["the", "in", "put", "back"]}, {"sent": "what does the green light mean ?", "words": ["the", "what", "does", "light", "green"]}, {"sent": "at the babysitters .", "words": ["the", "at"]}, {"sent": "not the workmen the bin men .", "words": ["the", "not"]}, {"sent": "we all went in the bath .", "words": ["the", "all", "in", "we"]}, {"sent": "the little boy and the little girl are sitting in the back with the dog arent they ?", "words": ["the", "back", "with", "and", "little", "they", "in", "are", "dog"]}, {"sent": "asleep on the hay .", "words": ["on", "asleep", "the"]}, {"sent": "which you like the carrot one .", "words": ["the", "which", "like", "you"]}, {"sent": "you cant we play crayons at the table thats where we play the crayons we dont walk around the house with the crayons .", "words": ["the", "where", "around", "table", "we", "play", "with", "you", "dont", "house", "at", "walk"]}, {"sent": "no its right here see theres the other ball lets throw the other ball .", "words": ["the", "throw", "other", "here", "ball", "see"]}, {"sent": "no camels live in the desert .", "words": ["the", "in"]}, {"sent": "as the lorry that collects the lorry on Mondays .", "words": ["the", "on", "that"]}, {"sent": "do you not want me to come round the back of the chair with you ?", "words": ["the", "do", "back", "of", "to", "with", "chair", "you", "me", "not"]}, {"sent": "and at the end of the story what did the prince and the princess and the dragon all do ?", "words": ["the", "what", "all", "do", "of", "and", "did", "at", "story"]}, {"sent": "yeah shoulda done that in the first place .", "words": ["the", "in", "first", "that"]}, {"sent": "we tried to teach him the putting away game .", "words": ["the", "game", "we", "to", "him", "away"]}, {"sent": "was that because she didnt cross the road ?", "words": ["the", "was", "that", "she", "because"]}, {"sent": "you just turn this and I will hold the nut .", "words": ["the", "hold", "and", "you", "this", "will"]}, {"sent": "the truck ?", "words": ["the", "truck"]}, {"sent": "things you bought in the sweetie shop ?", "words": ["the", "in", "you"]}, {"sent": "say thankyou Cathy for the sweeties .", "words": ["the", "for", "say"]}, {"sent": "the other way .", "words": ["the", "other"]}, {"sent": "Ive got to do some work in the house .", "words": ["the", "do", "work", "to", "in", "house", "some"]}, {"sent": "put him on here so the man can see as well .", "words": ["the", "can", "put", "so", "here", "him", "on", "see"]}, {"sent": "and the hairdresser says to me .", "words": ["the", "to", "me", "and"]}, {"sent": "what did the choochoo do ?", "words": ["the", "what", "do", "did"]}, {"sent": "yeah look at the toys .", "words": ["the", "at", "look"]}, {"sent": "does the orange block have a dog ?", "words": ["the", "does", "block", "have", "orange", "dog", "a"]}, {"sent": "and what was the goat doing ?", "words": ["was", "what", "the", "and"]}, {"sent": "Im going to read Ross the story of the Great Train Ride .", "words": ["the", "of", "to", "read", "story"]}, {"sent": "the car .", "words": ["the", "car"]}, {"sent": "on the top .", "words": ["on", "the"]}, {"sent": "we havent done the book about Daddy .", "words": ["the", "book", "about", "we"]}, {"sent": "I wish I could be the loin tamer .", "words": ["the", "be", "could", "wish"]}, {"sent": "and he really wanted the track yousee .", "words": ["the", "he", "and"]}, {"sent": "remember how the bull pawed the ground and looked real mean at us ?", "words": ["the", "how", "and", "us", "at"]}, {"sent": "cause thats the .", "words": ["the"]}, {"sent": "the bus the truck and the tractor ?", "words": ["the", "truck", "and", "tractor", "bus"]}, {"sent": "half of it feel into the sea the kitchens .", "words": ["of", "the", "it", "into"]}, {"sent": "jump over the .", "words": ["the", "jump", "over"]}, {"sent": "whats this in the middle of your stomach ?", "words": ["the", "your", "of", "this", "in"]}, {"sent": "Im happy to have the chocolate .", "words": ["the", "happy", "to", "chocolate", "have"]}, {"sent": "did you stay with Maggie while the painters were painting our house ?", "words": ["the", "with", "you", "our", "house", "did", "stay", "were"]}, {"sent": "the swimming class teacher .", "words": ["the"]}, {"sent": "the big one ?", "words": ["the", "big"]}, {"sent": "you cant see it real well with the yellow what letter is that ?", "words": ["the", "what", "is", "yellow", "with", "you", "it", "see", "that"]}, {"sent": "at the end of the story what did they do ?", "words": ["the", "what", "do", "of", "they", "did", "at", "story"]}, {"sent": "going to get some medicine from the doctors .", "words": ["the", "to", "medicine", "get", "some"]}, {"sent": "why dont you take the animals out and well have a veterinary hospital in the barn ?", "words": ["the", "you", "and", "dont", "have", "in", "out", "take", "why", "a"]}, {"sent": "why did they spread the blanket under the tree ?", "words": ["the", "tree", "they", "under", "did", "blanket", "why"]}, {"sent": "and Baloo comes along and he grabs the tiger by the tail doesnt he ?", "words": ["the", "by", "and", "he", "tiger"]}, {"sent": "Ben ate the burger but left the chips and peas .", "words": ["the", "peas", "but", "and"]}, {"sent": "with the police helicopter theyve got the same black dog .", "words": ["the", "helicopter", "with", "black", "same", "dog"]}, {"sent": "do you hit Mommy in the head with the ball ?", "words": ["the", "do", "head", "with", "you", "ball", "in", "hit"]}, {"sent": "wheres the theres the bear .", "words": ["the", "bear"]}, {"sent": "then he shut up the things in the box with a hook and the cat went away with a sad kind of look .", "words": ["up", "the", "box", "of", "with", "and", "cat", "in", "away", "he", "look", "sad", "then", "a"]}, {"sent": "mind the tea John .", "words": ["the"]}, {"sent": "oh we saw a spider yesterday didnt we in the bath ?", "words": ["the", "in", "a", "we"]}, {"sent": "well what did you guys do at the party ?", "words": ["the", "what", "do", "you", "party", "did", "at"]}, {"sent": "we had another party .", "words": ["another", "party", "we"]}, {"sent": "they going to go to a party ?", "words": ["go", "to", "party", "they", "a"]}, {"sent": "did you wear that at the party ?", "words": ["the", "you", "party", "did", "at", "that"]}, {"sent": "so then theres only three more weeks the week of the twenty ninth and then the week of the sixth of December which is getting ready for the Christmas party .", "words": ["the", "for", "is", "of", "so", "and", "party", "more", "then", "which"]}, {"sent": "its a party is it ?", "words": ["it", "party", "is", "a"]}, {"sent": "and then you wont be able to go to Jwwws party .", "words": ["go", "to", "be", "you", "and", "party", "then"]}, {"sent": "like at Rosss party .", "words": ["party", "like", "at"]}, {"sent": "okay what do we need for party Luna ?", "words": ["what", "do", "for", "need", "we", "party"]}, {"sent": "Mummys going to a party tonight isnt she ?", "words": ["to", "party", "she", "a"]}, {"sent": "but they wont be going to Isabels party on Saturday because theyre going to Spain .", "words": ["to", "be", "because", "party", "they", "on", "but"]}, {"sent": "with Trixies party bags werent they ?", "words": ["they", "with", "party"]}, {"sent": "does it does it look like a party now ?", "words": ["does", "like", "party", "it", "look", "a"]}, {"sent": "uhhuh what kind of party do you want ?", "words": ["what", "do", "of", "you", "party"]}, {"sent": "so you can use those both fer tables if youre gonna have a party and you need one big table and one little table .", "words": ["can", "those", "if", "need", "table", "so", "you", "have", "party", "and", "little", "big", "a"]}, {"sent": "xxx a birthday party ?", "words": ["party", "a"]}, {"sent": "Im having a party .", "words": ["party", "a"]}, {"sent": "when he was having a party he stuffed his cabinet full of little white ping pong balls .", "words": ["full", "of", "was", "party", "little", "he", "his", "white", "when", "a"]}, {"sent": "theres a party next Monday .", "words": ["party", "a"]}, {"sent": "and didnt you have one taken when you had you at your a birthday party ?", "words": ["your", "you", "have", "and", "party", "at", "when", "a"]}, {"sent": "are you gonna have a party ?", "words": ["party", "have", "you", "are", "a"]}, {"sent": "and whos gone to the party ?", "words": ["the", "to", "party", "and"]}, {"sent": "you know what its just gonna be a fun day cause youre going to have a party thats all .", "words": ["what", "all", "to", "be", "you", "have", "party", "a"]}, {"sent": "do you go to the party saturday ?", "words": ["the", "do", "go", "to", "you", "party"]}, {"sent": "and what did you do at the party .", "words": ["the", "what", "do", "you", "and", "party", "did", "at"]}, {"sent": "did you play games at your party ?", "words": ["your", "play", "you", "party", "did", "at"]}, {"sent": "Mummy went to a twelfth night party once and so it might well have had twelve people in the house with drums .", "words": ["the", "to", "so", "with", "party", "and", "have", "in", "it", "house", "a"]}, {"sent": "lets get them out and makebelieve were having a party .", "words": ["them", "and", "get", "party", "out", "were", "a"]}, {"sent": "oh she is getting ready for her party .", "words": ["for", "is", "party", "her", "she"]}, {"sent": "and are you gonna have a party ?", "words": ["you", "have", "and", "party", "are", "a"]}, {"sent": "my party .", "words": ["my", "party"]}, {"sent": "is that your pretty dress for Emilys party ?", "words": ["for", "your", "is", "party", "that", "pretty", "dress"]}, {"sent": "are you going to invite Jwww to your party ?", "words": ["your", "to", "party", "you", "are"]}, {"sent": "and uh Sonia did the party didnt she ?", "words": ["the", "party", "and", "did", "she"]}, {"sent": "whens your party ?", "words": ["your", "party"]}, {"sent": "has he got a party hat ?", "words": ["hat", "party", "he", "a"]}, {"sent": "have you been to Noddys party ?", "words": ["have", "to", "party", "you"]}, {"sent": "oh Id like to come to your party .", "words": ["to", "your", "like", "party"]}, {"sent": "oh another birthday party ?", "words": ["another", "party"]}, {"sent": "a happy party .", "words": ["party", "happy", "a"]}, {"sent": "did you enjoy your party Thomas ?", "words": ["party", "your", "did", "you"]}, {"sent": "well we bought some little bubbles to put in your party bags when it was your birthday didnt we ?", "words": ["put", "your", "we", "to", "was", "little", "party", "in", "it", "some", "when", "bubbles"]}, {"sent": "whos coming to this birthday party ?", "words": ["to", "this", "party"]}, {"sent": "uhhum nice time at the party .", "words": ["the", "party", "nice", "at"]}, {"sent": "so were going to a party .", "words": ["to", "so", "party", "were", "a"]}, {"sent": "did you go to a graduation party ?", "words": ["go", "to", "you", "party", "did", "a"]}, {"sent": "I wanna come to your birthday party .", "words": ["to", "your", "wanna", "party"]}, {"sent": "did you have some nice food to eat at the party ?", "words": ["the", "to", "nice", "you", "have", "eat", "food", "party", "did", "at", "some"]}, {"sent": "shes gonna have a party .", "words": ["have", "party", "a"]}, {"sent": "we had a party .", "words": ["party", "a", "we"]}, {"sent": "and at Joes party you got a uh .", "words": ["party", "you", "and", "at", "a"]}, {"sent": "how many days until your pretend party Abe ?", "words": ["pretend", "how", "your", "party"]}, {"sent": "so would you like to invite them all to the party ?", "words": ["the", "all", "to", "so", "them", "like", "you", "party", "would"]}, {"sent": "whats your magic trick going to be that youre making at the party .", "words": ["the", "your", "to", "be", "party", "at", "that"]}, {"sent": "at that party over the way .", "words": ["the", "party", "that", "at", "over"]}, {"sent": "its not a birthday party but her Mummy and Daddy are having a welcome to the year two thousand party arent they ?", "words": ["the", "to", "party", "and", "they", "are", "her", "not", "but", "a"]}, {"sent": "she said that she doesnt want her Mummy to be at the party just her Daddy .", "words": ["the", "to", "be", "party", "that", "at", "her", "she"]}, {"sent": "and weve no intention of inviting KathyMartins to your party have we ?", "words": ["your", "we", "of", "to", "party", "and", "have"]}, {"sent": "if you have if youre at a party and you go whwoo and it whloohwhalooh like that .", "words": ["if", "go", "like", "you", "have", "party", "and", "it", "at", "that", "a"]}, {"sent": "Im trying to think the last party he got asked to .", "words": ["the", "think", "to", "last", "party", "he"]}, {"sent": "but the other party wasnt very good sure it wasnt ?", "words": ["the", "party", "other", "it", "good", "but"]}, {"sent": "at the party ?", "words": ["the", "party", "at"]}, {"sent": "Pos coming to the party .", "words": ["the", "to", "party"]}, {"sent": "did we play any other games at the party ?", "words": ["the", "we", "play", "any", "other", "party", "did", "at"]}, {"sent": "we were having a lovely party werent we ?", "words": ["party", "were", "a", "we"]}, {"sent": "ah I think they all want to go to the party .", "words": ["the", "all", "think", "go", "to", "party", "they", "want to"]}, {"sent": "and whos got a party hat ?", "words": ["hat", "party", "a", "and"]}, {"sent": "and then we wont be able to go to Jwwws party .", "words": ["we", "go", "to", "be", "and", "party", "then"]}, {"sent": "animals would like to come to the party .", "words": ["the", "to", "like", "party", "would"]}, {"sent": "oh theres the party table .", "words": ["the", "table", "party"]}, {"sent": "theyre doing something that we had at your party there Thomas .", "words": ["your", "we", "party", "there", "at", "that"]}, {"sent": "well shall we have a party .", "words": ["have", "party", "a", "we"]}, {"sent": "was he at your party ?", "words": ["your", "was", "party", "he", "at"]}, {"sent": "you wouldnt say that at Charlottes party the other day would you ?", "words": ["the", "say", "would", "party", "you", "other", "at", "that"]}, {"sent": "no remember Grannys party ?", "words": ["party"]}, {"sent": "he says its Spots birthday this is the way to Spots birthday party lets play hide and seek .", "words": ["the", "hide", "is", "to", "play", "party", "and", "this", "he"]}, {"sent": "okay so thats some of our exciting things coming up and then you had your birthday party and Im trying to think of what else was interesting about your party .", "words": ["up", "what", "think", "your", "of", "so", "to", "was", "our", "and", "you", "party", "about", "some", "then"]}, {"sent": "did you tell Cathy about your party on Sunday ?", "words": ["your", "party", "you", "did", "about", "on"]}, {"sent": "we were going to take him one of the balloons left from your party .", "words": ["the", "your", "we", "of", "to", "party", "him", "take", "were"]}, {"sent": "oh well maybe we could have a pretend tea party .", "words": ["we", "party", "have", "pretend", "could", "a"]}, {"sent": "you still wanna have a party ?", "words": ["party", "have", "you", "wanna", "a"]}, {"sent": "is she having a party ?", "words": ["party", "she", "a", "is"]}, {"sent": "did we go and see Jake at the party ?", "words": ["the", "go", "we", "and", "party", "did", "at", "see"]}, {"sent": "and maybe they could have they could either have a birthday party or they could have a tea party .", "words": ["and", "have", "they", "party", "could", "a"]}, {"sent": "at your birthday party .", "words": ["party", "your", "at"]}, {"sent": "have you ever had a party ?", "words": ["have", "party", "a", "you"]}, {"sent": "were having a big party arent we Thomas ?", "words": ["we", "party", "big", "were", "a"]}, {"sent": "we last saw Matty at his party didnt we ?", "words": ["we", "last", "party", "at", "his"]}, {"sent": "Cyrils gone to the party .", "words": ["the", "to", "party"]}, {"sent": "youre having a party in school Tuesday huh ?", "words": ["school", "in", "party", "a"]}, {"sent": "ooh Ive been to a party .", "words": ["to", "party", "a"]}, {"sent": "well it wasnt Bens party .", "words": ["it", "party"]}, {"sent": "the birthday party youre going to have .", "words": ["the", "to", "have", "party"]}, {"sent": "we hafta have a little party on Jwww on the Sunday before Jwwws birthday .", "words": ["the", "we", "party", "have", "little", "on", "a"]}, {"sent": "its because its Joes birthday party .", "words": ["because", "party"]}, {"sent": "did you like Isabelles party on Saturday ?", "words": ["like", "party", "you", "did", "on"]}, {"sent": "oh is this your party hat here ?", "words": ["your", "is", "party", "here", "this", "hat"]}, {"sent": "she turns them all into gentlemen for the party afterwards ?", "words": ["the", "all", "for", "into", "them", "party", "she"]}, {"sent": "what time does the party start ?", "words": ["the", "what", "does", "party"]}, {"sent": "who else is going to come to the party ?", "words": ["the", "is", "to", "party", "who"]}, {"sent": "see .", "words": ["see"]}, {"sent": "let me see .", "words": ["see", "me"]}, {"sent": "what number can you see xxx .", "words": ["can", "what", "see", "you"]}, {"sent": "hey wanna see of theres any tomatos in the garden ?", "words": ["the", "of", "any", "in", "garden", "wanna", "see"]}, {"sent": "oh you didnt see him with the cars .", "words": ["the", "with", "you", "him", "see"]}, {"sent": "see I thought that was your nose .", "words": ["your", "was", "see", "that", "nose"]}, {"sent": "see ?", "words": ["see"]}, {"sent": "to see if we can find that picture is this it ?", "words": ["can", "if", "we", "is", "to", "picture", "this", "it", "find", "see", "that"]}, {"sent": "oh I see the red ball belongs in the digger does it ?", "words": ["the", "red", "does", "ball", "in", "it", "see"]}, {"sent": "oop see Amandas here we can leave it off .", "words": ["can", "off", "we", "here", "it", "see"]}, {"sent": "see these are both twos .", "words": ["see", "are", "these"]}, {"sent": "you see there are three beds here .", "words": ["you", "here", "there", "are", "see"]}, {"sent": "its very rude to put to put the television on when people come to see you .", "words": ["the", "put", "to", "you", "on", "see", "when"]}, {"sent": "lets see .", "words": ["see"]}, {"sent": "I dont know let me see .", "words": ["see", "dont", "me"]}, {"sent": "it is still on you see look look .", "words": ["is", "you", "it", "look", "on", "see"]}, {"sent": "you cant see the road .", "words": ["the", "see", "you"]}, {"sent": "what else do you see in there ?", "words": ["what", "do", "you", "there", "in", "see"]}, {"sent": "see that is what you do .", "words": ["what", "do", "is", "you", "see", "that"]}, {"sent": "I can see .", "words": ["can", "see"]}, {"sent": "I dont see do you see a spoon ?", "words": ["do", "you", "dont", "spoon", "see", "a"]}, {"sent": "lets see I will try to hold it for you .", "words": ["for", "hold", "to", "you", "it", "will", "see", "try", "try to"]}, {"sent": "I went for a walk tonight and what did I see ?", "words": ["what", "for", "and", "did", "see", "walk", "a"]}, {"sent": "see the baby going nightnights ?", "words": ["the", "see"]}, {"sent": "see you .", "words": ["see", "you"]}, {"sent": "can you see number three ?", "words": ["can", "see", "you"]}, {"sent": "I have two pictures see ?", "words": ["have", "see"]}, {"sent": "see the police cars .", "words": ["the", "see"]}, {"sent": "we hafta try to see if we can fix it somehow .", "words": ["can", "if", "we", "to", "fix", "it", "see", "try", "try to"]}, {"sent": "what can you see ?", "words": ["can", "what", "see", "you"]}, {"sent": "do you see something here ?", "words": ["see", "do", "here", "you"]}, {"sent": "I see you .", "words": ["see", "you"]}, {"sent": "are you ready ta see something different ?", "words": ["see", "are", "you"]}, {"sent": "lets see .", "words": ["see"]}, {"sent": "Abe do you wanna go see a movie ?", "words": ["do", "go", "you", "wanna", "see", "a"]}, {"sent": "can you see any flowers anywhere .", "words": ["can", "see", "any", "you"]}, {"sent": "see how tall .", "words": ["see", "how"]}, {"sent": "do you wanna see this telephone ?", "words": ["do", "you", "this", "wanna", "see", "telephone"]}, {"sent": "see it ?", "words": ["see", "it"]}, {"sent": "see what happened ?", "words": ["see", "what"]}, {"sent": "oh you know what we should do this morning we should go see our friend xxx .", "words": ["what", "do", "we", "go", "you", "our", "this", "see"]}, {"sent": "can you see ?", "words": ["can", "see", "you"]}, {"sent": "hey did you see this thing ?", "words": ["see", "did", "this", "you"]}, {"sent": "oh lets see here .", "words": ["here", "see"]}, {"sent": "maybe we see frank ?", "words": ["see", "we"]}, {"sent": "see his tail ?", "words": ["his", "see"]}, {"sent": "oh I see .", "words": ["see"]}, {"sent": "see ?", "words": ["see"]}, {"sent": "the little dog laughed ha ha ha to see such sport .", "words": ["the", "to", "little", "see", "dog"]}, {"sent": "I cant see a cat in the tree .", "words": ["the", "tree", "cat", "in", "see", "a"]}, {"sent": "go around and look and see if everybody has his right foot in .", "words": ["around", "if", "go", "and", "foot", "in", "look", "his", "see"]}, {"sent": "can you see if you can put it on ?", "words": ["can", "put", "if", "you", "it", "on", "see"]}, {"sent": "to see teapots being made .", "words": ["see", "to"]}, {"sent": "can you see some fruit ?", "words": ["can", "see", "some", "you"]}, {"sent": "well see the ducks in a minute .", "words": ["the", "see", "in", "a"]}, {"sent": "were going to see Grandma and Granddad in York on Sunday but were not going on the train like we did the last time .", "words": ["the", "train", "we", "to", "last", "like", "and", "in", "did", "on", "see", "were", "not", "but"]}, {"sent": "did you see this ?", "words": ["see", "did", "this", "you"]}, {"sent": "see you know what a spatula does ?", "words": ["what", "does", "you", "see", "a"]}, {"sent": "what do you see do you see someone over there ?", "words": ["what", "do", "you", "there", "see", "over"]}, {"sent": "she can share see .", "words": ["can", "see", "share", "she"]}, {"sent": "can you not see elephant ?", "words": ["can", "elephant", "you", "see", "not"]}, {"sent": "oh I see .", "words": ["see"]}, {"sent": "I cant see what that is .", "words": ["see", "what", "that", "is"]}, {"sent": "theres not a thing to see except all of my clothes .", "words": ["all", "my", "of", "to", "see", "not", "a"]}, {"sent": "what did you see ?", "words": ["see", "what", "did", "you"]}, {"sent": "ah lookit its pushing the car away see ?", "words": ["the", "car", "away", "see"]}, {"sent": "see so you can make extra room .", "words": ["can", "so", "you", "room", "see", "make"]}, {"sent": "oh I see .", "words": ["see"]}, {"sent": "you see these yellow things here ?", "words": ["these", "yellow", "you", "here", "see"]}, {"sent": "oh I see .", "words": ["see"]}, {"sent": "she look like shes gonna fall down thats right lets see .", "words": ["fall", "like", "look", "see", "down", "she"]}, {"sent": "lets see I got a couple .", "words": ["see", "a"]}, {"sent": "let me see darling .", "words": ["see", "me"]}, {"sent": "ah we can see him after uhn cant we ?", "words": ["can", "see", "him", "we"]}, {"sent": "let me take it out and see what is it please .", "words": ["what", "is", "see", "and", "it", "out", "me", "take"]}, {"sent": "lets see lets see if we can get that .", "words": ["can", "if", "we", "get", "see", "that"]}, {"sent": "can you see the shampoo ?", "words": ["can", "see", "the", "you"]}, {"sent": "theres some flags up there see them ?", "words": ["up", "see", "them", "there", "some"]}, {"sent": "that means that means that everyone was surprised see ?", "words": ["see", "was", "that"]}, {"sent": "let me have a look and see if I can see .", "words": ["can", "if", "see", "and", "have", "look", "me", "a"]}, {"sent": "*SI1: he be right back see but hes gonna go to basketball camp .", "words": ["back", "go", "be", "to", "he", "see", "but"]}, {"sent": "I see .", "words": ["see"]}, {"sent": "see Linda ?", "words": ["see"]}, {"sent": "aw why dontchu come over here so Mommy can see your squirrel book .", "words": ["can", "your", "so", "here", "book", "see", "over", "squirrel", "why"]}, {"sent": "on the trip for everybody to see when they come to New York mhm .", "words": ["the", "for", "to", "they", "on", "see", "when"]}, {"sent": "thats the way you open it see and then you close it again like that .", "words": ["the", "open", "like", "you", "and", "close", "it", "see", "then", "that"]}, {"sent": "I cant see them .", "words": ["see", "them"]}, {"sent": "you didnt see that one ?", "words": ["see", "that", "you"]}, {"sent": "but she doesnt see him .", "words": ["see", "him", "she", "but"]}, {"sent": "see the zebra ?", "words": ["the", "see", "zebra"]}, {"sent": "I see .", "words": ["see"]}, {"sent": "see ?", "words": ["see"]}, {"sent": "see if shell eat the icecream .", "words": ["eat", "see", "if", "the"]}, {"sent": "lets see what you wanna find .", "words": ["what", "you", "wanna", "find", "see"]}, {"sent": "went to see a friend .", "words": ["see", "to", "a"]}, {"sent": "see heres a grocery store .", "words": ["see", "store", "a"]}, {"sent": "well let us see .", "words": ["see", "us"]}, {"sent": "this stuff probably isnt dry yet lets see .", "words": ["see", "this", "dry"]}, {"sent": "I see you .", "words": ["see", "you"]}, {"sent": "look and see .", "words": ["see", "and", "look"]}, {"sent": "have we finished shopping now ?", "words": ["have", "we"]}, {"sent": "where shall we run away to ?", "words": ["where", "we", "to", "run", "away"]}, {"sent": "and we must leave it to set now .", "words": ["to", "it", "we", "and"]}, {"sent": "there we are .", "words": ["there", "are", "we"]}, {"sent": "shall we do the alpha .", "words": ["the", "do", "we"]}, {"sent": "we gotta get some on the way home .", "words": ["the", "we", "get", "on", "some", "home"]}, {"sent": "cause we figured itd be a boy you know .", "words": ["be", "you", "a", "we"]}, {"sent": "and maybe we should put together back the puzzle wanta do that ?", "words": ["the", "do", "put", "back", "we", "and", "puzzle", "that"]}, {"sent": "what shall we make ?", "words": ["what", "make", "we"]}, {"sent": "there we go .", "words": ["go", "there", "we"]}, {"sent": "maybe we will look for it later then .", "words": ["for", "we", "it", "look", "will", "then"]}, {"sent": "we can buy a lot of things that we can buy from the chemist when we go to the supermarket dont we ?", "words": ["can", "the", "when", "we", "of", "go", "to", "a lot", "dont", "that", "buy", "a"]}, {"sent": "because er we fixed that book .", "words": ["book", "that", "because", "we"]}, {"sent": "where have we been where people punch holes apart from the theater ?", "words": ["where", "the", "have", "we"]}, {"sent": "we had pizza for dinner .", "words": ["for", "pizza", "we"]}, {"sent": "and we played outside .", "words": ["outside", "we", "and"]}, {"sent": "maybe red is really my favorite but we havent got any red have we because I think the reds downstairs .", "words": ["red", "the", "my", "think", "we", "is", "any", "because", "have", "but"]}, {"sent": "what should we play with William ?", "words": ["what", "play", "with", "we"]}, {"sent": "shall we ?", "words": ["we"]}, {"sent": "I think we got everybody .", "words": ["think", "we"]}, {"sent": "shall we go downstairs then ?", "words": ["go", "then", "we"]}, {"sent": "shall we all have a whinge ?", "words": ["have", "all", "a", "we"]}, {"sent": "Im trying to see what else we have .", "words": ["what", "we", "to", "have", "see"]}, {"sent": "and we we count each thing one time .", "words": ["we", "each", "and"]}, {"sent": "we got you have the same cup as I do .", "words": ["the", "cup", "do", "we", "you", "have", "same"]}, {"sent": "so on she got and off we went .", "words": ["off", "we", "so", "and", "on", "she"]}, {"sent": "its just one just one in there we need Thomas .", "words": ["there", "in", "need", "we"]}, {"sent": "well hafta tidy up quickly wont we ?", "words": ["up", "we"]}, {"sent": "we could look at it together couldnt we ?", "words": ["we", "it", "at", "look", "could"]}, {"sent": "lets see see what we got masters of the universe .", "words": ["the", "what", "we", "of", "see"]}, {"sent": "what do we do with a pitcher ?", "words": ["what", "do", "we", "with", "a"]}, {"sent": "we had chinese food the other day didnt we ?", "words": ["the", "food", "other", "we"]}, {"sent": "because we wanna show Dada when Dada comes back .", "words": ["back", "we", "show", "wanna", "when", "because"]}, {"sent": "here we go .", "words": ["here", "go", "we"]}, {"sent": "we had a nice time didnt we ?", "words": ["nice", "a", "we"]}, {"sent": "and tomorrow we can open another one .", "words": ["can", "we", "open", "and", "another"]}, {"sent": "I thought we were going to go to the store .", "words": ["the", "go", "we", "to", "store", "were"]}, {"sent": "do you know what we did with that piece of toast ?", "words": ["do", "what", "toast", "we", "of", "with", "you", "did", "that"]}, {"sent": "shall we do a different game ?", "words": ["do", "game", "a", "we"]}, {"sent": "we hafta be careful with books dont we ?", "words": ["we", "be", "with", "dont", "careful"]}, {"sent": "right we pop that straight into there because I only wanted the egg yolk .", "words": ["the", "pop", "into", "egg", "we", "there", "that", "because"]}, {"sent": "we were looking for him before werent we ?", "words": ["him", "for", "were", "we"]}, {"sent": "where do we go when we go on an aeroplane ?", "words": ["where", "do", "go", "we", "an", "on", "when"]}, {"sent": "were finding the wedding arent we ?", "words": ["the", "were", "we"]}, {"sent": "we dont wanna we dont wanna trash it right off the bat .", "words": ["off", "the", "we", "trash", "dont", "it", "wanna", "bat"]}, {"sent": "here we are look .", "words": ["here", "look", "are", "we"]}, {"sent": "we went to see Cathy .", "words": ["see", "to", "we"]}, {"sent": "no we dont want him to hurt himself do we ?", "words": ["do", "we", "hurt", "to", "dont", "him"]}, {"sent": "are we finished ?", "words": ["are", "we"]}, {"sent": "give us the song and th then maybe we can figure out what word it is .", "words": ["the", "can", "what", "we", "is", "then", "and", "us", "it", "out", "give"]}, {"sent": "should we do that one too .", "words": ["do", "too", "that", "we"]}, {"sent": "we put money in .", "words": ["money", "in", "put", "we"]}, {"sent": "shall we take the rings out ?", "words": ["the", "take", "out", "we"]}, {"sent": "we went to AspinallStreet didnt we ?", "words": ["to", "we"]}, {"sent": "what are we having for dinner then ?", "words": ["what", "for", "we", "are", "then"]}, {"sent": "we dont need anymore cushions because we havent got anymore .", "words": ["dont", "need", "because", "we"]}, {"sent": "and whatre we going to do with him ?", "words": ["do", "we", "to", "with", "and", "him"]}, {"sent": "were going to bathe the doll arent we ?", "words": ["the", "we", "to", "doll", "were"]}, {"sent": "we hafta make little faces .", "words": ["make", "little", "we"]}, {"sent": "we could have a horse and pretend its a paddock .", "words": ["we", "pretend", "and", "have", "horse", "could", "a"]}, {"sent": "shall we do another one ?", "words": ["another", "do", "we"]}, {"sent": "shall we make a zoo ?", "words": ["zoo", "make", "a", "we"]}, {"sent": "if we do this .", "words": ["do", "if", "this", "we"]}, {"sent": "if we have lights at the front we can see where were going .", "words": ["the", "can", "where", "if", "we", "have", "at", "see", "were"]}, {"sent": "we oughta dry her with the towel .", "words": ["the", "we", "dry", "with", "towel", "her"]}, {"sent": "dont take it out with us any more do we ?", "words": ["do", "we", "with", "any", "dont", "us", "it", "more", "out", "take"]}, {"sent": "we found the hands to the clock .", "words": ["the", "to", "clock", "we"]}, {"sent": "we did ?", "words": ["did", "we"]}, {"sent": "well hm what should we do with the extra ?", "words": ["the", "what", "do", "we", "with"]}, {"sent": "we wait with baited breath to find out what you want .", "words": ["what", "wait", "we", "to", "with", "you", "out", "find"]}, {"sent": "why dont we look for pieces with the snake on ?", "words": ["the", "for", "we", "with", "dont", "look", "on", "why"]}, {"sent": "we like the pink pigs dont you ?", "words": ["the", "we", "like", "you", "dont"]}, {"sent": "were missing a car arent we ?", "words": ["car", "were", "we", "a"]}, {"sent": "yeah we put clothes on her .", "words": ["on", "put", "her", "we"]}, {"sent": "shall we find the flowers and grapes to put on there ?", "words": ["the", "put", "we", "to", "and", "there", "grapes", "find", "on"]}, {"sent": "and we can do some fingers .", "words": ["can", "do", "we", "and", "some"]}, {"sent": "yeah we have that at home .", "words": ["we", "have", "at", "that", "home"]}, {"sent": "shall we take .", "words": ["take", "we"]}, {"sent": "no when its hot we turn on the air conditioner .", "words": ["the", "we", "hot", "on", "when"]}, {"sent": "oh we got that in Pennsylvania .", "words": ["in", "that", "we"]}, {"sent": "we like to hop on top of pop .", "words": ["pop", "we", "of", "to", "like", "on"]}, {"sent": "we heard something noisy yesterday didnt we John ?", "words": ["noisy", "we"]}, {"sent": "we can hear them playing on the bell with the bell cant we ?", "words": ["can", "the", "we", "them", "with", "hear", "on"]}, {"sent": "we so I dont know if we really call it peekaboo we call it like Iseeyou .", "words": ["if", "we", "so", "like", "dont", "it"]}, {"sent": "we are running out .", "words": ["are", "out", "we"]}, {"sent": "we watched that on tv on bowling .", "words": ["on", "tv", "that", "we"]}, {"sent": "oh here it says we need to find Eeyores triangular house .", "words": ["need", "we", "need to", "to", "here", "it", "house", "find"]}, {"sent": "we could get your knife your knives and forks and things out and pretend to cook with these couldnt we ?", "words": ["these", "your", "we", "to", "knife", "with", "and", "get", "cook", "out", "pretend", "could"]}, {"sent": "okay lets stop the taping shall we ?", "words": ["the", "stop", "we"]}, {"sent": "we read storys and we have a cuddle and then you go to bed .", "words": ["we", "go", "to", "bed", "and", "have", "you", "read", "then", "a"]}, {"sent": "what else do we have ?", "words": ["have", "what", "do", "we"]}, {"sent": "yes we are going to give you some hot cereal .", "words": ["we", "to", "you", "hot", "are", "some", "give", "cereal"]}, {"sent": "and so we were joking with them .", "words": ["we", "so", "with", "them", "and", "were"]}, {"sent": "put this one on shall we ?", "words": ["on", "this", "put", "we"]}, {"sent": "there we go .", "words": ["go", "there", "we"]}, {"sent": "we picked him up by the tail ?", "words": ["up", "the", "we", "by", "him"]}, {"sent": "what about if we um .", "words": ["what", "if", "about", "we"]}, {"sent": "what else could we put on her huh ?", "words": ["what", "put", "we", "on", "could", "her"]}, {"sent": "do you think he liked any of the ones we have here ?", "words": ["the", "do", "think", "we", "of", "any", "you", "have", "here", "he"]}, {"sent": "we did it we saved him .", "words": ["him", "it", "did", "we"]}, {"sent": "how does it work ?", "words": ["does", "how", "it", "work"]}, {"sent": "does ducky sleep ?", "words": ["sleep", "does"]}, {"sent": "how does your garden grow ?", "words": ["garden", "does", "how", "your"]}, {"sent": "oh remember what does it have for eyes ?", "words": ["what", "does", "for", "have", "it"]}, {"sent": "does a bird fly ?", "words": ["does", "bird", "a"]}, {"sent": "does this match ?", "words": ["does", "this"]}, {"sent": "he does not have any ear infection so you can get him wet all you want .", "words": ["can", "does", "wet", "all", "so", "any", "you", "have", "get", "him", "he", "ear", "not"]}, {"sent": "yeah what does Thomas wanna share ?", "words": ["what", "does", "share", "wanna"]}, {"sent": "does that one go in there ?", "words": ["does", "go", "there", "in", "that"]}, {"sent": "where does he live ?", "words": ["where", "does", "he"]}, {"sent": "who does she find being a policeman ?", "words": ["does", "find", "who", "she", "a"]}, {"sent": "does it look like the the circles moving around it ?", "words": ["the", "does", "around", "like", "it", "look"]}, {"sent": "she just she how does she feel about the playground ?", "words": ["the", "does", "how", "about", "she"]}, {"sent": "so does that one and that one and that one and that one .", "words": ["so", "does", "that", "and"]}, {"sent": "he does say quack .", "words": ["does", "he", "say"]}, {"sent": "a lizard it does .", "words": ["does", "it", "a"]}, {"sent": "what does tiger say ?", "words": ["what", "does", "say", "tiger"]}, {"sent": "where does eleven and one half come in ?", "words": ["where", "does", "in", "and"]}, {"sent": "does not taste too good .", "words": ["does", "taste", "good", "too", "not"]}, {"sent": "what does an elephant do ?", "words": ["what", "does", "do", "elephant", "an"]}, {"sent": "does she catch good fish ?", "words": ["does", "catch", "good", "fish", "she"]}, {"sent": "what about his face does he have a beak ?", "words": ["what", "does", "face", "have", "he", "about", "his", "a"]}, {"sent": "where does that one go then ?", "words": ["where", "does", "go", "then", "that"]}, {"sent": "what does he do ?", "words": ["what", "do", "does", "he"]}, {"sent": "does she does she own the house ?", "words": ["the", "does", "house", "she"]}, {"sent": "then she said where does your Mother live .", "words": ["where", "does", "your", "then", "she"]}, {"sent": "what does ?", "words": ["what", "does"]}, {"sent": "um and did you notice anything about his play today that was different from what he normally does ?", "words": ["what", "does", "was", "play", "you", "and", "he", "did", "about", "his", "that"]}, {"sent": "what does the dairy mean ?", "words": ["the", "what", "does"]}, {"sent": "does the teddy want the bunnyrabbit ?", "words": ["the", "does"]}, {"sent": "I wont honey it its it does that because its so long but when we cut it off it wont make snarls anymore .", "words": ["off", "does", "when", "cut", "we", "so", "long", "it", "make", "but", "that", "because"]}, {"sent": "does it ?", "words": ["does", "it"]}, {"sent": "does that go with the big sheep ?", "words": ["the", "does", "go", "with", "big", "sheep", "that"]}, {"sent": "oh does she needta go for a nappie change ?", "words": ["does", "for", "go", "she", "a"]}, {"sent": "does that mean you dont wanna read it ?", "words": ["does", "you", "dont", "it", "read", "wanna", "that"]}, {"sent": "nobody does .", "words": ["does"]}, {"sent": "does it say ?", "words": ["does", "it", "say"]}, {"sent": "right what does that say ?", "words": ["what", "does", "that", "say"]}, {"sent": "does the cheese taste nice ?", "words": ["the", "does", "nice", "taste", "cheese"]}, {"sent": "does that sound like a good idea ?", "words": ["does", "like", "good", "that", "a"]}, {"sent": "does that lego go on like go there ?", "words": ["does", "go", "like", "there", "on", "that"]}, {"sent": "what does that car say ?", "words": ["what", "does", "car", "say", "that"]}, {"sent": "oh it does .", "words": ["does", "it"]}, {"sent": "yes it does it does .", "words": ["does", "it"]}, {"sent": "you know that xxx has an apron for when he does cooking ?", "words": ["does", "for", "when", "you", "an", "he", "that"]}, {"sent": "what does Oigan do ?", "words": ["what", "do", "does"]}, {"sent": "what does puddy mean .", "words": ["what", "does"]}, {"sent": "does the frog have eyes ?", "words": ["the", "does", "have", "frog"]}, {"sent": "does this go in the kitchen ?", "words": ["the", "does", "go", "this", "in", "kitchen"]}, {"sent": "what does the stork say when he eats a frog .", "words": ["the", "what", "does", "say", "frog", "he", "when", "a"]}, {"sent": "it does look like PostmanPats van .", "words": ["does", "it", "like", "look"]}, {"sent": "does that look nice ?", "words": ["does", "nice", "that", "look"]}, {"sent": "how does a butterfly go from one place to another ?", "words": ["butterfly", "does", "how", "go", "to", "another", "a"]}, {"sent": "like like Purdie does ?", "words": ["does", "like"]}, {"sent": "does she look just like the girls in your ballet class ?", "words": ["the", "does", "your", "like", "in", "look", "she"]}, {"sent": "he does what ?", "words": ["does", "what", "he"]}, {"sent": "what does a horsie say ?", "words": ["what", "does", "say", "a"]}, {"sent": "but for the most he ju he does his cooking and his and reading .", "words": ["the", "does", "for", "and", "he", "his", "but"]}, {"sent": "what sound does a chicken make ?", "words": ["what", "does", "chicken", "make", "a"]}, {"sent": "what does that say ?", "words": ["what", "does", "that", "say"]}, {"sent": "does your mummie say that to you sometimes ?", "words": ["does", "your", "to", "say", "you", "that"]}, {"sent": "but then Purdie doesnt get her clothes wet does she ?", "words": ["wet", "does", "get", "then", "her", "she", "but"]}, {"sent": "what does eh mean .", "words": ["what", "does"]}, {"sent": "he wears a little harness does he ?", "words": ["does", "he", "little", "a"]}, {"sent": "BullMoose what does a moose have ?", "words": ["what", "does", "moose", "have", "a"]}, {"sent": "he doesnt go in there does he ?", "words": ["does", "go", "there", "in", "he"]}, {"sent": "does Mummy make a really good postbox Thomas ?", "words": ["does", "make", "good", "a"]}, {"sent": "why does she need a bottle ?", "words": ["does", "need", "bottle", "she", "why", "a"]}, {"sent": "how does that feel ?", "words": ["does", "how", "that"]}, {"sent": "does that smell nice ?", "words": ["does", "nice", "that"]}, {"sent": "does he look after Simba ?", "words": ["does", "he", "look"]}, {"sent": "what does the milkman bring ?", "words": ["the", "what", "does", "bring"]}, {"sent": "does he say its soft ?", "words": ["does", "soft", "he", "say"]}, {"sent": "chips does yeah .", "words": ["does"]}, {"sent": "he does ?", "words": ["does", "he"]}, {"sent": "what does the nurse say ?", "words": ["the", "what", "does", "say"]}, {"sent": "he does look just like in the show .", "words": ["the", "does", "show", "like", "in", "he", "look"]}, {"sent": "no he sure doesnt does he ?", "words": ["does", "he"]}, {"sent": "where does it go ?", "words": ["where", "does", "it", "go"]}, {"sent": "lettuce doesnt hafta be very small does it ?", "words": ["be", "does", "it"]}, {"sent": "dad does shots momma .", "words": ["does"]}, {"sent": "well she always does what shes told .", "words": ["does", "what", "she"]}, {"sent": "like momma does it .", "words": ["does", "it", "like"]}, {"sent": "whats the first thing he does ?", "words": ["the", "does", "first", "he"]}, {"sent": "does it need some candles ?", "words": ["does", "it", "need", "some"]}, {"sent": "what does he always say ?", "words": ["what", "does", "he", "say"]}, {"sent": "where does the moon go ?", "words": ["where", "does", "the", "moon", "go"]}, {"sent": "what does a dinosaur say ?", "words": ["what", "does", "say", "a"]}, {"sent": "what does a lambie say ?", "words": ["what", "does", "say", "a"]}, {"sent": "does it hafta drive all over me ?", "words": ["does", "all", "drive", "it", "me", "over"]}, {"sent": "the box does it go in the fire truck ?", "words": ["the", "does", "box", "go", "truck", "in", "it"]}, {"sent": "huh where does UncleRoy live ?", "words": ["where", "does"]}, {"sent": "does it have a back door too ?", "words": ["does", "back", "have", "it", "too", "door", "a"]}, {"sent": "does that mean Im the winner ?", "words": ["the", "does", "that"]}, {"sent": "but how does it get in ?", "words": ["does", "how", "get", "in", "it", "but"]}, {"sent": "what does ?", "words": ["what", "does"]}, {"sent": "should we talk about gymnastics what Roman does ?", "words": ["does", "what", "we", "about", "talk"]}, {"sent": "what does it xxx on something .", "words": ["on", "what", "does", "it"]}, {"sent": "what does Mummy needta do to your nose ?", "words": ["what", "do", "does", "your", "to", "nose"]}, {"sent": "how does Fuzzylittletaillamb drink ?", "words": ["does", "how", "drink"]}, {"sent": "Mummy got your plastic cup and she poured a tiny tiny spot of tea into it and then filled the rest up with milk .", "words": ["cup", "tiny", "the", "up", "your", "into", "of", "with", "and", "it", "milk", "then", "she", "a"]}, {"sent": "she always stays overnight .", "words": ["she"]}, {"sent": "why dont we ask her when she comes back ?", "words": ["when", "back", "we", "dont", "her", "she", "why"]}, {"sent": "we blew them all up for your birthday and then Spot popped them all didnt she ?", "words": ["up", "all", "for", "your", "we", "them", "and", "then", "she"]}, {"sent": "does she want some of this nice food for her dinner ?", "words": ["does", "for", "of", "nice", "this", "food", "some", "her", "she"]}, {"sent": "she hit that pillow and she went out like a light .", "words": ["light", "like", "and", "out", "hit", "that", "she", "a", "pillow"]}, {"sent": "so thats the sort of thing that she does all the time .", "words": ["the", "does", "all", "of", "so", "that", "she"]}, {"sent": "here she comes .", "words": ["here", "she"]}, {"sent": "well she must be there somewhere .", "words": ["be", "she", "there"]}, {"sent": "and while they were running away from her she grabbed a torch from the wall and tried to set the scarecrow on fire and Dorothy grabbed a pail of water and threw it but it hit the witch all of a sudden the witch gave a loud cry .", "words": ["the", "all", "of", "to", "cry", "loud", "and", "they", "away", "it", "but", "water", "on", "hit", "her", "she", "were", "a"]}, {"sent": "so I was kinda trying to see if she .", "words": ["if", "was", "so", "to", "see", "she"]}, {"sent": "and she tried to pull some strings and that so she could get in .", "words": ["to", "so", "pull", "and", "get", "in", "some", "could", "that", "she"]}, {"sent": "and they come apart so she can bang them together .", "words": ["can", "so", "them", "and", "they", "she"]}, {"sent": "um but uh yeah I mean I I she she crawls over to books any time she sees them .", "words": ["to", "any", "them", "over", "she", "but"]}, {"sent": "is she a wee bit .", "words": ["she", "a", "is"]}, {"sent": "she hasnt said anything to me .", "words": ["to", "me", "she"]}, {"sent": "she had an awful cold .", "words": ["an", "she", "cold"]}, {"sent": "and shell just come and like when my shirt is is very loose shell just come up and she just hasta grab like my the fat on my belly .", "words": ["up", "the", "my", "is", "shirt", "like", "and", "on", "when", "she"]}, {"sent": "did she listen to your tummy ?", "words": ["your", "listen", "to", "did", "tummy", "she"]}, {"sent": "she grab Mommys hair .", "words": ["hair", "she"]}, {"sent": "the Bow ooh shes going to get to say to let her in to give some supper but she has gone .", "words": ["the", "to", "say", "get", "in", "some", "give", "her", "she", "but"]}, {"sent": "so she she was she and .", "words": ["was", "so", "she", "and"]}, {"sent": "she wanted them back this morning before she went to work .", "words": ["work", "back", "to", "them", "this", "she"]}, {"sent": "I think she was just telling me what she sometimes does .", "words": ["what", "think", "does", "was", "me", "she"]}, {"sent": "did she bring you a present ?", "words": ["present", "bring", "you", "did", "she", "a"]}, {"sent": "she said you can keep it .", "words": ["can", "it", "she", "you"]}, {"sent": "okay I say that and she says hey all the time .", "words": ["the", "all", "say", "and", "that", "she"]}, {"sent": "she had a bandage on her leg didnt she ?", "words": ["leg", "on", "her", "she", "a"]}, {"sent": "shes got little red slippers on hasnt she ?", "words": ["red", "on", "she", "little"]}, {"sent": "she lets me know what the crack is .", "words": ["the", "what", "is", "me", "she"]}, {"sent": "is is she like into cows or .", "words": ["into", "she", "like", "is"]}, {"sent": "like these has she seen these ?", "words": ["these", "like", "she"]}, {"sent": "and she has come back to stay for a few weeks on holiday .", "words": ["for", "back", "to", "and", "on", "stay", "she", "a"]}, {"sent": "okay oh she she already knows what a microphone is all about and everything .", "words": ["all", "what", "is", "and", "about", "she", "a"]}, {"sent": "because shes not very nice is she ?", "words": ["is", "not", "nice", "she", "because"]}, {"sent": "she cant throw it can she ?", "words": ["can", "throw", "it", "she"]}, {"sent": "she does doesnt she ?", "words": ["does", "she"]}, {"sent": "well Amanda has to bring the microphone home with her because she takes care of the microphones .", "words": ["the", "of", "to", "with", "bring", "her", "she", "because", "home"]}, {"sent": "what is she doing ?", "words": ["what", "she", "is"]}, {"sent": "she has a good night moon book .", "words": ["moon", "good", "book", "she", "a"]}, {"sent": "shes crying because she wants her milk .", "words": ["milk", "her", "she", "because"]}, {"sent": "and she also waves alot .", "words": ["she", "and"]}, {"sent": "this purple spoon the smaller of the two spoons she liked .", "words": ["the", "of", "this", "spoon", "she"]}, {"sent": "that she thinks some of the smaller apples hafta fall off .", "words": ["the", "off", "of", "fall", "some", "that", "she"]}, {"sent": "she knew all about Bilbo and Frodo .", "words": ["all", "she", "about", "and"]}, {"sent": "she did say that ?", "words": ["did", "that", "say", "she"]}, {"sent": "does she laugh ?", "words": ["does", "she"]}, {"sent": "she tried the mamma bears bed .", "words": ["the", "she", "bed"]}, {"sent": "xxx got this here from she was xxx .", "words": ["here", "this", "was", "she"]}, {"sent": "did she get did she get a call ?", "words": ["get", "did", "a", "she"]}, {"sent": "there she is .", "words": ["there", "she", "is"]}, {"sent": "she hasta stay there .", "words": ["stay", "she", "there"]}, {"sent": "where has she done where has she gone ?", "words": ["where", "she"]}, {"sent": "she gave you the ball .", "words": ["the", "ball", "she", "you"]}, {"sent": "would she like any of this stuff ?", "words": ["of", "any", "like", "this", "would", "she"]}, {"sent": "what about with the animals like did she recognize .", "words": ["the", "what", "with", "like", "did", "about", "she"]}, {"sent": "has she gone ?", "words": ["she"]}, {"sent": "she likes Cornflakes .", "words": ["she"]}, {"sent": "she was the one that brought it xxx house ?", "words": ["the", "was", "it", "house", "that", "she"]}, {"sent": "Sue had it didnt she ?", "words": ["it", "she"]}, {"sent": "she made a big pee on the floor .", "words": ["the", "on", "big", "she", "a"]}, {"sent": "and then she could youknow she could wear that on her first day and all .", "words": ["all", "first", "her", "and", "on", "then", "that", "she", "could"]}, {"sent": "was she having a party ?", "words": ["was", "party", "she", "a"]}, {"sent": "oh yeah she .", "words": ["she"]}, {"sent": "whatd she do ?", "words": ["do", "she"]}, {"sent": "she put him in a bathtub to see if he could swim .", "words": ["put", "if", "to", "swim", "in", "him", "bathtub", "he", "see", "could", "she", "a"]}, {"sent": "um and so what el what else does she seem to like to play with ?", "words": ["what", "does", "so", "to", "play", "with", "like", "and", "she"]}, {"sent": "taking a photograph uhn isnt she ?", "words": ["she", "a"]}, {"sent": "what does she do ?", "words": ["what", "do", "does", "she"]}, {"sent": "um at home she youknow like I said its like the we have the whole crib the stroller so the whole doll ensemble .", "words": ["the", "we", "crib", "so", "like", "doll", "have", "stroller", "at", "she", "home"]}, {"sent": "does she look like anybody ?", "words": ["does", "she", "like", "look"]}, {"sent": "leave Amy where she is sweetheart .", "words": ["where", "she", "is"]}, {"sent": "but literally within two weeks she was outof them day and night .", "words": ["was", "them", "and", "she", "but"]}, {"sent": "and then she realized instead of drinking his juice the top mustve come off and he had spilt it all over his blanket .", "words": ["the", "off", "all", "blanket", "of", "and", "it", "he", "his", "then", "juice", "over", "she"]}, {"sent": "and now whats she gonna do to go outside ?", "words": ["do", "go", "to", "outside", "and", "she"]}, {"sent": "um like the ones she recognized she was naming .", "words": ["the", "was", "like", "she"]}, {"sent": "she asked which .", "words": ["which", "she"]}, {"sent": "she said Matty she would like to what tomorrow ?", "words": ["what", "to", "like", "would", "she"]}, {"sent": "or for that matter she was the prettiest .", "words": ["the", "for", "was", "that", "she"]}, {"sent": "she has special food doesnt she ?", "words": ["food", "she"]}, {"sent": "does she hurt her feet without shoes ?", "words": ["hurt", "does", "her", "she"]}, {"sent": "oh I thought maybe she was nursing she looked like .", "words": ["was", "she", "like"]}, {"sent": "she dont wanna go to school .", "words": ["go", "to", "school", "dont", "wanna", "she"]}, {"sent": "here she comes again .", "words": ["here", "she"]}, {"sent": "did she make you go pick it up ?", "words": ["up", "go", "pick", "you", "it", "did", "make", "she"]}, {"sent": "was she crying Joseph ?", "words": ["was", "she"]}, {"sent": "here she comes .", "words": ["here", "she"]}, {"sent": "she does like her banana doesnt she ?", "words": ["does", "like", "banana", "her", "she"]}, {"sent": "I you know pointed out some other things but she didnt really going for it she definitely liked this frog .", "words": ["for", "other", "you", "this", "frog", "it", "out", "some", "she", "but"]}, {"sent": "oh here she is .", "words": ["here", "she", "is"]}, {"sent": "yeah I guess alot of them she wouldnt necessarily she might know a dog .", "words": ["of", "them", "she", "dog", "a"]}, {"sent": "boy she did that really well .", "words": ["did", "that", "she"]}, {"sent": "I dont remember why she got sticky but she did .", "words": ["sticky", "dont", "did", "why", "she", "but"]}, {"sent": "Hilda likes the music so much she forgets about the dentists drill .", "words": ["the", "so", "about", "much", "she"]}, {"sent": "she told my father inlaw last Sunday we were up there .", "words": ["up", "my", "we", "last", "there", "were", "she"]}, {"sent": "is she ?", "words": ["she", "is"]}, {"sent": "she stamped your hand with the Gymbo stamp .", "words": ["the", "your", "hand", "with", "she"]}, {"sent": "I was like look at how chubby she is she has the chubbiest cheeks .", "words": ["the", "how", "look", "is", "was", "like", "at", "she"]}, {"sent": "she gets this excited on days that we come here .", "words": ["we", "here", "this", "on", "that", "she"]}, {"sent": "she has green and white .", "words": ["white", "she", "green", "and"]}, {"sent": "come here Salley put that pig in the barn put him in the barn put that cow over there .", "words": ["the", "put", "pig", "here", "there", "in", "him", "over", "that", "cow"]}, {"sent": "where that thing was ?", "words": ["where", "was", "that"]}, {"sent": "oh that goes in there .", "words": ["there", "in", "that"]}, {"sent": "thats thats that .", "words": ["that"]}, {"sent": "you know what that is ?", "words": ["what", "that", "is", "you"]}, {"sent": "I see that you are Joey will always be your friend but during the day there will be times when you wanna do things you wanna do and there will be times when he wants to do what he wants to do .", "words": ["the", "do", "what", "your", "when", "be", "to", "you", "and", "there", "are", "he", "wanna", "will", "see", "that", "but"]}, {"sent": "there you press that .", "words": ["there", "that", "you"]}, {"sent": "that would hurt .", "words": ["hurt", "that", "would"]}, {"sent": "not that its very stable .", "words": ["that", "not"]}, {"sent": "oh youre struggling with that .", "words": ["that", "with"]}, {"sent": "basically when when the therapists come to our house we hafta remove all things with wheels from the room so that they can focus on something else .", "words": ["the", "all", "can", "we", "to", "so", "with", "our", "they", "that", "room", "house", "on", "when"]}, {"sent": "but he hasnt got eczema or anything like that .", "words": ["that", "he", "like", "but"]}, {"sent": "do you think maybe he recognizes food that you eat ?", "words": ["do", "think", "you", "eat", "food", "he", "that"]}, {"sent": "you want me to get that out .", "words": ["to", "you", "get", "out", "me", "that"]}, {"sent": "is that a tower ?", "words": ["that", "a", "is"]}, {"sent": "what is that ?", "words": ["what", "that", "is"]}, {"sent": "that one goes there .", "words": ["there", "that"]}, {"sent": "what is that ?", "words": ["what", "that", "is"]}, {"sent": "thats that one could .", "words": ["could", "that"]}, {"sent": "oh look at the little book oh dont that one doesnt open is the bunny sleeping ?", "words": ["the", "look", "bunny", "is", "open", "little", "dont", "at", "book", "that"]}, {"sent": "yeah yeah yeah lets pretend that we are having a show .", "words": ["we", "show", "are", "pretend", "that", "a"]}, {"sent": "you know who that is .", "words": ["who", "that", "is", "you"]}, {"sent": "youve got xxx that stool .", "words": ["that"]}, {"sent": "that tree is a painted tree .", "words": ["tree", "that", "a", "is"]}, {"sent": "look at that one .", "words": ["at", "that", "look"]}, {"sent": "youre right a toothbrush is another thing that has brushes on it .", "words": ["toothbrush", "is", "another", "it", "on", "that", "a"]}, {"sent": "is that easy that ?", "words": ["that", "is"]}, {"sent": "is that the fire station ?", "words": ["the", "that", "is"]}, {"sent": "is that your car ?", "words": ["your", "car", "that", "is"]}, {"sent": "that was her homework .", "words": ["was", "her", "that"]}, {"sent": "oh that would be nice .", "words": ["be", "nice", "that", "would"]}, {"sent": "can you find that one ?", "words": ["can", "that", "find", "you"]}, {"sent": "that was a good belch .", "words": ["was", "that", "good", "a"]}, {"sent": "theyre not that clicky are they ?", "words": ["they", "are", "that", "not"]}, {"sent": "is that the piece of the telephone that you talk into ?", "words": ["the", "into", "is", "of", "you", "talk", "that", "telephone"]}, {"sent": "youre saving it youre saving your voice please just tell me ever so quick before we read that book what you had .", "words": ["what", "your", "we", "so", "you", "it", "read", "book", "me", "that"]}, {"sent": "I wanna move that chair xxx .", "words": ["chair", "that", "wanna"]}, {"sent": "so like oh so before before you had the baby he didnt do that stuff but now that hes watching you do it ?", "words": ["the", "do", "so", "like", "you", "it", "he", "that", "but"]}, {"sent": "now whos that ?", "words": ["that"]}, {"sent": "does that mean theres fifteen ?", "words": ["does", "that"]}, {"sent": "that hat .", "words": ["hat", "that"]}, {"sent": "maybe that would help me stop coughing whaddya think ?", "words": ["help", "think", "stop", "would", "me", "that"]}, {"sent": "that says story .", "words": ["story", "that"]}, {"sent": "wow look at that thing .", "words": ["at", "that", "look"]}, {"sent": "what language is that ?", "words": ["what", "that", "is"]}, {"sent": "how about that ?", "words": ["how", "that", "about"]}, {"sent": "oh that tickles .", "words": ["that"]}, {"sent": "he says that the teacher tells him raw meat .", "words": ["the", "meat", "him", "he", "that"]}, {"sent": "and not only that the toys have germs on them .", "words": ["the", "them", "and", "have", "on", "that", "not"]}, {"sent": "oh Nomi if you stand on that basket the basketll tip .", "words": ["the", "if", "stand", "you", "basket", "on", "that"]}, {"sent": "that rabbit .", "words": ["that"]}, {"sent": "that we put things for the compost like tea bags and banana skins and apple peel and potato peel .", "words": ["the", "put", "for", "we", "like", "and", "banana", "apple", "that", "potato"]}, {"sent": "you can do that .", "words": ["can", "do", "that", "you"]}, {"sent": "that must be new .", "words": ["be", "new", "that"]}, {"sent": "Fraser dont stand on that .", "words": ["on", "dont", "that", "stand"]}, {"sent": "can you remember that ?", "words": ["can", "that", "you"]}, {"sent": "theres tons of animals that you see in real life .", "words": ["of", "you", "in", "see", "that"]}, {"sent": "oh honey when you yell like that I wont do anything .", "words": ["do", "when", "like", "you", "that"]}, {"sent": "oh no that goes here .", "words": ["here", "that"]}, {"sent": "what is that ?", "words": ["what", "that", "is"]}, {"sent": "will that make you grow up to be a big boy ?", "words": ["up", "to", "be", "you", "that", "big", "will", "make", "a"]}, {"sent": "that door .", "words": ["that", "door"]}, {"sent": "oh isnt that gorgeous .", "words": ["that"]}, {"sent": "now do you know what that is ?", "words": ["do", "what", "is", "you", "that"]}, {"sent": "is that what you see on the side of a lorry ?", "words": ["the", "what", "is", "of", "you", "on", "see", "that", "a"]}, {"sent": "and that one .", "words": ["that", "and"]}, {"sent": "that ones drinking water .", "words": ["that", "water"]}, {"sent": "gee I thought I needed that .", "words": ["that"]}, {"sent": "this po like anything that he could the stuff that was he could pick up .", "words": ["the", "up", "pick", "was", "like", "this", "he", "could", "that"]}, {"sent": "is that what you and Roman did ?", "words": ["what", "is", "and", "you", "did", "that"]}, {"sent": "is that a train ?", "words": ["train", "that", "a", "is"]}, {"sent": "I dont think that was wet enough to take it off huh ?", "words": ["off", "wet", "think", "was", "to", "dont", "it", "take", "that"]}, {"sent": "does that look like a wheel ?", "words": ["does", "like", "look", "that", "a"]}, {"sent": "howve you got all that .", "words": ["all", "that", "you"]}, {"sent": "whats that ?", "words": ["that"]}, {"sent": "is that daddy ?", "words": ["that", "is"]}, {"sent": "ah that horsies doing a somersault .", "words": ["that", "a"]}, {"sent": "that is a very nice circle .", "words": ["nice", "that", "a", "is"]}, {"sent": "no dont move that piece .", "words": ["dont", "that"]}, {"sent": "whats that ?", "words": ["that"]}, {"sent": "that yoghurt looks very nice .", "words": ["nice", "that"]}, {"sent": "oh that is good .", "words": ["that", "good", "is"]}, {"sent": "and Miss xxx read a story that we have at home today .", "words": ["we", "and", "have", "read", "at", "home", "story", "that", "a"]}, {"sent": "kids from Mommys old church that she useta go to .", "words": ["old", "go", "church", "to", "that", "she"]}, {"sent": "whats that ?", "words": ["that"]}, {"sent": "can I have that holder to carry it home with ?", "words": ["can", "to", "with", "have", "carry", "it", "that", "home"]}, {"sent": "Roman whats that right there what is that ?", "words": ["there", "what", "that", "is"]}, {"sent": "and whats that one ?", "words": ["that", "and"]}, {"sent": "that you put it in did not or xxx knock ?", "words": ["put", "you", "in", "it", "knock", "did", "that", "not"]}, {"sent": "is that like Murray ?", "words": ["that", "like", "is"]}, {"sent": "whats that ?", "words": ["that"]}, {"sent": "and that big xxx .", "words": ["big", "that", "and"]}, {"sent": "and we put that one on top that says one .", "words": ["put", "we", "and", "on", "that"]}, {"sent": "oh that was nice .", "words": ["was", "nice", "that"]}, {"sent": "okay that would be good I like a sandwich thats got baloney cheese tomatoes maybe some lettuce too .", "words": ["be", "like", "good", "would", "some", "too", "cheese", "that", "sandwich", "a"]}, {"sent": "is that Badger ?", "words": ["that", "is"]}, {"sent": "oh theres that hat again .", "words": ["hat", "that"]}, {"sent": "that looks just like a globe .", "words": ["that", "like", "a"]}, {"sent": "that is that is the sort of sound a helicopter makes isnt it ?", "words": ["the", "helicopter", "is", "of", "it", "that", "a"]}, {"sent": "this one on that pile .", "words": ["on", "this", "that"]}, {"sent": "you must be my friend .", "words": ["be", "my", "you"]}, {"sent": "and my sister AuntieJanet useta have a little green handbag .", "words": ["my", "little", "have", "and", "green", "a"]}, {"sent": "Gabriel wont let me play my ball .", "words": ["ball", "me", "my", "play"]}, {"sent": "I think my bear is big enough to go to the examination room by himself .", "words": ["the", "think", "my", "is", "go", "to", "bear", "by", "room", "big"]}, {"sent": "can Daddy not come in my house ?", "words": ["can", "my", "in", "house", "not"]}, {"sent": "Im twisting my head round that far so Im nowhere nowhere near the the food and I think it made me cough even more .", "words": ["the", "my", "think", "so", "head", "and", "food", "it", "more", "me", "that"]}, {"sent": "you just told me it was my go .", "words": ["my", "go", "was", "you", "it", "me"]}, {"sent": "smell my locker .", "words": ["my"]}, {"sent": "I have my crowbar .", "words": ["have", "my"]}, {"sent": "my name is Emily .", "words": ["my", "is"]}, {"sent": "I asked my daddy could I go out .", "words": ["go", "my", "out", "could"]}, {"sent": "my brothers gotten useta that shirt .", "words": ["shirt", "my", "that"]}, {"sent": "is my pick this now ?", "words": ["pick", "this", "my", "is"]}, {"sent": "Naima I want my present .", "words": ["my", "present"]}, {"sent": "when Dadas comes home you well you say to Dadas please will you put my calendar up on the wall please ?", "words": ["up", "the", "put", "my", "to", "say", "you", "on", "will", "when", "home"]}, {"sent": "your Grandma is my Mommy .", "words": ["my", "your", "is"]}, {"sent": "are you my baby ?", "words": ["my", "are", "you"]}, {"sent": "give me one of these at my real birthday at my real birthday .", "words": ["these", "my", "of", "at", "me", "give"]}, {"sent": "those arent my teeth .", "words": ["my", "those"]}, {"sent": "my tee shirt and my undies my tee shirt and my undies .", "words": ["shirt", "my", "and"]}, {"sent": "shes having some of my bread .", "words": ["of", "some", "my", "bread"]}, {"sent": "would you like to use my tissue ?", "words": ["my", "tissue", "to", "like", "you", "would"]}, {"sent": "is your mama a llama I asked my friend Dave .", "words": ["my", "your", "a", "is"]}, {"sent": "no my car is not broken .", "words": ["my", "is", "car", "broken", "not"]}, {"sent": "wash my hands and my face .", "words": ["my", "wash", "face", "and"]}, {"sent": "I wanna put some film in my new camera .", "words": ["my", "put", "camera", "in", "wanna", "some", "new"]}, {"sent": "this is my tool .", "words": ["this", "my", "is"]}, {"sent": "do you mind not wiping your nose on my jumper ?", "words": ["do", "my", "your", "you", "on", "not", "nose"]}, {"sent": "wind up walking around keeping tilting my head like this .", "words": ["up", "my", "head", "like", "this", "around", "wind"]}, {"sent": "you open my purse ?", "words": ["open", "my", "purse", "you"]}, {"sent": "Im Im afraid heres my ticket Ive remembered my ticket but Ive got a problem .", "words": ["my", "but", "a"]}, {"sent": "I wanna do my money .", "words": ["money", "do", "my", "wanna"]}, {"sent": "you get my hand ?", "words": ["get", "my", "hand", "you"]}, {"sent": "Jesus you are my friend .", "words": ["my", "are", "you"]}, {"sent": "I have a little sand in my eyes here .", "words": ["my", "little", "have", "here", "in", "a"]}, {"sent": "xxx and my expenses .", "words": ["my", "and"]}, {"sent": "this is like my geography lesson .", "words": ["this", "my", "like", "is"]}, {"sent": "got on the train and then oh Ive forgotten my purse .", "words": ["the", "my", "purse", "train", "and", "on", "then"]}, {"sent": "theres lotsof MunchingMikes in my name isnt there ?", "words": ["there", "in", "my"]}, {"sent": "its my fishing rod .", "words": ["my"]}, {"sent": "my straw .", "words": ["my"]}, {"sent": "put my feet up .", "words": ["up", "put", "my"]}, {"sent": "you pinched my pillow .", "words": ["my", "pillow", "you"]}, {"sent": "its my turn now .", "words": ["my"]}, {"sent": "or not to my knowledge anyway .", "words": ["to", "my", "not"]}, {"sent": "whats the matter my handsome fellow ?", "words": ["the", "my"]}, {"sent": "my toys are the washing up .", "words": ["the", "my", "are", "up"]}, {"sent": "oh my .", "words": ["my"]}, {"sent": "you want ta wear my barrette ?", "words": ["my", "you"]}, {"sent": "should I wear it on my head ?", "words": ["on", "my", "it", "head"]}, {"sent": "Ill just rinse my hands because theyre a bit sticky after eating that cheese .", "words": ["my", "sticky", "cheese", "that", "because", "a"]}, {"sent": "are you my baby ?", "words": ["my", "are", "you"]}, {"sent": "do you wanna sit on my knee ?", "words": ["do", "my", "knee", "you", "wanna", "on", "sit"]}, {"sent": "do you want me to get my husband to watch her ?", "words": ["do", "my", "watch", "to", "you", "get", "me", "her"]}, {"sent": "Ive gotta get it now because its over my side .", "words": ["my", "get", "it", "over", "because"]}, {"sent": "into my fingers .", "words": ["my", "into"]}, {"sent": "my what colors my toothpaste ?", "words": ["what", "my"]}, {"sent": "its for my wrist .", "words": ["my", "for"]}, {"sent": "oh my gosh .", "words": ["my"]}, {"sent": "you sit on my lap in the rockingchair .", "words": ["the", "my", "you", "in", "on", "sit"]}, {"sent": "thats where my cousins my girlfriend lives .", "words": ["where", "my"]}, {"sent": "I love my grandson Andrew .", "words": ["my", "love"]}, {"sent": "*SI1: what about me wheres my racket ?", "words": ["my", "what", "me", "about"]}, {"sent": "all my teenagers think Im silly too .", "words": ["too", "all", "my", "think"]}, {"sent": "my folder .", "words": ["my"]}, {"sent": "a chocolate surprise for my beautiful boy .", "words": ["my", "chocolate", "for", "a"]}, {"sent": "youve got to eat a little bit more before I get off my chair .", "words": ["off", "my", "to", "chair", "little", "eat", "get", "more", "a"]}, {"sent": "its alright my darling .", "words": ["my"]}, {"sent": "and you can polish my furniture if you like .", "words": ["can", "my", "if", "like", "you", "and"]}, {"sent": "Id like to buy this piece of toast for my Amy .", "words": ["my", "for", "toast", "of", "to", "like", "this", "buy"]}, {"sent": "can I have my shoes back ?", "words": ["can", "have", "back", "my"]}, {"sent": "right you xxx my body Kimberly ?", "words": ["my", "you"]}, {"sent": "oh just looking at my homework you know I didnt I missed the syllabus that week remember I was telling you that I thought I missed that paper ?", "words": ["the", "my", "was", "paper", "you", "at", "that"]}, {"sent": "that was my foot .", "words": ["was", "my", "that", "foot"]}, {"sent": "did my transformers get tired of watching the horses Kalie .", "words": ["the", "my", "of", "tired", "get", "did"]}, {"sent": "I dropped my purse .", "words": ["my", "purse"]}, {"sent": "wheres my Eleanor ?", "words": ["my"]}, {"sent": "my hand has gone red .", "words": ["red", "my", "hand"]}, {"sent": "I caught my tower .", "words": ["my"]}, {"sent": "there useta be a nightmare in my closet .", "words": ["my", "be", "closet", "there", "in", "a"]}, {"sent": "yes in my dress .", "words": ["in", "dress", "my"]}, {"sent": "oh my goodness .", "words": ["my"]}, {"sent": "were going to play my house game .", "words": ["my", "game", "to", "play", "house", "were"]}, {"sent": "my favorite .", "words": ["my"]}, {"sent": "now its my turn Mama .", "words": ["my"]}, {"sent": "oh you wanna comb my hair ?", "words": ["my", "comb", "you", "wanna", "hair"]}, {"sent": "oh mommy I smashed my hand .", "words": ["my", "hand"]}, {"sent": "and Id love pickles with my hamburger also .", "words": ["my", "with", "hamburger", "and", "love"]}, {"sent": "my parents came in from Florida .", "words": ["in", "my"]}, {"sent": "can I have some sugar in my tea Laura ?", "words": ["can", "my", "have", "in", "some"]}, {"sent": "are all the rest of the letters for my little sister ?", "words": ["the", "all", "my", "for", "of", "little", "are"]}, {"sent": "is it your drink or is it my drink ?", "words": ["my", "your", "drink", "is", "it"]}, {"sent": "Ill swear my transcriber to uh .", "words": ["to", "my"]}, {"sent": "my poor little baby .", "words": ["my", "little", "poor"]}, {"sent": "how are my shirts ?", "words": ["how", "are", "my"]}, {"sent": "my daddys coming seventy .", "words": ["my"]}, {"sent": "oh my god .", "words": ["my"]}, {"sent": "my little girl .", "words": ["my", "little"]}, {"sent": "this is my tshirt .", "words": ["this", "my", "is"]}, {"sent": "my turn .", "words": ["my"]}, {"sent": "well what would you like instead ?", "words": ["what", "like", "you", "would"]}, {"sent": "so do you think perhaps Grandad um would read about TeddyBrightEyes and TubbyToes TeddyTubbyToes ?", "words": ["do", "think", "so", "you", "and", "read", "about", "would"]}, {"sent": "would you like to blick one ?", "words": ["to", "like", "you", "would"]}, {"sent": "would you like something to play with while youre in the bath ?", "words": ["the", "to", "play", "with", "like", "you", "in", "would"]}, {"sent": "would you like to try some honey Thomas .", "words": ["to", "like", "you", "would", "some", "try"]}, {"sent": "what would you say ?", "words": ["what", "say", "you", "would"]}, {"sent": "would you like a cheese sandwich Thomas ?", "words": ["like", "you", "would", "cheese", "sandwich", "a"]}, {"sent": "would you like a drink of water ?", "words": ["water", "drink", "of", "like", "you", "would", "a"]}, {"sent": "no but he would get money right ?", "words": ["money", "get", "he", "would", "but"]}, {"sent": "well would you like to give it to prairie ?", "words": ["to", "like", "you", "it", "would", "give"]}, {"sent": "what would you what do you think well see at the park ?", "words": ["the", "what", "do", "think", "you", "at", "park", "would", "see"]}, {"sent": "and would you make me a magic ?", "words": ["you", "and", "would", "me", "make", "a"]}, {"sent": "what would you say would be good for us to remember and for what we would like to do in here ?", "words": ["what", "do", "for", "we", "be", "to", "say", "like", "you", "and", "us", "in", "here", "good", "would"]}, {"sent": "that would be two .", "words": ["be", "that", "would"]}, {"sent": "would you like to get down ?", "words": ["to", "like", "you", "get", "would", "down"]}, {"sent": "and what would you like for pudding ?", "words": ["what", "for", "like", "you", "and", "would", "pudding"]}, {"sent": "and then what would he say if I was hiding him ?", "words": ["what", "if", "was", "say", "and", "him", "he", "would", "then"]}, {"sent": "would you like me to getchoo some juice ?", "words": ["to", "like", "you", "would", "me", "juice", "some"]}, {"sent": "would you like a drink ?", "words": ["drink", "like", "you", "would", "a"]}, {"sent": "because she couldnt hear you sing because you wouldnt sing would you ?", "words": ["you", "sing", "hear", "would", "she", "because"]}, {"sent": "oo that would be fun .", "words": ["be", "that", "would"]}, {"sent": "what would you like to play with now ?", "words": ["what", "to", "play", "with", "like", "you", "would"]}, {"sent": "would you like to pick a page for Mommy to color ?", "words": ["for", "pick", "to", "like", "you", "would", "a"]}, {"sent": "that would be sad wouldnt it ?", "words": ["be", "it", "would", "sad", "that"]}, {"sent": "would it of hurt Dada ?", "words": ["of", "it", "hurt", "would"]}, {"sent": "what else would fit in the basket ?", "words": ["the", "what", "fit", "in", "basket", "would"]}, {"sent": "would you like to go for a liedown after lunch ?", "words": ["for", "go", "to", "like", "you", "would", "a"]}, {"sent": "what would Eleanor like to eat today ?", "words": ["what", "to", "like", "eat", "would"]}, {"sent": "what would you do ?", "words": ["what", "do", "you", "would"]}, {"sent": "would you like an apple ?", "words": ["like", "you", "an", "would", "apple"]}, {"sent": "I would really like it if you would sit and paint over there .", "words": ["paint", "if", "like", "you", "and", "there", "it", "would", "sit", "over"]}, {"sent": "do you think Snoopy would get scared of the witch ?", "words": ["the", "do", "think", "of", "scared", "you", "get", "would"]}, {"sent": "what color would you like ?", "words": ["what", "like", "you", "would"]}, {"sent": "oh well perhaps Margaret and Mary would like a tin of dog food for their dog wouldnt they ?", "words": ["for", "their", "of", "like", "and", "they", "food", "would", "dog", "a"]}, {"sent": "and then sure anybody else who would be coming in that would be over four from a playgroup or something .", "words": ["be", "would", "and", "in", "who", "over", "then", "that", "a"]}, {"sent": "what would you like from the shop now ?", "words": ["the", "what", "like", "you", "would"]}, {"sent": "what would you like for supper ?", "words": ["what", "for", "like", "you", "would"]}, {"sent": "that means that there would hafta be a Zamp in the lamp .", "words": ["the", "be", "there", "in", "lamp", "would", "that", "a"]}, {"sent": "would you like to swing on a star .", "words": ["swing", "to", "star", "like", "you", "would", "on", "a"]}, {"sent": "that would be a good idea .", "words": ["be", "good", "would", "that", "a"]}, {"sent": "well what would you like to play with ?", "words": ["what", "to", "play", "with", "like", "you", "would"]}, {"sent": "would you like to invite her to a party ?", "words": ["to", "like", "you", "party", "would", "her", "a"]}, {"sent": "what would you do then ?", "words": ["what", "do", "you", "would", "then"]}, {"sent": "that would never do .", "words": ["do", "that", "would"]}, {"sent": "now that would be a good idea .", "words": ["be", "good", "would", "that", "a"]}, {"sent": "how would you like to color ?", "words": ["how", "to", "like", "you", "would"]}, {"sent": "would you like it now ?", "words": ["it", "like", "you", "would"]}, {"sent": "you wouldnt do would you ?", "words": ["do", "would", "you"]}, {"sent": "would you like ?", "words": ["like", "you", "would"]}, {"sent": "would you like mommy to open the sticker ?", "words": ["the", "open", "to", "like", "you", "would"]}, {"sent": "what would he say ?", "words": ["what", "he", "say", "would"]}, {"sent": "would that would that work that far away ?", "words": ["work", "away", "that", "would"]}, {"sent": "thats where it would come out .", "words": ["where", "it", "out", "would"]}, {"sent": "would you like to hear those when they get that far ?", "words": ["those", "to", "like", "hear", "you", "they", "get", "that", "would", "when"]}, {"sent": "I would like to do the toadstools actually .", "words": ["the", "do", "to", "like", "would"]}, {"sent": "I think he would like it back .", "words": ["think", "back", "like", "it", "he", "would"]}, {"sent": "I would love to play toys with you .", "words": ["to", "play", "with", "you", "love", "would"]}, {"sent": "would you like to color in your book ?", "words": ["your", "to", "like", "you", "in", "would", "book"]}, {"sent": "what what flavor would you like ?", "words": ["what", "like", "you", "would"]}, {"sent": "Hopp would you like to make a picture ?", "words": ["to", "picture", "like", "you", "would", "make", "a"]}, {"sent": "I think the tiger would like some juice .", "words": ["the", "think", "like", "tiger", "would", "some", "juice"]}, {"sent": "well he would be covered in mud after hed pulled the tractor out of the pond wouldnt he ?", "words": ["the", "of", "be", "in", "he", "tractor", "out", "would"]}, {"sent": "I think it would be best if I took some toys I think .", "words": ["think", "if", "be", "it", "would", "some"]}, {"sent": "would they ?", "words": ["they", "would"]}, {"sent": "yep sometimes you would win sometimes I would win sometimes daddy would win .", "words": ["would", "you"]}, {"sent": "what would you like ?", "words": ["what", "like", "you", "would"]}, {"sent": "what what what would you like to do with them now ?", "words": ["what", "do", "to", "with", "like", "them", "you", "would"]}, {"sent": "would you like to weigh me some out ?", "words": ["to", "like", "you", "out", "would", "me", "some"]}, {"sent": "I would say the plastic things that she could chew on right now .", "words": ["the", "say", "would", "on", "could", "that", "she"]}, {"sent": "what would you like to buy ?", "words": ["what", "to", "like", "you", "would", "buy"]}, {"sent": "Thomas would like to put some clothes in the washing machine would he ?", "words": ["the", "put", "to", "like", "washing machine", "in", "he", "would", "some"]}, {"sent": "would you like to get this ?", "words": ["to", "like", "you", "get", "this", "would"]}, {"sent": "would that be fun ?", "words": ["be", "that", "would"]}, {"sent": "so I would like to buy this dog .", "words": ["dog", "to", "so", "like", "this", "would", "buy"]}, {"sent": "would you like something else to drink ?", "words": ["drink", "to", "like", "you", "would"]}, {"sent": "xxx would you like toast ?", "words": ["toast", "like", "you", "would"]}, {"sent": "would you like .", "words": ["like", "you", "would"]}, {"sent": "or you cut the skin it would hurt an awful lot .", "words": ["the", "cut", "hurt", "you", "an", "it", "would"]}, {"sent": "would you like that ?", "words": ["that", "like", "you", "would"]}, {"sent": "and if she threw a ball or something which hand would she use ?", "words": ["if", "hand", "and", "ball", "would", "which", "she", "a"]}, {"sent": "I whish your children would sit still when Kelly is talking to me .", "words": ["your", "is", "to", "would", "me", "sit", "when"]}, {"sent": "would you like that one ?", "words": ["that", "like", "you", "would"]}, {"sent": "would you like a cup of tea before you go ?", "words": ["cup", "go", "of", "like", "you", "would", "a"]}, {"sent": "probably would do .", "words": ["do", "would"]}, {"sent": "would you like to put the crayons back in the box ?", "words": ["the", "put", "box", "back", "to", "like", "you", "in", "would"]}, {"sent": "Nanette would you sit down please ?", "words": ["sit", "down", "you", "would"]}, {"sent": "would you like a sandwich or a biscuit or anything ?", "words": ["like", "you", "would", "sandwich", "a"]}, {"sent": "what would this be if I had this ?", "words": ["what", "if", "be", "this", "would"]}, {"sent": "why would ?", "words": ["why", "would"]}, {"sent": "she she would drop them and she would youknow shed have the big one and then shed switch to having all the little ones .", "words": ["the", "all", "to", "them", "and", "have", "little", "drop", "would", "then", "big", "she"]}, {"sent": "would you like to read it to me like you do to your sisters ?", "words": ["do", "your", "to", "like", "you", "it", "read", "would", "me"]}, {"sent": "I like peas and I like hamburgers but I dont think that would taste so good together .", "words": ["think", "so", "peas", "like", "taste", "and", "dont", "good", "would", "that", "but"]}, {"sent": "would you would you like to show her the back part of the the loft ?", "words": ["the", "back", "show", "to", "of", "like", "you", "would", "her"]}, {"sent": "would you like a spoon sweetheart ?", "words": ["like", "you", "spoon", "would", "a"]}, {"sent": "would you wanna have a gorilla in your house ?", "words": ["your", "you", "have", "in", "house", "wanna", "would", "a"]}, {"sent": "would you like some water ?", "words": ["like", "would", "you", "water", "some"]}, {"sent": "here would you like me to help stick the big one on .", "words": ["the", "help", "stick", "to", "like", "you", "here", "would", "on", "me", "big"]}, {"sent": "yes Francis I would like some cupcakes .", "words": ["some", "like", "would"]}, {"sent": "I thought we would make Emily Elizabeth I thought maybe you wanted to make her red .", "words": ["red", "we", "to", "her", "you", "would", "make"]}, {"sent": "would you like me to tell you a story ?", "words": ["to", "like", "you", "would", "me", "story", "a"]}, {"sent": "cock a doodle doo .", "words": ["a"]}, {"sent": "a big branch for Sarah a small branch for Percy and an old piece of ivy for Bill shell be back said Sarah back soon said Percy what did Bill say ?", "words": ["old", "what", "for", "back", "of", "be", "say", "and", "an", "did", "big", "a"]}, {"sent": "well find that bit in a minute .", "words": ["in", "that", "a", "find"]}, {"sent": "are you a lovely boy ?", "words": ["are", "a", "you"]}, {"sent": "shes a bit little to be having a cup .", "words": ["cup", "to", "be", "little", "a"]}, {"sent": "lets have a look .", "words": ["have", "look", "a"]}, {"sent": "you wanna to do a puzzle ?", "words": ["do", "to", "you", "puzzle", "wanna", "a"]}, {"sent": "its not a tree .", "words": ["tree", "not", "a"]}, {"sent": "is it a stairway ?", "words": ["it", "a", "is"]}, {"sent": "thats a very nice tree thats a big tree .", "words": ["tree", "nice", "big", "a"]}, {"sent": "thats a brush this is called blush .", "words": ["brush", "this", "is", "a"]}, {"sent": "are you a bad girl ?", "words": ["are", "bad", "a", "you"]}, {"sent": "but what we do alotof times we have a little cup under it .", "words": ["cup", "what", "do", "we", "little", "have", "under", "it", "but", "a"]}, {"sent": "this hats a little bit big he says .", "words": ["little", "this", "he", "big", "a"]}, {"sent": "just a different color .", "words": ["a"]}, {"sent": "youre a big girl .", "words": ["big", "a"]}, {"sent": "we pretended it was a plate and she just really just wanted to bite it .", "words": ["plate", "we", "was", "to", "and", "it", "bite", "she", "a"]}, {"sent": "they actually do a little MrBean Mini .", "words": ["they", "do", "little", "a"]}, {"sent": "whos a fishhead ?", "words": ["a"]}, {"sent": "it made a splash .", "words": ["it", "splash", "a"]}, {"sent": "theres a doggy on the side isnt it ?", "words": ["on", "the", "it", "a"]}, {"sent": "splendor a great brightness is around god like a tent a pavilion .", "words": ["around", "like", "is", "a"]}, {"sent": "a LittleMissMuffet LittleMissMuffet sat on her tuffet eating her curds and whey along came a .", "words": ["on", "her", "and", "a"]}, {"sent": "I was sleeping and she came crawling by me xxx clock and I told her tap xxx or play pen or make a xxx .", "words": ["clock", "was", "play", "play pen", "by", "and", "make", "pen", "me", "her", "she", "a"]}, {"sent": "we got a moon rock .", "words": ["rock", "moon", "a", "we"]}, {"sent": "theres a mommy and a daddy and some kids and a baby carriage and a van and a car and a playhouse and a pool and a couch and a chair and a slide .", "words": ["slide", "car", "pool", "couch", "chair", "and", "some", "a"]}, {"sent": "thats a bottle yes .", "words": ["bottle", "a"]}, {"sent": "its a little blue ball like that .", "words": ["like", "blue", "ball", "little", "that", "a"]}, {"sent": "thats the dripping water the faucets leaking a little .", "words": ["the", "little", "a", "water"]}, {"sent": "because you have eaten quite a lot of fruit havent you ?", "words": ["of", "a lot", "you", "have", "because", "a"]}, {"sent": "he needs to draw a balloon .", "words": ["to", "draw", "balloon", "he", "a"]}, {"sent": "um I have a lot of DoctorSeuss books I dont have that one though .", "words": ["of", "a lot", "have", "dont", "that", "a"]}, {"sent": "a drink in a box was it ?", "words": ["box", "drink", "was", "in", "it", "a"]}, {"sent": "I know well its a strange situation but uh dont feel bad about it .", "words": ["bad", "dont", "it", "about", "but", "a"]}, {"sent": "see it seems like almost a long time ago since we went that was only about a week or two ago but I dont remember our trip very well either hum .", "words": ["we", "was", "like", "long", "our", "dont", "it", "about", "see", "that", "but", "a"]}, {"sent": "that is a circle .", "words": ["that", "a", "is"]}, {"sent": "I used to make you giggle a lot when you were a little baby .", "words": ["when", "a lot", "to", "you", "little", "make", "were", "a"]}, {"sent": "and theyre all touch and feel books so every page has a little spot that you can feel .", "words": ["can", "all", "so", "touch", "and", "little", "you", "every", "that", "a"]}, {"sent": "thats not a penny thats just a piece of paper .", "words": ["of", "paper", "penny", "not", "a"]}, {"sent": "please may I have a piece .", "words": ["have", "a"]}, {"sent": "daddy got a puppet and mommy got a puppet .", "words": ["and", "a"]}, {"sent": "thats not a very good cow is it ?", "words": ["is", "it", "good", "not", "cow", "a"]}, {"sent": "I was going to buy you a swing but Nana and GrandadBealby came to stay and they gave us some money to buy the swing with .", "words": ["money", "the", "swing", "to", "was", "with", "you", "and", "they", "us", "some", "stay", "buy", "but", "a"]}, {"sent": "how about she hold a teacup ?", "words": ["how", "hold", "about", "she", "a"]}, {"sent": "a little Indian girl ?", "words": ["little", "a"]}, {"sent": "is a cutter tool .", "words": ["a", "is"]}, {"sent": "and a and do they get married ?", "words": ["do", "and", "get", "they", "a"]}, {"sent": "heres a dog who who do you want to have the dog ?", "words": ["the", "do", "to", "you", "have", "want to", "who", "dog", "a"]}, {"sent": "whats a a big party ?", "words": ["party", "big", "a"]}, {"sent": "take a little tiny piece like this .", "words": ["tiny", "like", "little", "this", "take", "a"]}, {"sent": "a sandwich ?", "words": ["sandwich", "a"]}, {"sent": "this is not a good pencil .", "words": ["is", "pencil", "this", "good", "not", "a"]}, {"sent": "but he was nervous and wobbled a bit .", "words": ["was", "and", "he", "but", "a"]}, {"sent": "thats a death defying look .", "words": ["look", "a"]}, {"sent": "that was a quick one huh ?", "words": ["was", "that", "a"]}, {"sent": "do you need a weewee ?", "words": ["do", "need", "a", "you"]}, {"sent": "and youre giving yourself a kiss .", "words": ["yourself", "kiss", "a", "and"]}, {"sent": "a different man .", "words": ["a"]}, {"sent": "right just a minute .", "words": ["a"]}, {"sent": "you gonna sing a song for Mommy ?", "words": ["sing", "for", "a", "you"]}, {"sent": "does he have a long tail ?", "words": ["does", "long", "have", "he", "a"]}, {"sent": "shall we draw the a ?", "words": ["the", "draw", "a", "we"]}, {"sent": "youre doing a loud trick ?", "words": ["loud", "a"]}, {"sent": "oh you gonna give the baby a drink ?", "words": ["the", "drink", "you", "give", "a"]}, {"sent": "there is a little teeny bunny right over here watching tv .", "words": ["bunny", "is", "little", "here", "there", "tv", "over", "a"]}, {"sent": "that is a lion .", "words": ["lion", "that", "a", "is"]}, {"sent": "that doesnt go there you dont put a radio in the garden you put a radio in the kitchen .", "words": ["the", "put", "go", "radio", "you", "dont", "there", "in", "garden", "kitchen", "that", "a"]}, {"sent": "well it might have been a squirrel or it could have been a bird .", "words": ["bird", "have", "it", "could", "squirrel", "a"]}, {"sent": "shh dont shout youve just had alright youve just had a big wee Thomas .", "words": ["dont", "big", "a"]}, {"sent": "youre going to put a blue one on .", "words": ["put", "to", "blue", "on", "a"]}, {"sent": "it is not a chain .", "words": ["it", "not", "a", "is"]}, {"sent": "hes just pulling his truck in a little bit further because hes blocking the road and the cars cant get past cant they ?", "words": ["the", "truck", "little", "and", "get", "in", "they", "his", "because", "a"]}, {"sent": "a boy .", "words": ["a"]}, {"sent": "youre pretending to be a duck now are you ?", "words": ["to", "be", "you", "are", "duck", "a"]}, {"sent": "could you gimme a napkin ?", "words": ["could", "napkin", "a", "you"]}, {"sent": "thats a squeaky scooter .", "words": ["a"]}, {"sent": "dont pull it a minute .", "words": ["dont", "it", "a", "pull"]}, {"sent": "what a big jump .", "words": ["what", "jump", "big", "a"]}, {"sent": "a triangle ?", "words": ["a"]}, {"sent": "well just do one more and then maybe well play a game .", "words": ["do", "game", "play", "and", "more", "then", "a"]}, {"sent": "is it a hen next ?", "words": ["it", "hen", "a", "is"]}, {"sent": "that is a slide .", "words": ["slide", "that", "a", "is"]}, {"sent": "I now what yall can build a house .", "words": ["can", "what", "house", "build", "a"]}, {"sent": "because we just decided to go on a boat .", "words": ["go", "we", "to", "on", "boat", "because", "a"]}, {"sent": "I dont want a vehicle driving on me at all .", "words": ["all", "dont", "at", "on", "me", "a"]}, {"sent": "thats a good elephant noise .", "words": ["elephant", "good", "a"]}, {"sent": "a party hat .", "words": ["hat", "party", "a"]}, {"sent": "youre building a tower ?", "words": ["a"]}, {"sent": "shall we read you a Maisie book for bedtime ?", "words": ["for", "we", "you", "read", "book", "a"]}, {"sent": "thats a pretty home .", "words": ["home", "pretty", "a"]}, {"sent": "just just wait a minute .", "words": ["wait", "a"]}, {"sent": "its a star .", "words": ["star", "a"]}, {"sent": "theres a quick ladder .", "words": ["ladder", "a"]}, {"sent": "with a book on one hand and a cup on my hat .", "words": ["cup", "my", "hand", "with", "and", "on", "hat", "book", "a"]}, {"sent": "are you a cheeky boy ?", "words": ["are", "a", "you"]}, {"sent": "um now I wuh a clerk .", "words": ["a"]}, {"sent": "a fine fable a folding xxx .", "words": ["fine", "a"]}, {"sent": "a seed and a pot .", "words": ["and", "a"]}, {"sent": "is Nicole going to give baby a bottle ?", "words": ["is", "to", "bottle", "give", "a"]}, {"sent": "right shall we have a look at another one ?", "words": ["we", "have", "another", "at", "look", "a"]}, {"sent": "do you wanna put some animals on your house now .", "words": ["do", "put", "your", "you", "house", "wanna", "on", "some"]}, {"sent": "no thats the fire house .", "words": ["the", "house"]}, {"sent": "that kind of a house is called a barn .", "words": ["is", "of", "house", "that", "a"]}, {"sent": "how big should the house be ?", "words": ["the", "how", "be", "house", "big"]}, {"sent": "and its not like bringing a playgroup to your house .", "words": ["your", "to", "like", "and", "house", "not", "a"]}, {"sent": "it was a house ?", "words": ["was", "it", "house", "a"]}, {"sent": "a little bit of window from the house .", "words": ["the", "of", "little", "house", "window", "a"]}, {"sent": "you you went to her house ?", "words": ["house", "to", "her", "you"]}, {"sent": "do you not want the house up ?", "words": ["the", "do", "up", "you", "house", "not"]}, {"sent": "okay Ill leave it open now go ahead look through the house and see what still needs to be put away .", "words": ["the", "what", "put", "go", "open", "to", "be", "and", "it", "house", "away", "look", "see"]}, {"sent": "where do you think you would see all of these things in your house ?", "words": ["where", "do", "think", "all", "these", "your", "of", "you", "in", "house", "would", "see"]}, {"sent": "we cant get to the house because youre sat in the way .", "words": ["the", "we", "to", "get", "in", "house", "because"]}, {"sent": "maybe its finally we get some fresh air in the house .", "words": ["the", "we", "get", "in", "house", "some"]}, {"sent": "no not at Craigs house .", "words": ["house", "not", "at"]}, {"sent": "you have never been to her house ?", "words": ["to", "you", "have", "house", "her"]}, {"sent": "house .", "words": ["house"]}, {"sent": "are you gonna make another house ?", "words": ["you", "another", "are", "house", "make"]}, {"sent": "I went down to my mothers house xxx you know dial the operator .", "words": ["the", "my", "to", "you", "house", "down"]}, {"sent": "but what do you think the vans doing at Sues house ?", "words": ["the", "what", "do", "think", "you", "house", "at", "but"]}, {"sent": "shes in her house .", "words": ["house", "in", "her"]}, {"sent": "Sonias house .", "words": ["house"]}, {"sent": "shes certainly seen sippy cups around the house for sure .", "words": ["the", "for", "house", "around"]}, {"sent": "she brought the house down .", "words": ["the", "down", "house", "she"]}, {"sent": "thats the house .", "words": ["the", "house"]}, {"sent": "is it a funny house ?", "words": ["it", "house", "a", "is"]}, {"sent": "this is inside inside the house .", "words": ["the", "is", "inside", "this", "house"]}, {"sent": "thats one reason why weve got strawberries in the house .", "words": ["the", "in", "house", "why"]}, {"sent": "weve got to go to Jos house because Mummy has just found a piece of paper that I think Jo would be interested in .", "words": ["think", "go", "of", "to", "paper", "be", "in", "house", "would", "that", "because", "a"]}, {"sent": "I aint going to dat boys house .", "words": ["to", "house"]}, {"sent": "this is the house .", "words": ["the", "this", "house", "is"]}, {"sent": "that means its a full house .", "words": ["house", "that", "full", "a"]}, {"sent": "is this Johns house hes building here ?", "words": ["here", "this", "house", "is"]}, {"sent": "he swims in his house ?", "words": ["his", "house", "in", "he"]}, {"sent": "you sit in the house and you can pop your head out to do the jigsaw .", "words": ["the", "can", "do", "pop", "your", "to", "head", "you", "and", "in", "house", "out", "sit"]}, {"sent": "you know who is coming to our house tomorrow ?", "words": ["is", "to", "our", "you", "house", "who"]}, {"sent": "that is all Daddy has in the house .", "words": ["the", "all", "is", "in", "house", "that"]}, {"sent": "and Ill blow your house in .", "words": ["your", "and", "in", "house", "blow"]}, {"sent": "this is a house .", "words": ["this", "house", "a", "is"]}, {"sent": "no were going to Nannys house .", "words": ["house", "to", "were"]}, {"sent": "watch you dont knock the house over .", "words": ["the", "watch", "you", "dont", "knock", "house", "over"]}, {"sent": "youre playing in your house .", "words": ["house", "in", "your"]}, {"sent": "when we go to their house .", "words": ["their", "go", "we", "to", "house", "when"]}, {"sent": "we can talk in here but were gonna stay in this side of the house .", "words": ["can", "the", "we", "of", "here", "this", "in", "house", "talk", "stay", "were", "but"]}, {"sent": "whats her grannys house look like ?", "words": ["house", "her", "like", "look"]}, {"sent": "you want an orange house ?", "words": ["an", "orange", "house", "you"]}, {"sent": "no they just have a little house .", "words": ["little", "have", "they", "house", "a"]}, {"sent": "each house can have a tree and a little person to live inside .", "words": ["can", "tree", "each", "to", "inside", "and", "have", "little", "house", "a"]}, {"sent": "grannys house and her daddys house .", "words": ["her", "house", "and"]}, {"sent": "right the way round there going over to Jos house and and um Kierans house wont it ?", "words": ["the", "to", "and", "there", "it", "house", "over"]}, {"sent": "they live at a house with a blue door .", "words": ["with", "blue", "they", "house", "at", "door", "a"]}, {"sent": "whose house has those lights on ?", "words": ["on", "house", "those"]}, {"sent": "well thats the other side would be the ate house .", "words": ["the", "be", "other", "house", "would"]}, {"sent": "thats a very tall house Carl .", "words": ["house", "a"]}, {"sent": "you go up into the house dont you ?", "words": ["the", "up", "into", "go", "you", "dont", "house"]}, {"sent": "you wanna build a house ?", "words": ["you", "house", "build", "wanna", "a"]}, {"sent": "how about the dog in the house ?", "words": ["the", "how", "in", "house", "about", "dog"]}, {"sent": "about to leave the house .", "words": ["the", "to", "house", "about"]}, {"sent": "lets go in the house .", "words": ["the", "in", "house", "go"]}, {"sent": "put Pinga in the house then .", "words": ["the", "put", "in", "house", "then"]}, {"sent": "no thats a house .", "words": ["house", "a"]}, {"sent": "Ill go back in the house and wait there .", "words": ["the", "wait", "back", "go", "and", "there", "in", "house"]}, {"sent": "who blew the house down ?", "words": ["the", "down", "house", "who"]}, {"sent": "thats Pierre and Maryses house .", "words": ["house", "and"]}, {"sent": "cooker in the house ?", "words": ["the", "in", "house"]}, {"sent": "your house didnt break down .", "words": ["house", "down", "break", "your"]}, {"sent": "are you going to make this house Dan ?", "words": ["to", "you", "this", "are", "house", "make"]}, {"sent": "part of the whole house .", "words": ["of", "the", "house"]}, {"sent": "who will live in the orange house Fraser ?", "words": ["the", "in", "orange", "house", "who", "will"]}, {"sent": "and then Tom can live in the house .", "words": ["can", "the", "and", "in", "house", "then"]}, {"sent": "yeah I think it goes in the house uhn do you ?", "words": ["the", "do", "think", "you", "in", "it", "house"]}, {"sent": "no he be in the house one uh them be he be in the xxx .", "words": ["the", "be", "them", "in", "he", "house"]}, {"sent": "whats happened to the house ?", "words": ["the", "to", "house"]}, {"sent": "its not a boat its a house .", "words": ["house", "boat", "not", "a"]}, {"sent": "Im sure that house will stand out for miles .", "words": ["for", "stand", "house", "out", "will", "that"]}, {"sent": "that says thats Otters house .", "words": ["house", "that"]}, {"sent": "well weve got plenty of chocolates actually in the house left from left over Christmas havent we ?", "words": ["the", "we", "of", "in", "house", "over"]}, {"sent": "you know youd love my house because Ive got a baby cat and then if you look out the window weve got a fire station opposite .", "words": ["the", "my", "if", "you", "and", "cat", "window", "house", "love", "look", "out", "then", "because", "a"]}, {"sent": "youve got a little house .", "words": ["house", "little", "a"]}, {"sent": "its going into your house .", "words": ["house", "your", "into"]}, {"sent": "well Felix looks like he might have jumped over the fence back to Linda and Brians house .", "words": ["the", "back", "to", "like", "and", "have", "he", "house", "over"]}, {"sent": "you want me to do a house ?", "words": ["do", "to", "you", "house", "me", "a"]}, {"sent": "yes youre in the house thats a funny ring .", "words": ["the", "in", "house", "a"]}, {"sent": "oh we have lotsof toys at our house .", "words": ["we", "our", "have", "house", "at"]}, {"sent": "um at Johns house the top of the house is Elizabeths room .", "words": ["the", "is", "of", "room", "house", "at"]}, {"sent": "theyre going to Sues house now .", "words": ["to", "house"]}, {"sent": "do you wanna do the house first ?", "words": ["the", "do", "first", "you", "house", "wanna"]}, {"sent": "yeah Ethan Im making a house here there .", "words": ["here", "there", "house", "a"]}, {"sent": "goes over here near my house .", "words": ["here", "house", "my", "over"]}, {"sent": "you have one on each floor of the house .", "words": ["the", "each", "of", "you", "have", "house", "on"]}, {"sent": "what is that thing Dingo brought into the house with him ?", "words": ["the", "what", "into", "is", "with", "him", "house", "that"]}, {"sent": "were in a house but I dont think were having a circus .", "words": ["think", "dont", "in", "house", "were", "but", "a"]}, {"sent": "its only a pretend house .", "words": ["pretend", "house", "a"]}, {"sent": "in the house and on the street how many different feet you meet .", "words": ["the", "how", "and", "you", "in", "house", "on", "street"]}, {"sent": "in the house .", "words": ["the", "in", "house"]}, {"sent": "Gregory xxx when can I go to your house ?", "words": ["can", "your", "go", "to", "house", "when"]}, {"sent": "hi you left your phone at the house .", "words": ["the", "your", "you", "house", "at"]}, {"sent": "shall we try another house ?", "words": ["another", "house", "try", "we"]}, {"sent": "well leave the banana right here in his house okay ?", "words": ["the", "here", "banana", "in", "house", "his"]}, {"sent": "oh there was a big house for the trains and there was a little house for the trains .", "words": ["the", "for", "was", "and", "little", "there", "house", "big", "a"]}, {"sent": "does your house look like that at home ?", "words": ["does", "your", "like", "at", "house", "look", "that", "home"]}, {"sent": "I dont think I did though did I ?", "words": ["dont", "think", "did"]}, {"sent": "I think Im gonna put this away actually .", "words": ["this", "think", "away", "put"]}, {"sent": "dont you think so ?", "words": ["dont", "think", "so", "you"]}, {"sent": "thats where I think half of your toys go .", "words": ["where", "think", "your", "go", "of"]}, {"sent": "what do you think I am your maid ?", "words": ["what", "do", "think", "your", "you", "am"]}, {"sent": "I dont think theres Douglas there .", "words": ["dont", "think", "there"]}, {"sent": "well I think he might slide off there Aran .", "words": ["off", "think", "slide", "there", "he"]}, {"sent": "I think youre starting with a cold arent you ?", "words": ["think", "with", "you", "cold", "a"]}, {"sent": "I think I got a cut on my back .", "words": ["think", "my", "cut", "back", "on", "a"]}, {"sent": "who is the baby think hes going to see on there ?", "words": ["the", "think", "is", "to", "there", "who", "on", "see"]}, {"sent": "but I think Ill be back again .", "words": ["be", "think", "back", "but"]}, {"sent": "I dont think theyre going to get cold any colder Thomas .", "words": ["think", "to", "any", "get", "dont", "cold"]}, {"sent": "whadyou think ?", "words": ["think"]}, {"sent": "so I think its as well to keep it inside the lorry .", "words": ["the", "think", "to", "so", "inside", "it"]}, {"sent": "I think its the one who who has finished all their cards first .", "words": ["the", "all", "think", "their", "first", "who"]}, {"sent": "I think were gonna hafta sew these .", "words": ["these", "think", "were"]}, {"sent": "now whaddya think we have in here ?", "words": ["think", "we", "have", "here", "in"]}, {"sent": "oh I dont think so poppet .", "words": ["dont", "think", "so"]}, {"sent": "do you think Bertiethebus is taking passengers to the zoo ?", "words": ["the", "do", "think", "zoo", "is", "to", "you"]}, {"sent": "I think Ill just go and get it Thomas .", "words": ["think", "go", "and", "get", "it"]}, {"sent": "but for this one I think I named it before he did .", "words": ["think", "for", "this", "it", "he", "did", "but"]}, {"sent": "I think wed better sit down here because well fall .", "words": ["better", "think", "fall", "here", "sit", "down", "because"]}, {"sent": "I think Daddy might be home .", "words": ["be", "think", "home"]}, {"sent": "I think when we next go shopping Thomas .", "words": ["go", "think", "when", "we"]}, {"sent": "you think its okay ?", "words": ["think", "you"]}, {"sent": "so I dont think hell be going home to his Mummy .", "words": ["think", "so", "be", "to", "dont", "his", "home"]}, {"sent": "and that one I think is a french golden delicious because .", "words": ["think", "is", "and", "that", "because", "a"]}, {"sent": "I dont think she knows who Rocky is .", "words": ["think", "is", "dont", "who", "she"]}, {"sent": "here I think hes getting angry with Mathilda .", "words": ["here", "think", "with"]}, {"sent": "what dya think about that ?", "words": ["what", "think", "that", "about"]}, {"sent": "I think you do .", "words": ["do", "think", "you"]}, {"sent": "dyou think hes happy because he read his book right ?", "words": ["think", "happy", "read", "he", "his", "book", "because"]}, {"sent": "you think so too ?", "words": ["too", "so", "think", "you"]}, {"sent": "what do you think her favorite thing was to play with today ?", "words": ["what", "do", "think", "was", "to", "play", "with", "you", "her"]}, {"sent": "now I think were missing the headlights .", "words": ["the", "think", "were"]}, {"sent": "I think we need more .", "words": ["think", "more", "need", "we"]}, {"sent": "yep that can too I think .", "words": ["can", "think", "that", "too"]}, {"sent": "you think Mommy had a good idea huh ?", "words": ["think", "good", "a", "you"]}, {"sent": "I dont think theres one in the house .", "words": ["the", "think", "dont", "in", "house"]}, {"sent": "I think Becky was the last person who had the brush .", "words": ["the", "think", "brush", "was", "last", "who"]}, {"sent": "I think he said .", "words": ["think", "he"]}, {"sent": "I dont think it would like being shut in the red box .", "words": ["the", "red", "think", "box", "like", "dont", "in", "it", "would"]}, {"sent": "do you think Purdie would like some of this cream ?", "words": ["do", "think", "of", "like", "you", "this", "would", "some"]}, {"sent": "oh I think were gonna make the best cookies Ive ever tasted .", "words": ["the", "think", "make", "were"]}, {"sent": "what do you think ?", "words": ["what", "do", "think", "you"]}, {"sent": "um which of these foods do you think shell be having in the future or do you guys have at home that she doesnt have ?", "words": ["the", "these", "do", "think", "of", "be", "you", "have", "in", "that", "at", "which", "she", "home"]}, {"sent": "they run I think they do run around .", "words": ["do", "think", "they", "run", "around"]}, {"sent": "what do you think is in the pitcher ?", "words": ["the", "what", "do", "think", "is", "you", "in"]}, {"sent": "xxx think its funny .", "words": ["think"]}, {"sent": "so I put it in a special pot and now I think I know what it is .", "words": ["what", "put", "think", "is", "so", "and", "in", "it", "a"]}, {"sent": "do you think it was the rabbit ?", "words": ["the", "do", "think", "was", "you", "it"]}, {"sent": "what you think ?", "words": ["what", "think", "you"]}, {"sent": "I think its got eight legs .", "words": ["think"]}, {"sent": "I think we should just carry on with the rest of it .", "words": ["the", "think", "we", "of", "with", "carry", "it", "on"]}, {"sent": "dont think there are very many in that .", "words": ["think", "dont", "in", "are", "there", "that"]}, {"sent": "it was one pound forty five I think to come .", "words": ["to", "was", "think", "it"]}, {"sent": "I think youre doing a good job all by yourself .", "words": ["all", "think", "yourself", "by", "good", "a"]}, {"sent": "think they hafta clean up the water from inside as the windows have been open ?", "words": ["clean", "up", "think", "the", "open", "inside", "have", "they", "water"]}, {"sent": "I think youve got this one sorted out havent you ?", "words": ["this", "think", "out", "you"]}, {"sent": "I think theyre all in the chicken coop that you made .", "words": ["the", "all", "think", "you", "in", "chicken", "that"]}, {"sent": "do you think theyre ready to go away yet ?", "words": ["do", "think", "go", "to", "you", "away"]}, {"sent": "but I think I made the strap a bit long but it wont matter theres a bit hanging out .", "words": ["the", "think", "long", "it", "out", "but", "a"]}, {"sent": "what do you think ?", "words": ["what", "do", "think", "you"]}, {"sent": "do you think ?", "words": ["do", "think", "you"]}, {"sent": "would the dog like to eat any of this food do you think ?", "words": ["the", "do", "think", "of", "to", "any", "like", "you", "eat", "this", "food", "would", "dog"]}, {"sent": "so I dont think Sam gets too upset when Julian tells him off .", "words": ["off", "think", "so", "dont", "him", "too", "when"]}, {"sent": "I dont think he can talk .", "words": ["can", "think", "dont", "he", "talk"]}, {"sent": "you wanna get some from mommys room I think shes got better books in her room .", "words": ["better", "think", "you", "get", "in", "room", "wanna", "some", "her"]}, {"sent": "I think the truck will be faster dont you ?", "words": ["the", "think", "be", "truck", "you", "dont", "will"]}, {"sent": "I dont know if there are any other words I dont think there are oh .", "words": ["think", "if", "any", "other", "dont", "there", "are"]}, {"sent": "ah you think youre so proud huh .", "words": ["so", "think", "you"]}, {"sent": "do you think he eats lunch in Boston ?", "words": ["do", "think", "you", "in", "he"]}, {"sent": "what do you think ?", "words": ["what", "do", "think", "you"]}, {"sent": "you see a dustbin man light you think ?", "words": ["think", "light", "you", "see", "a"]}, {"sent": "I think its sposta be a rooster .", "words": ["rooster", "be", "think", "a"]}, {"sent": "I dont think so .", "words": ["dont", "think", "so"]}, {"sent": "did not think so .", "words": ["so", "not", "did", "think"]}, {"sent": "we think .", "words": ["think", "we"]}, {"sent": "what do you think ?", "words": ["what", "do", "think", "you"]}, {"sent": "now Im just going to put the bread away while I think about it .", "words": ["the", "put", "bread", "think", "to", "away", "it", "about"]}, {"sent": "I didnt think youd want the skin on .", "words": ["the", "think", "on"]}, {"sent": "oh I think so .", "words": ["so", "think"]}, {"sent": "I think its cold .", "words": ["think", "cold"]}, {"sent": "do you think she knows that its a is it familiar do you have anything like that at home or ?", "words": ["do", "think", "is", "like", "you", "have", "it", "at", "home", "that", "she", "a"]}, {"sent": "do you think ?", "words": ["do", "think", "you"]}, {"sent": "anyoned think you was hungry .", "words": ["was", "think", "hungry", "you"]}, {"sent": "I think this is a great place .", "words": ["this", "think", "a", "is"]}, {"sent": "I dont really think you do .", "words": ["dont", "think", "do", "you"]}, {"sent": "think that goes there ?", "words": ["there", "think", "that"]}, {"sent": "what do you think this is for Thomas ?", "words": ["what", "do", "think", "for", "is", "you", "this"]}, {"sent": "I think this there .", "words": ["this", "think", "there"]}, {"sent": "I dont think Dumbo was in the movie .", "words": ["the", "think", "was", "dont", "in"]}, {"sent": "think what love ?", "words": ["what", "think", "love"]}, {"sent": "no I didnt think it would fit either .", "words": ["think", "it", "fit", "would"]}, {"sent": "he is I think hes eating plants Catherine .", "words": ["think", "he", "is"]}, {"sent": "you think so do you ?", "words": ["so", "think", "do", "you"]}, {"sent": "which toys do you think Amelia seemed most interested in ?", "words": ["do", "think", "you", "in", "which"]}, {"sent": "now I think you get the train in look .", "words": ["the", "think", "train", "you", "get", "in", "look"]}, {"sent": "I think he .", "words": ["think", "he"]}, {"sent": "I think this way .", "words": ["this", "think"]}, {"sent": "warmed it .", "words": ["it"]}, {"sent": "it doesnt hafta be for little girls but I think its um I think its a barbie doll one .", "words": ["think", "for", "be", "doll", "little", "it", "but", "a"]}, {"sent": "so perhaps shes going to iron it .", "words": ["to", "so", "it"]}, {"sent": "it would be a good fun thing to sort into colors .", "words": ["into", "be", "to", "it", "good", "would", "a"]}, {"sent": "look you can stick it up like this .", "words": ["can", "up", "stick", "like", "you", "this", "it", "look"]}, {"sent": "what is it ?", "words": ["what", "it", "is"]}, {"sent": "hold it .", "words": ["it", "hold"]}, {"sent": "and how will you fix it ?", "words": ["how", "fix", "you", "and", "it", "will"]}, {"sent": "it cant can it ?", "words": ["can", "it"]}, {"sent": "and its not very stable is it ?", "words": ["it", "not", "is", "and"]}, {"sent": "thats it .", "words": ["it"]}, {"sent": "and um you know the way when a babys born you buy it a xxx ?", "words": ["the", "you", "and", "it", "when", "buy", "a"]}, {"sent": "you dont want it anymore ?", "words": ["dont", "it", "you"]}, {"sent": "what is it ?", "words": ["what", "it", "is"]}, {"sent": "thats a house isnt it ?", "words": ["it", "house", "a"]}, {"sent": "well see it stopped turning because youre holding it and it cant turn .", "words": ["see", "it", "because", "and"]}, {"sent": "after a few minutes brewing pour it pour it into Mummys special cup .", "words": ["cup", "into", "pour", "it", "a"]}, {"sent": "yeah I see it .", "words": ["see", "it"]}, {"sent": "think it is a driver ?", "words": ["think", "it", "a", "is"]}, {"sent": "so it makes sense if Mummy can have a couple of hours on her own doing jobs you dont like .", "words": ["can", "if", "of", "so", "like", "you", "have", "dont", "it", "on", "her", "a"]}, {"sent": "it looks black to me .", "words": ["black", "to", "me", "it"]}, {"sent": "this is nasty isnt it ?", "words": ["this", "it", "is"]}, {"sent": "dont play with it .", "words": ["dont", "play", "with", "it"]}, {"sent": "it means blocks ?", "words": ["it"]}, {"sent": "what did you play in it the last day you were up ?", "words": ["the", "what", "up", "last", "play", "you", "in", "it", "did", "were"]}, {"sent": "it was the soap I said .", "words": ["the", "was", "it", "soap"]}, {"sent": "we keep it on .", "words": ["on", "it", "we"]}, {"sent": "can we make it again sometime ?", "words": ["can", "it", "make", "we"]}, {"sent": "you throw it to Mommy .", "words": ["to", "it", "throw", "you"]}, {"sent": "I thought it was a choochoo train .", "words": ["train", "was", "it", "a"]}, {"sent": "you gotta be really careful because Ryan brought this and you dont want it to get mushed up like it could get ripped right ?", "words": ["up", "be", "to", "like", "you", "and", "this", "dont", "it", "get", "careful", "could", "because"]}, {"sent": "will you read it to me ?", "words": ["to", "you", "it", "read", "will", "me"]}, {"sent": "lets stop and check it .", "words": ["it", "stop", "and"]}, {"sent": "thats very good isnt it ?", "words": ["it", "good"]}, {"sent": "do you want it now youve asked for it ?", "words": ["do", "it", "for", "you"]}, {"sent": "I cant get it to come out .", "words": ["get", "to", "it", "out"]}, {"sent": "you know when it jitters and misses out words .", "words": ["and", "you", "it", "out", "when"]}, {"sent": "is this it ?", "words": ["this", "it", "is"]}, {"sent": "what is it ?", "words": ["what", "it", "is"]}, {"sent": "you know why it was hard for me ?", "words": ["for", "was", "you", "it", "hard", "me", "why"]}, {"sent": "well look at it when we leave okay ?", "words": ["look", "we", "it", "at", "when"]}, {"sent": "you eat it .", "words": ["eat", "it", "you"]}, {"sent": "yeah it goes with the sand the tools for the sand .", "words": ["the", "it", "with", "for"]}, {"sent": "oh there it is .", "words": ["there", "it", "is"]}, {"sent": "youve put a big ding in it now .", "words": ["put", "in", "it", "big", "a"]}, {"sent": "you can finish it off .", "words": ["can", "off", "finish", "you", "it"]}, {"sent": "throw it toilet .", "words": ["it", "throw"]}, {"sent": "yes it is little one .", "words": ["it", "little", "is"]}, {"sent": "let us see if we can find it .", "words": ["can", "if", "we", "us", "it", "find", "see"]}, {"sent": "ah its not fair uhn is it ?", "words": ["it", "not", "is"]}, {"sent": "is it called Rover ?", "words": ["it", "is"]}, {"sent": "it is upside down .", "words": ["down", "it", "is"]}, {"sent": "I ate it .", "words": ["it"]}, {"sent": "thats it ?", "words": ["it"]}, {"sent": "if you turn it on .", "words": ["on", "it", "if", "you"]}, {"sent": "put it in the oven .", "words": ["the", "put", "in", "it", "oven"]}, {"sent": "well mix it up ?", "words": ["up", "it"]}, {"sent": "and the winds blowing it .", "words": ["the", "it", "and"]}, {"sent": "are you going to put it in your pan ?", "words": ["put", "your", "to", "you", "in", "are", "it"]}, {"sent": "Im doing it .", "words": ["it"]}, {"sent": "I dont know see you can turn it around and find where itll go Ill help ya .", "words": ["can", "where", "help", "go", "find", "you", "and", "dont", "it", "around", "see"]}, {"sent": "let mommy do it first .", "words": ["do", "it", "first"]}, {"sent": "where is it ?", "words": ["where", "it", "is"]}, {"sent": "you dont wanna touch it .", "words": ["touch", "you", "dont", "it", "wanna"]}, {"sent": "it keeps going very sunny and then it clouds over doesnt it ?", "words": ["then", "it", "over", "and"]}, {"sent": "that bunny ones nice isnt it ?", "words": ["nice", "it", "bunny", "that"]}, {"sent": "it must make it easier for you guys xxx .", "words": ["it", "make", "for", "you"]}, {"sent": "well what about it .", "words": ["what", "it", "about"]}, {"sent": "that was your sister wasnt it ?", "words": ["your", "was", "it", "that"]}, {"sent": "give it a quick shake .", "words": ["give", "it", "shake", "a"]}, {"sent": "it youre going to sit that way .", "words": ["to", "sit", "it", "that"]}, {"sent": "it was WinniethePooh on the other one .", "words": ["the", "was", "other", "it", "on"]}, {"sent": "where is the necklace with a heart on it ?", "words": ["where", "the", "necklace", "is", "with", "it", "on", "a"]}, {"sent": "I thought it was already locked .", "words": ["was", "it"]}, {"sent": "well actually it says green .", "words": ["it", "green"]}, {"sent": "what was it that you wanted to play ?", "words": ["what", "was", "to", "play", "you", "it", "that"]}, {"sent": "what was it again ?", "words": ["was", "what", "it"]}, {"sent": "it cant be that big .", "words": ["be", "big", "it", "that"]}, {"sent": "mop it up .", "words": ["up", "it", "mop"]}, {"sent": "is it big boots ?", "words": ["boots", "it", "big", "is"]}, {"sent": "its it melts doesnt it ?", "words": ["it"]}, {"sent": "I think you oughta eat it .", "words": ["eat", "think", "it", "you"]}, {"sent": "yeah it it will go on but youve got it the wrong way up .", "words": ["the", "up", "go", "on", "it", "will", "but"]}, {"sent": "what is it ?", "words": ["what", "it", "is"]}, {"sent": "whoa it rolled the wrong way .", "words": ["the", "it"]}, {"sent": "well put a tissue around it .", "words": ["put", "tissue", "it", "around", "a"]}, {"sent": "xxx leave it now .", "words": ["it"]}, {"sent": "it wont ?", "words": ["it"]}, {"sent": "I see it .", "words": ["see", "it"]}, {"sent": "oh tell me about it .", "words": ["me", "it", "about"]}, {"sent": "or is it Sooty ?", "words": ["it", "is"]}, {"sent": "good one for the beach because its got fish on it .", "words": ["the", "for", "beach", "it", "good", "fish", "on", "because"]}, {"sent": "just in case it falls off .", "words": ["off", "in", "it"]}, {"sent": "is it this one next ?", "words": ["this", "it", "is"]}, {"sent": "was it sposta bounce ?", "words": ["was", "it"]}, {"sent": "because it has a spout .", "words": ["it", "because", "a"]}, {"sent": "it must be mended now .", "words": ["be", "it"]}, {"sent": "I wrung it out and hung it to dry .", "words": ["to", "dry", "and", "it", "out"]}, {"sent": "does it fit ?", "words": ["does", "fit", "it"]}, {"sent": "thats a nice horse isnt it ?", "words": ["horse", "it", "nice", "a"]}, {"sent": "there was a lady waving at a little girl .", "words": ["was", "little", "there", "at", "a"]}, {"sent": "well that was time to get up anyway uhn wasnt it ?", "words": ["up", "to", "was", "get", "it", "that"]}, {"sent": "this is a story that I was a little bit a little bit I dont know if Id say angry about but I was a little bit hm remember when we were going to Chicago and I told the lady not to give you the toy .", "words": ["the", "about", "when", "not", "were", "that", "if", "is", "was", "to", "little", "and", "dont", "you", "toy", "a", "give", "we", "say", "this", "story", "but"]}, {"sent": "it really was .", "words": ["was", "it"]}, {"sent": "and if I was a fireman I wouldnt help you out in a fire if thats your attitude .", "words": ["help", "your", "if", "was", "and", "you", "in", "out", "a"]}, {"sent": "when somebody was hanging washing out on the line ?", "words": ["the", "was", "out", "on", "when"]}, {"sent": "oh no that was lovely .", "words": ["was", "that"]}, {"sent": "it was in your Christmas stocking I think Becky .", "words": ["think", "your", "was", "in", "it"]}, {"sent": "Mummy was going to wrap Jonathans present as well .", "words": ["to", "was", "present"]}, {"sent": "mm okay I think well start this up again later that was just seventeen minutes on Wednesday July ninth .", "words": ["up", "think", "was", "this", "on", "that"]}, {"sent": "um he thought the one thing was a puzzle the magnet board was a puzzle and when he found out it wasnt he didnt care .", "words": ["the", "was", "and", "it", "puzzle", "he", "out", "when", "a"]}, {"sent": "I was trying to figure out if you knew where cheese came from .", "words": ["where", "if", "to", "was", "you", "out", "cheese"]}, {"sent": "I xxx I wasnt I was talking to Jt .", "words": ["to", "was"]}, {"sent": "but all I meant was that it was all a very .", "words": ["all", "was", "it", "that", "but", "a"]}, {"sent": "was that a bit too heavy for that darling ?", "words": ["for", "was", "too", "heavy", "that", "a"]}, {"sent": "my story will be that once upon a time MissPiggy was sitting by the window with some balloons .", "words": ["the", "my", "be", "was", "with", "by", "window", "will", "some", "story", "that", "a"]}, {"sent": "where else was the dough ?", "words": ["where", "was", "the"]}, {"sent": "was there anything different from last time ?", "words": ["was", "last", "there"]}, {"sent": "what was that ?", "words": ["was", "what", "that"]}, {"sent": "and the other was a Christmas card from Jersey .", "words": ["the", "was", "other", "and", "a"]}, {"sent": "was the other water dirty ?", "words": ["the", "was", "other", "dirty", "water"]}, {"sent": "and then Isabel said she was too busy and I think Bethany was a little bit frightened by you both because you and Isabel know each other very well and you were running around werent you ?", "words": ["think", "around", "each", "was", "and", "little", "by", "you", "other", "too", "then", "were", "she", "because", "a"]}, {"sent": "perhaps it was that big tiger .", "words": ["was", "that", "it", "tiger", "big"]}, {"sent": "first to the wedding was a little tiny tick .", "words": ["the", "tiny", "to", "was", "first", "little", "a"]}, {"sent": "to you because it was your paper that she tore she didnt mean to tear it .", "words": ["your", "was", "to", "paper", "tear", "you", "it", "that", "she", "because"]}, {"sent": "my that was a short call .", "words": ["was", "my", "that", "a"]}, {"sent": "and he was fuming .", "words": ["was", "he", "and"]}, {"sent": "I was just giving it to him .", "words": ["to", "was", "him", "it"]}, {"sent": "and this is Grover wishing that he was a bird .", "words": ["is", "was", "bird", "and", "this", "he", "that", "a"]}, {"sent": "and so he would just crash them into eachother and crash xxx Imean like I actually hadta hide them for awhile because I was like I cant take the crashing anymore .", "words": ["the", "for", "into", "hide", "so", "was", "them", "like", "and", "he", "would", "take", "because"]}, {"sent": "yeah he was driving .", "words": ["was", "he"]}, {"sent": "lets see what was that ?", "words": ["was", "see", "what", "that"]}, {"sent": "who was driving the train ?", "words": ["the", "was", "train", "who"]}, {"sent": "yeah the little one he was holding .", "words": ["the", "was", "he", "little"]}, {"sent": "that was very good then Fraser .", "words": ["was", "then", "that", "good"]}, {"sent": "what was it ?", "words": ["was", "what", "it"]}, {"sent": "but he was in here .", "words": ["was", "here", "in", "he", "but"]}, {"sent": "but that might have been because he was facing them .", "words": ["was", "them", "have", "he", "but", "that", "because"]}, {"sent": "because it was an awful lot shorter on her than it was on the other wee girl .", "words": ["the", "was", "other", "an", "it", "on", "her", "because"]}, {"sent": "Mummy wasnt happy was she ?", "words": ["was", "she", "happy"]}, {"sent": "it was really big .", "words": ["was", "it", "big"]}, {"sent": "Smudge tried to get underneath the gate and got stuck because he was too fat wasnt he ?", "words": ["the", "to", "was", "and", "get", "stuck", "he", "too", "because"]}, {"sent": "know whose house that was at that was at xxx .", "words": ["was", "that", "house", "at"]}, {"sent": "I was just coming to eat it because it was in the egg box .", "words": ["the", "box", "egg", "to", "was", "eat", "in", "it", "because"]}, {"sent": "well he was asking for it .", "words": ["was", "for", "he", "it"]}, {"sent": "what color was a crow outside ?", "words": ["was", "what", "outside", "a"]}, {"sent": "what do you think he was doing ?", "words": ["what", "do", "think", "was", "you", "he"]}, {"sent": "she wasnt completely trained until she was .", "words": ["was", "she"]}, {"sent": "who was there this morning ?", "words": ["was", "there", "this", "who"]}, {"sent": "I was throwing up eight times a day xxx .", "words": ["up", "was", "a"]}, {"sent": "he was so hungry he was xxx in the food .", "words": ["the", "was", "so", "hungry", "in", "food", "he"]}, {"sent": "if this was more of a powder blue this would be UCLAs colors .", "words": ["if", "of", "was", "be", "blue", "this", "more", "would", "a"]}, {"sent": "and he was there .", "words": ["was", "he", "there", "and"]}, {"sent": "yup it was a happy ending .", "words": ["was", "it", "happy", "a"]}, {"sent": "but all I did was stepped on that little wire and the water just shot out .", "words": ["the", "all", "was", "little", "and", "did", "out", "water", "on", "that", "but"]}, {"sent": "it was nice .", "words": ["nice", "was", "it"]}, {"sent": "what was it ?", "words": ["was", "what", "it"]}, {"sent": "was it AuntyLiz ?", "words": ["was", "it"]}, {"sent": "she was there .", "words": ["was", "she", "there"]}, {"sent": "but what was it ?", "words": ["was", "what", "it", "but"]}, {"sent": "it was called a helmet .", "words": ["was", "it", "a"]}, {"sent": "it was too fast .", "words": ["was", "fast", "it", "too"]}, {"sent": "a party was held .", "words": ["was", "party", "a"]}, {"sent": "yeah she was kinda xxx different things this time .", "words": ["was", "she", "this"]}, {"sent": "Dipsy that was very naughty because that brick hit Po and it hurt her and Po is now crying .", "words": ["is", "hurt", "was", "her", "and", "it", "hit", "that", "naughty", "because"]}, {"sent": "what was that ?", "words": ["was", "what", "that"]}, {"sent": "was it not Jwwws birthday party ?", "words": ["was", "it", "not", "party"]}, {"sent": "yeah it was uh the red light kept coming on and and I th I didnt know why it was doing that .", "words": ["the", "red", "light", "was", "and", "it", "on", "that", "why"]}, {"sent": "that was like .", "words": ["was", "that", "like"]}, {"sent": "but now he obviously knows what it is because he was trying to get the liquid out haha .", "words": ["the", "what", "is", "was", "to", "because", "get", "it", "he", "out", "but"]}, {"sent": "because it was such a nice morning .", "words": ["was", "nice", "it", "because", "a"]}, {"sent": "it had a funny kind of a hat and was sitting all by itself on an old suitcase near the lost property office .", "words": ["old", "all", "the", "of", "was", "and", "by", "on", "it", "an", "hat", "a"]}, {"sent": "that was all I was after .", "words": ["all", "was", "that"]}, {"sent": "Tomkitten was quite unable to jump when walking upon his hind legs and trousers .", "words": ["to", "was", "and", "jump", "his", "when"]}, {"sent": "was that fun ?", "words": ["was", "that"]}, {"sent": "was it a present ?", "words": ["was", "it", "present", "a"]}, {"sent": "he was called Thomas .", "words": ["was", "he"]}, {"sent": "and I thought that it was so funny because Julia is Gabriellas best and oldest friend .", "words": ["is", "was", "so", "and", "it", "that", "because"]}, {"sent": "oh that was good .", "words": ["was", "that", "good"]}, {"sent": "juice was it ?", "words": ["was", "juice", "it"]}, {"sent": "it was just over .", "words": ["was", "it", "over"]}, {"sent": "now find the pink one because that was the next biggest .", "words": ["the", "was", "find", "that", "because"]}, {"sent": "and was she awake ?", "words": ["was", "awake", "she", "and"]}, {"sent": "was it sleeping ?", "words": ["was", "it"]}, {"sent": "there was something to straighten up the grass once it has been cut .", "words": ["up", "the", "cut", "was", "to", "there", "it", "grass"]}, {"sent": "I mean she was interested in the peekabye peekababy but after like the first few pages it was sort of like okay you know I mean she was sort of looking around for other things so .", "words": ["the", "for", "of", "was", "so", "first", "like", "you", "other", "in", "it", "around", "she", "but"]}, {"sent": "and thats how it was and thats how it still is .", "words": ["how", "is", "was", "and", "it"]}, {"sent": "well I was trying to be very smart the other day and um .", "words": ["the", "to", "was", "be", "other", "and"]}, {"sent": "it was coming up to his birthday .", "words": ["up", "was", "to", "it", "his"]}, {"sent": "oh I might have known what it was that was in .", "words": ["what", "was", "have", "in", "it", "that"]}, {"sent": "and she was gonna pick her up you see the days that I work .", "words": ["up", "the", "work", "pick", "was", "and", "you", "that", "see", "her", "she"]}, {"sent": "he was so little .", "words": ["was", "so", "he", "little"]}, {"sent": "mine was number five .", "words": ["was", "mine"]}, {"sent": "who was there ?", "words": ["was", "there", "who"]}, {"sent": "um hed never seen a donut he called the donut a bagel because that was his closet frame of reference .", "words": ["donut", "the", "of", "was", "closet", "he", "his", "that", "because", "a"]}, {"sent": "I was pretending this was the triangle .", "words": ["the", "was", "this"]}, {"sent": "they thought miss spider was dinner .", "words": ["was", "they"]}, {"sent": "what was it ?", "words": ["was", "what", "it"]}, {"sent": "thats what I was trying to do .", "words": ["was", "what", "do", "to"]}, {"sent": "have you finished reading your Noddy book ?", "words": ["have", "book", "your", "you"]}, {"sent": "the pudgy book of toy .", "words": ["the", "book", "toy", "of"]}, {"sent": "no its Mummys book .", "words": ["book"]}, {"sent": "you want me to move the book ?", "words": ["the", "to", "you", "book", "me"]}, {"sent": "xxx go get a book you wanna get mommy a book ?", "words": ["go", "you", "get", "wanna", "book", "a"]}, {"sent": "not really a book reader today no ?", "words": ["book", "not", "a"]}, {"sent": "you dont want the book ?", "words": ["the", "dont", "book", "you"]}, {"sent": "isnt that a funny book ?", "words": ["book", "that", "a"]}, {"sent": "okay lets go read the pancake book .", "words": ["the", "pancake", "go", "read", "book"]}, {"sent": "the book ?", "words": ["the", "book"]}, {"sent": "this is Laras book .", "words": ["book", "this", "is"]}, {"sent": "look at the book .", "words": ["the", "book", "at", "look"]}, {"sent": "what about the book ?", "words": ["the", "what", "book", "about"]}, {"sent": "this is a counting book .", "words": ["book", "this", "a", "is"]}, {"sent": "why dont you go get another book ?", "words": ["go", "you", "get", "dont", "another", "book", "why"]}, {"sent": "thats a bit like your Thomas book isnt it when theyre taking all the animals to the wildlife park .", "words": ["the", "all", "your", "to", "like", "it", "park", "book", "when", "a"]}, {"sent": "shall we show Caroline the LetterLand book ?", "words": ["show", "the", "book", "we"]}, {"sent": "I think theres a fox in that book .", "words": ["think", "in", "book", "that", "a"]}, {"sent": "I dont have this book at home why dont you show me this book ?", "words": ["show", "you", "have", "dont", "this", "at", "book", "me", "why", "home"]}, {"sent": "hell sit still for like half a book occasionally a whole book .", "words": ["for", "like", "book", "sit", "a"]}, {"sent": "wheres your special book ?", "words": ["book", "your"]}, {"sent": "please dont pull on the book .", "words": ["the", "pull", "dont", "on", "book"]}, {"sent": "were gonna read this book .", "words": ["book", "this", "were", "read"]}, {"sent": "would you like to read another book ?", "words": ["to", "like", "you", "another", "read", "would", "book"]}, {"sent": "I cant see your coloring book .", "words": ["book", "see", "your"]}, {"sent": "what the address book ?", "words": ["the", "what", "book"]}, {"sent": "George and Martha or Lyle or the mouse book or Williams dog or would I have a friend at school .", "words": ["the", "school", "and", "have", "at", "would", "book", "dog", "mouse", "a"]}, {"sent": "you reading the book ?", "words": ["the", "book", "you"]}, {"sent": "of the book .", "words": ["of", "the", "book"]}, {"sent": "let us go this way because this is the end of the book .", "words": ["the", "is", "go", "of", "us", "this", "book", "because"]}, {"sent": "oh how about a book ?", "words": ["book", "how", "about", "a"]}, {"sent": "look let us look in the book here .", "words": ["the", "here", "us", "in", "look", "book"]}, {"sent": "here well turn the table around sothat you can see the book with mummy .", "words": ["the", "can", "table", "with", "you", "here", "around", "book", "see"]}, {"sent": "how about you go get another book then ?", "words": ["how", "go", "you", "get", "another", "about", "book", "then"]}, {"sent": "do you wanna read the other Toot and Puddle book ?", "words": ["the", "do", "you", "other", "and", "read", "wanna", "book"]}, {"sent": "thats a book you seen the last time .", "words": ["the", "last", "you", "book", "a"]}, {"sent": "you read this book to me .", "words": ["to", "you", "this", "read", "book", "me"]}, {"sent": "the other book is on the floor .", "words": ["the", "is", "other", "on", "book"]}, {"sent": "book ?", "words": ["book"]}, {"sent": "are there any cars in that book that look like the two cars you have ?", "words": ["the", "any", "like", "you", "have", "there", "in", "are", "look", "book", "that"]}, {"sent": "whats that book ?", "words": ["book", "that"]}, {"sent": "whats whats that book about ?", "words": ["book", "that", "about"]}, {"sent": "this book ?", "words": ["book", "this"]}, {"sent": "yeah thats a very beautiful book .", "words": ["book", "a"]}, {"sent": "can you come and read this book for all your friends ?", "words": ["can", "all", "for", "your", "and", "you", "this", "read", "book"]}, {"sent": "you wanna read a train book ?", "words": ["train", "you", "read", "wanna", "book", "a"]}, {"sent": "do you wanna look at that book .", "words": ["do", "you", "at", "wanna", "look", "book", "that"]}, {"sent": "sounds yours book at home huh ?", "words": ["book", "home", "at"]}, {"sent": "last week with the book .", "words": ["the", "book", "last", "with"]}, {"sent": "xxx book .", "words": ["book"]}, {"sent": "he does like the book theres also bookbag I hear .", "words": ["the", "does", "like", "hear", "he", "book"]}, {"sent": "a baby in the book .", "words": ["the", "book", "in", "a"]}, {"sent": "you done with the book ?", "words": ["the", "book", "with", "you"]}, {"sent": "you wanna look at a book ?", "words": ["you", "at", "wanna", "look", "book", "a"]}, {"sent": "another book ?", "words": ["another", "book"]}, {"sent": "book .", "words": ["book"]}, {"sent": "you wanna get another book for me to read ?", "words": ["for", "to", "you", "get", "another", "read", "wanna", "book", "me"]}, {"sent": "the book makes sounds .", "words": ["the", "book"]}, {"sent": "okay lets put the crayons in the box and well take a look at a book .", "words": ["the", "put", "box", "and", "in", "at", "look", "book", "take", "a"]}, {"sent": "a book with five twelve piece puzzles .", "words": ["book", "with", "a"]}, {"sent": "but BearSnoresOn is sortof like his book thats like his nighttime cue .", "words": ["is", "like", "his", "book", "but"]}, {"sent": "its in that book over there .", "words": ["there", "in", "that", "book", "over"]}, {"sent": "time book ?", "words": ["book"]}, {"sent": "thats not a book .", "words": ["book", "not", "a"]}, {"sent": "what colors that book Fraser as well ?", "words": ["book", "what", "that"]}, {"sent": "wanna go get another book ?", "words": ["go", "get", "another", "wanna", "book"]}, {"sent": "go and get me a book .", "words": ["go", "and", "get", "book", "me", "a"]}, {"sent": "oh while youre making a mess Im going to have a look at this book because I can see a picture of a squirrel .", "words": ["can", "of", "to", "picture", "have", "this", "at", "look", "book", "see", "squirrel", "because", "a"]}, {"sent": "come on lets see the book .", "words": ["on", "see", "book", "the"]}, {"sent": "a book .", "words": ["book", "a"]}, {"sent": "Matty can I look at the picture book momma likes the pic oh what is this ?", "words": ["can", "the", "what", "look", "is", "picture", "this", "at", "book"]}, {"sent": "thats a book .", "words": ["book", "a"]}, {"sent": "youknow I thought youknow heres a child whos working on this and my book thing thats standardized .", "words": ["my", "and", "this", "on", "book", "a"]}, {"sent": "which book would he prefer to read or look at with you or which book did he prefer ?", "words": ["look", "to", "with", "you", "at", "he", "read", "did", "would", "book", "which"]}, {"sent": "whose story book ?", "words": ["book", "story"]}, {"sent": "look at the book .", "words": ["the", "book", "at", "look"]}, {"sent": "come on close the book if youre finished .", "words": ["the", "if", "on", "book", "close"]}, {"sent": "the vicar is excited hes got a book of Hymns so PostmanPat .", "words": ["the", "is", "of", "so", "book", "a"]}, {"sent": "you dont want the peekaboo book huh ?", "words": ["the", "dont", "book", "you"]}, {"sent": "well its not as easy saying it again without the book Thomas .", "words": ["the", "book", "it", "not"]}, {"sent": "you wanna read the book ?", "words": ["the", "you", "read", "wanna", "book"]}, {"sent": "does a book tell stories ?", "words": ["book", "does", "a"]}, {"sent": "theres this book mama .", "words": ["book", "this"]}, {"sent": "okay let us look at this book .", "words": ["look", "us", "this", "at", "book"]}, {"sent": "we should just look at our book .", "words": ["we", "our", "at", "look", "book"]}, {"sent": "do you wanna read the book ?", "words": ["the", "do", "you", "read", "wanna", "book"]}, {"sent": "its a lovely book that .", "words": ["book", "that", "a"]}, {"sent": "youre sitting on the book .", "words": ["on", "the", "book"]}, {"sent": "you wanna see that book ?", "words": ["you", "wanna", "book", "see", "that"]}, {"sent": "yes hes reading his book .", "words": ["his", "book"]}, {"sent": "get the Arthur book .", "words": ["get", "the", "book"]}, {"sent": "now did you see that whole book ?", "words": ["you", "did", "book", "see", "that"]}, {"sent": "its a book .", "words": ["book", "a"]}, {"sent": "because you havent even got a book out or anything .", "words": ["you", "out", "book", "because", "a"]}, {"sent": "the picture on this book has lotsof shapes .", "words": ["the", "picture", "this", "on", "book"]}, {"sent": "you reading the book ?", "words": ["the", "book", "you"]}, {"sent": "have you read this book before ?", "words": ["you", "have", "this", "read", "book"]}, {"sent": "you wanna read the book ?", "words": ["the", "you", "read", "wanna", "book"]}, {"sent": "oh are you going to look at the book ?", "words": ["the", "to", "you", "are", "at", "look", "book"]}, {"sent": "you go and get a nice book and Mama read it to you .", "words": ["go", "to", "nice", "and", "get", "you", "it", "read", "book", "a"]}, {"sent": "the water on first is it ?", "words": ["the", "is", "first", "it", "water", "on"]}, {"sent": "I got it first got it first got it first got it first .", "words": ["it", "first"]}, {"sent": "oh lets look out of the window first .", "words": ["the", "of", "first", "out", "look", "window"]}, {"sent": "when we first moved in Purdie useta come for walks with me .", "words": ["for", "we", "first", "with", "in", "me", "when"]}, {"sent": "well lets use them first .", "words": ["first", "them"]}, {"sent": "it was Molly mouses first Christmas .", "words": ["was", "it", "first"]}, {"sent": "what comes first on the train ?", "words": ["the", "what", "train", "first", "on"]}, {"sent": "weve just got some more things to do first havent we ?", "words": ["do", "we", "to", "first", "more", "some"]}, {"sent": "first they need to tighten a beam on Farmer Pickles barn .", "words": ["need", "need to", "to", "first", "they", "on", "a"]}, {"sent": "you open your mouth first .", "words": ["your", "open", "mouth", "first", "you"]}, {"sent": "I had it first or a crash ?", "words": ["it", "first", "a"]}, {"sent": "well lets do the animals first .", "words": ["the", "do", "first"]}, {"sent": "so I actually I was planning to be here for most of that time and I can be here for all of that time but I thought if you were going to be home anyway during like the first half hour to an hour .", "words": ["the", "of", "first", "like", "an", "can", "be", "were", "that", "home", "all", "for", "if", "was", "so", "to", "and", "here", "you", "but"]}, {"sent": "take them out first dont we ?", "words": ["we", "first", "them", "dont", "out", "take"]}, {"sent": "I think she picked up the frog first .", "words": ["up", "the", "think", "first", "frog", "she"]}, {"sent": "oh I see she cut it first .", "words": ["cut", "first", "it", "see", "she"]}, {"sent": "why dont you use up all the paint you have on your finger first ?", "words": ["up", "all", "the", "paint", "your", "first", "you", "have", "dont", "finger", "on", "why"]}, {"sent": "Ill help you punch first you fold him down like this .", "words": ["help", "first", "like", "you", "this", "him", "down"]}, {"sent": "lets do this one first .", "words": ["do", "first", "this"]}, {"sent": "he was in first grade with me .", "words": ["was", "first", "with", "in", "he", "me"]}, {"sent": "first we first we write AuntieMwww in here .", "words": ["we", "first", "write", "here", "in"]}, {"sent": "well can we tidy this up first because otherwise its gonna be too messy .", "words": ["can", "up", "we", "be", "first", "this", "too", "because"]}, {"sent": "but remember right before we went to the first mall you were not feeling well remember what you said to me ?", "words": ["the", "what", "we", "to", "first", "you", "me", "were", "not", "but"]}, {"sent": "jigsaw away then first .", "words": ["then", "away", "first"]}, {"sent": "you can climb up there and get on the horse first .", "words": ["can", "up", "the", "first", "you", "and", "there", "get", "horse", "on", "climb"]}, {"sent": "what what did you say first of all though ?", "words": ["what", "all", "of", "first", "say", "you", "did"]}, {"sent": "Michael first you finish what you have there .", "words": ["what", "first", "finish", "you", "have", "there"]}, {"sent": "thats gonna be a special treat for your first birthday .", "words": ["for", "your", "be", "first", "a"]}, {"sent": "alright well first we need to finish making our cookies honey .", "words": ["need", "we", "need to", "to", "first", "finish", "our"]}, {"sent": "and I would imagine the picture perhaps starts with the first letter of the alphabet that theyre showing .", "words": ["the", "of", "picture", "with", "first", "and", "would", "that"]}, {"sent": "aah Im trying to fix it first darling .", "words": ["to", "it", "first", "fix"]}, {"sent": "Ill to the first one then you do it .", "words": ["the", "do", "to", "first", "you", "it", "then"]}, {"sent": "oh I think Ill Ill let it cool off first .", "words": ["off", "think", "it", "first"]}, {"sent": "feet first .", "words": ["first"]}, {"sent": "no I think you oughta take this over here first .", "words": ["think", "first", "you", "here", "this", "take", "over"]}, {"sent": "that would have been the first game of the World Series tomorrow .", "words": ["the", "game", "of", "first", "have", "would", "that"]}, {"sent": "are you playing with your trains first or not ?", "words": ["your", "first", "with", "you", "are", "not"]}, {"sent": "okay but we hafta let it dry first .", "words": ["we", "dry", "first", "it", "but"]}, {"sent": "your turn first ?", "words": ["first", "your"]}, {"sent": "the first time ?", "words": ["the", "first"]}, {"sent": "well lets run a bath first Thomas .", "words": ["run", "first", "a"]}, {"sent": "you gotta cook them first .", "words": ["cook", "first", "them", "you"]}, {"sent": "well you hafta ask her first .", "words": ["first", "her", "you"]}, {"sent": "xxx first xxx .", "words": ["first"]}, {"sent": "youre gonna draw you first ?", "words": ["draw", "first", "you"]}, {"sent": "but first were gonna eat breakfast .", "words": ["eat", "first", "were", "but"]}, {"sent": "first the xxx .", "words": ["the", "first"]}, {"sent": "you wanna read the tiger book first or do dinos ?", "words": ["the", "do", "first", "you", "read", "tiger", "wanna", "book"]}, {"sent": "do you want me to take the tiger off you first ?", "words": ["the", "do", "off", "to", "first", "you", "tiger", "me", "take"]}, {"sent": "stop the car first .", "words": ["the", "car", "first", "stop"]}, {"sent": "youre going to give him his pie first .", "words": ["to", "first", "him", "his", "give"]}, {"sent": "my first prayer book .", "words": ["book", "my", "first"]}, {"sent": "uh today is September twenty fifth this is participant five six three zero and this is the seven month first visit .", "words": ["the", "is", "first", "and", "this"]}, {"sent": "no but hes in the first grade in a different school .", "words": ["the", "school", "first", "in", "but", "a"]}, {"sent": "trust you to have a qualified nurse to put your first plaster on .", "words": ["put", "your", "to", "first", "you", "have", "on", "a"]}, {"sent": "push the sides in first .", "words": ["the", "in", "first", "push"]}, {"sent": "first .", "words": ["first"]}, {"sent": "okay tell me first what it means to die .", "words": ["what", "to", "first", "it", "me"]}, {"sent": "first grade just like Robin is in .", "words": ["in", "first", "like", "is"]}, {"sent": "type your name first .", "words": ["first", "your"]}, {"sent": "lets lets you wanna set them all up first ?", "words": ["up", "all", "first", "them", "you", "wanna"]}, {"sent": "and its on the next page that she says first she scrubbed their faces .", "words": ["the", "their", "first", "and", "on", "that", "she"]}, {"sent": "and I mean Im only going to be looking at at first at the first at at the stop consonants just those consonants of a particular kind that occur at the beginning of a word so Im ignoring xxx .", "words": ["the", "those", "of", "to", "be", "first", "stop", "so", "and", "at", "that", "a"]}, {"sent": "shall I wait till you finish your grapes first ?", "words": ["wait", "your", "first", "finish", "you", "grapes"]}, {"sent": "what you hafta do is first of all take your legs out right ?", "words": ["what", "do", "all", "your", "is", "of", "first", "you", "out", "take"]}, {"sent": "and we let them go first in the shop didnt we ?", "words": ["the", "go", "we", "first", "them", "and", "in"]}, {"sent": "it is March the twenty first in the morning .", "words": ["the", "is", "first", "in", "it"]}, {"sent": "have you dried dried him off first ?", "words": ["off", "first", "you", "have", "him"]}, {"sent": "you do that one first .", "words": ["do", "first", "that", "you"]}, {"sent": "Zoe hang up your coat first .", "words": ["up", "first", "your", "coat"]}, {"sent": "do you want toast first ?", "words": ["do", "first", "toast", "you"]}, {"sent": "whats the first thing you do in the morning when you go to school Harry ?", "words": ["the", "do", "go", "to", "school", "first", "you", "in", "when"]}, {"sent": "and hes going the first way again .", "words": ["the", "first", "and"]}, {"sent": "thats the first time .", "words": ["the", "first"]}, {"sent": "tell daddy what happened first when we got to the car .", "words": ["the", "what", "we", "to", "car", "first", "when"]}, {"sent": "I do yours first .", "words": ["do", "first"]}, {"sent": "the burglars climbed across into the next tree and the next all round the garden until they were back in the first tree .", "words": ["the", "all", "tree", "into", "back", "first", "and", "they", "garden", "in", "were"]}, {"sent": "what do you hafta do first ?", "words": ["what", "do", "first", "you"]}, {"sent": "give her something to eat first though .", "words": ["to", "first", "eat", "give", "her"]}, {"sent": "arent you going to wind it up first ?", "words": ["up", "to", "first", "you", "it", "wind"]}, {"sent": "well take your turn if youre first .", "words": ["your", "first", "take", "if"]}, {"sent": "we gotta take it all out first ?", "words": ["all", "we", "first", "it", "out", "take"]}, {"sent": "lets just get the nappie on first .", "words": ["get", "the", "on", "first"]}, {"sent": "well we hafta get this box out first .", "words": ["box", "we", "first", "get", "this", "out"]}, {"sent": "are we gonna do Harryhippo first ?", "words": ["do", "are", "first", "we"]}, {"sent": "hafta shut the door first to cook them .", "words": ["the", "to", "first", "them", "cook", "door"]}, {"sent": "well you sit down first .", "words": ["sit", "down", "first", "you"]}, {"sent": "but weve got to just pick one or two things up first .", "words": ["up", "pick", "to", "first", "but"]}, {"sent": "give me the first letter .", "words": ["the", "me", "give", "first"]}, {"sent": "who do you think is gonna saw through the branch first ?", "words": ["the", "do", "think", "is", "first", "you", "who"]}, {"sent": "that one I think she knows because we always used that was her first food .", "words": ["think", "we", "was", "first", "her", "food", "that", "she", "because"]}, {"sent": "shall we look at these first ?", "words": ["these", "we", "first", "at", "look"]}, {"sent": "put the lid back on first Jordan .", "words": ["the", "put", "back", "first", "on"]}, {"sent": "head first .", "words": ["head", "first"]}, {"sent": "oh but I needta know who much they cost first .", "words": ["first", "they", "who", "much", "but"]}, {"sent": "well we hafta get you dressed first .", "words": ["get", "first", "you", "we"]}, {"sent": "okay you go first .", "words": ["go", "first", "you"]}, {"sent": "but I dont know the first one .", "words": ["the", "dont", "first", "but"]}, {"sent": "you wanna play first ?", "words": ["play", "first", "wanna", "you"]}, {"sent": "well lets wash your hands and face first .", "words": ["your", "face", "first", "and", "wash"]}, {"sent": "I dropped those pipes on the tracks .", "words": ["on", "the", "those"]}, {"sent": "come on over here and you can sing .", "words": ["can", "you", "here", "and", "sing", "on", "over"]}, {"sent": "come on Willary .", "words": ["on"]}, {"sent": "well just put them on like that .", "words": ["put", "them", "like", "on", "that"]}, {"sent": "I thought I saw scratches on your arms last night when I was getting you ready for bed .", "words": ["for", "your", "was", "last", "bed", "you", "on", "when"]}, {"sent": "something on my leg it seemed to me .", "words": ["my", "to", "it", "leg", "on", "me"]}, {"sent": "do you think thats whats going on ?", "words": ["on", "do", "think", "you"]}, {"sent": "is he on the beach ?", "words": ["the", "is", "beach", "he", "on"]}, {"sent": "Im going to start on piggy .", "words": ["on", "to"]}, {"sent": "pull it all the way on .", "words": ["the", "all", "pull", "it", "on"]}, {"sent": "come on .", "words": ["on"]}, {"sent": "wanna sit on the pottie do you ?", "words": ["the", "do", "you", "wanna", "on", "sit"]}, {"sent": "a slide and what can you do on that ?", "words": ["can", "what", "do", "slide", "and", "you", "on", "that", "a"]}, {"sent": "can you make this fit on here can you make that fit on ?", "words": ["can", "fit", "you", "here", "this", "that", "on", "make"]}, {"sent": "do they live on the moon ?", "words": ["the", "do", "moon", "they", "on"]}, {"sent": "to go bowling yeah because thats the thing on Sunday afternoons they have little pins .", "words": ["the", "go", "to", "little", "have", "they", "on", "because"]}, {"sent": "or should I put the car on top ?", "words": ["the", "car", "put", "on"]}, {"sent": "come on .", "words": ["on"]}, {"sent": "youre not going up the slide on the bike .", "words": ["up", "the", "slide", "on", "not"]}, {"sent": "are you putting the spotty outfit on then ?", "words": ["the", "you", "are", "on", "then"]}, {"sent": "do you wanna put them on ?", "words": ["do", "put", "them", "you", "wanna", "on"]}, {"sent": "and what color are these socks were putting on today ?", "words": ["what", "these", "and", "are", "on", "were"]}, {"sent": "you want this on my head now ?", "words": ["my", "head", "you", "this", "on"]}, {"sent": "go on then .", "words": ["on", "then", "go"]}, {"sent": "he fell on the ice and then ?", "words": ["the", "ice", "and", "he", "on", "then"]}, {"sent": "youve got to make sure youre getting on the right one havent you ?", "words": ["the", "to", "you", "on", "make"]}, {"sent": "come on Fraser .", "words": ["on"]}, {"sent": "cmon see whats on the next page .", "words": ["on", "see", "the"]}, {"sent": "thats Jwww playing on the computer .", "words": ["on", "the"]}, {"sent": "hang on .", "words": ["on"]}, {"sent": "so come on clap your hands and sing .", "words": ["your", "clap", "so", "and", "sing", "on"]}, {"sent": "Im gonna put some clothes on her .", "words": ["on", "some", "put", "her"]}, {"sent": "yes youve got your green tshirt on .", "words": ["on", "your", "green"]}, {"sent": "remember theyre on at noontime ?", "words": ["on", "at"]}, {"sent": "they are going to go on a .", "words": ["go", "to", "they", "are", "on", "a"]}, {"sent": "on that little corner behind a pillar .", "words": ["behind", "little", "on", "that", "a"]}, {"sent": "laying on the bench ?", "words": ["on", "the", "bench"]}, {"sent": "right come on then .", "words": ["on", "then"]}, {"sent": "wanna sit on my lap .", "words": ["on", "sit", "wanna", "my"]}, {"sent": "do you wanna put it on ?", "words": ["do", "put", "you", "it", "wanna", "on"]}, {"sent": "youre jumping on the couch .", "words": ["on", "the", "couch"]}, {"sent": "what does she have on her ?", "words": ["what", "does", "have", "on", "her", "she"]}, {"sent": "Abe and I saw MickeyMouse and the bears talk on tv .", "words": ["the", "and", "tv", "talk", "on"]}, {"sent": "we cant have anything very noisy on the tape .", "words": ["the", "we", "noisy", "tape", "have", "on"]}, {"sent": "do you wanna come up on duh loft ?", "words": ["up", "do", "you", "wanna", "on"]}, {"sent": "with Dipsy on it .", "words": ["on", "it", "with"]}, {"sent": "I think some wheels go on here uhn dont they ?", "words": ["think", "go", "here", "dont", "they", "on", "some"]}, {"sent": "now what are we left with on the piece of paper ?", "words": ["the", "what", "we", "of", "paper", "with", "are", "on"]}, {"sent": "come on .", "words": ["on"]}, {"sent": "sockies on ?", "words": ["on"]}, {"sent": "with a house stuck on your head ?", "words": ["your", "head", "with", "house", "stuck", "on", "a"]}, {"sent": "I think its on the floor John .", "words": ["on", "the", "think"]}, {"sent": "theres no baby on this picture .", "words": ["on", "this", "picture"]}, {"sent": "come on .", "words": ["on"]}, {"sent": "oh whats that on top of the train ?", "words": ["the", "train", "of", "on", "that"]}, {"sent": "can you put your play clothes on all by yourself ?", "words": ["can", "all", "put", "yourself", "your", "play", "you", "by", "on"]}, {"sent": "whats going on on this farm ?", "words": ["on", "this"]}, {"sent": "that is neat it is got strawberrys on it .", "words": ["on", "it", "that", "is"]}, {"sent": "you hung up on me .", "words": ["up", "me", "on", "you"]}, {"sent": "gonna put this on my motha bed ?", "words": ["put", "my", "bed", "this", "on"]}, {"sent": "now she can put her lipstick on cant she ?", "words": ["can", "put", "on", "her", "she"]}, {"sent": "and put them on the plate .", "words": ["the", "put", "plate", "them", "and", "on"]}, {"sent": "but here peanut butter on raisin peanut butter and jelly on raisin toast ?", "words": ["jelly", "toast", "and", "here", "butter", "peanut butter", "on", "raisin", "but"]}, {"sent": "come on .", "words": ["on"]}, {"sent": "the wheels on the bus go round and round .", "words": ["the", "go", "and", "on", "bus"]}, {"sent": "say jump on .", "words": ["on", "jump", "say"]}, {"sent": "but did you hafta have it put down on his wee tag xxx ?", "words": ["put", "you", "have", "his", "it", "did", "on", "down", "but"]}, {"sent": "you gonna sit on my lap ?", "words": ["on", "sit", "my", "you"]}, {"sent": "what does she have on her hair ?", "words": ["what", "does", "have", "on", "hair", "her", "she"]}, {"sent": "do you want to pretend to be a mourning dove on a nest ?", "words": ["do", "to", "be", "you", "want to", "on", "pretend", "a"]}, {"sent": "on my spoon ?", "words": ["on", "my", "spoon"]}, {"sent": "come on Hilda shouts Huckle catch it .", "words": ["on", "it", "catch"]}, {"sent": "that was the mans voice on the answering machine but I dont understand it because Mummy was nowhere near the phone .", "words": ["the", "was", "because", "dont", "it", "on", "that", "but"]}, {"sent": "but I dont think we got juice on too many things .", "words": ["think", "we", "dont", "too", "on", "juice", "but"]}, {"sent": "Im only putting butter on it and .", "words": ["on", "it", "butter", "and"]}, {"sent": "Annes sitting on there ?", "words": ["on", "there"]}, {"sent": "yeah can I put it on ?", "words": ["can", "put", "it", "on"]}, {"sent": "dont you draw on that mind .", "words": ["draw", "you", "dont", "on", "that"]}, {"sent": "so its obviously the colors on the sign that you recognize isnt it ?", "words": ["the", "so", "you", "it", "on", "that"]}, {"sent": "xxx on this way .", "words": ["on", "this"]}, {"sent": "come on now .", "words": ["on"]}, {"sent": "dont do it on your hands .", "words": ["do", "your", "dont", "it", "on"]}, {"sent": "now put the top on .", "words": ["the", "put", "on"]}, {"sent": "thats on your arm .", "words": ["on", "your", "arm"]}, {"sent": "next week Im gonna bring you a tape thats gonna have you talking on it .", "words": ["bring", "tape", "you", "have", "it", "on", "a"]}, {"sent": "you want some ketchup on it ?", "words": ["on", "some", "it", "you"]}, {"sent": "put one on your bed ?", "words": ["on", "put", "your", "bed"]}, {"sent": "do you think Ive set the kitchen on fire ?", "words": ["the", "do", "think", "you", "kitchen", "on"]}, {"sent": "now if we put it on the outside .", "words": ["the", "put", "if", "we", "outside", "it", "on"]}, {"sent": "go on then .", "words": ["on", "then", "go"]}, {"sent": "are the wheels a bit stuck on this train ?", "words": ["the", "train", "this", "are", "stuck", "on", "a"]}, {"sent": "come on Puss .", "words": ["on"]}, {"sent": "or put their suitcases on the thing and pick up their suitcases .", "words": ["the", "up", "put", "their", "pick", "and", "on"]}, {"sent": "move it back on the paper .", "words": ["the", "back", "paper", "it", "on"]}, {"sent": "hang on .", "words": ["on"]}, {"sent": "well bring it on Friday when we come to see you .", "words": ["we", "to", "bring", "you", "it", "on", "see", "when"]}, {"sent": "all this stuff going on jeez .", "words": ["on", "all", "this"]}, {"sent": "there the cow on the seesaw put the cow on the seesaw .", "words": ["the", "put", "there", "on", "cow"]}, {"sent": "come on sweethearts .", "words": ["on"]}, {"sent": "are you drawing on that ?", "words": ["on", "are", "that", "you"]}, {"sent": "good girl .", "words": ["good"]}, {"sent": "very good .", "words": ["good"]}, {"sent": "good .", "words": ["good"]}, {"sent": "good boy .", "words": ["good"]}, {"sent": "thats good .", "words": ["good"]}, {"sent": "does it taste good ?", "words": ["does", "it", "taste", "good"]}, {"sent": "is it good ?", "words": ["it", "good", "is"]}, {"sent": "very good .", "words": ["good"]}, {"sent": "and hes got a couple in each hand just for good luck .", "words": ["for", "each", "hand", "and", "in", "good", "a"]}, {"sent": "thats good .", "words": ["good"]}, {"sent": "thats good and this is all the book of all the different ones .", "words": ["the", "all", "is", "of", "and", "this", "good", "book"]}, {"sent": "good girl .", "words": ["good"]}, {"sent": "thank you that was a good kiss .", "words": ["was", "you", "good", "kiss", "that", "a"]}, {"sent": "oh thats good .", "words": ["good"]}, {"sent": "very good .", "words": ["good"]}, {"sent": "water sounds like a good idea .", "words": ["good", "like", "a", "water"]}, {"sent": "this is very good tea .", "words": ["this", "good", "is"]}, {"sent": "mhm good .", "words": ["good"]}, {"sent": "good bye .", "words": ["good"]}, {"sent": "oh youre very good at it .", "words": ["it", "good", "at"]}, {"sent": "good morning FiremanSam .", "words": ["good"]}, {"sent": "good girl yes .", "words": ["good"]}, {"sent": "you did a very good job .", "words": ["good", "did", "a", "you"]}, {"sent": "hey listen partner that was good .", "words": ["was", "that", "good", "listen"]}, {"sent": "thats a good idea .", "words": ["good", "a"]}, {"sent": "good girl .", "words": ["good"]}, {"sent": "yes did you have a good time ?", "words": ["you", "have", "did", "good", "a"]}, {"sent": "good .", "words": ["good"]}, {"sent": "ah very good .", "words": ["good"]}, {"sent": "good .", "words": ["good"]}, {"sent": "that would be good fun .", "words": ["be", "that", "good", "would"]}, {"sent": "thats good .", "words": ["good"]}, {"sent": "is that good ?", "words": ["that", "good", "is"]}, {"sent": "oh very good .", "words": ["good"]}, {"sent": "is your daddy not a good cook ?", "words": ["your", "is", "cook", "good", "not", "a"]}, {"sent": "okay good .", "words": ["good"]}, {"sent": "we know our body parts pretty good now .", "words": ["our", "pretty", "good", "we"]}, {"sent": "well if youre a good girl shell go .", "words": ["go", "if", "good", "a"]}, {"sent": "good .", "words": ["good"]}, {"sent": "thats a good .", "words": ["good", "a"]}, {"sent": "okay good idea .", "words": ["good"]}, {"sent": "I know that probably doesnt feel good .", "words": ["that", "good"]}, {"sent": "good fun .", "words": ["good"]}, {"sent": "thats very good writing John .", "words": ["good"]}, {"sent": "good .", "words": ["good"]}, {"sent": "youre doing a good job theres the Barney purple one .", "words": ["the", "good", "a"]}, {"sent": "it tastes good .", "words": ["it", "good"]}, {"sent": "oh what a good boy for getting in your seat .", "words": ["what", "for", "your", "in", "good", "a"]}, {"sent": "it makes a good noise .", "words": ["it", "good", "a"]}, {"sent": "very good .", "words": ["good"]}, {"sent": "mm look at all this good food .", "words": ["all", "look", "this", "food", "good", "at"]}, {"sent": "okay good .", "words": ["good"]}, {"sent": "if shes a good girl while shes having her hair cut are you going to give her a lollipop ?", "words": ["if", "cut", "to", "you", "lollipop", "are", "good", "give", "hair", "her", "a"]}, {"sent": "good .", "words": ["good"]}, {"sent": "does that taste good you wanna give your friend some ?", "words": ["does", "your", "taste", "you", "good", "wanna", "some", "give", "that"]}, {"sent": "very good sweetie .", "words": ["good"]}, {"sent": "squash is good good .", "words": ["good", "is"]}, {"sent": "good .", "words": ["good"]}, {"sent": "oh good job .", "words": ["good"]}, {"sent": "apparently she says its really good .", "words": ["she", "good"]}, {"sent": "say good .", "words": ["say", "good"]}, {"sent": "good .", "words": ["good"]}, {"sent": "be good .", "words": ["be", "good"]}, {"sent": "good .", "words": ["good"]}, {"sent": "now heres a good one .", "words": ["good", "a"]}, {"sent": "good .", "words": ["good"]}, {"sent": "you have a good nap ?", "words": ["have", "good", "a", "you"]}, {"sent": "is it good ?", "words": ["it", "good", "is"]}, {"sent": "its no good Thomas .", "words": ["good"]}, {"sent": "oh thats not good William .", "words": ["not", "good"]}, {"sent": "good .", "words": ["good"]}, {"sent": "oh you look like youve been having a good time .", "words": ["like", "you", "good", "look", "a"]}, {"sent": "thats a good boy .", "words": ["good", "a"]}, {"sent": "well not so sour just a cherry cherry vinaigrette do you think that would be good on our chicken salad ?", "words": ["do", "think", "so", "be", "you", "our", "good", "would", "on", "chicken", "that", "not", "a"]}, {"sent": "good job .", "words": ["good"]}, {"sent": "good .", "words": ["good"]}, {"sent": "I knew it kept spinning and it wasnt doing any good .", "words": ["it", "any", "good", "and"]}, {"sent": "good job buddy .", "words": ["good"]}, {"sent": "good deal minus all those coupons .", "words": ["all", "those", "good"]}, {"sent": "thats good isnt it .", "words": ["it", "good"]}, {"sent": "Im not Im not very good at it .", "words": ["it", "not", "good", "at"]}, {"sent": "good job .", "words": ["good"]}, {"sent": "good throw .", "words": ["throw", "good"]}, {"sent": "wee good one thats a nice one .", "words": ["nice", "good", "a"]}, {"sent": "good .", "words": ["good"]}, {"sent": "that was such a good story .", "words": ["was", "good", "story", "that", "a"]}, {"sent": "its not a good place to put it .", "words": ["put", "to", "it", "good", "not", "a"]}, {"sent": "good concentrating .", "words": ["good"]}, {"sent": "good .", "words": ["good"]}, {"sent": "good boy .", "words": ["good"]}, {"sent": "that looks good .", "words": ["that", "good"]}, {"sent": "well have you had a good look at them ?", "words": ["them", "you", "have", "at", "good", "look", "a"]}, {"sent": "good boy .", "words": ["good"]}, {"sent": "thats thats good .", "words": ["good"]}, {"sent": "yay good throw .", "words": ["throw", "good"]}, {"sent": "when did I ever sing you a good night song ?", "words": ["you", "sing", "did", "good", "when", "a"]}, {"sent": "you put it on a plate good job .", "words": ["put", "plate", "you", "it", "good", "on", "a"]}, {"sent": "Jwwws very good isnt she ?", "words": ["she", "good"]}, {"sent": "plum good .", "words": ["good"]}, {"sent": "that was for good measure was it ?", "words": ["for", "was", "it", "good", "that"]}, {"sent": "can you open the door ?", "words": ["can", "the", "open", "you", "door"]}, {"sent": "do you want me to open the book ?", "words": ["the", "do", "open", "to", "you", "book", "me"]}, {"sent": "why do you want me to open the door ?", "words": ["the", "do", "open", "to", "you", "me", "why", "door"]}, {"sent": "open the mailbox .", "words": ["open", "the"]}, {"sent": "if you want uh the liberry to stay open you hafta sign a petition .", "words": ["the", "if", "open", "to", "you", "stay", "a"]}, {"sent": "okay open it up .", "words": ["open", "it", "up"]}, {"sent": "you wanna make him open his mouth ?", "words": ["open", "mouth", "you", "him", "wanna", "his", "make"]}, {"sent": "open it up .", "words": ["open", "it", "up"]}, {"sent": "open .", "words": ["open"]}, {"sent": "open the window .", "words": ["open", "the", "window"]}, {"sent": "can you open it up ?", "words": ["can", "up", "open", "you", "it"]}, {"sent": "and just open one little house and see whats inside .", "words": ["open", "inside", "little", "and", "house", "see"]}, {"sent": "open it out ya mouth .", "words": ["open", "it", "out", "mouth"]}, {"sent": "say open mommy .", "words": ["open", "say"]}, {"sent": "open open the door .", "words": ["open", "the", "door"]}, {"sent": "well you can see he has got his drivers cab open .", "words": ["can", "open", "you", "he", "his", "see"]}, {"sent": "open the door no its cold and rainy out if you open the door .", "words": ["the", "if", "open", "and", "you", "out", "cold", "door"]}, {"sent": "shall we open another present ?", "words": ["open", "present", "another", "we"]}, {"sent": "yeah now make it open .", "words": ["open", "it", "make"]}, {"sent": "it willnt open now .", "words": ["open", "it"]}, {"sent": "you willnt be able to pull it open Becky .", "words": ["open", "to", "be", "pull", "you", "it"]}, {"sent": "open .", "words": ["open"]}, {"sent": "you want me to open ?", "words": ["to", "me", "open", "you"]}, {"sent": "you see the door is open ?", "words": ["the", "is", "open", "you", "see", "door"]}, {"sent": "open the box ?", "words": ["open", "the", "box"]}, {"sent": "what are you going to do if I open it ?", "words": ["what", "do", "if", "open", "to", "you", "are", "it"]}, {"sent": "open the door .", "words": ["open", "the", "door"]}, {"sent": "open .", "words": ["open"]}, {"sent": "on the front door theyve got a beautiful wreath and a little sign that says open .", "words": ["the", "open", "and", "little", "that", "on", "door", "a"]}, {"sent": "that dont open baby .", "words": ["open", "dont", "that"]}, {"sent": "open it ?", "words": ["open", "it"]}, {"sent": "can I open their moutheses ?", "words": ["can", "their", "open"]}, {"sent": "lets open it to see whats inside .", "words": ["open", "to", "inside", "it", "see"]}, {"sent": "open the drawer .", "words": ["open", "the", "drawer"]}, {"sent": "can you open ?", "words": ["can", "open", "you"]}, {"sent": "fish dont open .", "words": ["open", "dont", "fish"]}, {"sent": "well shall I open it ?", "words": ["open", "it"]}, {"sent": "whats open ?", "words": ["open"]}, {"sent": "say open now .", "words": ["open", "say"]}, {"sent": "xxx open the book up .", "words": ["open", "the", "book", "up"]}, {"sent": "open it .", "words": ["open", "it"]}, {"sent": "do you want it open ?", "words": ["open", "do", "it", "you"]}, {"sent": "open ?", "words": ["open"]}, {"sent": "can you open this door here ?", "words": ["can", "open", "you", "here", "this", "door"]}, {"sent": "oh did you get it open open ?", "words": ["open", "you", "get", "it", "did"]}, {"sent": "well you closed it you want me to open it ?", "words": ["open", "to", "you", "it", "me"]}, {"sent": "shall I open it up ?", "words": ["open", "it", "up"]}, {"sent": "we gotta open a new container .", "words": ["open", "new", "a", "we"]}, {"sent": "can you open that ?", "words": ["can", "that", "open", "you"]}, {"sent": "as soon as youre open Ill be up .", "words": ["open", "be", "up"]}, {"sent": "yours is xxx closed or open ?", "words": ["open", "is"]}, {"sent": "can you open it ?", "words": ["can", "it", "open", "you"]}, {"sent": "what does this open ?", "words": ["open", "what", "does", "this"]}, {"sent": "are you going to open the lid ?", "words": ["the", "open", "to", "you", "are"]}, {"sent": "yeah that door doesnt open .", "words": ["open", "that", "door"]}, {"sent": "here Ill open it you shut it .", "words": ["here", "it", "you", "open"]}, {"sent": "each morning theres a fight over whos going to open the front door and whos going to open the elevator .", "words": ["the", "door", "each", "open", "to", "and", "over", "a"]}, {"sent": "open the door and lets see what happens .", "words": ["the", "what", "open", "and", "see", "door"]}, {"sent": "it was it was open when we walked past yesterday but we didnt want any did we ?", "words": ["we", "open", "was", "any", "it", "did", "when", "but"]}, {"sent": "it pops open .", "words": ["open", "it"]}, {"sent": "windows are open .", "words": ["open", "are"]}, {"sent": "do you want me to open ?", "words": ["do", "open", "to", "you", "me"]}, {"sent": "I have a brandnew one thats not even cut open .", "words": ["cut", "open", "have", "not", "a"]}, {"sent": "open ?", "words": ["open"]}, {"sent": "can you open it ?", "words": ["can", "it", "open", "you"]}, {"sent": "shall I open it ?", "words": ["open", "it"]}, {"sent": "a bit tricky to open these ones arent they ?", "words": ["these", "open", "to", "they", "a"]}, {"sent": "open shut open shut open shut open shut open shut open shut open shut open shut open shut ah .", "words": ["open"]}, {"sent": "you open that .", "words": ["open", "that", "you"]}, {"sent": "just open the flap and you will see .", "words": ["the", "open", "you", "and", "will", "see"]}, {"sent": "you open .", "words": ["open", "you"]}, {"sent": "can you open it ?", "words": ["can", "it", "open", "you"]}, {"sent": "if I open my purse do wanna see if we can afford a cup of tea .", "words": ["can", "do", "my", "cup", "if", "purse", "we", "open", "of", "wanna", "see", "a"]}, {"sent": "well shall Mummy open one and Warren open the other ?", "words": ["open", "the", "other", "and"]}, {"sent": "oh its open .", "words": ["open"]}, {"sent": "does the trunk open I mean the hood of the car .", "words": ["the", "does", "of", "car", "open"]}, {"sent": "open .", "words": ["open"]}, {"sent": "okay now shall we open it up ?", "words": ["open", "it", "up", "we"]}, {"sent": "no that doesnt open .", "words": ["open", "that"]}, {"sent": "let me open it up to a good page .", "words": ["up", "open", "to", "it", "good", "me", "a"]}, {"sent": "hes just going to open the door ?", "words": ["open", "to", "door", "the"]}, {"sent": "push it open .", "words": ["open", "it", "push"]}, {"sent": "my si my nephew gave him his old crash and go cars I dont know if youve ever seen them but they theyre these cars that when you crash them the doors pop open an the hood pops up .", "words": ["the", "go", "an", "when", "that", "up", "these", "pop", "if", "and", "you", "dont", "him", "old", "my", "open", "them", "they", "his", "but"]}, {"sent": "open it .", "words": ["open", "it"]}, {"sent": "but once we open it .", "words": ["open", "it", "but", "we"]}, {"sent": "open up the flaps .", "words": ["open", "the", "up"]}, {"sent": "is it open ?", "words": ["open", "it", "is"]}, {"sent": "make it open .", "words": ["open", "it", "make"]}, {"sent": "I dont think you can get that open .", "words": ["can", "think", "open", "you", "get", "dont", "that"]}, {"sent": "no because pat dont want it open it .", "words": ["open", "dont", "it", "because"]}, {"sent": "got to open the bag first .", "words": ["open", "to", "first", "the"]}, {"sent": "want those open Greta ?", "words": ["open", "those"]}, {"sent": "and the man was open wasnt he ?", "words": ["the", "open", "was", "and", "he"]}, {"sent": "alright so open it and put it in .", "words": ["put", "open", "so", "and", "in", "it"]}, {"sent": "open your mouth .", "words": ["open", "mouth", "your"]}, {"sent": "you hafta leave this part open .", "words": ["open", "this", "you"]}, {"sent": "Im gonna open it .", "words": ["open", "it"]}, {"sent": "open up basically open up the tupperware cabinet so I think stuff like this .", "words": ["the", "up", "think", "open", "so", "like", "this"]}, {"sent": "open .", "words": ["open"]}, {"sent": "should we leave the gate open ?", "words": ["the", "open", "we"]}, {"sent": "youre not to go up there Thomas .", "words": ["up", "go", "to", "there", "not"]}, {"sent": "up down then make a tiny dot here and a tiny dot here .", "words": ["up", "tiny", "and", "here", "then", "down", "make", "a"]}, {"sent": "and well hang those trousers up for later on .", "words": ["up", "for", "those", "and", "on"]}, {"sent": "wanna finish up your little bit of banana you have left ?", "words": ["up", "your", "of", "finish", "little", "you", "banana", "have", "wanna"]}, {"sent": "Tara you gonna clean it up ?", "words": ["clean", "it", "up", "you"]}, {"sent": "now pick up a yellow .", "words": ["pick", "yellow", "up", "a"]}, {"sent": "Dadas up .", "words": ["up"]}, {"sent": "thats up to you Nomi .", "words": ["up", "to", "you"]}, {"sent": "Eleanor you cant youve got to tidy your shop up .", "words": ["up", "to", "your", "you"]}, {"sent": "please dont eat me up .", "words": ["eat", "dont", "me", "up"]}, {"sent": "pick it up .", "words": ["pick", "it", "up"]}, {"sent": "and Ill lift you up .", "words": ["up", "you", "and"]}, {"sent": "because Im just folding up some things that I washed yesterday .", "words": ["up", "some", "that", "because"]}, {"sent": "then up goes the big crate .", "words": ["up", "the", "then", "big"]}, {"sent": "whats up Nathan are you tired ?", "words": ["up", "are", "tired", "you"]}, {"sent": "mix it up .", "words": ["up", "it"]}, {"sent": "shut up Elspeth .", "words": ["up"]}, {"sent": "because see that waterfall go up ?", "words": ["up", "go", "see", "that", "because"]}, {"sent": "why cant I have it filled up to there .", "words": ["up", "to", "have", "there", "it", "why"]}, {"sent": "can you see him going up the stairs ?", "words": ["can", "up", "the", "you", "him", "see", "stairs"]}, {"sent": "see you just lift it up like this .", "words": ["up", "like", "you", "this", "it", "see"]}, {"sent": "say up up up up up up up up up .", "words": ["up", "say"]}, {"sent": "did he pick up ?", "words": ["pick", "up", "he", "did"]}, {"sent": "I dont think he wants to stand up .", "words": ["up", "think", "to", "stand", "dont", "he"]}, {"sent": "lets go and tidy up the study .", "words": ["up", "the", "and", "go"]}, {"sent": "stand up .", "words": ["up", "stand"]}, {"sent": "you gonna mess up that diaper ?", "words": ["up", "diaper", "that", "you"]}, {"sent": "pausing here for salt and they carefully trained a real smart dog named Daniel to serve as our countrys first gun toting Spaniel then Daniel the kick a poo Spaniel and I marched toward the wall with our heads held up high while they roared and cheered and their cheers filled the sky fight .", "words": ["the", "up", "for", "their", "sky", "to", "first", "with", "and", "here", "they", "salt", "our", "kick", "high", "then", "dog", "a"]}, {"sent": "stand up because these are too big for you .", "words": ["up", "these", "for", "stand", "you", "are", "too", "big", "because"]}, {"sent": "cover up the bunny .", "words": ["up", "the", "bunny", "cover"]}, {"sent": "mummie just do the washing up .", "words": ["the", "do", "up"]}, {"sent": "he could probably lift you up with one hand .", "words": ["up", "hand", "with", "you", "he", "could"]}, {"sent": "now we hafta pick up all the dirt .", "words": ["the", "all", "up", "we", "pick"]}, {"sent": "oh mummy lets go in the cafe Im fed up of shopping .", "words": ["the", "up", "go", "of", "in"]}, {"sent": "put another one on the fishing hook thing got tangled up .", "words": ["the", "up", "put", "another", "on"]}, {"sent": "if you wanna stay up there thats fine if you come down here .", "words": ["up", "if", "fine", "you", "here", "there", "wanna", "stay", "down"]}, {"sent": "and when I come back up itll be time to get out .", "words": ["up", "back", "be", "to", "and", "get", "out", "when"]}, {"sent": "her eyes are all covered up .", "words": ["up", "all", "are", "her"]}, {"sent": "whos coming whos who can climb up buildings ?", "words": ["can", "climb", "up", "who"]}, {"sent": "you pick her up ?", "words": ["pick", "up", "her", "you"]}, {"sent": "you gonna set em all standing up ?", "words": ["up", "all", "you"]}, {"sent": "put them up .", "words": ["up", "put", "them"]}, {"sent": "so its like shes so easy and like its gonna be the situation where as they grow up hell be li shell be like hey you gonna finish that ?", "words": ["the", "where", "up", "so", "be", "like", "finish", "and", "you", "they", "that"]}, {"sent": "Sarah whatre you up to ?", "words": ["up", "to", "you"]}, {"sent": "and we dont want them to join up .", "words": ["up", "we", "to", "them", "and", "dont"]}, {"sent": "lets put our sleeve up .", "words": ["up", "put", "our"]}, {"sent": "you wake up ?", "words": ["up", "wake", "you"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "lets clean up .", "words": ["clean", "up"]}, {"sent": "well eat it all up .", "words": ["eat", "all", "it", "up"]}, {"sent": "lotsof tea cups hanging up there .", "words": ["up", "there"]}, {"sent": "so you can actually fill cars up with petrol right at the top ?", "words": ["can", "up", "the", "so", "with", "you", "at"]}, {"sent": "I noticed that she has a bunch of clean up routines .", "words": ["clean", "up", "of", "that", "she", "a"]}, {"sent": "Im just picking them up so they dont go on the floor .", "words": ["up", "the", "go", "so", "them", "dont", "they", "on"]}, {"sent": "after Joey gets done Im going up to see my new friend ?", "words": ["up", "my", "to", "see", "new"]}, {"sent": "chew it up nice .", "words": ["up", "it", "nice"]}, {"sent": "gingerbread Ill write boy up here boy .", "words": ["up", "here", "write"]}, {"sent": "xxx you wanna hear a new one Jesus christ almighty the rat went the rat went up my nighty he bit my and then made me spit Jesus christ almighty .", "words": ["the", "up", "my", "hear", "you", "and", "he", "wanna", "me", "then", "new", "a"]}, {"sent": "oo youd better close it up .", "words": ["up", "better", "close", "it"]}, {"sent": "because we were up in Portrush on Saturday there .", "words": ["up", "we", "there", "in", "on", "were", "because"]}, {"sent": "can you pick that up ?", "words": ["can", "up", "pick", "you", "that"]}, {"sent": "roll up your sleeve .", "words": ["up", "your"]}, {"sent": "and Im going to sit up and then put her on your knee .", "words": ["up", "put", "knee", "your", "to", "then", "and", "on", "sit", "her"]}, {"sent": "they pulled and pulled again and up came the .", "words": ["up", "they", "the", "and"]}, {"sent": "Im gonna just clean up the kitchen for a quick second .", "words": ["clean", "the", "up", "for", "kitchen", "a"]}, {"sent": "just like you blew up the duck .", "words": ["up", "the", "like", "you", "duck"]}, {"sent": "can you pick up the bell with two fingers ?", "words": ["can", "the", "up", "pick", "with", "you"]}, {"sent": "is your arm up there ?", "words": ["up", "your", "is", "there", "arm"]}, {"sent": "as soon as Ive fed Amy and washed up .", "words": ["up", "and"]}, {"sent": "do her shirt up .", "words": ["up", "do", "her", "shirt"]}, {"sent": "shall we do your laces up again ?", "words": ["up", "do", "your", "we"]}, {"sent": "with velcro he can put pieces together and cut it up .", "words": ["can", "up", "put", "cut", "with", "and", "it", "he"]}, {"sent": "because he is growing up .", "words": ["up", "he", "because", "is"]}, {"sent": "up was this the dog ?", "words": ["up", "the", "was", "this", "dog"]}, {"sent": "youre fed up with your toys .", "words": ["up", "with", "your"]}, {"sent": "wake up .", "words": ["up", "wake"]}, {"sent": "up up and down and up and down and up and down .", "words": ["up", "down", "and"]}, {"sent": "and look when you tip it up it doesnt come off .", "words": ["up", "off", "you", "and", "it", "look", "when"]}, {"sent": "uh Ive got the end sorted and weve ended up with the middle twisted .", "words": ["the", "up", "with", "and"]}, {"sent": "whats coming up in a couple of weeks that we have been talking about ?", "words": ["up", "we", "of", "have", "in", "about", "that", "a"]}, {"sent": "are you going to pick all those up ?", "words": ["up", "all", "those", "pick", "to", "you", "are"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "and I saw just assoonas I stepped back on land so were all the girls of the butter up band the chief drum majorette Miss Ukie Ann Sue said that was a pretty sour flight that you flew and the chief Ukeroo has been looking for you I raced to his office the place was a sight have no fear said the chief everything is alright my bright back room boys have been brighter then bright they thought up a gadget thats newer than new a mysterious moo lack moo and it can blow up those Zukes clear to Solamagoo theyve invented the bitsy big boy boomaroo .", "words": ["the", "of", "can", "those", "have", "it", "on", "then", "that", "were", "big", "up", "all", "for", "is", "so", "was", "to", "and", "you", "butter", "new", "blow", "a", "my", "back", "they", "room", "pretty", "his"]}, {"sent": "filled up ?", "words": ["up"]}, {"sent": "the itsy bitsy spider walks up to Dillons head .", "words": ["the", "to", "head", "up"]}, {"sent": "cut it up you want me to cut it up ?", "words": ["up", "cut", "to", "you", "it", "me"]}, {"sent": "eat up then .", "words": ["eat", "then", "up"]}, {"sent": "now the mailman is gonna pick it up .", "words": ["the", "up", "is", "pick", "it"]}, {"sent": "and Lwww can pick you up from baby school ?", "words": ["can", "up", "pick", "school", "you", "and"]}, {"sent": "hed be up there for days wouldnt he ?", "words": ["up", "for", "be", "there", "he"]}, {"sent": "even though you were playing you were cleaning up .", "words": ["up", "were", "you"]}, {"sent": "before you break it up .", "words": ["up", "it", "break", "you"]}, {"sent": "were gonna mix em all up .", "words": ["up", "all", "were"]}, {"sent": "did John wake you up ?", "words": ["up", "wake", "did", "you"]}, {"sent": "eat it up then .", "words": ["eat", "then", "it", "up"]}, {"sent": "would you like to do the poppers up so it doesnt fall off there ?", "words": ["the", "do", "up", "off", "to", "so", "like", "fall", "you", "there", "it", "would"]}, {"sent": "hold it up higher .", "words": ["up", "it", "hold"]}, {"sent": "I havent got up another one .", "words": ["up", "another"]}, {"sent": "theyve got a good view from up there havent they ?", "words": ["up", "there", "they", "good", "a"]}, {"sent": "stand up whos hungry ?", "words": ["up", "stand", "hungry"]}, {"sent": "what is that ?", "words": ["what", "that", "is"]}, {"sent": "this is just like it .", "words": ["this", "it", "like", "is"]}, {"sent": "bear is in your house .", "words": ["your", "is", "bear", "in", "house"]}, {"sent": "all take a bow and the dance is .", "words": ["the", "all", "is", "dance", "and", "take", "a"]}, {"sent": "is that Rabbits house ?", "words": ["house", "that", "is"]}, {"sent": "all I do is wipe my shoe off .", "words": ["off", "all", "do", "my", "is", "shoe", "wipe"]}, {"sent": "because grass is green isnt it ?", "words": ["is", "because", "it", "green", "grass"]}, {"sent": "is it very sad ?", "words": ["sad", "it", "is"]}, {"sent": "and this is a calf .", "words": ["this", "a", "is", "and"]}, {"sent": "is it rolling around ?", "words": ["around", "it", "is"]}, {"sent": "Child is placing his men at the airport .", "words": ["his", "the", "at", "is"]}, {"sent": "you see this is the problem .", "words": ["the", "is", "you", "this", "see"]}, {"sent": "what is this what is this car do ?", "words": ["what", "do", "is", "car", "this"]}, {"sent": "so Oma is gonna bring him back to you .", "words": ["back", "is", "so", "to", "bring", "you", "him"]}, {"sent": "is that what you mean ?", "words": ["what", "that", "you", "is"]}, {"sent": "no I dim a dis is rice .", "words": ["is", "a"]}, {"sent": "this is Maisys bedtime .", "words": ["this", "is"]}, {"sent": "is that right ?", "words": ["that", "is"]}, {"sent": "is she ?", "words": ["she", "is"]}, {"sent": "is it ?", "words": ["it", "is"]}, {"sent": "xxx this is a knee right here .", "words": ["knee", "is", "here", "this", "a"]}, {"sent": "yes and what is this cat doing ?", "words": ["what", "is", "and", "cat", "this"]}, {"sent": "is she ?", "words": ["she", "is"]}, {"sent": "it is darling .", "words": ["it", "is"]}, {"sent": "this xxx listing thing is all messed up .", "words": ["all", "this", "up", "is"]}, {"sent": "oh there he is .", "words": ["there", "he", "is"]}, {"sent": "what is this ?", "words": ["what", "this", "is"]}, {"sent": "is your finger a spoon ?", "words": ["your", "is", "finger", "spoon", "a"]}, {"sent": "there she is and there is the spider hello spider I dont think thats good to eat is it ?", "words": ["the", "think", "is", "to", "and", "eat", "there", "dont", "it", "good", "she"]}, {"sent": "the house of bricks is a good house it says .", "words": ["the", "is", "of", "it", "house", "good", "a"]}, {"sent": "is it a tractor ?", "words": ["it", "tractor", "a", "is"]}, {"sent": "no this is a ssss .", "words": ["this", "a", "is"]}, {"sent": "its not as good as the cars that are at grandma grandpas is it ?", "words": ["the", "is", "are", "it", "good", "at", "that", "not"]}, {"sent": "what da ya think is in here ?", "words": ["what", "think", "is", "here", "in"]}, {"sent": "is she going to sing row row row the boat ?", "words": ["the", "is", "to", "sing", "boat", "she"]}, {"sent": "is he going in the house ?", "words": ["the", "is", "in", "house", "he"]}, {"sent": "Mommy is just trying to get you something to drink okay ?", "words": ["drink", "is", "to", "you", "get"]}, {"sent": "monkey is jumping on the settee .", "words": ["on", "the", "monkey", "is"]}, {"sent": "he is gonna get you .", "words": ["get", "he", "you", "is"]}, {"sent": "is it always Jwww ?", "words": ["it", "is"]}, {"sent": "mm my kitchen is one a those wall things you know ?", "words": ["my", "those", "is", "you", "kitchen", "a"]}, {"sent": "she knows this is probably a horse I know its a donkey .", "words": ["is", "this", "donkey", "horse", "she", "a"]}, {"sent": "Priya is a pretty little girl .", "words": ["little", "pretty", "a", "is"]}, {"sent": "yes FruitoftheLoom is sposta be in the front .", "words": ["the", "be", "in", "is"]}, {"sent": "and Ive never heard anyone say anything bad about it which is unusual .", "words": ["is", "say", "bad", "and", "it", "about", "which"]}, {"sent": "is it fun to hit ?", "words": ["to", "it", "hit", "is"]}, {"sent": "is that funny ?", "words": ["that", "is"]}, {"sent": "what is that ?", "words": ["what", "that", "is"]}, {"sent": "who is driving ?", "words": ["is", "who"]}, {"sent": "Tinkerbell the horse is getting hungry I think we should get going to the store .", "words": ["the", "think", "is", "we", "to", "store", "hungry", "get", "horse"]}, {"sent": "is that a football ?", "words": ["that", "a", "is"]}, {"sent": "do they have snow inside the building or is the snow outside ?", "words": ["the", "do", "is", "outside", "inside", "snow", "have", "they"]}, {"sent": "the green one is in your hair .", "words": ["the", "your", "is", "in", "green", "hair"]}, {"sent": "is our is our bedroom green ?", "words": ["our", "bedroom", "green", "is"]}, {"sent": "where is the train coming to see Alex .", "words": ["where", "the", "train", "is", "to", "see"]}, {"sent": "because this is her first time to see so many people in her castle .", "words": ["is", "to", "so", "first", "this", "in", "see", "her", "because"]}, {"sent": "it is isnt it ?", "words": ["it", "is"]}, {"sent": "oh mommy sheep is called an ewe can you feel that .", "words": ["can", "is", "you", "an", "sheep", "that"]}, {"sent": "why is he naughty ?", "words": ["he", "naughty", "why", "is"]}, {"sent": "what else is green ?", "words": ["what", "green", "is"]}, {"sent": "oh I think it is a bow .", "words": ["think", "it", "a", "is"]}, {"sent": "is there any in the freezer ?", "words": ["the", "is", "any", "there", "in"]}, {"sent": "is it ?", "words": ["it", "is"]}, {"sent": "what is that ?", "words": ["what", "that", "is"]}, {"sent": "is that a black teddy bear Jilly ?", "words": ["is", "bear", "black", "that", "a"]}, {"sent": "is it sunny ?", "words": ["it", "is"]}, {"sent": "is he coming Becky ?", "words": ["he", "is"]}, {"sent": "what is what ?", "words": ["what", "is"]}, {"sent": "well who is going to rescue them if the the police car .", "words": ["the", "if", "is", "to", "car", "them", "who"]}, {"sent": "what is he doing ?", "words": ["what", "he", "is"]}, {"sent": "its very difficult is it ?", "words": ["it", "is"]}, {"sent": "it is a long zipper .", "words": ["is", "long", "zipper", "it", "a"]}, {"sent": "oh what is that ?", "words": ["what", "that", "is"]}, {"sent": "show me what it is .", "words": ["what", "is", "show", "it", "me"]}, {"sent": "what is she doing ?", "words": ["what", "she", "is"]}, {"sent": "oh this is the back of it .", "words": ["the", "back", "is", "of", "this", "it"]}, {"sent": "what color is it ?", "words": ["what", "it", "is"]}, {"sent": "this is a toughies .", "words": ["this", "a", "is"]}, {"sent": "where is your awie .", "words": ["where", "your", "is"]}, {"sent": "my name is Freddy .", "words": ["my", "is"]}, {"sent": "do you know what that picture is ?", "words": ["do", "what", "is", "picture", "you", "that"]}, {"sent": "I know it is but its got a its got a a sort of a picture on the side hasnt it ?", "words": ["the", "is", "of", "picture", "it", "on", "but", "a"]}, {"sent": "is that nice ?", "words": ["nice", "that", "is"]}, {"sent": "what is Cookie Monster doing ?", "words": ["what", "is"]}, {"sent": "its not rough is it ?", "words": ["it", "not", "is"]}, {"sent": "and Gra James is cross and Gordons grumpy .", "words": ["is", "and"]}, {"sent": "is he sleeping ?", "words": ["he", "is"]}, {"sent": "is he coming to do the repair work ?", "words": ["the", "do", "work", "is", "to", "he"]}, {"sent": "Mommy is tying it up right now .", "words": ["up", "it", "is"]}, {"sent": "is your scratch better ?", "words": ["better", "your", "is"]}, {"sent": "is that right ?", "words": ["that", "is"]}, {"sent": "now is there a bunny oh yes theres a bunny now .", "words": ["there", "bunny", "a", "is"]}, {"sent": "you know what this one is called ?", "words": ["what", "is", "this", "you"]}, {"sent": "well see Washington is a city where people do important things .", "words": ["where", "do", "is", "see", "a"]}, {"sent": "is that right ?", "words": ["that", "is"]}, {"sent": "what color is that ?", "words": ["what", "that", "is"]}, {"sent": "it just it is hard to open .", "words": ["is", "open", "to", "it", "hard"]}, {"sent": "oh here is our paper .", "words": ["here", "our", "paper", "is"]}, {"sent": "we just dont know where he is uhn do we ?", "words": ["where", "do", "is", "we", "dont", "he"]}, {"sent": "is there a television ?", "words": ["there", "a", "is"]}, {"sent": "yeah Im really hurt because Ive been run over about three times now .", "words": ["hurt", "run", "about", "over", "because"]}, {"sent": "you always think about the day we were walking past theater and it collapsed .", "words": ["the", "think", "we", "you", "and", "it", "about", "were"]}, {"sent": "how about .", "words": ["how", "about"]}, {"sent": "about there .", "words": ["there", "about"]}, {"sent": "shall we talk about father Jean ?", "words": ["about", "talk", "we"]}, {"sent": "who else can we look at and talk about ?", "words": ["can", "we", "and", "at", "talk", "look", "about", "who"]}, {"sent": "what was it about ?", "words": ["was", "what", "it", "about"]}, {"sent": "um what does Aurora like to sing about ?", "words": ["what", "does", "to", "like", "sing", "about"]}, {"sent": "how about that ?", "words": ["how", "that", "about"]}, {"sent": "and how about follow that bird ?", "words": ["how", "bird", "and", "about", "that"]}, {"sent": "what about some of these other things over here huh ?", "words": ["what", "these", "of", "other", "here", "about", "some", "over"]}, {"sent": "and how about this ?", "words": ["this", "how", "about", "and"]}, {"sent": "we wanna the aquarium and saw the movie about fish .", "words": ["the", "we", "and", "about", "wanna", "fish"]}, {"sent": "okay how about an orange one ?", "words": ["how", "orange", "about", "an"]}, {"sent": "lets make lets make sure about that first .", "words": ["that", "first", "make", "about"]}, {"sent": "so whenever we have a babysitter come over were like okay if she goes like this just make up some song about whatever shes holding .", "words": ["up", "if", "we", "so", "like", "have", "this", "make", "about", "some", "over", "she", "were", "a"]}, {"sent": "what about the other chair ?", "words": ["the", "what", "chair", "other", "about"]}, {"sent": "whats good about Goldilocks ?", "words": ["about", "good"]}, {"sent": "you ever thought about that ?", "words": ["that", "about", "you"]}, {"sent": "how about that one piece red thing ?", "words": ["red", "how", "that", "about"]}, {"sent": "what about the octopus ?", "words": ["the", "what", "about"]}, {"sent": "how about this one ?", "words": ["this", "how", "about"]}, {"sent": "he gets up about five oclock in the morning .", "words": ["up", "the", "in", "he", "about"]}, {"sent": "what about this page ?", "words": ["what", "about", "this"]}, {"sent": "this book is about numbers .", "words": ["book", "this", "about", "is"]}, {"sent": "thats a book about telling the time .", "words": ["book", "the", "about", "a"]}, {"sent": "can you tell me anything about this package ?", "words": ["can", "you", "this", "about", "me"]}, {"sent": "okay what about the other foot .", "words": ["the", "what", "other", "foot", "about"]}, {"sent": "how about these Henry ?", "words": ["these", "how", "about"]}, {"sent": "tell me about that picture .", "words": ["me", "picture", "that", "about"]}, {"sent": "tell me about this park .", "words": ["me", "park", "about", "this"]}, {"sent": "what about these whales ?", "words": ["what", "these", "about"]}, {"sent": "what about the dragon ?", "words": ["the", "what", "about"]}, {"sent": "I dont know about that .", "words": ["dont", "that", "about"]}, {"sent": "whats it about ?", "words": ["it", "about"]}, {"sent": "about to drive off into the water .", "words": ["off", "the", "into", "to", "drive", "about", "water"]}, {"sent": "there you see about getting pizzas .", "words": ["see", "there", "about", "you"]}, {"sent": "what about this foot ?", "words": ["what", "foot", "about", "this"]}, {"sent": "whoa talk about dumping out .", "words": ["out", "about", "talk"]}, {"sent": "what about Dimitra ?", "words": ["what", "about"]}, {"sent": "I mean hes ya know hes he really gets specific about stuff .", "words": ["he", "about"]}, {"sent": "what about that one ?", "words": ["what", "that", "about"]}, {"sent": "how about that guy ?", "words": ["how", "that", "about"]}, {"sent": "did you ever dream about anything ?", "words": ["did", "about", "you"]}, {"sent": "what do you wanna tell me about Gordon ?", "words": ["what", "do", "you", "wanna", "about", "me"]}, {"sent": "how about finding the same ?", "words": ["the", "how", "same", "about"]}, {"sent": "how about this one ?", "words": ["this", "how", "about"]}, {"sent": "why do you keep talking about Daddy Rods big tummy ?", "words": ["do", "you", "about", "tummy", "big", "why"]}, {"sent": "remember the book about the helicopter ?", "words": ["the", "book", "helicopter", "about"]}, {"sent": "youve been thinking about eating jello .", "words": ["about"]}, {"sent": "how about the food dont I hafta pay for the food ?", "words": ["the", "how", "for", "dont", "food", "about"]}, {"sent": "Ethan out and about .", "words": ["about", "out", "and"]}, {"sent": "what about Thursday ?", "words": ["what", "about"]}, {"sent": "you keep talking about big bears .", "words": ["big", "about", "you"]}, {"sent": "its been about fifteen minutes .", "words": ["about"]}, {"sent": "xxx Mia just leave it and go about your business .", "words": ["your", "go", "and", "it", "about"]}, {"sent": "oh what about those ?", "words": ["what", "those", "about"]}, {"sent": "how can I make it if I dont know what youre talking about ?", "words": ["can", "what", "how", "if", "dont", "it", "about", "make"]}, {"sent": "she was complaining a little about her ears .", "words": ["was", "little", "about", "her", "she", "a"]}, {"sent": "how about this this will be my two dollars ?", "words": ["how", "my", "be", "this", "about", "will"]}, {"sent": "how about Andy ?", "words": ["how", "about"]}, {"sent": "what about like that then ?", "words": ["what", "like", "about", "then", "that"]}, {"sent": "alright how about putting some of these things in there ?", "words": ["these", "how", "of", "there", "in", "about", "some"]}, {"sent": "do you know one thing about Purdie ?", "words": ["do", "about", "you"]}, {"sent": "how about this tree ?", "words": ["tree", "this", "how", "about"]}, {"sent": "and what about this one ?", "words": ["what", "this", "about", "and"]}, {"sent": "and does she sing a little song about her backpack ?", "words": ["does", "and", "sing", "little", "about", "her", "she", "a"]}, {"sent": "youre saying something about last night .", "words": ["last", "about"]}, {"sent": "and what about poor mousie ?", "words": ["what", "poor", "about", "and"]}, {"sent": "alright talk about it .", "words": ["it", "about", "talk"]}, {"sent": "shall we sing our song about shoes ?", "words": ["sing", "our", "about", "we"]}, {"sent": "what can you tell me about Pokemon ?", "words": ["can", "what", "you", "about", "me"]}, {"sent": "um left them in stores and whatnot shes been pretty good about it .", "words": ["them", "and", "in", "it", "pretty", "good", "about"]}, {"sent": "where about in the dresser do you think well put them ?", "words": ["where", "the", "do", "think", "put", "them", "you", "in", "about"]}, {"sent": "how about in the Tweenies cup ?", "words": ["the", "cup", "how", "in", "about"]}, {"sent": "what about .", "words": ["what", "about"]}, {"sent": "that we were talking about before .", "words": ["about", "that", "were", "we"]}, {"sent": "and its all about colors and shapes and animals and things like that .", "words": ["all", "like", "and", "about", "that"]}, {"sent": "although theyre about the same .", "words": ["the", "same", "about"]}, {"sent": "can you tell me a story about the picture ?", "words": ["can", "the", "picture", "you", "about", "me", "story", "a"]}, {"sent": "what about a book ?", "words": ["book", "what", "about", "a"]}, {"sent": "um he seemed to be excited about the ice cream cones um in the beginning and then they got his interest again so I would hafta say thats probably the one that sticks in my mind .", "words": ["the", "ice cream", "my", "to", "be", "so", "say", "and", "his", "they", "in", "he", "about", "would", "ice", "then", "that"]}, {"sent": "how about you just put the box down little man .", "words": ["the", "how", "put", "box", "you", "little", "about", "down"]}, {"sent": "lets put this somewhere how about well put it right there .", "words": ["put", "how", "this", "there", "it", "about"]}, {"sent": "he dont care about this .", "words": ["dont", "he", "about", "this"]}, {"sent": "what about .", "words": ["what", "about"]}, {"sent": "how about Linda ?", "words": ["how", "about"]}, {"sent": "ah what are you going to dream about crocodile ?", "words": ["what", "to", "you", "are", "about"]}, {"sent": "what about the giraffe ?", "words": ["the", "what", "giraffe", "about"]}, {"sent": "what about your people they wanna go for a ride .", "words": ["what", "for", "your", "go", "they", "wanna", "about", "ride", "a"]}, {"sent": "oh okay what about this ?", "words": ["what", "about", "this"]}, {"sent": "no you dont care about the dinosaur anymore ?", "words": ["the", "dont", "about", "you"]}, {"sent": "how about this ?", "words": ["this", "how", "about"]}, {"sent": "how about this one ?", "words": ["this", "how", "about"]}, {"sent": "how about we draw on this ?", "words": ["how", "we", "draw", "this", "about", "on"]}, {"sent": "do you know a song about a sheep ?", "words": ["do", "you", "sheep", "about", "a"]}, {"sent": "that led us into talking about things that are shaped like circles cause weve been working on shapes at home .", "words": ["into", "like", "us", "are", "about", "at", "on", "that", "home"]}, {"sent": "was it about a little boy ?", "words": ["was", "little", "it", "about", "a"]}, {"sent": "for about a week ?", "words": ["for", "about", "a"]}, {"sent": "I think you smile with your top teeth .", "words": ["think", "your", "with", "you", "smile"]}, {"sent": "panda was wanting can she come to sleep over at your house ?", "words": ["can", "your", "sleep", "was", "to", "house", "at", "over", "she"]}, {"sent": "did Myrna wash your lunchbox .", "words": ["your", "wash", "did"]}, {"sent": "because youre putting Mummys hair into your ears .", "words": ["hair", "your", "into", "because"]}, {"sent": "okay put your sticker on the chair .", "words": ["the", "put", "your", "chair", "on"]}, {"sent": "who are you going to give your new racoon to ?", "words": ["your", "to", "you", "are", "who", "give", "new"]}, {"sent": "you did such a good job and Sue said your back teeth are your smily teeth .", "words": ["your", "back", "and", "you", "are", "did", "good", "a"]}, {"sent": "your wheels broken off has it ?", "words": ["off", "it", "your", "broken"]}, {"sent": "you have your hands washed and you can eat .", "words": ["can", "your", "and", "have", "you", "eat"]}, {"sent": "you cant have socks in your head .", "words": ["your", "head", "you", "have", "in"]}, {"sent": "look at your foot funny looking foot .", "words": ["foot", "at", "your", "look"]}, {"sent": "but I hafta find your other bootie .", "words": ["other", "your", "but", "find"]}, {"sent": "your sisters a little girl .", "words": ["your", "little", "a"]}, {"sent": "right so you dont have one of those in your room .", "words": ["those", "your", "of", "so", "you", "have", "dont", "in", "room"]}, {"sent": "is your strap bigger than mine ?", "words": ["mine", "your", "is"]}, {"sent": "you were going to ask Jeannine to come to your birthday party .", "words": ["your", "to", "party", "you", "were"]}, {"sent": "put your car down .", "words": ["car", "put", "down", "your"]}, {"sent": "go get your doll and get out your rockingchair and you can sing .", "words": ["can", "your", "go", "doll", "and", "get", "you", "sing", "out"]}, {"sent": "hows your jellybelly doing ?", "words": ["your"]}, {"sent": "lets do your hair .", "words": ["do", "hair", "your"]}, {"sent": "xxx where did your little granola bites go ?", "words": ["where", "your", "go", "little", "did"]}, {"sent": "the tray is out in the in the kitchen with your highchair .", "words": ["the", "your", "is", "with", "in", "out", "kitchen", "tray"]}, {"sent": "I like your new shoes .", "words": ["new", "your", "like"]}, {"sent": "are your hands sticky ?", "words": ["sticky", "are", "your"]}, {"sent": "how about giving me your place mat ?", "words": ["me", "how", "your", "about"]}, {"sent": "it sleeps in your cot .", "words": ["in", "it", "your"]}, {"sent": "thats for playing all your music ?", "words": ["all", "for", "your"]}, {"sent": "well hafta find your little ambulance Thomas .", "words": ["your", "little", "find"]}, {"sent": "lets put him over there by your juice .", "words": ["put", "your", "by", "there", "him", "juice", "over"]}, {"sent": "is that your Daddy Frosty ?", "words": ["your", "that", "is"]}, {"sent": "wheres your other queen ?", "words": ["your", "other"]}, {"sent": "Imean the only thing I really need was right Conors name your name address phone number Conors date of birth which is the fifth of the first .", "words": ["the", "need", "your", "is", "of", "was", "first", "which"]}, {"sent": "and then you eat your peach .", "words": ["your", "you", "eat", "and", "then"]}, {"sent": "eat your cookies .", "words": ["eat", "your"]}, {"sent": "move your foot .", "words": ["foot", "your"]}, {"sent": "your arm ?", "words": ["your", "arm"]}, {"sent": "mm that handles in your way .", "words": ["your", "in", "that"]}, {"sent": "your beads .", "words": ["your", "beads"]}, {"sent": "your Mamas busy tidying up .", "words": ["up", "your"]}, {"sent": "shall I get the hoover upstairs and well hoover your room ?", "words": ["the", "your", "and", "get", "room"]}, {"sent": "Nathaniel why dont you bring your good book your new book ?", "words": ["your", "bring", "you", "dont", "good", "book", "new", "why"]}, {"sent": "Im protecting myself against your maniac mother .", "words": ["your", "myself"]}, {"sent": "Ill beep your nose .", "words": ["your", "nose"]}, {"sent": "where is your foot .", "words": ["where", "foot", "your", "is"]}, {"sent": "stuck on your tutu ?", "words": ["on", "stuck", "your"]}, {"sent": "it has gone in your tummy .", "words": ["in", "it", "your", "tummy"]}, {"sent": "which way will your airplane go ?", "words": ["your", "go", "airplane", "will", "which"]}, {"sent": "when you were a little baby Nathaniel we useta hang your nappies in the sun to dry because they were all yellow .", "words": ["the", "all", "your", "we", "to", "sun", "dry", "because", "little", "you", "they", "in", "yellow", "when", "were", "a"]}, {"sent": "you want me to do your trousers ?", "words": ["do", "your", "to", "you", "me"]}, {"sent": "what else did you try there with the spoon what did you eat with the spoon xxx okay come on did you eat maple syrup xxx microphone sweetie you hafta hold onto this sweetie if you dont want to hold it were gonna put it back in your backpack xxx put your backpack on okay which would you rather do wear your backpack or hold this microphone ?", "words": ["the", "what", "put", "hold", "eat", "spoon", "which", "with", "there", "it", "want to", "on", "were", "if", "to", "you", "dont", "would", "try", "do", "your", "back", "this", "in", "did"]}, {"sent": "Becca could I make one of your .", "words": ["of", "could", "make", "your"]}, {"sent": "do you like your pillow ?", "words": ["do", "your", "like", "you", "pillow"]}, {"sent": "where wouldjou like to sit in your chair or the big chair ?", "words": ["where", "the", "your", "to", "like", "chair", "in", "sit", "big"]}, {"sent": "well take your coat off here .", "words": ["off", "your", "here", "take", "coat"]}, {"sent": "put my hand on your hand and then your hand on my hand .", "words": ["put", "my", "your", "hand", "and", "on", "then"]}, {"sent": "because youre going to have your dinner soon .", "words": ["have", "to", "your", "because"]}, {"sent": "can you shall we play with your PlayDoh ?", "words": ["can", "your", "we", "play", "with", "you"]}, {"sent": "you having your botbot again are you ?", "words": ["are", "your", "you"]}, {"sent": "you want your granola in your cup or do you want me to put it in a bowl ?", "words": ["cup", "do", "put", "bowl", "your", "to", "you", "in", "it", "me", "a"]}, {"sent": "when youve finished wiping with your cloth you can have Emilys picture .", "words": ["can", "your", "picture", "with", "you", "have", "when"]}, {"sent": "thats gonna be your room ?", "words": ["be", "room", "your"]}, {"sent": "there is the buttons on your box .", "words": ["the", "your", "box", "is", "there", "on"]}, {"sent": "come on Sam said Stella put on your skates not right now said Sam Im listening to the frogs snore .", "words": ["the", "put", "your", "to", "on", "not"]}, {"sent": "your coat makes you think we have more toys ?", "words": ["think", "your", "we", "you", "have", "more", "coat"]}, {"sent": "is this your foot ?", "words": ["this", "foot", "your", "is"]}, {"sent": "can you put your doctors stuff away ?", "words": ["can", "put", "your", "you", "away"]}, {"sent": "your last name ?", "words": ["last", "your"]}, {"sent": "show John your books .", "words": ["show", "your"]}, {"sent": "and do you remember your lips turned blue ?", "words": ["do", "your", "blue", "you", "and"]}, {"sent": "wheres your ankle ?", "words": ["ankle", "your"]}, {"sent": "has your that one got black hair ?", "words": ["black", "that", "hair", "your"]}, {"sent": "okay here is your next thing Mag .", "words": ["here", "your", "is"]}, {"sent": "wheres your ears ?", "words": ["your"]}, {"sent": "your what ?", "words": ["what", "your"]}, {"sent": "dyou remember what else is in your book ?", "words": ["what", "your", "is", "in", "book"]}, {"sent": "you want to put your books in ?", "words": ["put", "your", "to", "you", "in", "want to"]}, {"sent": "it says put put your hand in the fuzzy mitten .", "words": ["the", "put", "your", "hand", "in", "it"]}, {"sent": "Mummys having your juice .", "words": ["juice", "your"]}, {"sent": "its shampoo in your hair .", "words": ["in", "hair", "your"]}, {"sent": "your mean you could scratch it ?", "words": ["could", "it", "your", "you"]}, {"sent": "put your baba the other way round .", "words": ["the", "put", "your", "other"]}, {"sent": "wheres your cup ?", "words": ["cup", "your"]}, {"sent": "thats your little pet name .", "words": ["your", "little"]}, {"sent": "I dont think weve anything else thatd fit on your finger .", "words": ["think", "your", "fit", "dont", "finger", "on"]}, {"sent": "I wondered what the silly look was on your face .", "words": ["the", "what", "your", "face", "was", "look", "on"]}, {"sent": "is that your favorite page ?", "words": ["your", "that", "is"]}, {"sent": "your starfish in there ?", "words": ["there", "in", "your"]}, {"sent": "but youre in deep concentration when you play with your train .", "words": ["your", "train", "play", "with", "you", "in", "when", "but"]}, {"sent": "do your ears hurt love ?", "words": ["hurt", "do", "your", "love"]}, {"sent": "are you going out xxx youre putting your shoes on ?", "words": ["your", "you", "are", "out", "on"]}, {"sent": "we can just put your old nappie on .", "words": ["can", "old", "put", "your", "we", "on"]}, {"sent": "you just wanna walk around in your vest do you ?", "words": ["do", "your", "you", "in", "wanna", "around", "walk"]}, {"sent": "found your mittens ?", "words": ["mittens", "your"]}, {"sent": "xxx unable to answer your call right now please leave your name number and message after the tone .", "words": ["the", "to", "your", "and"]}, {"sent": "your bunny is so big .", "words": ["bunny", "your", "is", "so", "big"]}, {"sent": "stung your butt ?", "words": ["your"]}, {"sent": "thats for your Daddy is it ?", "words": ["it", "for", "your", "is"]}, {"sent": "can you say your welcome ?", "words": ["can", "say", "your", "you"]}, {"sent": "your new costume .", "words": ["new", "your"]}, {"sent": "I like your family xxx .", "words": ["your", "like"]}, {"sent": "everybodys ears are hurting them Baby Cassie had an ear infection too .", "words": ["them", "an", "are", "too", "ear"]}, {"sent": "put them on yeah .", "words": ["on", "put", "them"]}, {"sent": "open them out .", "words": ["open", "them", "out"]}, {"sent": "and Im going to hide them .", "words": ["to", "them", "hide", "and"]}, {"sent": "hafta look for them .", "words": ["for", "them", "look"]}, {"sent": "just let them play .", "words": ["play", "them"]}, {"sent": "can you match them up ?", "words": ["can", "up", "them", "you"]}, {"sent": "theyre not going under because you just put them on the board .", "words": ["the", "put", "them", "you", "under", "on", "not", "because"]}, {"sent": "where are you going to take them ?", "words": ["where", "to", "them", "you", "are", "take"]}, {"sent": "you like them toy ?", "words": ["toy", "them", "like", "you"]}, {"sent": "er better put them in here to take them then .", "words": ["better", "put", "to", "them", "here", "in", "then", "take"]}, {"sent": "I think weve found them .", "words": ["think", "them"]}, {"sent": "did you ride them ?", "words": ["them", "ride", "did", "you"]}, {"sent": "youre going to try and catch them ?", "words": ["to", "catch", "them", "and", "try"]}, {"sent": "we dont we didnt all get them each day did we ?", "words": ["all", "each", "we", "them", "get", "dont", "did"]}, {"sent": "yeah but have you done them right ?", "words": ["have", "them", "but", "you"]}, {"sent": "oh youre putting them outside the truck .", "words": ["the", "outside", "truck", "them"]}, {"sent": "and now you might might be able to get them undone .", "words": ["to", "be", "them", "you", "get", "and"]}, {"sent": "and you collect all the dead flowers and then you put them in the compost bin for me dont you ?", "words": ["the", "all", "put", "for", "them", "you", "and", "dont", "in", "me", "then"]}, {"sent": "you hand me the pictures when youre finished with them .", "words": ["the", "hand", "with", "them", "you", "me", "when"]}, {"sent": "youre just gonna eat them huh ?", "words": ["eat", "them"]}, {"sent": "you could actually pick them up with your hands .", "words": ["up", "your", "pick", "them", "with", "you", "could"]}, {"sent": "no you hafta put them out like this .", "words": ["put", "them", "like", "you", "this", "out"]}, {"sent": "put them all in the box .", "words": ["the", "all", "put", "box", "them", "in"]}, {"sent": "or do you wanna join them ?", "words": ["do", "them", "wanna", "you"]}, {"sent": "aw maybe youll get them for your birthday .", "words": ["get", "for", "them", "your"]}, {"sent": "have they got smarties on them ?", "words": ["have", "they", "on", "them"]}, {"sent": "and then you can look at them later on .", "words": ["can", "them", "you", "and", "at", "look", "on", "then"]}, {"sent": "just put them back in .", "words": ["in", "put", "them", "back"]}, {"sent": "for them to stand on .", "words": ["for", "to", "stand", "them", "on"]}, {"sent": "but June to September I hope were not in Spain when they start flowering or well miss them .", "words": ["when", "to", "them", "they", "in", "were", "not", "but"]}, {"sent": "look at them all coming .", "words": ["all", "at", "them", "look"]}, {"sent": "the wonderful thing about dolphins is hearing them try to speak .", "words": ["the", "is", "to", "them", "about", "try", "try to"]}, {"sent": "dont lose them .", "words": ["dont", "them"]}, {"sent": "he said he said to bring them and let Conor see them .", "words": ["to", "them", "bring", "and", "he", "see"]}, {"sent": "alright now we do needta turn them over .", "words": ["over", "do", "them", "we"]}, {"sent": "shall I put them on the table ?", "words": ["the", "put", "table", "them", "on"]}, {"sent": "its much easier breaking them up isnt it ?", "words": ["up", "it", "much", "them"]}, {"sent": "pick them up and put them pop them in .", "words": ["up", "put", "pop", "pick", "them", "and", "in"]}, {"sent": "should we put them back ?", "words": ["put", "them", "back", "we"]}, {"sent": "Mommy put them away over here .", "words": ["put", "them", "here", "away", "over"]}, {"sent": "Ill eat them .", "words": ["eat", "them"]}, {"sent": "Ill get them shall I ?", "words": ["get", "them"]}, {"sent": "can you pick them up ?", "words": ["can", "up", "pick", "them", "you"]}, {"sent": "see if you can throw them in from there .", "words": ["can", "throw", "if", "them", "you", "there", "in", "see"]}, {"sent": "yes but she did mean play with them Aran .", "words": ["play", "with", "them", "did", "she", "but"]}, {"sent": "be careful now throwing them .", "words": ["be", "them", "careful"]}, {"sent": "you smell them .", "words": ["them", "you"]}, {"sent": "no we just play with them .", "words": ["them", "play", "with", "we"]}, {"sent": "go open them wide .", "words": ["open", "them", "go"]}, {"sent": "we pick them up and spin them round .", "words": ["up", "we", "pick", "them", "and"]}, {"sent": "okay and we roll them up like this there he goes .", "words": ["up", "we", "them", "like", "and", "this", "there", "he"]}, {"sent": "you can reach them there .", "words": ["can", "there", "them", "you"]}, {"sent": "can you put them back in ?", "words": ["can", "put", "back", "them", "you", "in"]}, {"sent": "lets try putting them back .", "words": ["try", "back", "them"]}, {"sent": "take them both and put them in .", "words": ["put", "them", "and", "in", "take"]}, {"sent": "why do you wear them if they dont help ?", "words": ["do", "help", "if", "them", "you", "they", "dont", "why"]}, {"sent": "do they need the straps on to secure them in ?", "words": ["the", "do", "need", "to", "them", "they", "in", "on"]}, {"sent": "unless you hafta put them up .", "words": ["up", "put", "them", "you"]}, {"sent": "we waved them off at Manchester airport .", "words": ["off", "at", "them", "we"]}, {"sent": "I dont want you smashing them .", "words": ["dont", "them", "you"]}, {"sent": "shall I tip them out ?", "words": ["them", "out"]}, {"sent": "because you like them .", "words": ["them", "like", "because", "you"]}, {"sent": "she will continue to rotate around the same circle with them .", "words": ["the", "to", "with", "them", "same", "around", "will", "she"]}, {"sent": "because they were too heavy to carry all of them uhn werent they ?", "words": ["all", "of", "to", "them", "carry", "they", "too", "heavy", "were", "because"]}, {"sent": "weve had them in the house .", "words": ["the", "house", "in", "them"]}, {"sent": "Ill do this and you do them round there and Ill turn them round to meet up right ?", "words": ["up", "do", "to", "them", "you", "and", "this", "there"]}, {"sent": "yeah clever girl you are so clever out of all them which ones the smallest one ?", "words": ["the", "all", "of", "so", "them", "you", "are", "out", "which"]}, {"sent": "are you mixing them up ?", "words": ["up", "are", "them", "you"]}, {"sent": "I was just about to say Ill take the jelly babies off you for saying silly woman but youve eaten them havent you ?", "words": ["the", "jelly", "off", "for", "to", "was", "say", "them", "you", "about", "take", "but"]}, {"sent": "its a little bit difficult to hook them isnt it ?", "words": ["to", "them", "little", "it", "a"]}, {"sent": "oh but can you bring them out and show them to me ?", "words": ["can", "show", "to", "them", "bring", "you", "and", "out", "me", "but"]}, {"sent": "they just buy one stock and I guess they figure if youre gonna buy them youre gonna buy them early when .", "words": ["if", "them", "and", "they", "when", "buy"]}, {"sent": "that is a bunch of very ah look at those nails theyre so dirty I havta clean them ah .", "words": ["clean", "those", "is", "of", "so", "them", "at", "dirty", "look", "that", "a"]}, {"sent": "no lets not get them muddled .", "words": ["get", "them", "not"]}, {"sent": "do you want them on your plate then ?", "words": ["do", "your", "plate", "them", "you", "on", "then"]}, {"sent": "take all of them out though .", "words": ["all", "of", "them", "out", "take"]}, {"sent": "Id better put them in because its kind of far across and its hot .", "words": ["better", "put", "of", "them", "and", "in", "hot", "because"]}, {"sent": "what color should I use to draw them with ?", "words": ["what", "to", "draw", "them", "with"]}, {"sent": "you were telling them about the train you had just seen going under the bridge .", "words": ["the", "train", "them", "you", "under", "about", "were"]}, {"sent": "you walk outof the way of them ?", "words": ["the", "of", "them", "you", "walk"]}, {"sent": "make them .", "words": ["make", "them"]}, {"sent": "hit them all over .", "words": ["over", "all", "hit", "them"]}, {"sent": "Ive never seen them before .", "words": ["them"]}, {"sent": "he is more interested in books with real um babies in them .", "words": ["is", "with", "them", "in", "more", "he"]}, {"sent": "what happened to them ?", "words": ["to", "what", "them"]}, {"sent": "okay let us put them back on now .", "words": ["put", "back", "them", "us", "on"]}, {"sent": "in a fence can look at them .", "words": ["can", "them", "in", "at", "look", "a"]}, {"sent": "no I could take them outside .", "words": ["outside", "could", "take", "them"]}, {"sent": "what we needta do I needta cut them out .", "words": ["what", "do", "cut", "we", "them", "out"]}, {"sent": "uh who are you posting them to ?", "words": ["to", "them", "you", "are", "who"]}, {"sent": "oh somebodys mixed them up havent they ?", "words": ["up", "they", "them"]}, {"sent": "youll be able to see them wont you ?", "words": ["to", "be", "them", "you", "see"]}, {"sent": "whos getting a story read to them ?", "words": ["to", "them", "read", "story", "a"]}, {"sent": "wait okay let Melissa count them too .", "words": ["wait", "them", "too"]}, {"sent": "well Ive hinted often enough about the two of them the way they get on about things and .", "words": ["the", "of", "them", "and", "get", "they", "about", "on"]}, {"sent": "I know you hit them all over .", "words": ["all", "them", "you", "hit", "over"]}, {"sent": "did Roz put them away ?", "words": ["put", "away", "them", "did"]}, {"sent": "she doesnt like them .", "words": ["them", "she", "like"]}, {"sent": "are you messing them all up again ?", "words": ["up", "all", "them", "you", "are"]}, {"sent": "I think the men when we hear them they must be coming to the top of the road and they must go right down to the bottom end of Streetname Drive and pick up the bags there because we always seem to be waiting a long long time for them to come dont we ?", "words": ["the", "think", "go", "of", "when", "be", "there", "down", "up", "bottom", "for", "pick", "to", "long", "hear", "and", "dont", "because", "a", "we", "them", "they"]}, {"sent": "xxx one of them is it ?", "words": ["of", "it", "them", "is"]}, {"sent": "Purdie doesnt like the noise of them does she ?", "words": ["the", "does", "of", "them", "like", "she"]}, {"sent": "wanna finish your picture of Nan ?", "words": ["your", "of", "picture", "finish", "wanna"]}, {"sent": "and well just put a drop of milk into this saucer for Jess .", "words": ["put", "for", "into", "of", "and", "this", "drop", "milk", "a"]}, {"sent": "have you had some of these as well ?", "words": ["these", "of", "you", "have", "some"]}, {"sent": "lot of cars outside today .", "words": ["of", "outside"]}, {"sent": "so I guess I could take down a few of those boxes .", "words": ["those", "of", "so", "could", "down", "take", "a"]}, {"sent": "a slice of cheese .", "words": ["of", "cheese", "a"]}, {"sent": "you like to pull the clothes out of the basket ?", "words": ["the", "of", "to", "like", "pull", "you", "basket", "out"]}, {"sent": "two different shades of oranges .", "words": ["of"]}, {"sent": "theres always noise of some sort isnt there ?", "words": ["of", "some", "there"]}, {"sent": "it is not a cup of tea and even if it was you wouldnt eat it .", "words": ["cup", "if", "is", "of", "was", "and", "you", "eat", "it", "not", "a"]}, {"sent": "look at this tree full of green leaves .", "words": ["tree", "look", "full", "of", "this", "green", "at"]}, {"sent": "like this kind of cereal ?", "words": ["of", "this", "cereal", "like"]}, {"sent": "so youve had a few RiceKrispies half a piece of toast and a whole apple .", "words": ["toast", "of", "so", "and", "apple", "a"]}, {"sent": "if the n were in front of the o then of course it would be no .", "words": ["the", "if", "of", "be", "in", "it", "would", "then", "were"]}, {"sent": "youre gonna hafta find another way of getting them arent you Liz ?", "words": ["of", "them", "you", "another", "find"]}, {"sent": "and theres such a difference between the two of them .", "words": ["the", "of", "them", "and", "a"]}, {"sent": "when Daddy comes home are you going to tell him youve had hold of his cheque book ?", "words": ["hold", "of", "to", "you", "him", "are", "his", "book", "when", "home"]}, {"sent": "it says nothing about what the contents of a Big Mac .", "words": ["the", "what", "of", "it", "about", "a"]}, {"sent": "what kind of animal ?", "words": ["of", "what", "animal"]}, {"sent": "weve got plenty of tissues .", "words": ["of"]}, {"sent": "be careful of that xxx because you can fall over that .", "words": ["can", "of", "be", "fall", "you", "careful", "over", "that", "because"]}, {"sent": "you can read the book to daddy instead of daddy reading to you .", "words": ["can", "the", "of", "to", "you", "read", "book"]}, {"sent": "no I think she thinks its just full of chipmunks .", "words": ["of", "think", "she", "full"]}, {"sent": "Nick Londain have you heard of that its just about a woman a journalist .", "words": ["of", "you", "have", "about", "that", "a"]}, {"sent": "now have you built anything out of all the stones and the pebbles that we were playing with at the beginning ?", "words": ["the", "all", "we", "of", "with", "you", "have", "and", "out", "at", "that", "were"]}, {"sent": "do you want gravy on top of it ?", "words": ["do", "of", "you", "it", "on"]}, {"sent": "theres three different kinds of horsies .", "words": ["of"]}, {"sent": "so Ill get you a clean one out of the drawer .", "words": ["clean", "the", "of", "so", "you", "get", "out", "drawer", "a"]}, {"sent": "can you find some of your animals ?", "words": ["can", "your", "of", "you", "find", "some"]}, {"sent": "this could be the army of hamburgers .", "words": ["the", "of", "be", "this", "could"]}, {"sent": "I think were gonna hafta do a hunt of all the cars and pile them all together because theres a few missing .", "words": ["the", "do", "think", "all", "of", "them", "and", "were", "because", "a"]}, {"sent": "pocket full of posy .", "words": ["of", "full"]}, {"sent": "and is Fraser scared of that frog ?", "words": ["is", "of", "scared", "and", "frog", "that"]}, {"sent": "would you like one of these ?", "words": ["these", "of", "like", "you", "would"]}, {"sent": "and a bad gang of boys tried to capture his balloon from him .", "words": ["of", "to", "bad", "and", "him", "balloon", "his", "a"]}, {"sent": "weve seen them a couple of times in pictures but its been a while .", "words": ["of", "them", "in", "but", "a"]}, {"sent": "see heres another one of these things too .", "words": ["these", "of", "another", "too", "see"]}, {"sent": "I think I might just gather up few things of clothing items of clothing for you .", "words": ["up", "think", "for", "of", "you"]}, {"sent": "speaking of which do you want some of the scrumptious cookies that Mommy made the other day ?", "words": ["the", "do", "of", "you", "other", "that", "some", "which"]}, {"sent": "you want it on a peace of bread ?", "words": ["bread", "of", "you", "it", "on", "a"]}, {"sent": "a rare moment of sleep .", "words": ["of", "sleep", "a"]}, {"sent": "well youve got some of your bricks here .", "words": ["of", "some", "here", "your"]}, {"sent": "xxx you can take it out of the box now if you want .", "words": ["can", "the", "box", "if", "of", "you", "it", "out", "take"]}, {"sent": "are you going to watch Grandma get out of the car ?", "words": ["the", "watch", "of", "to", "car", "you", "get", "are", "out"]}, {"sent": "what sort of machine ?", "words": ["of", "what"]}, {"sent": "shout the Grand Old Duke of York .", "words": ["the", "of"]}, {"sent": "thats why Mummy has so many of those herbs and things .", "words": ["those", "of", "so", "and", "why"]}, {"sent": "Thomas had a lot of exercise today .", "words": ["of", "a lot", "a"]}, {"sent": "ah thats part of Wendys head .", "words": ["of", "head"]}, {"sent": "I missed half of it .", "words": ["of", "it"]}, {"sent": "while he was out of the room I did some painting to help him .", "words": ["the", "help", "of", "was", "to", "him", "room", "he", "did", "out", "some"]}, {"sent": "teddy can have this one of you like .", "words": ["can", "of", "like", "you", "have", "this"]}, {"sent": "you can take a picture of it if you want .", "words": ["can", "if", "of", "picture", "you", "it", "take", "a"]}, {"sent": "why dont we put xxx xxx on the back of Thomas .", "words": ["the", "put", "back", "we", "of", "dont", "on", "why"]}, {"sent": "or we could make it a necklace with a piece of yarn .", "words": ["necklace", "we", "of", "with", "it", "could", "make", "a"]}, {"sent": "you sending the piece of paper down the slide ?", "words": ["the", "slide", "of", "paper", "you", "down"]}, {"sent": "thats another part of a barn .", "words": ["another", "of", "a"]}, {"sent": "maybe Ill sit on this side of you so you can get this out of your way .", "words": ["can", "your", "of", "so", "you", "get", "this", "out", "on", "sit"]}, {"sent": "because theres plenty of blue in there .", "words": ["of", "blue", "there", "in", "because"]}, {"sent": "cup of tea ?", "words": ["cup", "of"]}, {"sent": "you can put all of the stuff on top of here .", "words": ["can", "all", "put", "the", "of", "you", "here", "on"]}, {"sent": "yeah neither of them won .", "words": ["of", "them"]}, {"sent": "a part of him is resting on the bed .", "words": ["the", "is", "of", "bed", "him", "on", "a"]}, {"sent": "um there are alot of children in that school .", "words": ["of", "school", "there", "in", "are", "that"]}, {"sent": "alright lets get out of this window .", "words": ["of", "get", "this", "out", "window"]}, {"sent": "whats a type of two person work ?", "words": ["of", "work", "a"]}, {"sent": "heres one of the men .", "words": ["of", "the"]}, {"sent": "a banana on top of his head ?", "words": ["of", "head", "his", "banana", "on", "a"]}, {"sent": "it has a different kind of bark ?", "words": ["of", "it", "a"]}, {"sent": "kind of just gave them a passing she liked this one best I think .", "words": ["think", "of", "them", "this", "she", "a"]}, {"sent": "dyou recognize any of these colors ?", "words": ["of", "these", "any"]}, {"sent": "wow thats a lot of people .", "words": ["of", "a lot", "a"]}, {"sent": "shes doing a lot of jumping around this morning isnt she ?", "words": ["of", "a lot", "this", "around", "she", "a"]}, {"sent": "no youve thought better of it havent you ?", "words": ["of", "better", "it", "you"]}, {"sent": "I see youre not listening the cow and little cow ran past all full of fear .", "words": ["the", "all", "full", "of", "and", "little", "see", "not", "cow"]}, {"sent": "oh yes you got that you got every one of them .", "words": ["of", "them", "you", "every", "that"]}, {"sent": "in fact Brendas one of the most helpful people I know .", "words": ["of", "the", "in"]}, {"sent": "I think one of these would be good .", "words": ["these", "think", "of", "be", "good", "would"]}, {"sent": "Jilly do you think its okay if I draw some pictures of stories ?", "words": ["do", "think", "if", "of", "draw", "you", "some"]}, {"sent": "well have two of those please .", "words": ["have", "of", "those"]}, {"sent": "look at some of these hm ?", "words": ["these", "look", "of", "at", "some"]}, {"sent": "you know that clamp on the head of the bed ?", "words": ["the", "of", "head", "bed", "you", "on", "that"]}, {"sent": "I saw you move put that little p on the end of it .", "words": ["the", "put", "of", "little", "you", "it", "on", "that"]}, {"sent": "on top of trees ?", "words": ["on", "of"]}, {"sent": "draw a picture of Jack and Jane .", "words": ["of", "draw", "picture", "and", "a"]}, {"sent": "the names of them ?", "words": ["the", "of", "them"]}, {"sent": "need a piece of wood ?", "words": ["of", "need", "a"]}, {"sent": "of mhm .", "words": ["of"]}, {"sent": "its green the color of peas .", "words": ["the", "of", "peas", "green"]}, {"sent": "you dont really get all rid of all the shadows .", "words": ["the", "all", "of", "you", "get", "dont"]}, {"sent": "there was dere was some noise in the machine Tony that got that got in the way of your voice .", "words": ["the", "your", "of", "was", "there", "in", "some", "that"]}, {"sent": "thats a different kind of green isnt it ?", "words": ["of", "it", "green", "a"]}, {"sent": "um so you mentioned he liked the food alot but he doesnt have those types of things at home but do you think that he recognized any of them as the food like does he eat any of those .", "words": ["the", "think", "of", "any", "like", "eat", "does", "those", "have", "food", "that", "home", "so", "you", "do", "them", "he", "at", "but"]}, {"sent": "he plays a lot of cars and trucks .", "words": ["of", "a lot", "and", "he", "a"]}, {"sent": "thats yeah I was curious about that um are any of these things um .", "words": ["these", "of", "was", "any", "are", "about", "that"]}, {"sent": "instead of your big one .", "words": ["of", "big", "your"]}, {"sent": "have a good bite of egg .", "words": ["egg", "of", "have", "good", "bite", "a"]}, {"sent": "what does he say ?", "words": ["what", "does", "he", "say"]}, {"sent": "is he talking on the telephone ?", "words": ["the", "is", "he", "on", "telephone"]}, {"sent": "has he ?", "words": ["he"]}, {"sent": "you think he can make a house outof that ?", "words": ["can", "think", "you", "that", "he", "house", "make", "a"]}, {"sent": "he said she was really really set on it .", "words": ["was", "it", "he", "on", "she"]}, {"sent": "he said .", "words": ["he"]}, {"sent": "he says .", "words": ["he"]}, {"sent": "is he going too ?", "words": ["too", "he", "is"]}, {"sent": "he has stuffed animals .", "words": ["he"]}, {"sent": "he has already been this morning .", "words": ["this", "he"]}, {"sent": "he always hasta check his outfits in the mirror .", "words": ["his", "the", "in", "he"]}, {"sent": "where is he going ?", "words": ["where", "he", "is"]}, {"sent": "he has .", "words": ["he"]}, {"sent": "uhhuh hes hiding behind that tree isnt he ?", "words": ["tree", "he", "that", "behind"]}, {"sent": "now he was ready for bed .", "words": ["was", "for", "he", "bed"]}, {"sent": "well markie might but markie he needs it .", "words": ["it", "he", "but"]}, {"sent": "he would go over and see them every now and again .", "words": ["go", "them", "and", "he", "every", "would", "see", "over"]}, {"sent": "hes a funny dinosaur isnt he ?", "words": ["he", "a"]}, {"sent": "that duck is swimming isnt he ?", "words": ["duck", "he", "that", "is"]}, {"sent": "he wants to doesnt he ?", "words": ["to", "he"]}, {"sent": "oh no and whats he gonna do if he gets me ?", "words": ["do", "if", "and", "he", "me"]}, {"sent": "who did he hafta ask if he could do it ?", "words": ["do", "if", "it", "he", "did", "who", "could"]}, {"sent": "he doesnt fit .", "words": ["fit", "he"]}, {"sent": "might might er bite you with his teeth mightnt he ?", "words": ["with", "you", "his", "he", "bite"]}, {"sent": "well he needs some petrol first from over there .", "words": ["first", "there", "he", "some", "over"]}, {"sent": "he said theyre marching .", "words": ["he"]}, {"sent": "he would like to know if you could sing .", "words": ["if", "to", "like", "you", "sing", "he", "would", "could"]}, {"sent": "first he was mad and then he was happy and now shes mad why is that ?", "words": ["happy", "is", "mad", "was", "first", "and", "he", "then", "that", "why"]}, {"sent": "he said he useta be a BoyScout ?", "words": ["be", "he", "a"]}, {"sent": "shall we press daddys head down and see if he jumps up ?", "words": ["up", "if", "we", "head", "and", "he", "see", "down"]}, {"sent": "look and on that he had some sheep .", "words": ["and", "he", "sheep", "look", "on", "some", "that"]}, {"sent": "thats okay that he doesnt know he doesnt hafta know .", "words": ["he", "that"]}, {"sent": "um but he also likes the play he likes the play foo .", "words": ["the", "play", "he", "but"]}, {"sent": "and he says .", "words": ["he", "and"]}, {"sent": "uhoh he fell over .", "words": ["over", "he"]}, {"sent": "has he ?", "words": ["he"]}, {"sent": "he caught you back there by the dogs ?", "words": ["the", "back", "by", "you", "there", "he"]}, {"sent": "then what did he put on ?", "words": ["what", "put", "he", "did", "on", "then"]}, {"sent": "no he wont get those animals .", "words": ["get", "he", "those"]}, {"sent": "has he got a name ?", "words": ["he", "a"]}, {"sent": "whats he doing now ?", "words": ["he"]}, {"sent": "he wasnt xxx there was he ?", "words": ["was", "there", "he"]}, {"sent": "I bet he hasnt got out of bed yet .", "words": ["of", "he", "bed", "out"]}, {"sent": "because he was going to get a big elastic band wasnt he and put it on ?", "words": ["put", "to", "was", "and", "get", "it", "he", "on", "big", "because", "a"]}, {"sent": "he has got a tail hasnt he ?", "words": ["he", "a"]}, {"sent": "how did he look ?", "words": ["how", "he", "did", "look"]}, {"sent": "is he green ?", "words": ["he", "green", "is"]}, {"sent": "he walk okay .", "words": ["walk", "he"]}, {"sent": "does he like it ?", "words": ["does", "it", "he", "like"]}, {"sent": "whats he doing ?", "words": ["he"]}, {"sent": "they have like a hole in one and and then like something sticking out of the other and he still loves playing with those so Imean sometimes its just simple stuff like that .", "words": ["the", "those", "of", "so", "with", "like", "and", "have", "they", "in", "other", "he", "out", "then", "that", "a"]}, {"sent": "and what do you say to daddy when he gets when he goes off to the train ?", "words": ["off", "what", "do", "the", "train", "to", "say", "you", "and", "he", "when"]}, {"sent": "and he turned round you know .", "words": ["he", "you", "and"]}, {"sent": "so she found he found all her friends .", "words": ["all", "so", "he", "her", "she"]}, {"sent": "does he have a name ?", "words": ["have", "does", "he", "a"]}, {"sent": "you said he fell down in the basket .", "words": ["the", "you", "in", "he", "basket", "down"]}, {"sent": "and he could use it to lay a new path .", "words": ["to", "and", "it", "he", "could", "new", "a"]}, {"sent": "blue with red spots isnt he ?", "words": ["red", "he", "with", "blue"]}, {"sent": "he died of uh instant uh no quick pneumonia .", "words": ["of", "he"]}, {"sent": "like when he sees a cow he goes moo or a dog he well he says dog because we have a dog so he says doggy he sa he barks too .", "words": ["we", "so", "like", "because", "have", "he", "too", "when", "dog", "cow", "a"]}, {"sent": "maybe because what would be a reason why he couldnt land yet ?", "words": ["what", "be", "he", "would", "why", "because", "a"]}, {"sent": "has no real love for them he does youknow hell take them and sortof youknow toss them across the so I was like just like Sharons doll .", "words": ["the", "does", "for", "so", "was", "them", "like", "doll", "and", "he", "love", "take"]}, {"sent": "whats he doing ?", "words": ["he"]}, {"sent": "is he ?", "words": ["he", "is"]}, {"sent": "he is really a good flier isnt he ?", "words": ["he", "good", "a", "is"]}, {"sent": "now hes got bigger so he goes to big school now .", "words": ["to", "so", "school", "he", "big"]}, {"sent": "does does he go in the kitchen with you ?", "words": ["the", "does", "go", "with", "you", "in", "he", "kitchen"]}, {"sent": "now he has hair and a hat .", "words": ["and", "he", "hat", "hair", "a"]}, {"sent": "he was called James .", "words": ["was", "he"]}, {"sent": "licking his hand is he ?", "words": ["his", "he", "is", "hand"]}, {"sent": "where did he go ?", "words": ["where", "he", "did", "go"]}, {"sent": "he calls you Dad because he hears me call you Dad .", "words": ["me", "he", "because", "you"]}, {"sent": "is he your teddy ?", "words": ["your", "he", "is"]}, {"sent": "he has a stuffed bear that he sleeps with .", "words": ["with", "bear", "he", "that", "a"]}, {"sent": "but I I like youknow different foods that he eats he seems to be excited about .", "words": ["to", "be", "like", "he", "about", "that", "but"]}, {"sent": "think he might get wet ?", "words": ["get", "wet", "think", "he"]}, {"sent": "he has a flat tire .", "words": ["he", "a"]}, {"sent": "he was walking around and then he melted or something didnt he ?", "words": ["was", "and", "he", "around", "then"]}, {"sent": "oh he has little prickly things .", "words": ["he", "little"]}, {"sent": "he needs a wash .", "words": ["he", "wash", "a"]}, {"sent": "oh the other things he has that are kind of like um babies are little sesame street characters that I like wrap them up in blankets and youknow show him like that how to nurture a baby .", "words": ["the", "up", "how", "of", "show", "to", "them", "like", "other", "little", "and", "in", "are", "he", "him", "street", "that", "a"]}, {"sent": "remember Ben was frightened of it wasnt he ?", "words": ["of", "was", "it", "he"]}, {"sent": "does he know those colors ?", "words": ["does", "he", "those"]}, {"sent": "he had a van like this didnt he ?", "words": ["this", "he", "like", "a"]}, {"sent": "but he has he has specific .", "words": ["he", "but"]}, {"sent": "Thomas Humptys going to fall right through isnt he ?", "words": ["to", "he", "fall"]}, {"sent": "he likes skating doesnt he ?", "words": ["he"]}, {"sent": "isnt he lovely ?", "words": ["he"]}, {"sent": "so does he get that one for free or does Ryan hafta pay for that last one ?", "words": ["does", "for", "so", "last", "get", "he", "that"]}, {"sent": "bit scary was he ?", "words": ["was", "he"]}, {"sent": "but he is getting there .", "words": ["there", "he", "but", "is"]}, {"sent": "he looks like hes dancing .", "words": ["he", "like"]}, {"sent": "he fell out .", "words": ["he", "out"]}, {"sent": "he couldnt bend it .", "words": ["it", "he"]}, {"sent": "he hasta be careful .", "words": ["be", "he", "careful"]}, {"sent": "I saw the way he carry he handling you he takes Jerry by his hair and just pulls it up pulling him straight up in the air ya know xxx .", "words": ["the", "up", "by", "carry", "you", "and", "it", "he", "him", "in", "his", "hair"]}, {"sent": "do you know that crabs walk backwards so he looks at you .", "words": ["do", "so", "you", "that", "he", "at", "walk"]}, {"sent": "where is he ?", "words": ["where", "he", "is"]}, {"sent": "because he keeps scaring everybody .", "words": ["he", "because"]}, {"sent": "so you can put these in them .", "words": ["can", "these", "put", "so", "them", "you", "in"]}, {"sent": "who else is in there ?", "words": ["there", "in", "is", "who"]}, {"sent": "when Mummy goes in the water you watch Pippin on the video dont you ?", "words": ["the", "watch", "you", "dont", "in", "water", "on", "when"]}, {"sent": "I cant get it in .", "words": ["get", "in", "it"]}, {"sent": "I think somehow the balls fall in there .", "words": ["the", "think", "fall", "there", "in"]}, {"sent": "theres lotsof holes in the basket .", "words": ["the", "in", "basket"]}, {"sent": "are you gonna pick that up for Jeannine and put it in the bin ?", "words": ["up", "the", "put", "for", "pick", "you", "and", "in", "are", "it", "that"]}, {"sent": "what do we have in there ?", "words": ["what", "do", "we", "have", "there", "in"]}, {"sent": "can you put it in ?", "words": ["can", "put", "you", "in", "it"]}, {"sent": "he is in the back seat .", "words": ["the", "back", "is", "in", "he"]}, {"sent": "here here I think its in the drawer .", "words": ["the", "think", "here", "in", "drawer"]}, {"sent": "I think the bus is in that box .", "words": ["the", "think", "box", "is", "in", "that", "bus"]}, {"sent": "and pack them in a box .", "words": ["box", "them", "and", "in", "a"]}, {"sent": "everybody in the house ?", "words": ["the", "in", "house"]}, {"sent": "oops you in your food .", "words": ["in", "your", "food", "you"]}, {"sent": "can you draw in here ?", "words": ["can", "draw", "you", "here", "in"]}, {"sent": "UncleBrian is outside banging in the garden .", "words": ["the", "is", "outside", "in", "garden"]}, {"sent": "like hes interested in it for a whole day and then he wont be interested in it for multiple days right ?", "words": ["for", "be", "like", "and", "in", "it", "he", "then", "a"]}, {"sent": "is she in the medicine cabinet ?", "words": ["the", "is", "medicine", "in", "she"]}, {"sent": "the lady in the shop said that the HushPup is called Jason .", "words": ["the", "in", "that", "is"]}, {"sent": "it is eight oclock in the morning .", "words": ["the", "in", "it", "is"]}, {"sent": "oh no dont put the zebra in the pot lets put the cupcake in the pot .", "words": ["the", "zebra", "put", "dont", "in"]}, {"sent": "what is in there ?", "words": ["what", "in", "there", "is"]}, {"sent": "thats a jack in the box .", "words": ["the", "in", "box", "a"]}, {"sent": "hes got a big ball in there hasnt he ?", "words": ["ball", "there", "in", "he", "big", "a"]}, {"sent": "shall we put the lady in the house ?", "words": ["the", "put", "we", "in", "house"]}, {"sent": "hes in .", "words": ["in"]}, {"sent": "well hes in the he doesnt like getting clean .", "words": ["the", "clean", "like", "in", "he"]}, {"sent": "in the potty .", "words": ["the", "in", "potty"]}, {"sent": "go in there naughty Charmanda hm ?", "words": ["there", "in", "naughty", "go"]}, {"sent": "youre having fish in your sandwich ?", "words": ["in", "your", "sandwich", "fish"]}, {"sent": "three oclock this xxx before you come in I dozing off in the ladies room ?", "words": ["off", "the", "you", "this", "in", "room"]}, {"sent": "cause hes got a shoe in each hand .", "words": ["each", "hand", "shoe", "in", "a"]}, {"sent": "here well put it back in the refrigerator until you invite all your friends .", "words": ["the", "all", "put", "your", "refrigerator", "back", "you", "here", "in", "it"]}, {"sent": "put it in the grinder till its ground coffee and put it in the xxx .", "words": ["the", "put", "and", "in", "it", "coffee"]}, {"sent": "do you think that should be in bed ?", "words": ["do", "think", "be", "bed", "you", "in", "that"]}, {"sent": "thats not a very nice way to treat somebody who has been in a crash is it ?", "words": ["is", "to", "nice", "in", "it", "who", "not", "a"]}, {"sent": "your friends in here .", "words": ["here", "in", "your"]}, {"sent": "oh whos in there ?", "words": ["there", "in"]}, {"sent": "in this hole .", "words": ["this", "in"]}, {"sent": "well maybe I should put you in there so ya could take a nap .", "words": ["put", "so", "you", "there", "in", "could", "take", "a"]}, {"sent": "he gots a battery in his belly .", "words": ["his", "in", "he", "a"]}, {"sent": "you like looking out for bears in the trees dont you ?", "words": ["the", "for", "like", "you", "dont", "in", "out"]}, {"sent": "and Jackll be fine in there wont he ?", "words": ["fine", "be", "and", "there", "in", "he"]}, {"sent": "so in the near the tub I would say .", "words": ["the", "so", "say", "in", "would"]}, {"sent": "I think Griz is in the kitchen sitting in a chair lets go in cause I wanna start my water anyway .", "words": ["the", "water", "think", "my", "is", "go", "chair", "in", "kitchen", "wanna", "a"]}, {"sent": "but now that color would be beautiful for in here .", "words": ["for", "be", "here", "in", "would", "that", "but"]}, {"sent": "put it in over here .", "words": ["put", "here", "in", "it", "over"]}, {"sent": "videos not in it .", "words": ["in", "it", "not"]}, {"sent": "here look in this one .", "words": ["here", "this", "in", "look"]}, {"sent": "do you know I love coming shopping in your shops .", "words": ["do", "your", "you", "in", "love"]}, {"sent": "he had the largest longest car in BigTown .", "words": ["the", "car", "in", "he"]}, {"sent": "oh the diggers up in the air .", "words": ["the", "in", "up"]}, {"sent": "can you find whats in there ?", "words": ["can", "you", "there", "in", "find"]}, {"sent": "were not going to get the tiger in are we ?", "words": ["the", "we", "to", "get", "in", "are", "tiger", "were", "not"]}, {"sent": "and what were you and Isabel doing in that bed ?", "words": ["what", "bed", "you", "and", "in", "that", "were"]}, {"sent": "they could fit in there .", "words": ["fit", "they", "in", "there", "could"]}, {"sent": "Im not sure if its in that one or not .", "words": ["if", "in", "that", "not"]}, {"sent": "are you gonna come or do I hafta carry you in ?", "words": ["do", "you", "carry", "in", "are"]}, {"sent": "so well have em go for a walk in the woods .", "words": ["the", "for", "go", "so", "have", "in", "walk", "a"]}, {"sent": "well hes in the right place Mummy .", "words": ["the", "in"]}, {"sent": "do you feel as if youve done a good days work in the garden ?", "words": ["the", "do", "work", "if", "you", "in", "garden", "good", "a"]}, {"sent": "lets have a look at the other one because weve got some good ones in this one havent we ?", "words": ["the", "we", "other", "have", "this", "in", "at", "good", "look", "some", "because", "a"]}, {"sent": "standing in wet concrete ?", "words": ["wet", "in"]}, {"sent": "you might just put your foot on it and grind it in .", "words": ["put", "your", "you", "and", "foot", "it", "in", "on"]}, {"sent": "shes got a lot of bubbles in her bath .", "words": ["of", "a lot", "in", "her", "a", "bubbles"]}, {"sent": "you what what is in here ?", "words": ["what", "is", "you", "here", "in"]}, {"sent": "is there a xxx in his name xxx xxx xxx ?", "words": ["is", "there", "in", "his", "a"]}, {"sent": "in the door ?", "words": ["the", "in", "door"]}, {"sent": "what is he in ?", "words": ["what", "in", "he", "is"]}, {"sent": "Im going to throw it up in the air and then hit it with the racket so .", "words": ["up", "the", "throw", "to", "so", "with", "and", "in", "it", "then", "hit"]}, {"sent": "because otherwise we wouldnt have got in .", "words": ["have", "in", "because", "we"]}, {"sent": "yeah theres a goat in your jigsaw puzzle isnt there ?", "words": ["your", "there", "in", "puzzle", "a"]}, {"sent": "be careful because Mummy has got notes written in there .", "words": ["be", "there", "in", "careful", "because"]}, {"sent": "is your banana in your stomach ?", "words": ["banana", "in", "your", "is"]}, {"sent": "er she felt a lump in the easy chair .", "words": ["the", "chair", "in", "she", "a"]}, {"sent": "you go in where the ambulance goes in .", "words": ["the", "where", "go", "you", "in"]}, {"sent": "daddys in his truck .", "words": ["his", "in", "truck"]}, {"sent": "she wasnt interested in having me read to her she wanted to play .", "words": ["to", "play", "in", "read", "me", "her", "she"]}, {"sent": "Ill be down in a sec .", "words": ["be", "in", "down", "a"]}, {"sent": "what else is there in here to play with ?", "words": ["what", "is", "to", "play", "with", "here", "there", "in"]}, {"sent": "what else did you do in school today Nick ?", "words": ["what", "do", "school", "you", "in", "did"]}, {"sent": "lets put a little bit of this in .", "words": ["put", "of", "little", "this", "in", "a"]}, {"sent": "why do you put everything in your mouth ?", "words": ["do", "put", "your", "mouth", "you", "in", "why"]}, {"sent": "so she ate up all the beef soup in the little bitty bowl .", "words": ["up", "all", "the", "bowl", "so", "little", "in", "soup", "she"]}, {"sent": "ha ha she was in my shirt she was hiding in my shirt Im ganna go here and steal some cardboard looks like Daddy pulled that cardboard already ha ha ha ha ha ha ha I just steal something here alright were done in the closet right now .", "words": ["the", "my", "go", "was", "shirt", "closet", "like", "and", "here", "in", "some", "that", "she", "were"]}, {"sent": "can you put your sandals in the box ?", "words": ["can", "the", "put", "your", "box", "you", "in"]}, {"sent": "you keep getting caught in these .", "words": ["these", "in", "you"]}, {"sent": "go in there .", "words": ["there", "in", "go"]}, {"sent": "well get some more in a bit shall we ?", "words": ["we", "get", "more", "in", "some", "a"]}, {"sent": "and then well wrap her present in this blue paper too .", "words": ["present", "paper", "and", "blue", "this", "in", "too", "then", "her"]}, {"sent": "please would you like to sit in this chair .", "words": ["to", "like", "chair", "you", "this", "in", "would", "sit"]}, {"sent": "good morning said the wasp in a very friendly way .", "words": ["the", "in", "good", "a"]}, {"sent": "put it in the box .", "words": ["the", "put", "box", "in", "it"]}, {"sent": "whynt why dont you put some of your dollies back in your toy box ?", "words": ["put", "your", "box", "back", "of", "you", "dont", "in", "toy", "some", "why"]}, {"sent": "uh lets pick em up and put em in here .", "words": ["up", "put", "pick", "and", "here", "in"]}, {"sent": "a hair in your mouth .", "words": ["your", "mouth", "in", "hair", "a"]}, {"sent": "xxx in here .", "words": ["here", "in"]}, {"sent": "in case someone puts their hand on it .", "words": ["their", "hand", "in", "it", "on"]}, {"sent": "in there and the baby .", "words": ["the", "there", "in", "and"]}, {"sent": "let me get the door .", "words": ["get", "me", "door", "the"]}, {"sent": "wanna close the door .", "words": ["the", "close", "wanna", "door"]}, {"sent": "toot toot says Percy thanks for opening the door .", "words": ["the", "for", "door"]}, {"sent": "my mother locked the door though right ?", "words": ["the", "my", "door"]}, {"sent": "and think about closing an oven door .", "words": ["think", "and", "an", "about", "oven", "door"]}, {"sent": "whos behind that door ?", "words": ["that", "behind", "door"]}, {"sent": "this is the same child that shuts every door .", "words": ["the", "is", "this", "same", "every", "that", "door"]}, {"sent": "he wished that the lady next door would sing like a big horn .", "words": ["the", "like", "sing", "big", "he", "would", "that", "a", "door"]}, {"sent": "door closed ?", "words": ["door"]}, {"sent": "open the door Mandy .", "words": ["open", "the", "door"]}, {"sent": "close door ?", "words": ["close", "door"]}, {"sent": "Caroline open door .", "words": ["open", "door"]}, {"sent": "well we can pretend this is a door right here .", "words": ["can", "is", "we", "here", "this", "pretend", "door", "a"]}, {"sent": "shut the door .", "words": ["the", "door"]}, {"sent": "we useta live door to another Sue didnt we ?", "words": ["another", "to", "door", "we"]}, {"sent": "suddenly theres a knock at the door .", "words": ["the", "knock", "at", "a", "door"]}, {"sent": "stick it in the back door .", "words": ["the", "stick", "back", "in", "it", "door"]}, {"sent": "I wish I had a front door xxx .", "words": ["door", "a", "wish"]}, {"sent": "Im gonna shut the door .", "words": ["the", "door"]}, {"sent": "just let me shut the door Purdie before you jump on me .", "words": ["the", "you", "jump", "on", "me", "door"]}, {"sent": "like George next door ?", "words": ["like", "door"]}, {"sent": "wanna push the door to ?", "words": ["the", "push", "to", "wanna", "door"]}, {"sent": "that can be a door yeah .", "words": ["can", "be", "that", "a", "door"]}, {"sent": "the garage door .", "words": ["the", "garage", "door"]}, {"sent": "what door ?", "words": ["what", "door"]}, {"sent": "open the door .", "words": ["open", "the", "door"]}, {"sent": "mhm and then you have a door on it .", "words": ["you", "have", "and", "it", "on", "then", "door", "a"]}, {"sent": "bet shes probably worked out now its the door .", "words": ["the", "out", "door"]}, {"sent": "theyd come in the door and theyd look at her and look her all over and then theyd go out .", "words": ["the", "all", "go", "and", "in", "at", "out", "look", "over", "then", "her", "door"]}, {"sent": "what does Maggie have on her door for halloween ?", "words": ["what", "does", "for", "have", "on", "her", "door"]}, {"sent": "I think the door is in here .", "words": ["the", "think", "is", "here", "in", "door"]}, {"sent": "were bigger than the the little were big than the little guys down I mean next door .", "words": ["the", "little", "down", "big", "were", "door"]}, {"sent": "on top of the door ?", "words": ["on", "of", "door", "the"]}, {"sent": "open the door heres a rectangle close the door its a rectangle .", "words": ["the", "open", "close", "door", "a"]}, {"sent": "oh you got his tongue in the door .", "words": ["the", "you", "in", "his", "tongue", "door"]}, {"sent": "when we go up to your bedroom this teddy hangs on the door handle .", "words": ["up", "the", "door", "your", "go", "we", "to", "this", "on", "when", "bedroom"]}, {"sent": "my mother locked the door though right ?", "words": ["the", "my", "door"]}, {"sent": "knock at the door .", "words": ["the", "knock", "door", "at"]}, {"sent": "so I was only in the door and Bill on his heels .", "words": ["the", "was", "so", "and", "his", "in", "on", "door"]}, {"sent": "I think she must be next door .", "words": ["be", "think", "she", "door"]}, {"sent": "gonna shut the door ?", "words": ["the", "door"]}, {"sent": "will you just come in here so I can shut the door .", "words": ["can", "the", "so", "you", "here", "in", "will", "door"]}, {"sent": "can I just borrow your tape measure and I can measure I can measure the whats because Ive got it marked on the door .", "words": ["can", "the", "your", "tape", "and", "it", "on", "because", "door"]}, {"sent": "the door opened yeah .", "words": ["the", "door"]}, {"sent": "push the door to .", "words": ["the", "to", "push", "door"]}, {"sent": "because Jwww wanted to play next door didnt he ?", "words": ["to", "play", "he", "because", "door"]}, {"sent": "no dont open the door .", "words": ["open", "dont", "door", "the"]}, {"sent": "now youre banging your head on the door .", "words": ["the", "your", "head", "on", "door"]}, {"sent": "Lorna what did I tell you about this door ?", "words": ["what", "you", "this", "did", "about", "door"]}, {"sent": "who goes in the door ?", "words": ["the", "in", "door", "who"]}, {"sent": "hes next door too ?", "words": ["too", "door"]}, {"sent": "let me close the door here .", "words": ["the", "here", "me", "close", "door"]}, {"sent": "xxx do at the door ?", "words": ["the", "do", "door", "at"]}, {"sent": "xxx my brother broke my door .", "words": ["my", "door"]}, {"sent": "I dont see the door .", "words": ["see", "dont", "door", "the"]}, {"sent": "to the door .", "words": ["the", "to", "door"]}, {"sent": "door .", "words": ["door"]}, {"sent": "why dont we go in this door ?", "words": ["go", "we", "dont", "in", "this", "why", "door"]}, {"sent": "so Im going to leave outside the door .", "words": ["the", "to", "so", "outside", "door"]}, {"sent": "now close the green door .", "words": ["the", "close", "green", "door"]}, {"sent": "which door ?", "words": ["which", "door"]}, {"sent": "wed better move it from there because if Daddy comes home hes not going to be able to get through the door is he ?", "words": ["the", "better", "if", "is", "to", "be", "get", "there", "it", "he", "door", "not", "because", "home"]}, {"sent": "close the door right ?", "words": ["the", "close", "door"]}, {"sent": "but a hose pipe on the door .", "words": ["the", "hose", "on", "door", "but", "a"]}, {"sent": "out the front door then .", "words": ["the", "then", "out", "door"]}, {"sent": "shut the door .", "words": ["the", "door"]}, {"sent": "no I dont think the door opens does it ?", "words": ["the", "does", "think", "dont", "it", "door"]}, {"sent": "oh I see well do you think we should shut the door maybe ?", "words": ["the", "do", "think", "we", "you", "see", "door"]}, {"sent": "shes going to put it through the door in a moment .", "words": ["the", "put", "to", "in", "it", "a", "door"]}, {"sent": "its its its by the door .", "words": ["the", "door", "by"]}, {"sent": "the door is white thats right .", "words": ["the", "white", "is", "door"]}, {"sent": "lets open the door .", "words": ["open", "the", "door"]}, {"sent": "something has happened to the door .", "words": ["the", "to", "door"]}, {"sent": "its the door when you open the door .", "words": ["the", "open", "you", "when", "door"]}, {"sent": "but what is over there at the door ?", "words": ["the", "what", "is", "there", "at", "over", "but", "door"]}, {"sent": "door .", "words": ["door"]}, {"sent": "yes but you cant pull that door open by yourself .", "words": ["yourself", "open", "pull", "you", "by", "that", "but", "door"]}, {"sent": "its just the back door banging isnt it ?", "words": ["the", "it", "back", "door"]}, {"sent": "there are two people at the door Thomas .", "words": ["the", "there", "are", "at", "door"]}, {"sent": "he went in the door .", "words": ["the", "in", "he", "door"]}, {"sent": "yeah be careful they we just doesnt have a door .", "words": ["we", "be", "have", "they", "careful", "door", "a"]}, {"sent": "Ill open the door for ya .", "words": ["open", "the", "for", "door"]}, {"sent": "are these the keys to the door ?", "words": ["the", "these", "to", "are", "keys", "door"]}, {"sent": "can you go and shut the door please Warren ?", "words": ["can", "the", "go", "and", "you", "door"]}, {"sent": "dont bang the door darling .", "words": ["the", "dont", "door"]}, {"sent": "but we might actually put it on the conservatory door .", "words": ["the", "put", "we", "it", "on", "but", "door"]}, {"sent": "theyll get trapped in the door .", "words": ["get", "the", "in", "door"]}, {"sent": "say knock knock knock on the door .", "words": ["the", "say", "knock", "on", "door"]}, {"sent": "did that sound like somebody was at our door ?", "words": ["was", "like", "our", "did", "at", "that", "door"]}, {"sent": "I think that was a a door opening in one of the classrooms .", "words": ["the", "think", "of", "was", "in", "that", "a", "door"]}, {"sent": "Bows at the door .", "words": ["the", "door", "at"]}, {"sent": "thats a door .", "words": ["door", "a"]}, {"sent": "shall I open the door for him ?", "words": ["the", "for", "open", "him", "door"]}, {"sent": "is he behind the door ?", "words": ["the", "is", "behind", "he", "door"]}, {"sent": "that cant go in the door .", "words": ["the", "go", "in", "that", "door"]}, {"sent": "xxx open this door .", "words": ["open", "this", "door"]}, {"sent": "when people like Melissa opened up the door .", "words": ["up", "the", "like", "when", "door"]}, {"sent": "one door .", "words": ["door"]}, {"sent": "oh whos sleeping in front of the door ?", "words": ["of", "the", "in", "door"]}, {"sent": "I hope youre not pulling the door off the hinges .", "words": ["the", "not", "off", "door"]}, {"sent": "oh we dont take it down honey we can play it up there .", "words": ["can", "up", "we", "play", "dont", "there", "it", "down", "take"]}, {"sent": "you knocked the house down .", "words": ["the", "down", "house", "you"]}, {"sent": "no youll hafta put one down .", "words": ["put", "down"]}, {"sent": "slow down .", "words": ["down", "slow"]}, {"sent": "whyre you turning upside down ?", "words": ["down", "you"]}, {"sent": "you sit down and eat your xxx .", "words": ["your", "and", "eat", "you", "sit", "down"]}, {"sent": "and our fish came down too and fell into a pot .", "words": ["into", "our", "and", "too", "down", "a", "fish"]}, {"sent": "and our fish came down too .", "words": ["our", "and", "too", "down", "fish"]}, {"sent": "sit down .", "words": ["sit", "down"]}, {"sent": "okay zip it down pull down .", "words": ["down", "it", "pull"]}, {"sent": "gotta push down .", "words": ["down", "push"]}, {"sent": "put your hand down .", "words": ["put", "down", "your", "hand"]}, {"sent": "no get down .", "words": ["get", "down"]}, {"sent": "come down here and put xxx .", "words": ["here", "put", "down", "and"]}, {"sent": "until it goes down .", "words": ["down", "it"]}, {"sent": "did you wanna lay down and take a nap ?", "words": ["and", "you", "did", "wanna", "down", "take", "a"]}, {"sent": "you are the one who wants the fox put down your gun .", "words": ["the", "put", "your", "you", "are", "who", "down"]}, {"sent": "its upside down .", "words": ["down"]}, {"sent": "put the knife down now please .", "words": ["the", "put", "down", "knife"]}, {"sent": "here sit down xxx .", "words": ["here", "sit", "down"]}, {"sent": "set down to the table .", "words": ["the", "to", "down", "table"]}, {"sent": "I put that down .", "words": ["put", "down", "that"]}, {"sent": "does he fall down ?", "words": ["does", "down", "he", "fall"]}, {"sent": "maybe Ill cool this xxx piece down .", "words": ["this", "down"]}, {"sent": "head down .", "words": ["head", "down"]}, {"sent": "down boing .", "words": ["down"]}, {"sent": "no you dont needta bring it down .", "words": ["bring", "you", "dont", "it", "down"]}, {"sent": "goes up and down .", "words": ["up", "down", "and"]}, {"sent": "you ready to get down ?", "words": ["get", "to", "down", "you"]}, {"sent": "there she goes yeah theyre both going down down this big steep mountain she has that long hat on .", "words": ["long", "there", "this", "that", "on", "hat", "down", "big", "she"]}, {"sent": "you wanna get down there ?", "words": ["you", "get", "there", "wanna", "down"]}, {"sent": "sit down in your chair .", "words": ["your", "chair", "in", "sit", "down"]}, {"sent": "down there .", "words": ["there", "down"]}, {"sent": "put them down ?", "words": ["put", "down", "them"]}, {"sent": "and snow falling down .", "words": ["down", "snow", "and"]}, {"sent": "shall we do it down here ?", "words": ["do", "we", "here", "it", "down"]}, {"sent": "but we brought them down .", "words": ["down", "them", "but", "we"]}, {"sent": "and down and down and down .", "words": ["down", "and"]}, {"sent": "youre sitting down near Becca arent you ?", "words": ["down", "you"]}, {"sent": "is it low enough down do you think ?", "words": ["do", "think", "is", "you", "it", "down"]}, {"sent": "no this is the explosion down here .", "words": ["the", "is", "here", "this", "down"]}, {"sent": "because then shell come down and go Thomas you clever boy .", "words": ["go", "and", "you", "then", "down", "because"]}, {"sent": "please go sit down and learn about it .", "words": ["go", "and", "it", "about", "sit", "down"]}, {"sent": "what fell down ?", "words": ["what", "down"]}, {"sent": "you hafta turn it upside down to get it out .", "words": ["to", "you", "get", "it", "out", "down"]}, {"sent": "put that down .", "words": ["put", "down", "that"]}, {"sent": "maybe I put this in upside down .", "words": ["this", "put", "down", "in"]}, {"sent": "do you wanna get down ?", "words": ["do", "you", "get", "wanna", "down"]}, {"sent": "I think in a minute well have a sit down .", "words": ["think", "have", "in", "sit", "down", "a"]}, {"sent": "youll knock that down .", "words": ["down", "knock", "that"]}, {"sent": "youre gonna fall down and go boom .", "words": ["go", "down", "fall", "and"]}, {"sent": "can you see down there with all the wood in it ?", "words": ["can", "all", "the", "with", "you", "there", "in", "it", "see", "down"]}, {"sent": "upside down .", "words": ["down"]}, {"sent": "you gonna put her down ?", "words": ["put", "down", "her", "you"]}, {"sent": "Aggie was chopping down a branch that was sick ?", "words": ["was", "sick", "down", "that", "a"]}, {"sent": "time to calm down .", "words": ["to", "down"]}, {"sent": "come sit down by mommy come sit down by mommy .", "words": ["sit", "down", "by"]}, {"sent": "turn it down a bit now .", "words": ["down", "it", "a"]}, {"sent": "the monkey fell down ?", "words": ["the", "monkey", "down"]}, {"sent": "lay her down .", "words": ["down", "her"]}, {"sent": "gently down the stream .", "words": ["the", "down"]}, {"sent": "down the steps .", "words": ["the", "down"]}, {"sent": "all fall down .", "words": ["all", "down", "fall"]}, {"sent": "sit down .", "words": ["sit", "down"]}, {"sent": "that can fall down .", "words": ["can", "down", "that", "fall"]}, {"sent": "now hold my hand incase you fall going down the steps down the stairs now .", "words": ["the", "my", "hold", "hand", "fall", "you", "stairs", "down"]}, {"sent": "she wont sit down .", "words": ["sit", "down", "she"]}, {"sent": "down down baby down by the roller coaster .", "words": ["the", "down", "by"]}, {"sent": "look and hes obviously hes dangling upside down from the tree isnt he ?", "words": ["the", "tree", "and", "he", "look", "down"]}, {"sent": "is his hair falling down ?", "words": ["his", "down", "hair", "is"]}, {"sent": "and fall down the the the telephone .", "words": ["the", "fall", "and", "down", "telephone"]}, {"sent": "sit down .", "words": ["sit", "down"]}, {"sent": "oh it went down .", "words": ["down", "it"]}, {"sent": "Ill move it down for you .", "words": ["down", "it", "for", "you"]}, {"sent": "should we have Ernie sit down ?", "words": ["have", "sit", "down", "we"]}, {"sent": "thing two and thing one they ran up they ran down on the string of one kite we saw mothers new gown the gown with the dots that are pink white and red .", "words": ["up", "the", "new", "red", "we", "of", "white", "with", "and", "they", "are", "on", "down", "that"]}, {"sent": "sit down .", "words": ["sit", "down"]}, {"sent": "sit down here and read a book .", "words": ["and", "here", "read", "book", "sit", "down", "a"]}, {"sent": "you betuh not put no more paint down here on this .", "words": ["put", "paint", "you", "here", "this", "more", "on", "down", "not"]}, {"sent": "Ill try and sit him down a bit .", "words": ["and", "him", "sit", "down", "try", "a"]}, {"sent": "you hold this down .", "words": ["this", "down", "hold", "you"]}, {"sent": "get down please .", "words": ["get", "down"]}, {"sent": "put down put down those .", "words": ["put", "down", "those"]}, {"sent": "can you see all of Mummys plants down there ?", "words": ["can", "all", "of", "you", "there", "see", "down"]}, {"sent": "shall I kneel down again then ?", "words": ["then", "down"]}, {"sent": "they fall down .", "words": ["they", "down", "fall"]}, {"sent": "put it down nicely Lucca .", "words": ["down", "put", "it"]}, {"sent": "up and down Streetname Drive .", "words": ["up", "down", "and"]}, {"sent": "get down .", "words": ["get", "down"]}, {"sent": "it knocked it right down ?", "words": ["down", "it"]}, {"sent": "cause when you cut down a tree then you get logs and we have a log on your train table somewhere .", "words": ["tree", "your", "cut", "table", "we", "train", "you", "get", "and", "have", "on", "then", "down", "when", "a"]}, {"sent": "you have tears down your eyes quite quickly dont you ?", "words": ["your", "you", "have", "dont", "down"]}, {"sent": "down there yes .", "words": ["there", "down"]}, {"sent": "down by the bay .", "words": ["the", "down", "by"]}, {"sent": "get dow get down oh .", "words": ["get", "down"]}, {"sent": "I dont think you jump down .", "words": ["think", "you", "dont", "jump", "down"]}, {"sent": "come on put your feet down .", "words": ["on", "put", "down", "your"]}, {"sent": "xxx she turns it over when somethings hanging upside down .", "words": ["when", "it", "down", "over", "she"]}, {"sent": "hes upside down .", "words": ["down"]}, {"sent": "I dont wanna go down the slide .", "words": ["the", "slide", "go", "dont", "wanna", "down"]}, {"sent": "hi Ethan how are you ?", "words": ["how", "are", "you"]}, {"sent": "how many rabbits ?", "words": ["how"]}, {"sent": "how can he get in ?", "words": ["can", "how", "get", "in", "he"]}, {"sent": "come here lets play show me how .", "words": ["how", "show", "play", "here", "me"]}, {"sent": "oh he needs how do they want you to oh like this .", "words": ["do", "how", "to", "like", "you", "they", "this", "he"]}, {"sent": "how about this one how about Mommy help you a little bit .", "words": ["how", "help", "you", "little", "this", "about", "a"]}, {"sent": "you know how to do that .", "words": ["do", "how", "to", "you", "that"]}, {"sent": "remember how we do it in gym class ?", "words": ["do", "how", "we", "in", "it"]}, {"sent": "how about milking them ?", "words": ["how", "them", "about"]}, {"sent": "how come ?", "words": ["how"]}, {"sent": "stand up and see how tall it is .", "words": ["up", "how", "is", "stand", "and", "it", "see"]}, {"sent": "tell me how many balls you have in there .", "words": ["how", "you", "have", "there", "in", "me"]}, {"sent": "how about doing what we did with Mary ?", "words": ["what", "how", "we", "with", "did", "about"]}, {"sent": "wait until I see how many weeks it is until you .", "words": ["wait", "how", "is", "you", "it", "see"]}, {"sent": "xxx see how long you can control it .", "words": ["can", "how", "long", "you", "it", "see"]}, {"sent": "how about .", "words": ["how", "about"]}, {"sent": "well how about you hold the pen and Ill help you write .", "words": ["the", "how", "help", "hold", "write", "you", "and", "about", "pen"]}, {"sent": "can you show her how big you are ?", "words": ["can", "how", "show", "you", "big", "are", "her"]}, {"sent": "Im not sure quite how well you do them when you do them on your own .", "words": ["do", "how", "your", "them", "you", "on", "when", "not"]}, {"sent": "well how about Dabees house .", "words": ["how", "house", "about"]}, {"sent": "how did you spot that ?", "words": ["how", "that", "did", "you"]}, {"sent": "and how yes how do we find him now the book is nearly done ?", "words": ["the", "do", "how", "we", "is", "and", "him", "find", "book"]}, {"sent": "how about orange chicken nuggets ?", "words": ["chicken", "how", "orange", "about"]}, {"sent": "for another hour is that how long it takes ?", "words": ["how", "for", "is", "long", "another", "it", "that"]}, {"sent": "how about your fingernails ?", "words": ["how", "your", "about"]}, {"sent": "you see how its tied together ?", "words": ["see", "how", "you"]}, {"sent": "so you dont know how long youre going to keep him in then ?", "words": ["how", "so", "to", "long", "you", "dont", "him", "in", "then"]}, {"sent": "then how the reindeer loved him .", "words": ["the", "then", "how", "him"]}, {"sent": "how about getting in your chair for breakfast because where where are we going this morning ?", "words": ["where", "how", "for", "your", "we", "chair", "this", "in", "are", "about", "because"]}, {"sent": "how did you know hes called Fred ?", "words": ["how", "did", "you"]}, {"sent": "do you think he knows how to turn it on ?", "words": ["do", "think", "how", "to", "you", "it", "he", "on"]}, {"sent": "and you know how to swim .", "words": ["how", "to", "swim", "you", "and"]}, {"sent": "can you remember how old was Rachel ?", "words": ["can", "old", "how", "was", "you"]}, {"sent": "how about giving mom the circle ?", "words": ["the", "how", "about"]}, {"sent": "how about getting your train set ?", "words": ["how", "your", "about", "train"]}, {"sent": "well how d how do you stop the baby from crying ?", "words": ["the", "do", "how", "stop", "you"]}, {"sent": "show me how its done .", "words": ["show", "me", "how"]}, {"sent": "how does this go ?", "words": ["does", "how", "this", "go"]}, {"sent": "how many animals are there Fraser ?", "words": ["there", "how", "are"]}, {"sent": "how bout this one ?", "words": ["this", "how"]}, {"sent": "how do you cook Oreos ?", "words": ["do", "how", "cook", "you"]}, {"sent": "how many slices ?", "words": ["how"]}, {"sent": "how many tigers rar .", "words": ["how"]}, {"sent": "Im going to take the price tag off that because I dont want them knowing how much we spent .", "words": ["the", "off", "how", "we", "to", "them", "much", "dont", "take", "that", "because"]}, {"sent": "how much ?", "words": ["how", "much"]}, {"sent": "how does that feel ?", "words": ["does", "how", "that"]}, {"sent": "how bout his hat ?", "words": ["his", "how", "hat"]}, {"sent": "how about these other stuffed animals ?", "words": ["these", "how", "about", "other"]}, {"sent": "how come ?", "words": ["how"]}, {"sent": "how about how about his mommy ?", "words": ["his", "how", "about"]}, {"sent": "is that how that story goes ?", "words": ["how", "that", "story", "is"]}, {"sent": "how many mouthfulls ?", "words": ["how"]}, {"sent": "they told you how to use a toilet .", "words": ["how", "to", "you", "they", "a"]}, {"sent": "how about OldMcdonald ?", "words": ["how", "about"]}, {"sent": "ew how soft .", "words": ["how", "soft"]}, {"sent": "do you know how to do that ?", "words": ["do", "how", "to", "you", "that"]}, {"sent": "how about that ?", "words": ["how", "that", "about"]}, {"sent": "how can it smell dirty ?", "words": ["can", "how", "it", "dirty"]}, {"sent": "see how you go let me know you know .", "words": ["how", "go", "me", "you", "see"]}, {"sent": "how many humps has the camel got ?", "words": ["the", "how"]}, {"sent": "shall we see how it goes ?", "words": ["see", "how", "it", "we"]}, {"sent": "Nomi how come you stopped talking ?", "words": ["how", "you"]}, {"sent": "wow how bout this ?", "words": ["this", "how"]}, {"sent": "how did it get up here ?", "words": ["up", "how", "get", "here", "it", "did"]}, {"sent": "hm how about that ?", "words": ["how", "that", "about"]}, {"sent": "so how many people in your family ?", "words": ["so", "how", "your", "in"]}, {"sent": "how do you know its baby food ?", "words": ["do", "how", "food", "you"]}, {"sent": "how much is that ?", "words": ["that", "how", "much", "is"]}, {"sent": "how about if we build with the red blocks ?", "words": ["the", "red", "how", "if", "we", "with", "build", "about"]}, {"sent": "are you going to show us how you can jump ?", "words": ["can", "how", "show", "to", "you", "us", "jump", "are"]}, {"sent": "how many legs has a spider got ?", "words": ["how", "a"]}, {"sent": "how bout there ?", "words": ["there", "how"]}, {"sent": "how much is four and one ?", "words": ["how", "much", "and", "is"]}, {"sent": "do you know how to blow your nose ?", "words": ["do", "how", "your", "to", "you", "blow", "nose"]}, {"sent": "show me how you make tea .", "words": ["how", "show", "you", "me", "make"]}, {"sent": "do you know how to spell blue ?", "words": ["do", "how", "to", "blue", "you"]}, {"sent": "Ronnie knows how tuh play chess .", "words": ["how", "play"]}, {"sent": "you just forget how tiny they are eventhough youve had them yourself .", "words": ["tiny", "how", "yourself", "them", "you", "they", "are"]}, {"sent": "thats how you sold him you sold him to me and he had blood all over him ?", "words": ["all", "how", "to", "you", "and", "him", "he", "me", "over"]}, {"sent": "I wonder how he gets in and out .", "words": ["how", "and", "in", "he", "out"]}, {"sent": "how do you like that ?", "words": ["do", "how", "like", "you", "that"]}, {"sent": "how many pieces are there ?", "words": ["there", "how", "are"]}, {"sent": "yeah how many shells does he pick does he lift there ?", "words": ["does", "how", "pick", "there", "he"]}, {"sent": "how many minutes should we do it for ?", "words": ["do", "how", "for", "we", "it"]}, {"sent": "um how is it different than uh playing at home ?", "words": ["how", "is", "it", "at", "home"]}, {"sent": "how are you ?", "words": ["how", "are", "you"]}, {"sent": "how do you know ?", "words": ["do", "how", "you"]}, {"sent": "how does the birdie in the clock go ?", "words": ["the", "does", "how", "clock", "go", "in"]}, {"sent": "how many tins have we got now ?", "words": ["have", "how", "we"]}, {"sent": "how many ?", "words": ["how"]}, {"sent": "how would she say .", "words": ["how", "say", "she", "would"]}, {"sent": "how is that ?", "words": ["how", "that", "is"]}, {"sent": "how many fiddlers ?", "words": ["how"]}, {"sent": "how many times a day do you brush your teeth ?", "words": ["do", "how", "your", "brush", "you", "a"]}, {"sent": "how soon did you need because next so my schedule really because I guess it hasta be me it cant be her dad because they said since Im the one thats been bringing her ?", "words": ["the", "how", "my", "need", "so", "be", "you", "they", "it", "did", "me", "her", "because"]}, {"sent": "how can that one sit ?", "words": ["can", "sit", "how", "that"]}, {"sent": "how how long ago was she here ?", "words": ["how", "was", "long", "here", "she"]}, {"sent": "how about in here ?", "words": ["here", "in", "how", "about"]}, {"sent": "how does that seem ?", "words": ["does", "how", "that"]}, {"sent": "we all eat our meals and make the plates dirty and I magic all the dirt away and get nice clean plates again .", "words": ["the", "all", "clean", "we", "nice", "and", "eat", "our", "get", "away", "dirty", "make"]}, {"sent": "lets see if I got what I was sposta get .", "words": ["what", "if", "was", "get", "see"]}, {"sent": "Ill get it but I want you to sit down .", "words": ["to", "you", "get", "it", "sit", "down", "but"]}, {"sent": "wanna get down ?", "words": ["get", "down", "wanna"]}, {"sent": "whered you get Teddy from ?", "words": ["get", "you"]}, {"sent": "Im a little too tall I cant get my head in there .", "words": ["my", "head", "little", "get", "there", "in", "too", "a"]}, {"sent": "go get it .", "words": ["get", "it", "go"]}, {"sent": "Ill go and get it then .", "words": ["go", "and", "get", "it", "then"]}, {"sent": "lets see lets see the Wendy out here lets get her .", "words": ["the", "here", "get", "out", "see", "her"]}, {"sent": "are you gonna get the other one ?", "words": ["the", "other", "get", "you", "are"]}, {"sent": "managed to get all fourteen biscuits .", "words": ["get", "to", "all"]}, {"sent": "did we get you all dry ?", "words": ["all", "we", "dry", "you", "get", "did"]}, {"sent": "and get it pulled ?", "words": ["get", "it", "and"]}, {"sent": "whatre you gonna get ?", "words": ["get", "you"]}, {"sent": "do you wanna get this .", "words": ["do", "you", "get", "this", "wanna"]}, {"sent": "because you asked me to get you peanut butter and jelly .", "words": ["jelly", "to", "you", "get", "and", "butter", "peanut butter", "me", "because"]}, {"sent": "boy theyre really hard to get out .", "words": ["get", "to", "out", "hard"]}, {"sent": "will we get the other bits of the engine ?", "words": ["the", "we", "of", "other", "get", "will"]}, {"sent": "Ill give you a shot so you wont get pneumonia okay .", "words": ["so", "you", "get", "give", "a"]}, {"sent": "well get you to Candyland .", "words": ["get", "to", "you"]}, {"sent": "yeah get it its my aunt .", "words": ["get", "my", "it"]}, {"sent": "well you cant get down until you leave those on the table .", "words": ["the", "those", "table", "you", "get", "on", "down"]}, {"sent": "uh time to get up .", "words": ["get", "to", "up"]}, {"sent": "how can you get those kids out ?", "words": ["can", "how", "those", "you", "get", "out"]}, {"sent": "well get you some chocolate milk well get you some .", "words": ["chocolate", "you", "get", "milk", "some"]}, {"sent": "get the towel .", "words": ["get", "the", "towel"]}, {"sent": "Eddy and you get someone in the corner for the night ?", "words": ["the", "for", "you", "get", "and", "in"]}, {"sent": "if you get it out for me .", "words": ["for", "if", "you", "get", "it", "out", "me"]}, {"sent": "come back and get it .", "words": ["get", "it", "back", "and"]}, {"sent": "do you get up in the morning and make it flap its wings ?", "words": ["up", "do", "the", "you", "get", "and", "in", "it", "make"]}, {"sent": "get her .", "words": ["get", "her"]}, {"sent": "did you get it ?", "words": ["get", "it", "did", "you"]}, {"sent": "look can you go get that ?", "words": ["can", "go", "you", "get", "look", "that"]}, {"sent": "you better get it because Im gonna put all that stuff up .", "words": ["up", "better", "put", "all", "you", "get", "it", "that", "because"]}, {"sent": "can you get my book ?", "words": ["can", "my", "you", "get", "book"]}, {"sent": "and we get the picture of the tree again .", "words": ["the", "tree", "we", "of", "picture", "and", "get"]}, {"sent": "I think theyre gonna needta get rid of that one .", "words": ["get", "of", "think", "that"]}, {"sent": "out you get .", "words": ["get", "out", "you"]}, {"sent": "oh you gonna get frenchfrys ?", "words": ["get", "you"]}, {"sent": "can you get this one ?", "words": ["can", "this", "get", "you"]}, {"sent": "Fraser if you dont get dressed you cant go to grandmas house .", "words": ["if", "go", "to", "you", "get", "dont", "house"]}, {"sent": "I think Ill get you some water .", "words": ["think", "you", "get", "water", "some"]}, {"sent": "yeah its not tables dont get hot do they ?", "words": ["do", "get", "dont", "hot", "they", "not"]}, {"sent": "get a drink for you .", "words": ["for", "drink", "you", "get", "a"]}, {"sent": "shall we get the box out and put it out in the middle of the room ?", "words": ["the", "put", "box", "we", "of", "and", "get", "in", "it", "room", "out"]}, {"sent": "wheres you get this eleventeen from ?", "words": ["get", "this", "you"]}, {"sent": "Johnathan you wanna get sick ?", "words": ["get", "sick", "wanna", "you"]}, {"sent": "shall we go and get a picture of Bella ?", "words": ["go", "we", "of", "picture", "and", "get", "a"]}, {"sent": "can I get off now ?", "words": ["can", "get", "off"]}, {"sent": "go get it .", "words": ["get", "it", "go"]}, {"sent": "Im gonna get you .", "words": ["get", "you"]}, {"sent": "get yourself ready to pajama .", "words": ["get", "to", "yourself"]}, {"sent": "Im gonna get ya boy .", "words": ["get"]}, {"sent": "okay get a hold of it there .", "words": ["hold", "of", "get", "there", "it", "a"]}, {"sent": "Mickey Mouse gonna get all wet .", "words": ["get", "all", "wet"]}, {"sent": "maybe in fifteen minutes or so well get ourselves ready to go do that .", "words": ["do", "go", "so", "to", "get", "in", "that"]}, {"sent": "why dont we get dolly out before you go to sleep under there .", "words": ["we", "go", "to", "sleep", "you", "get", "dont", "under", "there", "out", "why"]}, {"sent": "she um got a pair of rain boots for her birthday and loves them because she can get them on and off very easily .", "words": ["can", "off", "for", "rain", "of", "them", "and", "get", "boots", "on", "her", "she", "because", "a"]}, {"sent": "Stuart get off the bed .", "words": ["get", "the", "off", "bed"]}, {"sent": "Im going to get Daddy to bake me a cake with a key in it .", "words": ["cake", "to", "with", "get", "in", "it", "me", "a"]}, {"sent": "Ill just get my tea .", "words": ["get", "my"]}, {"sent": "okay can you get some tea for him now ?", "words": ["can", "for", "you", "get", "him", "some"]}, {"sent": "careful get up .", "words": ["get", "up", "careful"]}, {"sent": "dont I get any ?", "words": ["get", "dont", "any"]}, {"sent": "shall we get a big bin bag and put them all in ?", "words": ["all", "put", "we", "them", "and", "get", "in", "big", "a"]}, {"sent": "lets just get your onesie in .", "words": ["get", "in", "your"]}, {"sent": "you get a green one out and a pink one out please .", "words": ["you", "get", "and", "out", "green", "a"]}, {"sent": "get your belly .", "words": ["get", "your"]}, {"sent": "lemme get your cheese .", "words": ["get", "cheese", "your"]}, {"sent": "lets get a dark crayon .", "words": ["get", "dark", "crayon", "a"]}, {"sent": "where did you get those ?", "words": ["where", "those", "you", "get", "did"]}, {"sent": "okay lets get are gonna get into bed very soon ?", "words": ["get", "are", "into", "bed"]}, {"sent": "do you think you can throw it and get it in ?", "words": ["can", "do", "think", "throw", "you", "and", "get", "in", "it"]}, {"sent": "get the ball .", "words": ["get", "the", "ball"]}, {"sent": "get over here .", "words": ["get", "here", "over"]}, {"sent": "sit down and go get it .", "words": ["go", "and", "get", "it", "sit", "down"]}, {"sent": "dont get carried away with that water .", "words": ["with", "get", "dont", "away", "water", "that"]}, {"sent": "and they get some because he got big like Winnie the Pooh .", "words": ["the", "like", "and", "get", "they", "he", "some", "big", "because"]}, {"sent": "I must say once you get her to sleep she wouldnt .", "words": ["sleep", "to", "say", "you", "get", "her", "she"]}, {"sent": "got to get all these bits into the compost bin havent you ?", "words": ["the", "all", "these", "into", "to", "you", "get"]}, {"sent": "go get it .", "words": ["get", "it", "go"]}, {"sent": "get run away .", "words": ["get", "run", "away"]}, {"sent": "you wanna get it all out ?", "words": ["all", "you", "get", "it", "wanna", "out"]}, {"sent": "did you get did you ?", "words": ["get", "did", "you"]}, {"sent": "no Daddy wont get you .", "words": ["get", "you"]}, {"sent": "wanna get Mister Frog ?", "words": ["get", "wanna"]}, {"sent": "Ill go Ill get one when Ive finished feeding Amy .", "words": ["get", "when", "go"]}, {"sent": "hey let me see if I can get them both in .", "words": ["can", "if", "me", "them", "get", "in", "see"]}, {"sent": "lets get em up and get em ready for school okay ?", "words": ["up", "for", "school", "and", "get"]}, {"sent": "do you wanna give get those clothes off the radiator for me near you ?", "words": ["off", "do", "the", "for", "those", "you", "get", "wanna", "me", "give"]}, {"sent": "Daddy get a diaper .", "words": ["get", "diaper", "a"]}, {"sent": "you cant get away now .", "words": ["get", "away", "you"]}, {"sent": "get bread from the bakers .", "words": ["get", "the", "bread"]}, {"sent": "tell him to go and get the smartie .", "words": ["the", "go", "to", "and", "get", "him"]}, {"sent": "and youll get to know it .", "words": ["get", "to", "it", "and"]}, {"sent": "we can get rid of those pretty soon .", "words": ["can", "those", "we", "of", "get", "pretty"]}, {"sent": "where did he get you honey ?", "words": ["where", "you", "get", "he", "did"]}, {"sent": "get somebody to see my child .", "words": ["get", "to", "my", "see"]}, {"sent": "dont dont get yours out because um theyll get all mixed up .", "words": ["up", "all", "get", "dont", "out", "because"]}, {"sent": "you will hafta get rid of .", "words": ["will", "of", "get", "you"]}, {"sent": "next time wont you sing with me ?", "words": ["sing", "me", "with", "you"]}, {"sent": "could get some lions and tigers and things to go with them .", "words": ["go", "to", "with", "them", "and", "get", "some", "could"]}, {"sent": "you can play with all your other toys .", "words": ["can", "all", "your", "play", "with", "you", "other"]}, {"sent": "actually thats like a typical one playing with cars versus a doll .", "words": ["with", "like", "doll", "a"]}, {"sent": "and the otherll fit with a bit of help .", "words": ["the", "help", "of", "fit", "with", "and", "a"]}, {"sent": "are you going to build with the blocks ?", "words": ["the", "to", "with", "you", "are", "build"]}, {"sent": "you either get pins or collect them and well give them to someone with a little baby .", "words": ["to", "them", "with", "and", "get", "you", "little", "give", "a"]}, {"sent": "Ive finished with the purple now .", "words": ["the", "with"]}, {"sent": "shall we take Purdie with us ?", "words": ["us", "take", "with", "we"]}, {"sent": "oh well Mummyll play with the tube and you wont .", "words": ["the", "play", "with", "you", "and"]}, {"sent": "with a spoon ?", "words": ["with", "spoon", "a"]}, {"sent": "okay were playing with the next toy okay ?", "words": ["the", "toy", "were", "with"]}, {"sent": "why dont you play with your dollies .", "words": ["your", "play", "with", "you", "dont", "why"]}, {"sent": "thats cakes and look theres a drink with a straw .", "words": ["drink", "with", "and", "look", "a"]}, {"sent": "with the camera .", "words": ["the", "camera", "with"]}, {"sent": "remember when we do our rhyme with Elmo ?", "words": ["do", "we", "with", "our", "when"]}, {"sent": "juice with a straw .", "words": ["juice", "with", "a"]}, {"sent": "lets play peekaboo with Bear .", "words": ["play", "with"]}, {"sent": "well try with this .", "words": ["this", "try", "with"]}, {"sent": "and then you can go out on the sledge with Awww and you can make a snowman as well .", "words": ["can", "the", "go", "with", "you", "and", "snowman", "out", "on", "then", "make", "a"]}, {"sent": "you hafta find the letters that would go with the numbers .", "words": ["the", "go", "with", "find", "you", "would", "that"]}, {"sent": "whats the matter with his legs ?", "words": ["the", "his", "with"]}, {"sent": "and theres ones with four wheels as against two wheels on the front and two rollers .", "words": ["on", "the", "with", "and"]}, {"sent": "yeah he started to practice with the spoon and he he can feed himself with his fingers .", "words": ["the", "can", "to", "with", "and", "he", "spoon", "feed", "his"]}, {"sent": "whatll we play with now then ?", "words": ["then", "play", "with", "we"]}, {"sent": "we have one doll it doesnt look anything like that I would say its its not its a soft doll with a flat face thats much more realistic .", "words": ["soft", "face", "we", "say", "like", "doll", "would", "have", "with", "much", "it", "more", "look", "that", "not", "a"]}, {"sent": "that was the picture with all the brown circles and he stayed inside for more than two weeks and then what happened ?", "words": ["the", "all", "what", "for", "brown", "was", "inside", "picture", "with", "and", "more", "he", "then", "that"]}, {"sent": "you wanna play with with what with this ?", "words": ["what", "play", "with", "you", "this", "wanna"]}, {"sent": "you wanna play with something else ?", "words": ["play", "with", "wanna", "you"]}, {"sent": "youre really starting to play with me .", "words": ["to", "me", "play", "with"]}, {"sent": "you close it with the string .", "words": ["the", "with", "you", "it", "close"]}, {"sent": "punky with a lunky .", "words": ["with", "a"]}, {"sent": "but thats Manuelas so we dont wanna play with that .", "words": ["we", "so", "play", "with", "dont", "wanna", "that", "but"]}, {"sent": "okay come with mommy .", "words": ["with"]}, {"sent": "we havent got one with WinniethePooh on there .", "words": ["on", "there", "with", "we"]}, {"sent": "and shell play with that but hes good with both of them .", "words": ["of", "play", "with", "them", "and", "good", "that", "but"]}, {"sent": "um I guess anything she played with today that ya know was new for us or something she never played with ?", "words": ["for", "was", "with", "us", "new", "that", "she"]}, {"sent": "oh youll make out the dues with me but you hafta decide in advance what candy youre gonna buy .", "words": ["the", "what", "with", "you", "in", "out", "me", "candy", "make", "buy", "but"]}, {"sent": "dont do it with green do it with one of these darker ones .", "words": ["do", "these", "of", "with", "dont", "it", "green"]}, {"sent": "and then you can play with some lorrys on the table while I finish the rest of my breakfast .", "words": ["can", "the", "my", "table", "of", "play", "with", "finish", "you", "and", "on", "some", "then"]}, {"sent": "he has a saw the froggie has a motor with a saw in him to cut the street ?", "words": ["the", "cut", "to", "with", "in", "him", "he", "street", "a"]}, {"sent": "armadillos beat their chests with their fists .", "words": ["with", "their"]}, {"sent": "and see if you can find something for me to eat with .", "words": ["can", "for", "if", "to", "me", "with", "you", "and", "eat", "find", "see"]}, {"sent": "what did Mommy do with your eggnog ?", "words": ["what", "do", "your", "with", "did"]}, {"sent": "um I mean he doesnt sit and play with it but like you know like pretend its but he just he likes to look at it that caught his attention .", "words": ["to", "play", "with", "like", "and", "you", "his", "it", "he", "at", "look", "pretend", "sit", "that", "but"]}, {"sent": "with his friend .", "words": ["his", "with"]}, {"sent": "with who ?", "words": ["with", "who"]}, {"sent": "you havent really played with dolls but wanna lets see what happens .", "words": ["what", "with", "you", "wanna", "see", "but"]}, {"sent": "because Rwwws waiting to play with you at PlanetPlay .", "words": ["to", "play", "with", "you", "at", "because"]}, {"sent": "are you doing some more rolling with the ink ?", "words": ["the", "with", "you", "more", "are", "some"]}, {"sent": "if youre okay with it .", "words": ["it", "with", "if"]}, {"sent": "dont go away with the pencil because I wouldnt want Jenny to get it because it would hurt her .", "words": ["the", "go", "hurt", "to", "with", "pencil", "get", "dont", "away", "it", "would", "her", "because"]}, {"sent": "hm play with the toys please .", "words": ["the", "play", "with"]}, {"sent": "the one with the long white dress ?", "words": ["the", "with", "long", "white", "dress"]}, {"sent": "we can also play with the truck okay ?", "words": ["can", "the", "we", "play", "with", "truck"]}, {"sent": "Gla well look at something else next after you finish that with mummy and daddy .", "words": ["look", "with", "finish", "you", "and", "at", "that"]}, {"sent": "yes I know you want to play with Luna .", "words": ["to", "play", "with", "you", "want to"]}, {"sent": "which you can certainly fill with money if you feel you must .", "words": ["can", "money", "if", "with", "you", "which"]}, {"sent": "Im having trouble understanding you with your fist in your mouth June .", "words": ["your", "mouth", "with", "you", "in"]}, {"sent": "and then Cwww came over as well with her .", "words": ["her", "with", "and", "then", "over"]}, {"sent": "so how are you coping with it then ?", "words": ["how", "so", "with", "you", "are", "it", "then"]}, {"sent": "what did you do with your fingers ?", "words": ["what", "do", "your", "with", "you", "did"]}, {"sent": "with MrsHwww .", "words": ["with"]}, {"sent": "starts with a king k .", "words": ["with", "a"]}, {"sent": "and the blue boy goes with the blue button .", "words": ["the", "with", "blue", "and", "button"]}, {"sent": "you wanna draw a picture with crayons ?", "words": ["picture", "draw", "with", "you", "wanna", "a"]}, {"sent": "I dont think thats with it .", "words": ["dont", "think", "it", "with"]}, {"sent": "because of that youre coming with me now .", "words": ["of", "with", "me", "that", "because"]}, {"sent": "and is there anything she didnt seem interested in playing with at all ?", "words": ["all", "is", "with", "and", "there", "in", "at", "she"]}, {"sent": "do you wanna play with Hannah ?", "words": ["do", "play", "with", "you", "wanna"]}, {"sent": "shall I come with this big grabber ?", "words": ["this", "big", "with"]}, {"sent": "what would you like to play with ?", "words": ["what", "to", "play", "with", "like", "you", "would"]}, {"sent": "this is the one with the long hair .", "words": ["the", "is", "with", "long", "this", "hair"]}, {"sent": "I dont want a two year old with a pacifier so haha .", "words": ["old", "so", "with", "dont", "a"]}, {"sent": "you want ta play with this ?", "words": ["this", "play", "with", "you"]}, {"sent": "with the other balls .", "words": ["the", "with", "other"]}, {"sent": "what are ya gonna do with it ?", "words": ["what", "do", "with", "are", "it"]}, {"sent": "this is a ball with red spots .", "words": ["red", "is", "with", "ball", "this", "a"]}, {"sent": "Martin went with us didnt he ?", "words": ["he", "us", "with"]}, {"sent": "with your red crayon ?", "words": ["your", "red", "with", "crayon"]}, {"sent": "with a bow on it huh ?", "words": ["on", "it", "with", "a"]}, {"sent": "what shall I do with you ?", "words": ["what", "do", "with", "you"]}, {"sent": "those are Mrs Woods dogs but shes letting Nina play with them .", "words": ["those", "play", "with", "them", "are", "but"]}, {"sent": "were you playing with Kwww ?", "words": ["were", "with", "you"]}, {"sent": "what do they play with you ?", "words": ["what", "do", "play", "with", "you", "they"]}, {"sent": "youre trying and trying to get them with those toes .", "words": ["those", "to", "them", "with", "and", "get"]}, {"sent": "why dont you give them a hotdog with this sausage ?", "words": ["them", "with", "you", "dont", "this", "give", "why", "a"]}, {"sent": "all the better to eat you with .", "words": ["the", "all", "better", "to", "with", "you", "eat"]}, {"sent": "I see a problem with my bird already .", "words": ["my", "bird", "with", "see", "a"]}, {"sent": "is he coming to sit with me ?", "words": ["is", "to", "with", "he", "me", "sit"]}, {"sent": "what did you play with yesterday Eleanor what did you play with darling ?", "words": ["what", "play", "with", "you", "did"]}, {"sent": "we like to find and play with things were not sposta play with .", "words": ["we", "to", "play", "with", "like", "and", "find", "were", "not"]}, {"sent": "have you seen anywhere anybody on television with a snake round their neck ?", "words": ["their", "with", "you", "have", "on", "a"]}, {"sent": "what are you gonna do with them ?", "words": ["what", "do", "with", "them", "you", "are"]}, {"sent": "how about the giraffe xxx with the green background ?", "words": ["the", "how", "giraffe", "with", "about", "green"]}, {"sent": "well really Imean between thats exactly what Roberts doing with the ActionMan .", "words": ["the", "what", "with"]}, {"sent": "are you playing with her ear ?", "words": ["with", "you", "are", "ear", "her"]}, {"sent": "this is what you write with .", "words": ["what", "is", "with", "write", "you", "this"]}, {"sent": "well what do you wanna do with it ?", "words": ["what", "do", "with", "you", "it", "wanna"]}, {"sent": "the little dog laughed to see such sport and the dish ran away with the spoon .", "words": ["the", "to", "with", "little", "and", "dish", "away", "spoon", "see", "dog"]}, {"sent": "all done .", "words": ["all"]}, {"sent": "okay Mamall put all your dinosaurs in there .", "words": ["all", "put", "your", "there", "in"]}, {"sent": "theyre all touch and feel books .", "words": ["all", "touch", "and"]}, {"sent": "thats all ?", "words": ["all"]}, {"sent": "take all the stuff off the floor .", "words": ["the", "all", "take", "off"]}, {"sent": "catching all the fish uhn arent they ?", "words": ["the", "all", "they", "fish"]}, {"sent": "all done .", "words": ["all"]}, {"sent": "so he put a special mark upon your feet and fingers too and of all the children everyone everywhere no one has that mark but you .", "words": ["the", "all", "put", "your", "of", "so", "and", "you", "he", "too", "that", "but", "a"]}, {"sent": "whoa all the boats .", "words": ["the", "all"]}, {"sent": "all about birds .", "words": ["all", "about"]}, {"sent": "oh now you see Ive made a little house for all the animals so they can all go in there .", "words": ["the", "all", "can", "for", "go", "so", "little", "you", "they", "in", "there", "house", "see", "a"]}, {"sent": "uhoh look at all the animals .", "words": ["the", "all", "at", "look"]}, {"sent": "its all .", "words": ["all"]}, {"sent": "oh I see how he gets piano because all the different .", "words": ["the", "all", "how", "he", "see", "because"]}, {"sent": "all aboard .", "words": ["all"]}, {"sent": "are you picking it all up ?", "words": ["up", "all", "you", "it", "are"]}, {"sent": "and all the snows falling down .", "words": ["the", "all", "down", "and"]}, {"sent": "put all the pennys in one .", "words": ["the", "all", "put", "in"]}, {"sent": "I dont think you did because it is all hanging up or its .", "words": ["up", "all", "think", "is", "you", "dont", "it", "did", "because"]}, {"sent": "are you unpacking all the shopping for Daddy ?", "words": ["the", "all", "for", "you", "are"]}, {"sent": "all done Fraser .", "words": ["all"]}, {"sent": "okay you want all those people in their seats .", "words": ["all", "those", "their", "you", "in"]}, {"sent": "youve been up all morning .", "words": ["up", "all"]}, {"sent": "well they looked up into the sky first of all .", "words": ["up", "the", "all", "into", "sky", "of", "first", "they"]}, {"sent": "here we are then leave them all like this get them all the same can you see M those are all the same arent they ?", "words": ["the", "all", "can", "those", "we", "them", "like", "you", "here", "this", "get", "are", "same", "they", "see", "then"]}, {"sent": "here comes all the Duplo .", "words": ["here", "all", "the"]}, {"sent": "wanna go with Fraser and all those children ?", "words": ["all", "those", "go", "with", "and", "wanna"]}, {"sent": "its all over everybody .", "words": ["all", "over"]}, {"sent": "all .", "words": ["all"]}, {"sent": "the wee boys got all the pots and pans out .", "words": ["the", "all", "out", "and"]}, {"sent": "and he learned all about ice the hard way didnt he ?", "words": ["the", "all", "and", "he", "about", "hard", "ice"]}, {"sent": "theyre all sitting down arent they ?", "words": ["all", "down", "they"]}, {"sent": "I dont like us to be on the phone all the time .", "words": ["the", "all", "to", "be", "like", "dont", "us", "on"]}, {"sent": "you dont know but you say yes all the time dont you ?", "words": ["the", "all", "say", "you", "dont", "but"]}, {"sent": "does she not need the slippers at all do you think ?", "words": ["the", "does", "all", "do", "need", "think", "you", "at", "she", "not"]}, {"sent": "all things considered .", "words": ["all"]}, {"sent": "I need all .", "words": ["all", "need"]}, {"sent": "all the time dirty face and hands .", "words": ["the", "all", "face", "and", "dirty"]}, {"sent": "should we put all the people in the dentists office ?", "words": ["the", "all", "put", "we", "in"]}, {"sent": "all right .", "words": ["all"]}, {"sent": "no the birdie cant because everybody there is all is hungry right .", "words": ["the", "all", "is", "hungry", "there", "because"]}, {"sent": "do you remember all those little houses ?", "words": ["all", "do", "those", "little", "you"]}, {"sent": "okay were all done .", "words": ["all", "were"]}, {"sent": "no neither of my kids have been into stuffed animals at all .", "words": ["all", "my", "into", "of", "have", "at"]}, {"sent": "and when we go upstairs we can take all that with us cant we ?", "words": ["can", "all", "go", "we", "with", "and", "us", "that", "take", "when"]}, {"sent": "theres snow all over Horton .", "words": ["all", "snow", "over"]}, {"sent": "oh all the blocks are on the floor .", "words": ["the", "all", "on", "are"]}, {"sent": "eventually he gets to all of them .", "words": ["all", "of", "to", "them", "he"]}, {"sent": "look now all his spots are blue .", "words": ["all", "blue", "are", "look", "his"]}, {"sent": "which is your favorite one outof all of yours ?", "words": ["all", "your", "is", "of", "which"]}, {"sent": "just straightening up a little bit thats all .", "words": ["up", "all", "little", "a"]}, {"sent": "Griz do you have enough gas in your box to take you to all the way to Maine ?", "words": ["the", "do", "all", "your", "box", "to", "you", "have", "in", "take"]}, {"sent": "what did you have all over your face yesterday Gail .", "words": ["what", "all", "your", "face", "you", "have", "did", "over"]}, {"sent": "Im not going to sweep Purdie all up .", "words": ["up", "all", "to", "sweep", "not"]}, {"sent": "how come you like all the noisy toys hm ?", "words": ["the", "all", "how", "like", "noisy", "you"]}, {"sent": "to stay strong and healthy the brown horse eats all day .", "words": ["the", "all", "brown", "to", "and", "horse", "stay"]}, {"sent": "oh and Ive got all three .", "words": ["all", "and"]}, {"sent": "we all had suits didnt we ?", "words": ["all", "we"]}, {"sent": "we dont want it all messed up right ?", "words": ["up", "all", "we", "dont", "it"]}, {"sent": "hes all frozen .", "words": ["all"]}, {"sent": "all fall down .", "words": ["all", "down", "fall"]}, {"sent": "ohmygoodness look at all these toys .", "words": ["all", "at", "these", "look"]}, {"sent": "yeah hes gone out with all the other little doggies .", "words": ["the", "all", "with", "other", "little", "out"]}, {"sent": "look at all those presents under the christmastree .", "words": ["the", "all", "look", "those", "under", "at"]}, {"sent": "the lion was scared all the time .", "words": ["the", "all", "was", "scared", "lion"]}, {"sent": "gonna walk all over Gods heaven .", "words": ["over", "all", "walk"]}, {"sent": "and what about all those hungry elephants and the lions ?", "words": ["the", "all", "what", "those", "hungry", "and", "about"]}, {"sent": "well I needta just tidy this up because otherwise its all going to the waters going to drip on the floor .", "words": ["up", "all", "the", "to", "this", "on", "because"]}, {"sent": "thats because you blew all the air out .", "words": ["the", "all", "you", "out", "because"]}, {"sent": "so should we line all the animals up ?", "words": ["the", "all", "up", "we", "so"]}, {"sent": "its all their birthdays ?", "words": ["all", "their"]}, {"sent": "all over my dress and I had ta iron it yeah .", "words": ["all", "my", "and", "it", "over", "dress"]}, {"sent": "all this stuff .", "words": ["all", "this"]}, {"sent": "xxx all done .", "words": ["all"]}, {"sent": "and AuntieVal said you can come again because you tidied up all the paper plates .", "words": ["can", "up", "all", "the", "paper", "you", "and", "because"]}, {"sent": "get all the bits .", "words": ["get", "all", "the"]}, {"sent": "got to be careful banging all these boxes and things down .", "words": ["all", "these", "to", "be", "and", "careful", "down"]}, {"sent": "we can all hug the bear .", "words": ["can", "all", "the", "we", "bear", "hug"]}, {"sent": "no were not finished but thats um all that we can do .", "words": ["can", "all", "do", "we", "but", "that", "not", "were"]}, {"sent": "no say Rabbit all gone .", "words": ["all", "all gone", "say"]}, {"sent": "theyre all in the bath uhn arent they ?", "words": ["the", "all", "in", "they"]}, {"sent": "all right time to go back to class .", "words": ["to", "all", "back", "go"]}, {"sent": "and Im going to come next week because youre gonna show all the mums and dads next week uhn are you ?", "words": ["the", "all", "show", "to", "and", "you", "are", "because"]}, {"sent": "do you not remember folding up all the clothes to go inside last night ?", "words": ["up", "do", "all", "the", "go", "to", "inside", "last", "you", "not"]}, {"sent": "did you see all your friends outside ?", "words": ["all", "your", "outside", "you", "did", "see"]}, {"sent": "and youknow the lead and all like he says .", "words": ["the", "all", "like", "and", "he"]}, {"sent": "ooh buddy whoa now Im all tangled up in your thing .", "words": ["up", "all", "in", "your"]}, {"sent": "the monkeys doing it all on his own .", "words": ["the", "all", "his", "it", "on"]}, {"sent": "all broke .", "words": ["all"]}, {"sent": "yeah we clean all the toys between .", "words": ["clean", "all", "the", "we"]}, {"sent": "I dont think llamas have all of those things .", "words": ["all", "think", "those", "of", "have", "dont"]}, {"sent": "hes got it all on now hasnt he ?", "words": ["on", "all", "it", "he"]}, {"sent": "look at all that together .", "words": ["all", "at", "that", "look"]}, {"sent": "do you get the toy food and all ?", "words": ["the", "do", "all", "you", "get", "and", "toy", "food"]}, {"sent": "all day long .", "words": ["all", "long"]}, {"sent": "lets see if youve cleared up all that mess yet .", "words": ["up", "all", "if", "see", "that"]}, {"sent": "shall we all go to bed then ?", "words": ["all", "go", "we", "to", "bed", "then"]}, {"sent": "thats a wee cupboard for putting all their their brushes and things in .", "words": ["all", "for", "their", "and", "in", "a"]}, {"sent": "its all done .", "words": ["all"]}, {"sent": "do you wanna see this birds nest Fraser ?", "words": ["do", "you", "this", "wanna", "see"]}, {"sent": "you let me do your finger nails but you willnt let me do your toy nails will you ?", "words": ["do", "your", "you", "finger", "toy", "will", "me", "but"]}, {"sent": "so do you know what happens ?", "words": ["so", "do", "what", "you"]}, {"sent": "do you want it the other way ?", "words": ["the", "do", "other", "you", "it"]}, {"sent": "they wanna do some nursery rhymes .", "words": ["they", "do", "wanna", "some"]}, {"sent": "do you ?", "words": ["do", "you"]}, {"sent": "oh hes got his teddy bear what happens do you think what do you think he says to the lady when he goes into the lost property office ?", "words": ["the", "what", "do", "think", "into", "to", "you", "bear", "he", "his", "when"]}, {"sent": "do you wanna do the counting one ?", "words": ["the", "do", "wanna", "you"]}, {"sent": "you sure do .", "words": ["do", "you"]}, {"sent": "which dogs do you know ?", "words": ["do", "which", "you"]}, {"sent": "oh I see but did you know what was on that picture do you know what that picture is ?", "words": ["what", "do", "is", "was", "picture", "you", "did", "on", "see", "that", "but"]}, {"sent": "heres what were gonna do were gonna put it right on the table .", "words": ["the", "what", "do", "put", "table", "it", "on", "were"]}, {"sent": "do it there see .", "words": ["see", "do", "it", "there"]}, {"sent": "and I try to do that to help her understand that food is for feeding not for playing and throwing and whatever .", "words": ["do", "help", "for", "is", "to", "her", "and", "that", "food", "try to", "try", "not"]}, {"sent": "so I can do it to you or you can do it to me and I can see you .", "words": ["can", "do", "see", "so", "to", "you", "and", "it", "me"]}, {"sent": "do you think they have ?", "words": ["do", "think", "you", "have", "they"]}, {"sent": "do you tell Nadia stories ?", "words": ["do", "you"]}, {"sent": "we do .", "words": ["do", "we"]}, {"sent": "do you have any newspapers in your tea shop ?", "words": ["do", "your", "any", "you", "have", "in"]}, {"sent": "what do you wanna play with these ?", "words": ["what", "do", "these", "play", "with", "you", "wanna"]}, {"sent": "do you ?", "words": ["do", "you"]}, {"sent": "what do you think happens .", "words": ["what", "do", "think", "you"]}, {"sent": "do I have everything now ?", "words": ["have", "do"]}, {"sent": "are you going to do that .", "words": ["do", "to", "you", "are", "that"]}, {"sent": "for what what do you need it for ?", "words": ["what", "do", "for", "need", "you", "it"]}, {"sent": "do you wanna help her put some clothes on ?", "words": ["do", "help", "put", "you", "wanna", "on", "some", "her"]}, {"sent": "do you have any french fries ?", "words": ["do", "french fries", "any", "you", "have"]}, {"sent": "I bet you do .", "words": ["do", "you"]}, {"sent": "how do you know when youre happy ?", "words": ["do", "how", "happy", "you", "when"]}, {"sent": "oh you still want the shark to eat her do you ?", "words": ["the", "do", "to", "you", "eat", "her"]}, {"sent": "why do I look funny ?", "words": ["do", "why", "look"]}, {"sent": "do you know ?", "words": ["do", "you"]}, {"sent": "you do it .", "words": ["do", "it", "you"]}, {"sent": "what do the babys on the bus do ?", "words": ["the", "what", "do", "on", "bus"]}, {"sent": "do you not want the present ?", "words": ["the", "do", "present", "you", "not"]}, {"sent": "what do you do with Thumbkin sometimes ?", "words": ["what", "do", "with", "you"]}, {"sent": "come over here and show her how you do the puzzle .", "words": ["the", "do", "how", "show", "her", "and", "here", "you", "puzzle", "over"]}, {"sent": "wonder what this one will do .", "words": ["will", "what", "do", "this"]}, {"sent": "thats what it says what do you wanna show Erin today ?", "words": ["what", "do", "show", "you", "it", "wanna"]}, {"sent": "do you think we should find them all look ?", "words": ["do", "think", "all", "look", "we", "them", "you", "find"]}, {"sent": "oh do you remember what this is called ?", "words": ["do", "what", "is", "you", "this"]}, {"sent": "and what did Lynn do ?", "words": ["what", "do", "did", "and"]}, {"sent": "do they ?", "words": ["do", "they"]}, {"sent": "what do you like in your coffee ?", "words": ["what", "do", "your", "like", "you", "in", "coffee"]}, {"sent": "well do it next time then youll be able to do it better .", "words": ["do", "better", "to", "be", "it", "then"]}, {"sent": "well do one more fish .", "words": ["do", "more", "fish"]}, {"sent": "okay where do you wanna sit ?", "words": ["where", "do", "you", "wanna", "sit"]}, {"sent": "what could you do ?", "words": ["what", "could", "do", "you"]}, {"sent": "we wont do that .", "words": ["do", "that", "we"]}, {"sent": "what did you do when you caught them ?", "words": ["what", "do", "them", "you", "did", "when"]}, {"sent": "okay lets do it together can you help here you do it too there whos that ?", "words": ["can", "do", "help", "you", "here", "there", "it", "too", "that"]}, {"sent": "d do you want it ?", "words": ["do", "it", "you"]}, {"sent": "can you do your bottom half ?", "words": ["can", "do", "bottom", "your", "you"]}, {"sent": "oh you are gonna do that one .", "words": ["do", "are", "that", "you"]}, {"sent": "do you remember whos Sarah ?", "words": ["do", "you"]}, {"sent": "xxx do you wanna um put these down ?", "words": ["do", "put", "these", "you", "wanna", "down"]}, {"sent": "you can do some of this aswell can we ?", "words": ["can", "do", "we", "of", "you", "this", "some"]}, {"sent": "what do you do with the cars ?", "words": ["the", "what", "do", "with", "you"]}, {"sent": "I do like you .", "words": ["do", "like", "you"]}, {"sent": "see so the first thing you hafta do for the puzzle is take these out right ?", "words": ["the", "do", "these", "for", "is", "so", "first", "you", "puzzle", "out", "see", "take"]}, {"sent": "do you feel full ?", "words": ["do", "full", "you"]}, {"sent": "do you wanna come and do it over here so I can see ?", "words": ["can", "do", "so", "and", "you", "here", "it", "wanna", "see", "over"]}, {"sent": "what you would like to try to do ?", "words": ["what", "do", "to", "like", "you", "would", "try", "try to"]}, {"sent": "what did dada just do ?", "words": ["what", "do", "did"]}, {"sent": "whats the name of the book do you think ?", "words": ["the", "do", "think", "of", "you", "book"]}, {"sent": "do you like being outside when its windy ?", "words": ["do", "outside", "like", "windy", "you", "when"]}, {"sent": "no let me do it .", "words": ["me", "do", "it"]}, {"sent": "do you want some salt ?", "words": ["do", "salt", "some", "you"]}, {"sent": "so I do that .", "words": ["so", "do", "that"]}, {"sent": "no I cant do it .", "words": ["do", "it"]}, {"sent": "what do you do at Nwwws house ?", "words": ["what", "do", "you", "house", "at"]}, {"sent": "we do have some play food at home .", "words": ["do", "we", "play", "have", "food", "at", "some", "home"]}, {"sent": "do you want Mummy to do it ?", "words": ["to", "do", "it", "you"]}, {"sent": "then what did he do ?", "words": ["what", "do", "he", "did", "then"]}, {"sent": "what do they do ?", "words": ["what", "do", "they"]}, {"sent": "Cathy do not or youre not getting in .", "words": ["do", "in", "not"]}, {"sent": "do you remember ?", "words": ["do", "you"]}, {"sent": "do you think so ?", "words": ["do", "think", "so", "you"]}, {"sent": "what did her mummie what did her mummie and daddy do for her birthday ?", "words": ["what", "do", "for", "and", "did", "her"]}, {"sent": "would you like to do that ?", "words": ["do", "to", "like", "you", "would", "that"]}, {"sent": "do you have cousins that you see when you go there ?", "words": ["do", "when", "go", "you", "have", "there", "see", "that"]}, {"sent": "what songs do you sing in school ?", "words": ["what", "do", "school", "you", "sing", "in"]}, {"sent": "what do you say ?", "words": ["what", "do", "say", "you"]}, {"sent": "do you play puzzle ?", "words": ["do", "play", "puzzle", "you"]}, {"sent": "what color shall we do ?", "words": ["what", "do", "we"]}, {"sent": "I like to sit and do them now .", "words": ["do", "to", "them", "like", "and", "sit"]}, {"sent": "now I was going to say do you remember in Spain where we had air conditioning ?", "words": ["where", "do", "we", "to", "was", "say", "you", "in"]}, {"sent": "why dont you come and sit at the table and do some sticking .", "words": ["the", "do", "table", "and", "you", "dont", "at", "some", "sit", "why"]}, {"sent": "are we gonna build lets see do you think theres any way that we could build a track from here that stretches all the way to the wall over there ?", "words": ["the", "do", "think", "all", "we", "to", "any", "you", "here", "there", "are", "build", "over", "see", "could", "that", "a"]}, {"sent": "do you remember crawling through that tunnel .", "words": ["do", "that", "you"]}, {"sent": "yes I do .", "words": ["do"]}, {"sent": "do you ?", "words": ["do", "you"]}, {"sent": "who do you think you are ?", "words": ["do", "think", "you", "are", "who"]}, {"sent": "do you have a favorite game ?", "words": ["do", "game", "you", "have", "a"]}, {"sent": "ah you want some more of that uhn do you ?", "words": ["do", "of", "you", "more", "some", "that"]}, {"sent": "do you have any books here ?", "words": ["do", "any", "you", "have", "here"]}, {"sent": "and do it on the line like that .", "words": ["the", "do", "like", "and", "it", "on", "that"]}, {"sent": "why are you gonna do that ?", "words": ["do", "you", "are", "that", "why"]}, {"sent": "do you want Mommy to help you .", "words": ["to", "do", "help", "you"]}, {"sent": "what did you do to him ?", "words": ["what", "do", "to", "you", "him", "did"]}, {"sent": "for what ?", "words": ["what", "for"]}, {"sent": "are you going for a walk with your Mama ?", "words": ["for", "your", "with", "you", "are", "walk", "a"]}, {"sent": "heres one for you for the apron .", "words": ["the", "for", "you"]}, {"sent": "looking for Fraser ?", "words": ["for"]}, {"sent": "I got awfully tired while looking for sheep .", "words": ["for", "sheep", "tired"]}, {"sent": "it is for your hair .", "words": ["for", "your", "is", "it", "hair"]}, {"sent": "can you turn the light on for me Sweetie Pie ?", "words": ["can", "the", "for", "light", "you", "on", "me"]}, {"sent": "sign language for elephant right xxx ?", "words": ["for", "elephant"]}, {"sent": "a prize for my Alex .", "words": ["my", "for", "a"]}, {"sent": "Mrs Bear and Sam waited for cakes .", "words": ["for", "and"]}, {"sent": "a for ?", "words": ["for", "a"]}, {"sent": "more for Daddy ?", "words": ["more", "for"]}, {"sent": "whats daddy having for tea ?", "words": ["for"]}, {"sent": "is it for Amy ?", "words": ["it", "for", "is"]}, {"sent": "want me to go and get a tissue for you ?", "words": ["for", "go", "tissue", "to", "and", "get", "you", "me", "a"]}, {"sent": "fill something up for your baby ?", "words": ["up", "for", "your"]}, {"sent": "go and get your plate and Ill put it in the dishwasher for you .", "words": ["the", "put", "for", "your", "plate", "go", "and", "get", "you", "in", "it"]}, {"sent": "youve been playing in the bath for a while so at least youre now pretty clean werent you ?", "words": ["the", "clean", "for", "so", "you", "in", "pretty", "at", "a"]}, {"sent": "youre putting some nice sugar strands on and it will be finished wont it for me .", "words": ["for", "be", "me", "nice", "will", "and", "it", "on", "some"]}, {"sent": "youre looking for the d now see any more with ducks on ?", "words": ["the", "for", "any", "with", "more", "on", "see"]}, {"sent": "silent trees in silent forests long for parrots so .", "words": ["so", "in", "for", "long"]}, {"sent": "its for Purdies food is it ?", "words": ["it", "food", "for", "is"]}, {"sent": "oh what did you do out here for me ?", "words": ["what", "do", "for", "you", "here", "did", "out", "me"]}, {"sent": "that ones for scooping .", "words": ["for", "that"]}, {"sent": "well call Sarah and Julian and say were going to the zoo for the day .", "words": ["the", "zoo", "for", "to", "say", "and", "were"]}, {"sent": "surprise one for me too .", "words": ["me", "for", "too"]}, {"sent": "no not for you huh ?", "words": ["for", "not", "you"]}, {"sent": "you know what I think youd need glue for that wouldnt you ?", "words": ["what", "think", "for", "need", "you", "glue", "that"]}, {"sent": "put some wheat on a spoonful of food can I have the spoon for a second ?", "words": ["can", "the", "put", "for", "of", "have", "food", "spoon", "on", "some", "a"]}, {"sent": "what are you looking for ?", "words": ["what", "are", "for", "you"]}, {"sent": "for bringing me a pear .", "words": ["me", "for", "a"]}, {"sent": "twelve for tea with GrannyDryden and one for lunch at one oclock .", "words": ["at", "for", "with", "and"]}, {"sent": "youve got all the pieces in this thing just for a change arent they ?", "words": ["the", "all", "for", "this", "in", "they", "a"]}, {"sent": "this is handy because you know we say p for passengers ?", "words": ["for", "we", "is", "say", "you", "this", "because"]}, {"sent": "it has gone for naughty boys ?", "words": ["it", "naughty", "for"]}, {"sent": "or would you like to come on the train for a ride ?", "words": ["the", "for", "train", "to", "like", "you", "would", "on", "ride", "a"]}, {"sent": "weve got let me think what were doing for the party .", "words": ["the", "what", "think", "for", "party", "me", "were"]}, {"sent": "you havent got time to do a house for Buzzlightyear .", "words": ["do", "for", "to", "you", "house", "a"]}, {"sent": "a paper for me .", "words": ["me", "paper", "for", "a"]}, {"sent": "no mommy can open it for you .", "words": ["can", "for", "open", "you", "it"]}, {"sent": "big pot for a little Maggie .", "words": ["for", "big", "little", "a"]}, {"sent": "Ill throw it for him .", "words": ["him", "it", "throw", "for"]}, {"sent": "is that for me ?", "words": ["me", "for", "that", "is"]}, {"sent": "and then you got this Po for your birthday didnt you ?", "words": ["for", "your", "you", "and", "this", "then"]}, {"sent": "for coffee .", "words": ["for", "coffee"]}, {"sent": "thats the right train for the track .", "words": ["the", "for", "train"]}, {"sent": "for the dog to eat .", "words": ["the", "for", "to", "eat", "dog"]}, {"sent": "theres your ginger ale waiting for you .", "words": ["for", "your", "you"]}, {"sent": "you brought that especially for Mummy didnt you ?", "words": ["for", "that", "you"]}, {"sent": "this one was for a hair xxx from a hair dryer for dat when I was taking it outof the hair dryer I forgot to take the other pieces out dat make xxx .", "words": ["the", "dryer", "for", "was", "to", "other", "this", "it", "make", "out", "take", "hair", "when", "a"]}, {"sent": "wheres the nipple for that bottle ?", "words": ["the", "bottle", "for", "that"]}, {"sent": "get some more soup for me ?", "words": ["for", "me", "get", "more", "soup", "some"]}, {"sent": "we paid for two .", "words": ["for", "we"]}, {"sent": "you gonna make something for Mommy ?", "words": ["for", "make", "you"]}, {"sent": "xxx oh maybe she used it for a recipe ?", "words": ["it", "she", "for", "a"]}, {"sent": "cuddle her and give her a kiss and tell her youre sorry for pulling her hair .", "words": ["for", "her", "and", "give", "hair", "kiss", "a"]}, {"sent": "well shes gonna bring the food for the pigs now .", "words": ["the", "food", "for", "bring"]}, {"sent": "for Christmas ?", "words": ["for"]}, {"sent": "what for you ?", "words": ["what", "for", "you"]}, {"sent": "I think she needs to rest for a long time .", "words": ["think", "for", "to", "long", "she", "a"]}, {"sent": "well hotels are usually for people to go to on holiday arent they ?", "words": ["for", "go", "to", "they", "are", "on"]}, {"sent": "okay but I need a flipper for the fish .", "words": ["the", "for", "need", "fish", "but", "a"]}, {"sent": "I think well hafta have a good look for them later on wont we because I dont know where they are now .", "words": ["where", "think", "for", "we", "them", "have", "dont", "they", "are", "good", "look", "on", "because", "a"]}, {"sent": "oh thats the leaf that you brought back for me when you went for a walk the other day with Grandma and Granddad didnt you ?", "words": ["the", "for", "when", "back", "with", "you", "other", "and", "walk", "me", "that", "a"]}, {"sent": "what are you looking for ?", "words": ["what", "are", "for", "you"]}, {"sent": "Daddy bought that for you at the bonfire didnt he ?", "words": ["the", "for", "you", "he", "at", "that"]}, {"sent": "shall we go and get some new shoes for daddy today ?", "words": ["for", "go", "we", "and", "get", "some", "new"]}, {"sent": "Ill have a look in the box for the policeman .", "words": ["the", "for", "box", "have", "in", "look", "a"]}, {"sent": "Weener you just talked so well for a while today .", "words": ["so", "for", "a", "you"]}, {"sent": "right like that for doing what shes not suppose to do .", "words": ["what", "do", "for", "to", "like", "that", "not"]}, {"sent": "lemme see wanna lemme clean it for you ?", "words": ["clean", "for", "you", "it", "wanna", "see"]}, {"sent": "thats for quite young children but there are some xxx .", "words": ["for", "there", "are", "some", "but"]}, {"sent": "well I was trying to hold it for you .", "words": ["for", "hold", "to", "was", "you", "it"]}, {"sent": "you gonna make it for me ?", "words": ["for", "you", "it", "me", "make"]}, {"sent": "thats the piece I was saying youve got to save for your Grandad when he comes .", "words": ["the", "for", "your", "was", "to", "he", "when"]}, {"sent": "or a bow for her hair .", "words": ["hair", "for", "her", "a"]}, {"sent": "and theres no grandparents round here to do your jobs for you .", "words": ["do", "for", "your", "to", "and", "here", "you"]}, {"sent": "I forgot to look at the menu for today .", "words": ["the", "for", "look", "to", "at"]}, {"sent": "heres the food for the animals look .", "words": ["the", "food", "for", "look"]}, {"sent": "why dont you sing em a song and play for them ?", "words": ["for", "play", "them", "you", "sing", "dont", "and", "why", "a"]}, {"sent": "so she was cutting up a waffle we cut up her waffles so she was cutting up a waffle for the baby .", "words": ["up", "the", "for", "cut", "we", "was", "so", "her", "she", "a"]}, {"sent": "I think you deserve a star for that .", "words": ["think", "for", "star", "you", "that", "a"]}, {"sent": "yeah you knew what that was for .", "words": ["what", "for", "was", "you", "that"]}, {"sent": "youre too big for a baby bottle .", "words": ["for", "too", "bottle", "big", "a"]}, {"sent": "go um go get a napkin for your fingers .", "words": ["for", "your", "go", "get", "napkin", "a"]}, {"sent": "but then you always ask for another one .", "words": ["for", "you", "another", "then", "but"]}, {"sent": "thank you for sharing .", "words": ["for", "you"]}, {"sent": "do you wanna eat it now or do you wanna stick it for later ?", "words": ["do", "stick", "for", "you", "eat", "it", "wanna"]}, {"sent": "let her sit in this one for a while .", "words": ["for", "this", "in", "sit", "her", "a"]}, {"sent": "a for ambulance .", "words": ["for", "a"]}, {"sent": "och shed need that for the daughter really .", "words": ["the", "that", "for", "need"]}, {"sent": "ask daddy what he wants for lunch .", "words": ["what", "for", "he"]}, {"sent": "thats for making mashed potato .", "words": ["for", "potato"]}, {"sent": "how much do I owe you for this small one MisterPebbleman ?", "words": ["do", "how", "for", "you", "this", "much"]}, {"sent": "and Ill hang it up for you .", "words": ["up", "for", "you", "and", "it"]}, {"sent": "when is that for ?", "words": ["that", "for", "when", "is"]}, {"sent": "theyre for the postbox .", "words": ["the", "for"]}, {"sent": "thats e for elbow goes there .", "words": ["there", "for"]}, {"sent": "for happy ?", "words": ["for", "happy"]}, {"sent": "are you looking for one of the people ?", "words": ["the", "for", "of", "you", "are"]}, {"sent": "I didnt say throw him did I ?", "words": ["did", "him", "say", "throw"]}, {"sent": "no we wont kill him .", "words": ["him", "we"]}, {"sent": "this thing tripped him .", "words": ["this", "him"]}, {"sent": "yeah but you splashed him .", "words": ["him", "but", "you"]}, {"sent": "color him .", "words": ["him"]}, {"sent": "this isnt him is it .", "words": ["this", "him", "it", "is"]}, {"sent": "did he how long did it take for him to get there ?", "words": ["how", "for", "to", "long", "get", "there", "him", "it", "he", "did", "take"]}, {"sent": "Ill put him back on the chair .", "words": ["the", "put", "back", "chair", "him", "on"]}, {"sent": "I let him kinda have free reign .", "words": ["have", "him"]}, {"sent": "no leave him alone .", "words": ["him"]}, {"sent": "we waved at him .", "words": ["him", "at", "we"]}, {"sent": "I help him bounce his great big ?", "words": ["his", "help", "big", "him"]}, {"sent": "um which book did you would you say your your child prefers to that you read to him or which .", "words": ["your", "to", "say", "you", "that", "him", "read", "did", "would", "book", "which"]}, {"sent": "what else did you ask him for ?", "words": ["what", "for", "you", "him", "did"]}, {"sent": "find him .", "words": ["him", "find"]}, {"sent": "when he flies up youll see him .", "words": ["up", "him", "he", "see", "when"]}, {"sent": "okay Im gonna leave you down here so you can wait for him right in here okay ?", "words": ["can", "wait", "for", "so", "you", "here", "him", "in", "down"]}, {"sent": "can you see him in that white box ?", "words": ["can", "box", "you", "him", "in", "see", "white", "that"]}, {"sent": "and he goes round and round and you watch him dont you ?", "words": ["watch", "you", "and", "dont", "him", "he"]}, {"sent": "Dyeisha probably running him around .", "words": ["him", "around"]}, {"sent": "you just say nightnight to him .", "words": ["to", "him", "say", "you"]}, {"sent": "sometimes I do when hes really busting and I hafta nurse him .", "words": ["do", "him", "when", "and"]}, {"sent": "and mommys gonna give him a little cup and what is he gonna do ?", "words": ["cup", "what", "do", "is", "little", "and", "him", "he", "give", "a"]}, {"sent": "oh you hafta buy him a birthday card aswell .", "words": ["him", "buy", "a", "you"]}, {"sent": "youre going to hug him ?", "words": ["hug", "to", "him"]}, {"sent": "what are you going to do with him ?", "words": ["what", "do", "to", "with", "you", "him", "are"]}, {"sent": "put him up here .", "words": ["up", "put", "here", "him"]}, {"sent": "turn him round so his foots facing me .", "words": ["his", "so", "him", "me"]}, {"sent": "dont drop him .", "words": ["dont", "him", "drop"]}, {"sent": "give him a glass of milk with that .", "words": ["of", "with", "him", "glass", "milk", "give", "that", "a"]}, {"sent": "well leave him up there to drain until we need him .", "words": ["up", "need", "we", "to", "there", "him"]}, {"sent": "Rachels just putting him in the car .", "words": ["the", "in", "him", "car"]}, {"sent": "but mostly just getting him to .", "words": ["to", "him", "but"]}, {"sent": "so I ordered him the Pingu .", "words": ["the", "so", "him"]}, {"sent": "look at him kick it .", "words": ["look", "him", "kick", "it", "at"]}, {"sent": "lets just not forget about Peanuts and let him in the house .", "words": ["the", "and", "in", "him", "house", "about", "not"]}, {"sent": "back him up the to the thing .", "words": ["up", "the", "back", "to", "him"]}, {"sent": "your pencil will keep him from what ?", "words": ["what", "your", "pencil", "him", "will"]}, {"sent": "that kitty lets you pet him .", "words": ["him", "kitty", "that", "you"]}, {"sent": "hafta take him to the doctor .", "words": ["the", "to", "him", "take"]}, {"sent": "you wanna put him away ?", "words": ["put", "you", "him", "away", "wanna"]}, {"sent": "can you not find him Fraser ?", "words": ["can", "you", "him", "find", "not"]}, {"sent": "when Daddy rings back weve got to read the numbers out to him havent we ?", "words": ["the", "back", "we", "to", "him", "read", "out", "when"]}, {"sent": "well hafta give him some oil .", "words": ["some", "give", "him"]}, {"sent": "will I hurt him ?", "words": ["will", "him", "hurt"]}, {"sent": "what makes him a grown up ?", "words": ["up", "what", "him", "a"]}, {"sent": "well he means that itll give him a tummy ache .", "words": ["him", "he", "tummy", "give", "that", "a"]}, {"sent": "what happens when you brush him ?", "words": ["what", "brush", "you", "him", "when"]}, {"sent": "are you going to wind him up ?", "words": ["up", "to", "you", "him", "are", "wind"]}, {"sent": "throw him in the air and catch him .", "words": ["the", "throw", "catch", "and", "in", "him"]}, {"sent": "so he gets him out of the fire ?", "words": ["the", "of", "so", "him", "he", "out"]}, {"sent": "whats whats he going to do to me if I touch him ?", "words": ["do", "if", "to", "touch", "him", "he", "me"]}, {"sent": "apart from him not sleeping .", "words": ["him", "not"]}, {"sent": "no because I wont give him the box .", "words": ["the", "box", "him", "give", "because"]}, {"sent": "hm make him all clean .", "words": ["clean", "all", "him", "make"]}, {"sent": "you like him with blue stripes ?", "words": ["with", "like", "blue", "you", "him"]}, {"sent": "do you know him ?", "words": ["do", "him", "you"]}, {"sent": "thats why Ive never seen him before .", "words": ["him", "why"]}, {"sent": "take him .", "words": ["him", "take"]}, {"sent": "got him ?", "words": ["him"]}, {"sent": "you saved him .", "words": ["him", "you"]}, {"sent": "can you check him out ?", "words": ["can", "him", "out", "you"]}, {"sent": "are you gonna put him on the train ?", "words": ["the", "put", "train", "you", "him", "are", "on"]}, {"sent": "so what about him ?", "words": ["so", "what", "him", "about"]}, {"sent": "hit him .", "words": ["him", "hit"]}, {"sent": "you take him .", "words": ["him", "take", "you"]}, {"sent": "okay take him .", "words": ["him", "take"]}, {"sent": "were there any animals you maybe tried to show him for the first time he hadnt seen before ?", "words": ["the", "for", "show", "to", "first", "any", "you", "there", "him", "he", "were"]}, {"sent": "do you think we should wrap it up for him ?", "words": ["up", "do", "think", "for", "we", "you", "him", "it"]}, {"sent": "leaving her out or leaving him out .", "words": ["him", "her", "out"]}, {"sent": "you see him ?", "words": ["see", "him", "you"]}, {"sent": "come on get him .", "words": ["on", "him", "get"]}, {"sent": "watch his mummy goes and pick him up .", "words": ["up", "watch", "pick", "and", "him", "his"]}, {"sent": "stand him up then .", "words": ["up", "stand", "him", "then"]}, {"sent": "do you wanna make him tell us a story ?", "words": ["do", "you", "us", "him", "wanna", "story", "make", "a"]}, {"sent": "it because youknow if he if he wants it hell scoff that into him to get out .", "words": ["if", "into", "to", "get", "him", "it", "he", "out", "that", "because"]}, {"sent": "it must be PostmanPat because I saw him coming up the path .", "words": ["up", "the", "be", "him", "it", "because"]}, {"sent": "get him in there .", "words": ["get", "in", "him", "there"]}, {"sent": "whatd you buy him ?", "words": ["him", "buy", "you"]}, {"sent": "just as long as Im holding him .", "words": ["him", "long"]}, {"sent": "smaller than him .", "words": ["him"]}, {"sent": "there drive him around the circle .", "words": ["the", "drive", "there", "him", "around"]}, {"sent": "I have a ball for him .", "words": ["for", "have", "ball", "him", "a"]}, {"sent": "did he let you pick him up ?", "words": ["up", "pick", "you", "him", "he", "did"]}, {"sent": "I saw him yeah .", "words": ["him"]}, {"sent": "why dont cha give him a bath ?", "words": ["dont", "him", "give", "why", "a"]}, {"sent": "get him up .", "words": ["get", "him", "up"]}, {"sent": "bonk him .", "words": ["him"]}, {"sent": "just leave him there like that .", "words": ["there", "him", "that", "like"]}, {"sent": "she couldnt see him could she ?", "words": ["see", "him", "she", "could"]}, {"sent": "you can pat him gently .", "words": ["can", "him", "you"]}, {"sent": "we call him the ancient of days because he is god forever and ever .", "words": ["the", "we", "of", "is", "and", "him", "he", "because"]}, {"sent": "uhoh you better get him .", "words": ["get", "better", "him", "you"]}, {"sent": "oh you have to show him something go head .", "words": ["go", "show", "to", "head", "you", "have", "him"]}, {"sent": "there you go you can hang him from there .", "words": ["can", "go", "you", "there", "him"]}, {"sent": "can I have him please ?", "words": ["can", "have", "him"]}, {"sent": "do you see him ?", "words": ["see", "do", "him", "you"]}, {"sent": "can you make the door open for him ?", "words": ["can", "the", "for", "open", "you", "him", "make", "door"]}, {"sent": "stop him running away .", "words": ["him", "away", "stop"]}, {"sent": "its hurting him .", "words": ["him"]}, {"sent": "you must be hungry Thomas .", "words": ["be", "hungry", "you"]}, {"sent": "two hungry caterpillars .", "words": ["hungry"]}, {"sent": "the boys are hungry huh ?", "words": ["the", "are", "hungry"]}, {"sent": "I think that um the mother is hungry and wants some lunch .", "words": ["the", "think", "is", "hungry", "and", "some", "that"]}, {"sent": "the alligators hungry .", "words": ["the", "hungry"]}, {"sent": "knock knock Lily should we should we feed the zebra you think the zebra is hungry ?", "words": ["the", "zebra", "think", "we", "is", "hungry", "you", "knock", "feed"]}, {"sent": "youre not hungry .", "words": ["hungry", "not"]}, {"sent": "Jeremy begins to feel hungry he reaches in his basket for his delicious lunch .", "words": ["for", "to", "hungry", "in", "he", "basket", "his"]}, {"sent": "are you hungry ?", "words": ["are", "hungry", "you"]}, {"sent": "still hungry .", "words": ["hungry"]}, {"sent": "I think these are hungry baby noises .", "words": ["these", "think", "are", "hungry"]}, {"sent": "youre a bit hungry so you needta blow your nose ?", "words": ["your", "nose", "so", "hungry", "you", "blow", "a"]}, {"sent": "are you hungry ?", "words": ["are", "hungry", "you"]}, {"sent": "you hungry too ?", "words": ["too", "hungry", "you"]}, {"sent": "are you hungry ?", "words": ["are", "hungry", "you"]}, {"sent": "every day when Mary gets hungry she .", "words": ["hungry", "when", "every", "she"]}, {"sent": "Harry the hungry caterpillar .", "words": ["the", "hungry"]}, {"sent": "youre probably hungry .", "words": ["hungry"]}, {"sent": "Im so hungry .", "words": ["so", "hungry"]}, {"sent": "I think youre hungry buddy .", "words": ["think", "hungry"]}, {"sent": "are you hungry ?", "words": ["are", "hungry", "you"]}, {"sent": "well Im going to start my toast because Im hungry and Ive got things to do today .", "words": ["do", "my", "toast", "to", "hungry", "and", "because"]}, {"sent": "because Hannah and Ellen are hungry .", "words": ["are", "hungry", "because", "and"]}, {"sent": "are you just not hungry ?", "words": ["are", "hungry", "not", "you"]}, {"sent": "is she hungry ?", "words": ["hungry", "she", "is"]}, {"sent": "well shes not gonna go hungry today .", "words": ["hungry", "not", "go"]}, {"sent": "what does he want then if hes hungry ?", "words": ["what", "does", "if", "hungry", "he", "then"]}, {"sent": "on Friday he ate through five oranges and still he was hungry .", "words": ["was", "hungry", "and", "he", "on"]}, {"sent": "who is hungry besides me ?", "words": ["me", "hungry", "is", "who"]}, {"sent": "are you hungry Fraser ?", "words": ["are", "hungry", "you"]}, {"sent": "*SI1: um and sometimes when when um when theres um food in the house and sometimes were hungry so we .", "words": ["the", "we", "so", "hungry", "and", "in", "food", "house", "when", "were"]}, {"sent": "you hungry ?", "words": ["hungry", "you"]}, {"sent": "Im so hungry Thomas .", "words": ["so", "hungry"]}, {"sent": "you hungry ?", "words": ["hungry", "you"]}, {"sent": "youre hungry .", "words": ["hungry"]}, {"sent": "are you hungry ?", "words": ["are", "hungry", "you"]}, {"sent": "if shes hungry she usually smiles so .", "words": ["so", "hungry", "if", "she"]}, {"sent": "she did read the very hungry caterpillar I think she read .", "words": ["the", "think", "hungry", "read", "did", "she"]}, {"sent": "and dont wake up and tell me youre hungry .", "words": ["up", "hungry", "and", "dont", "me", "wake"]}, {"sent": "but he was still hungry .", "words": ["was", "he", "hungry", "but"]}, {"sent": "are you hungry ?", "words": ["are", "hungry", "you"]}, {"sent": "was he just very hungry ?", "words": ["was", "he", "hungry"]}, {"sent": "why are you getting hungry ?", "words": ["are", "hungry", "why", "you"]}, {"sent": "hes not hungry .", "words": ["hungry", "not"]}, {"sent": "is Anna hungry ?", "words": ["hungry", "is"]}, {"sent": "babys hungry .", "words": ["hungry"]}, {"sent": "are you hungry ?", "words": ["are", "hungry", "you"]}, {"sent": "were you hungry ?", "words": ["were", "hungry", "you"]}, {"sent": "but um but OinkOink youknow sometimes we hafta explain to her that OinkOink cant come to the table or hell get dirty and she says hes hungry .", "words": ["the", "table", "we", "to", "her", "hungry", "and", "get", "dirty", "that", "she", "but"]}, {"sent": "youre making Fraser feel hungry Eve .", "words": ["hungry"]}, {"sent": "she says Im ever so hungry .", "words": ["so", "hungry", "she"]}, {"sent": "hungry people .", "words": ["hungry"]}, {"sent": "theyre hungry ?", "words": ["hungry"]}, {"sent": "Morags very hungry .", "words": ["hungry"]}, {"sent": "do you think hes hungry ?", "words": ["do", "think", "hungry", "you"]}, {"sent": "poor Anthony was ever so hungry and he had worked really hard hadnt he ?", "words": ["was", "so", "hungry", "and", "poor", "he", "hard"]}, {"sent": "he seems a bit hungry .", "words": ["he", "hungry", "a"]}, {"sent": "because he might be hungry .", "words": ["be", "he", "hungry", "because"]}, {"sent": "what if youre still hungry and you wanna have xxx baguette and xxx ?", "words": ["what", "if", "hungry", "you", "and", "have", "wanna"]}, {"sent": "are you hungry ?", "words": ["are", "hungry", "you"]}, {"sent": "oh now thats one red ripe strawberry the big hungry bear will never get .", "words": ["red", "the", "strawberry", "hungry", "bear", "get", "will", "big"]}, {"sent": "hungry .", "words": ["hungry"]}, {"sent": "Im hungry .", "words": ["hungry"]}, {"sent": "hungry ?", "words": ["hungry"]}, {"sent": "youre hungry arent you darling ?", "words": ["hungry", "you"]}, {"sent": "Im hungry .", "words": ["hungry"]}, {"sent": "hungry .", "words": ["hungry"]}, {"sent": "you must be hungry .", "words": ["be", "hungry", "you"]}, {"sent": "is he not hungry ?", "words": ["hungry", "he", "not", "is"]}, {"sent": "shes hungry come feed her .", "words": ["her", "hungry", "feed"]}, {"sent": "are you hungry Lara ?", "words": ["are", "hungry", "you"]}, {"sent": "Im always hungry in the morning .", "words": ["the", "in", "hungry"]}, {"sent": "are you hungry ?", "words": ["are", "hungry", "you"]}, {"sent": "are you hungry ?", "words": ["are", "hungry", "you"]}, {"sent": "ah are you hungry again ?", "words": ["are", "hungry", "you"]}, {"sent": "Pattys hungry .", "words": ["hungry"]}, {"sent": "do you think Buddys hungry ?", "words": ["do", "think", "hungry", "you"]}, {"sent": "are you still hungry ?", "words": ["are", "hungry", "you"]}, {"sent": "its made her a bit hungry .", "words": ["her", "hungry", "a"]}, {"sent": "the very hungry caterpillar .", "words": ["the", "hungry"]}, {"sent": "Im hungry .", "words": ["hungry"]}, {"sent": "Im getting hungry .", "words": ["hungry"]}, {"sent": "then you shall be hungry no more said the old woman .", "words": ["the", "old", "be", "hungry", "you", "more", "then"]}, {"sent": "Tyranosaurus Rex is gonna eat Bambi up because Tyranosaurus Rex is hungry .", "words": ["up", "is", "hungry", "eat", "because"]}, {"sent": "ooh we didnt give Dorothy breakfast shes probably hungry .", "words": ["give", "hungry", "we"]}, {"sent": "Mummys hungry .", "words": ["hungry"]}, {"sent": "are you hungry ?", "words": ["are", "hungry", "you"]}, {"sent": "horsie says tiger is hungry .", "words": ["hungry", "tiger", "is"]}, {"sent": "I think youre quite hungry today .", "words": ["think", "hungry"]}, {"sent": "oh Im hungry .", "words": ["hungry"]}, {"sent": "youll got to bed hungry .", "words": ["to", "hungry", "bed"]}, {"sent": "Thomas was not hungry .", "words": ["was", "hungry", "not"]}, {"sent": "in fact Im hoping shell fall asleep a wee bit in the car and then waken up a wee bit more hungry and be in better form .", "words": ["the", "up", "better", "car", "be", "fall", "hungry", "and", "asleep", "in", "more", "then", "a"]}, {"sent": "*SI1: so so Maria started drinking it down because she was very hungry .", "words": ["so", "was", "hungry", "it", "down", "she", "because"]}, {"sent": "hes hungry ?", "words": ["hungry"]}, {"sent": "but he came and he was so hungry that he took a chunk out of my face .", "words": ["my", "face", "of", "was", "so", "hungry", "and", "he", "out", "that", "but", "a"]}, {"sent": "you hungry ?", "words": ["hungry", "you"]}, {"sent": "theyre all hungry .", "words": ["all", "hungry"]}, {"sent": "Im hungry .", "words": ["hungry"]}, {"sent": "because he is hungry .", "words": ["he", "hungry", "because", "is"]}, {"sent": "because Im thirsty .", "words": ["because", "thirsty"]}, {"sent": "so Im thinking shes thirsty .", "words": ["so", "thirsty"]}, {"sent": "the bunny was really thirsty .", "words": ["the", "was", "bunny", "thirsty"]}, {"sent": "is she thirsty ?", "words": ["thirsty", "she", "is"]}, {"sent": "well if he has been digging Smarties perhaps he has worked very hard and hes thirsty now Thomas .", "words": ["if", "thirsty", "and", "he", "hard"]}, {"sent": "oh yeah you wanna share with the baby the baby probably is thirsty too .", "words": ["the", "share", "is", "with", "you", "thirsty", "wanna", "too"]}, {"sent": "Im thirsty .", "words": ["thirsty"]}, {"sent": "cause this lawn is thirsty .", "words": ["this", "thirsty", "is"]}, {"sent": "I was really thirsty .", "words": ["was", "thirsty"]}, {"sent": "if youre eating juicy oranges and grapes you dont feel quite so thirsty .", "words": ["if", "so", "you", "and", "dont", "thirsty", "grapes"]}, {"sent": "thirsty Violet .", "words": ["thirsty"]}, {"sent": "you must have been thirsty .", "words": ["have", "thirsty", "you"]}, {"sent": "boy you must be really thirsty .", "words": ["be", "thirsty", "you"]}, {"sent": "so you can tell Daddy when youre thirsty .", "words": ["can", "so", "thirsty", "you", "when"]}, {"sent": "are you a little thirsty ?", "words": ["thirsty", "little", "you", "are", "a"]}, {"sent": "xxx really thirsty huh ?", "words": ["thirsty"]}, {"sent": "so Ill hafta go home thirsty .", "words": ["so", "thirsty", "home", "go"]}, {"sent": "and if you get thirsty then Grandma and Granddad can give you a drink cant they ?", "words": ["can", "give", "if", "drink", "thirsty", "get", "you", "and", "they", "then", "a"]}, {"sent": "sure shes thirsty and hungry just like Eve .", "words": ["like", "hungry", "and", "thirsty"]}, {"sent": "are you thirsty .", "words": ["thirsty", "are", "you"]}, {"sent": "well I hope that the next time youre thirsty youll realize where the juice has gone to when you want it .", "words": ["the", "where", "when", "to", "thirsty", "you", "it", "juice", "that"]}, {"sent": "camels live in the desert where theyre always thirsty .", "words": ["the", "in", "where", "thirsty"]}, {"sent": "youre a thirsty guy huh you a thirsty guy .", "words": ["thirsty", "you", "a"]}, {"sent": "hum thirsty guy .", "words": ["thirsty"]}, {"sent": "you thirsty ?", "words": ["thirsty", "you"]}, {"sent": "do you think those people are thirsty ?", "words": ["do", "think", "those", "you", "thirsty", "are"]}, {"sent": "baby is still thirsty .", "words": ["thirsty", "is"]}, {"sent": "he looks like he might be thirsty .", "words": ["be", "he", "like", "thirsty"]}, {"sent": "thats a thirsty wirsty camel .", "words": ["thirsty", "a"]}, {"sent": "oh what i what are are you thirsty ?", "words": ["what", "i", "thirsty", "you", "are"]}, {"sent": "are you thirsty ?", "words": ["thirsty", "are", "you"]}, {"sent": "were very thirsty arent we ?", "words": ["were", "we", "thirsty"]}, {"sent": "youre thirsty thats not for you .", "words": ["for", "not", "you", "thirsty"]}, {"sent": "yes you know what I think theyre thirsty .", "words": ["what", "think", "thirsty", "you"]}, {"sent": "are you thirsty ?", "words": ["thirsty", "are", "you"]}, {"sent": "okay Im really thirsty .", "words": ["thirsty"]}, {"sent": "but Im thirsty .", "words": ["but", "thirsty"]}, {"sent": "the babys very thirsty Lee .", "words": ["the", "thirsty"]}, {"sent": "are you thirsty ?", "words": ["thirsty", "are", "you"]}, {"sent": "you are thirsty huh ?", "words": ["thirsty", "are", "you"]}, {"sent": "theyre looking very thirsty to me .", "words": ["to", "me", "thirsty"]}, {"sent": "now you can eat it Im thirsty so Im gonna take a little drink of tab .", "words": ["can", "drink", "of", "so", "thirsty", "eat", "you", "little", "it", "take", "a"]}, {"sent": "are you a thirsty guy ?", "words": ["thirsty", "are", "a", "you"]}, {"sent": "Naima are you thirsty .", "words": ["thirsty", "are", "you"]}, {"sent": "are you thirsty ?", "words": ["thirsty", "are", "you"]}, {"sent": "youre thirsty today .", "words": ["thirsty"]}, {"sent": "because theyre thirsty after a long day on the beat ?", "words": ["the", "long", "thirsty", "on", "because", "a"]}, {"sent": "dyou think shes thirsty ?", "words": ["think", "thirsty"]}, {"sent": "and she probably wants something to drink because she might get thirsty when she eats .", "words": ["drink", "to", "and", "get", "thirsty", "when", "she", "because"]}, {"sent": "you think hes thirsty for juice ?", "words": ["think", "for", "thirsty", "you", "juice"]}, {"sent": "why are you thirsty ?", "words": ["thirsty", "are", "why", "you"]}, {"sent": "Im thirsty .", "words": ["thirsty"]}, {"sent": "thats good that baby might be thirsty .", "words": ["be", "that", "good", "thirsty"]}, {"sent": "just some water would be fine Im so thirsty .", "words": ["water", "fine", "be", "so", "thirsty", "would", "some"]}, {"sent": "are you thirsty ?", "words": ["thirsty", "are", "you"]}, {"sent": "are you thirsty ?", "words": ["thirsty", "are", "you"]}, {"sent": "she must have been thirsty .", "words": ["have", "she", "thirsty"]}, {"sent": "he always tries to take the spoon out of my hand and play with it and then and hes he hasnt had anything to drink in a little bit so hes probably just hungry and thirsty .", "words": ["the", "drink", "of", "spoon", "hand", "with", "it", "then", "take", "to", "so", "play", "hungry", "and", "little", "a", "my", "thirsty", "in", "he", "out"]}, {"sent": "they might be very thirsty .", "words": ["they", "be", "thirsty"]}, {"sent": "youre a thirsty guy .", "words": ["thirsty", "a"]}, {"sent": "you were thirsty .", "words": ["thirsty", "were", "you"]}, {"sent": "Mommys thirsty .", "words": ["thirsty"]}, {"sent": "Im so thirsty .", "words": ["so", "thirsty"]}, {"sent": "somebody is thirsty this morning .", "words": ["this", "thirsty", "is"]}, {"sent": "I know youre thirsty .", "words": ["thirsty"]}, {"sent": "are you very thirsty ?", "words": ["thirsty", "are", "you"]}, {"sent": "are you thirsty ?", "words": ["thirsty", "are", "you"]}, {"sent": "youre thirsty ?", "words": ["thirsty"]}, {"sent": "you are thirsty .", "words": ["thirsty", "are", "you"]}, {"sent": "are they very thirsty ?", "words": ["they", "are", "thirsty"]}, {"sent": "and another another thirsty person there look .", "words": ["thirsty", "another", "there", "and", "look"]}, {"sent": "would you like a drink are you thirsty ?", "words": ["drink", "like", "you", "thirsty", "are", "would", "a"]}, {"sent": "are you thirsty ?", "words": ["thirsty", "are", "you"]}, {"sent": "are you thirsty ?", "words": ["thirsty", "are", "you"]}, {"sent": "are you thirsty ?", "words": ["thirsty", "are", "you"]}, {"sent": "thirsty yes .", "words": ["thirsty"]}, {"sent": "you were thirsty .", "words": ["thirsty", "were", "you"]}, {"sent": "thirsty thirsty .", "words": ["thirsty"]}, {"sent": "because Im thirsty and Ive made a drink .", "words": ["drink", "and", "thirsty", "because", "a"]}, {"sent": "are you thirsty ?", "words": ["thirsty", "are", "you"]}, {"sent": "oh Bobs drinking it is Bob thirsty ?", "words": ["thirsty", "it", "is"]}, {"sent": "thirsty .", "words": ["thirsty"]}, {"sent": "Bobs very thirsty .", "words": ["thirsty"]}, {"sent": "youre very thirsty ?", "words": ["thirsty"]}, {"sent": "you get thirsty a lot .", "words": ["a lot", "thirsty", "get", "you", "a"]}, {"sent": "thirsty ?", "words": ["thirsty"]}, {"sent": "yeah theyre thirsty .", "words": ["thirsty"]}, {"sent": "we needta get a word for thirsty .", "words": ["for", "we", "thirsty", "get", "a"]}, {"sent": "I was thirsty .", "words": ["was", "thirsty"]}, {"sent": "if you get thirsty .", "words": ["get", "thirsty", "if", "you"]}, {"sent": "you seem to be very thirsty tonight .", "words": ["thirsty", "to", "be", "you"]}, {"sent": "oh Snoopys thirsty ?", "words": ["thirsty"]}, {"sent": "are you thirsty ?", "words": ["thirsty", "are", "you"]}, {"sent": "oh hes a thirsty Bo .", "words": ["thirsty", "a"]}, {"sent": "youre just thirsty huh ?", "words": ["thirsty"]}, {"sent": "you must be very thirsty .", "words": ["be", "thirsty", "you"]}, {"sent": "xxx thirsty in a minute .", "words": ["in", "a", "thirsty"]}, {"sent": "our friend is thirsty .", "words": ["thirsty", "is", "our"]}, {"sent": "Im thirsty .", "words": ["thirsty"]}, {"sent": "Im not thirsty right now .", "words": ["not", "thirsty"]}, {"sent": "water .", "words": ["water"]}, {"sent": "youre not to go in the water .", "words": ["the", "go", "to", "in", "water", "not"]}, {"sent": "mice are not fond of the water .", "words": ["the", "of", "are", "water", "not"]}, {"sent": "yes there is water on there .", "words": ["water", "there", "on", "is"]}, {"sent": "wheres the water gone ?", "words": ["the", "water"]}, {"sent": "well no it says these are their head the orange part is their heads worms whose heads stick out of their white tubelike casings take hydrogen sulfide from the water and bacteria in their bodies make food from it .", "words": ["the", "these", "stick", "their", "is", "of", "head", "and", "in", "it", "are", "orange", "out", "water", "make", "food", "white", "take"]}, {"sent": "this is a turtle that slid into the pond that ate the snake that dropped from a branch that swallowed the fish that swam after the frog that was under the fly that climbed outta the water .", "words": ["the", "water", "into", "is", "was", "frog", "this", "under", "fish", "turtle", "that", "a"]}, {"sent": "oh well if the water has run out youll hafta stop being a duck wont you ?", "words": ["the", "if", "stop", "you", "run", "out", "water", "duck", "a"]}, {"sent": "shes just running some water in the washingup bowl .", "words": ["the", "bowl", "in", "water", "some"]}, {"sent": "you want water ?", "words": ["water", "you"]}, {"sent": "oh yes shall we wipe up the pool of water ?", "words": ["up", "the", "we", "of", "pool", "water", "wipe"]}, {"sent": "would you like some more water ?", "words": ["water", "like", "you", "more", "would", "some"]}, {"sent": "W A T E R is water you gotta find W A T E R .", "words": ["water", "find", "you", "is"]}, {"sent": "you done the water .", "words": ["the", "water", "you"]}, {"sent": "his arm is in the water .", "words": ["the", "is", "in", "arm", "water", "his"]}, {"sent": "is that the water ?", "words": ["the", "water", "that", "is"]}, {"sent": "hes floating on the water isnt he ?", "words": ["on", "the", "he", "water"]}, {"sent": "he tipped some water in the hole and what did it do ?", "words": ["the", "what", "do", "and", "in", "it", "he", "did", "water", "some"]}, {"sent": "thats so Purdie can have a drink of water because when its hot she gets extra thirsty .", "words": ["can", "drink", "of", "so", "thirsty", "have", "hot", "water", "when", "she", "because", "a"]}, {"sent": "and water comes onto this wheel .", "words": ["water", "this", "and"]}, {"sent": "what do you get outof a water fountain ?", "words": ["what", "do", "you", "get", "water", "a"]}, {"sent": "dont put water in it again though will you ?", "words": ["put", "you", "dont", "in", "it", "water", "will"]}, {"sent": "I think hes throwing water everywhere .", "words": ["think", "water"]}, {"sent": "he just fed the animals some water .", "words": ["the", "some", "he", "water"]}, {"sent": "and I looked at the container and thought it was all dirty water .", "words": ["the", "all", "water", "was", "and", "it", "dirty", "at"]}, {"sent": "would you pour the water back into the measuring cup for me ?", "words": ["the", "cup", "for", "into", "back", "pour", "would", "you", "water", "me"]}, {"sent": "the water is all muddy because the doggie was in it .", "words": ["the", "all", "is", "was", "in", "it", "water", "because"]}, {"sent": "where is water ?", "words": ["where", "water", "is"]}, {"sent": "put some water in .", "words": ["some", "put", "in", "water"]}, {"sent": "just be careful about splashing water .", "words": ["water", "be", "about", "careful"]}, {"sent": "so they buy it just thinking theyre buying ordinary coffee beans and then when theyre getting home and pour the water in it looks like ink ?", "words": ["the", "so", "pour", "like", "and", "they", "in", "coffee", "it", "beans", "water", "then", "when", "buy", "home"]}, {"sent": "okay put it in the water .", "words": ["the", "put", "in", "it", "water"]}, {"sent": "so the water goes into the hole .", "words": ["the", "so", "into", "water"]}, {"sent": "and heres the water .", "words": ["the", "water", "and"]}, {"sent": "put water in it ?", "words": ["in", "put", "it", "water"]}, {"sent": "all this is water .", "words": ["water", "all", "this", "is"]}, {"sent": "in the water ?", "words": ["the", "in", "water"]}, {"sent": "if fish get outof the water ?", "words": ["the", "if", "get", "water", "fish"]}, {"sent": "is it water or what ?", "words": ["water", "what", "it", "is"]}, {"sent": "but dont I have the same amount of water even if I put it in the other cup ?", "words": ["the", "cup", "put", "if", "of", "other", "have", "dont", "in", "it", "same", "water", "but"]}, {"sent": "Spots not in the water though is he ?", "words": ["the", "is", "in", "he", "water", "not"]}, {"sent": "no she fall in the water .", "words": ["the", "fall", "in", "water", "she"]}, {"sent": "are you drinking water from the cup ?", "words": ["the", "cup", "you", "are", "water"]}, {"sent": "cold water or hot water ?", "words": ["hot", "cold", "water"]}, {"sent": "wanna join me for some crock water he asked Jay Jay .", "words": ["for", "he", "wanna", "water", "me", "some"]}, {"sent": "have you put have you put water in that ?", "words": ["put", "you", "have", "in", "water", "that"]}, {"sent": "a dish of water huh ?", "words": ["of", "dish", "water", "a"]}, {"sent": "its swimming in the water thats right .", "words": ["the", "in", "water"]}, {"sent": "could I possibly have a glass of water ?", "words": ["of", "have", "glass", "water", "could", "a"]}, {"sent": "w a t e r water .", "words": ["water", "a"]}, {"sent": "he took off his hat and began using it to bail out the water .", "words": ["off", "the", "water", "to", "and", "it", "he", "hat", "out", "his"]}, {"sent": "the trouble is now youve put my jumper out with all that water my jumpers wet .", "words": ["the", "all", "put", "my", "wet", "is", "with", "out", "water", "that"]}, {"sent": "you saw a monster in the water at the museum ?", "words": ["the", "you", "in", "at", "water", "a"]}, {"sent": "did you jump in the water ?", "words": ["the", "you", "jump", "in", "did", "water"]}, {"sent": "does he hafta live in water with the whale ?", "words": ["the", "does", "with", "in", "he", "water"]}, {"sent": "do you wanna add that amount of water ?", "words": ["do", "of", "you", "wanna", "water", "that"]}, {"sent": "lets empty the water down the sink .", "words": ["the", "empty", "sink", "water", "down"]}, {"sent": "because if its just water you spill it doesnt matter .", "words": ["if", "you", "it", "spill", "water", "because"]}, {"sent": "oh Eleanor likes water .", "words": ["water"]}, {"sent": "just a little water .", "words": ["water", "little", "a"]}, {"sent": "it is theres no more water ?", "words": ["water", "more", "it", "is"]}, {"sent": "water ?", "words": ["water"]}, {"sent": "then he jumps out of the water .", "words": ["the", "of", "he", "out", "water", "then"]}, {"sent": "oh I dont want water on it because itll ruin the play dough .", "words": ["the", "play", "dont", "it", "water", "on", "play dough", "because"]}, {"sent": "he swallowed water and he got a cough .", "words": ["a", "he", "and", "water"]}, {"sent": "that good water ?", "words": ["that", "good", "water"]}, {"sent": "well when we put water into the water the plant so they could grow .", "words": ["the", "put", "into", "we", "so", "they", "water", "plant", "when", "could"]}, {"sent": "theyre going under the plumbing in the water ?", "words": ["the", "under", "in", "water"]}, {"sent": "you want some water ?", "words": ["water", "some", "you"]}, {"sent": "Mommy is gonna get you a cup of water .", "words": ["cup", "is", "of", "you", "get", "water", "a"]}, {"sent": "it took him rather a long time and when he looked around he found to his surprise that the bath was so full of water it was starting to run over the side .", "words": ["the", "of", "water", "when", "it", "that", "full", "to", "was", "so", "long", "and", "him", "a", "run", "he", "around", "his", "over"]}, {"sent": "and I forgot to turn the water off .", "words": ["the", "off", "to", "and", "water"]}, {"sent": "and you asked for some breakfast then you had a little sip of water and what happened ?", "words": ["what", "for", "of", "you", "and", "little", "water", "some", "then", "a"]}, {"sent": "oh hes gone hes gone on the water .", "words": ["on", "the", "water"]}, {"sent": "its a kind of water pump .", "words": ["of", "water", "a"]}, {"sent": "see this glass of water .", "words": ["of", "this", "glass", "water", "see"]}, {"sent": "its wet in the water is it ?", "words": ["the", "wet", "is", "in", "it", "water"]}, {"sent": "there may be fish in the water .", "words": ["the", "water", "be", "there", "in", "fish"]}, {"sent": "and your water is has run out too a little bit .", "words": ["your", "is", "and", "little", "run", "out", "water", "too", "a"]}, {"sent": "xxx water .", "words": ["water"]}, {"sent": "so its not quite in the water .", "words": ["the", "so", "in", "water", "not"]}, {"sent": "did you see the water ?", "words": ["the", "you", "did", "water", "see"]}, {"sent": "are you referring to the elephants when they spilt water out of their trunk .", "words": ["the", "their", "of", "to", "you", "they", "are", "out", "water", "when"]}, {"sent": "and the water .", "words": ["the", "water", "and"]}, {"sent": "do you want a drink of water ?", "words": ["do", "drink", "of", "you", "water", "a"]}, {"sent": "Im just gonna put the water in Amys bottle to cool it down .", "words": ["the", "put", "to", "in", "it", "water", "bottle", "down"]}, {"sent": "well how would you go about asking for that water ?", "words": ["water", "how", "for", "go", "you", "about", "would", "that"]}, {"sent": "and I think we are going to need a little bit more water dont you ?", "words": ["think", "need", "we", "to", "and", "little", "dont", "more", "are", "you", "water", "a"]}, {"sent": "well I think well give him some bread and water .", "words": ["bread", "think", "and", "him", "water", "some", "give"]}, {"sent": "and Granddad and Grandma said you were very helpful in the garden because theyve got a lot of plants to water .", "words": ["the", "of", "to", "a lot", "you", "and", "in", "garden", "water", "were", "because", "a"]}, {"sent": "look at the water .", "words": ["the", "at", "water", "look"]}, {"sent": "yeah wanna change the fishies water ?", "words": ["the", "wanna", "water"]}, {"sent": "so whenever you want a drink throughout the day we know weve always got some water ready dont we ?", "words": ["the", "drink", "we", "so", "you", "dont", "water", "some", "a"]}, {"sent": "and whats he doing in the water Barbara ?", "words": ["the", "and", "in", "he", "water"]}, {"sent": "you want some water for those hiccups ?", "words": ["for", "those", "you", "water", "some"]}, {"sent": "here pour some water in there .", "words": ["pour", "here", "there", "in", "water", "some"]}, {"sent": "okay Ill go turn on the water .", "words": ["on", "the", "water", "go"]}, {"sent": "water .", "words": ["water"]}, {"sent": "with some water .", "words": ["some", "with", "water"]}, {"sent": "a pail of good fresh milk skedaddle .", "words": ["of", "milk", "good", "a"]}, {"sent": "want some milk ?", "words": ["milk", "some"]}, {"sent": "how much is your chocolate milk that you get ?", "words": ["how", "your", "is", "chocolate", "you", "get", "that", "milk", "much"]}, {"sent": "she cant eat bread and she cant have milk .", "words": ["bread", "and", "eat", "have", "milk", "she"]}, {"sent": "but we dont have any milk .", "words": ["we", "any", "have", "dont", "milk", "but"]}, {"sent": "milk helps you grow .", "words": ["milk", "you"]}, {"sent": "oh put some milk in that one then .", "words": ["put", "in", "milk", "some", "then", "that"]}, {"sent": "where have you had strawberry milk before ?", "words": ["where", "strawberry", "you", "have", "milk"]}, {"sent": "I think theyre making a cup of coffee because they came to see if we had any milk didnt they ?", "words": ["cup", "think", "if", "we", "of", "to", "any", "they", "coffee", "milk", "see", "because", "a"]}, {"sent": "xxx you want more milk ?", "words": ["milk", "more", "you"]}, {"sent": "do you remember when Grandma puts the chocolate in your milk ?", "words": ["the", "do", "your", "chocolate", "you", "in", "milk", "when"]}, {"sent": "oh you love strawberery milk dont you ?", "words": ["milk", "dont", "love", "you"]}, {"sent": "some milk on there I should say .", "words": ["say", "there", "on", "milk", "some"]}, {"sent": "shall I make up one for milk ?", "words": ["up", "milk", "for", "make"]}, {"sent": "you know what its like when we put milk on and it all goes soft .", "words": ["what", "put", "all", "soft", "we", "like", "you", "and", "on", "it", "milk", "when"]}, {"sent": "shall I write milk ?", "words": ["milk", "write"]}, {"sent": "give him some milk .", "words": ["milk", "some", "give", "him"]}, {"sent": "asks for milk .", "words": ["milk", "for"]}, {"sent": "Im going to pour the milk thats left from the cereal bowl into Purdies saucer and she can drink it .", "words": ["the", "bowl", "can", "into", "drink", "to", "pour", "and", "it", "milk", "cereal", "she"]}, {"sent": "a little more milk .", "words": ["milk", "more", "little", "a"]}, {"sent": "do you want some soy milk ?", "words": ["milk", "do", "some", "you"]}, {"sent": "can I have some milk my bud ?", "words": ["can", "my", "have", "milk", "some"]}, {"sent": "you want milk do you ?", "words": ["milk", "do", "you"]}, {"sent": "or is it just going to be milk and sugar ?", "words": ["is", "to", "be", "and", "it", "milk"]}, {"sent": "theres your milk .", "words": ["milk", "your"]}, {"sent": "how do they get milk from a cow ?", "words": ["do", "how", "get", "they", "milk", "cow", "a"]}, {"sent": "lets get you some milk .", "words": ["get", "some", "milk", "you"]}, {"sent": "is that milk ?", "words": ["milk", "that", "is"]}, {"sent": "heres your milk .", "words": ["milk", "your"]}, {"sent": "its not time for your bottle of milk yet John .", "words": ["for", "your", "of", "milk", "bottle", "not"]}, {"sent": "you dont take a bottle but you get milk who gets the bottle at Susans house ?", "words": ["the", "you", "get", "dont", "at", "house", "who", "milk", "bottle", "take", "but", "a"]}, {"sent": "nice cool milk .", "words": ["milk", "nice"]}, {"sent": "milk ?", "words": ["milk"]}, {"sent": "oh daddy is gonna have some milk .", "words": ["have", "some", "milk", "is"]}, {"sent": "but I just thought if youre being served milk .", "words": ["milk", "if", "but"]}, {"sent": "this man is carrying some milk bottles .", "words": ["milk", "this", "some", "is"]}, {"sent": "shall we have a little break now and drink your milk ?", "words": ["your", "drink", "we", "little", "have", "and", "milk", "break", "a"]}, {"sent": "heres the glass of milk well put the glass of milk right here .", "words": ["the", "put", "of", "here", "glass", "milk"]}, {"sent": "what happened after she spilled the milk ?", "words": ["the", "what", "she", "milk"]}, {"sent": "fishing milk ?", "words": ["milk"]}, {"sent": "Ill get you some milk Laura .", "words": ["get", "some", "milk", "you"]}, {"sent": "does he just like milk ones ?", "words": ["milk", "does", "he", "like"]}, {"sent": "who drinks milk ?", "words": ["milk", "who"]}, {"sent": "and some milk and now hes jumping .", "words": ["milk", "some", "and"]}, {"sent": "did you finish your milk ?", "words": ["your", "finish", "you", "did", "milk"]}, {"sent": "have a nice bath and a drink of milk .", "words": ["drink", "of", "nice", "and", "have", "milk", "a"]}, {"sent": "come and get your xxx milk .", "words": ["get", "milk", "your", "and"]}, {"sent": "do you want some more milk ?", "words": ["do", "you", "more", "milk", "some"]}, {"sent": "yeah throwing some milk too .", "words": ["milk", "some", "too"]}, {"sent": "you spilled his milk ?", "words": ["his", "milk", "you"]}, {"sent": "milk .", "words": ["milk"]}, {"sent": "you dont want milk .", "words": ["milk", "dont", "you"]}, {"sent": "here let us take a milk break .", "words": ["break", "here", "us", "milk", "take", "a"]}, {"sent": "milk and cookies .", "words": ["milk", "and"]}, {"sent": "a little bit of milk .", "words": ["of", "milk", "little", "a"]}, {"sent": "can I have some milk ?", "words": ["can", "some", "have", "milk"]}, {"sent": "right you drink your milk .", "words": ["milk", "your", "drink", "you"]}, {"sent": "no sugar and just a wee drop of milk lovely .", "words": ["of", "and", "drop", "milk", "a"]}, {"sent": "more milk ?", "words": ["milk", "more"]}, {"sent": "Ill give you milk .", "words": ["milk", "give", "you"]}, {"sent": "you got your milk backwards .", "words": ["milk", "your", "you"]}, {"sent": "we leave the we dont take drinks of milk into the supermarket .", "words": ["the", "into", "we", "of", "dont", "milk", "take"]}, {"sent": "without milk .", "words": ["milk"]}, {"sent": "can you say milk ?", "words": ["can", "milk", "say", "you"]}, {"sent": "oh Daddys milk .", "words": ["milk"]}, {"sent": "do you really want some milk ?", "words": ["milk", "do", "some", "you"]}, {"sent": "I drink my water you drink your milk .", "words": ["my", "your", "drink", "you", "water", "milk"]}, {"sent": "baby cows just having some milk .", "words": ["milk", "some"]}, {"sent": "I didnt drink any milk and William and Deedee .", "words": ["milk", "any", "drink", "and"]}, {"sent": "you got a milk mustache .", "words": ["milk", "a", "you"]}, {"sent": "oh youve got milk all over .", "words": ["milk", "all", "over"]}, {"sent": "they dont crunch quite as much when theyve got milk on as when you have them dry .", "words": ["when", "dry", "them", "you", "have", "dont", "they", "on", "milk", "much"]}, {"sent": "what about the chocolate milk ?", "words": ["the", "what", "chocolate", "about", "milk"]}, {"sent": "and milk .", "words": ["milk", "and"]}, {"sent": "oh do you have enough milk ?", "words": ["have", "do", "milk", "you"]}, {"sent": "I think once weve finished what weve got to do you can have a drink of milk .", "words": ["can", "what", "think", "do", "drink", "of", "to", "you", "have", "milk", "a"]}, {"sent": "got milk .", "words": ["milk"]}, {"sent": "go and sit down and have some milk .", "words": ["go", "and", "have", "milk", "some", "sit", "down"]}, {"sent": "you want milk ?", "words": ["milk", "you"]}, {"sent": "milk ?", "words": ["milk"]}, {"sent": "is it hot milk or cold milk ?", "words": ["is", "hot", "it", "milk", "cold"]}, {"sent": "you put milk on cereal .", "words": ["put", "you", "on", "milk", "cereal"]}, {"sent": "corn flakes lovely daddy did we need milk you know you got to go out and buy milk .", "words": ["need", "we", "go", "to", "you", "and", "did", "corn", "out", "milk", "buy"]}, {"sent": "Im not sure there was milk in or shops in um those days .", "words": ["those", "was", "there", "in", "milk", "not"]}, {"sent": "you know milk is liquid ?", "words": ["milk", "is", "you"]}, {"sent": "is she is the baby having milk on it ?", "words": ["the", "is", "on", "it", "milk", "she"]}, {"sent": "is that how a cow gives milk ?", "words": ["how", "is", "milk", "that", "cow", "a"]}, {"sent": "is this your milk ?", "words": ["milk", "this", "your", "is"]}, {"sent": "do you want milk ?", "words": ["milk", "do", "you"]}, {"sent": "you just squirted milk all in your clean face .", "words": ["clean", "all", "your", "face", "you", "in", "milk"]}, {"sent": "what did she do to the milk ?", "words": ["the", "what", "do", "to", "did", "milk", "she"]}, {"sent": "youve fetched some more milk have you for the doll ?", "words": ["the", "for", "doll", "you", "have", "more", "milk", "some"]}, {"sent": "is he getting any milk outof there ?", "words": ["is", "any", "there", "he", "milk"]}, {"sent": "no this is white milk .", "words": ["milk", "this", "white", "is"]}, {"sent": "did your milk fall down ?", "words": ["your", "fall", "did", "milk", "down"]}, {"sent": "you make milk ?", "words": ["milk", "make", "you"]}, {"sent": "could I have some warm milk in my tea please ?", "words": ["my", "have", "in", "milk", "some", "could"]}, {"sent": "he had milk ?", "words": ["milk", "he"]}, {"sent": "my hot milk .", "words": ["milk", "hot", "my"]}, {"sent": "want a drink a milk ?", "words": ["milk", "drink", "a"]}, {"sent": "do you like the bumblebee ?", "words": ["the", "do", "like", "you"]}, {"sent": "torpedo thats how Daddy takes you around the yard like a torpedo .", "words": ["the", "how", "like", "you", "around", "a"]}, {"sent": "just like Jwww ?", "words": ["like"]}, {"sent": "um things like that so .", "words": ["so", "that", "like"]}, {"sent": "would you like to make snakes ?", "words": ["to", "like", "you", "would", "make"]}, {"sent": "I like those sweets .", "words": ["those", "like"]}, {"sent": "it goes like that .", "words": ["it", "that", "like"]}, {"sent": "thats what we would look like .", "words": ["what", "we", "like", "would", "look"]}, {"sent": "its like a football isnt it ?", "words": ["it", "like", "a"]}, {"sent": "what does it taste like ?", "words": ["what", "does", "taste", "like", "it"]}, {"sent": "look like that .", "words": ["that", "like", "look"]}, {"sent": "make it go like that .", "words": ["go", "like", "it", "make", "that"]}, {"sent": "why I can take that cat ring off tubs just like that .", "words": ["can", "off", "like", "cat", "take", "that", "why"]}, {"sent": "looks like bricks .", "words": ["like"]}, {"sent": "lets see she actually her dad has an ipad and shell play theres like animal games where you hit the card and animal will make a sound or theres matching games on there that shell play like I mean like she can navigate her way through the different games on the ipad .", "words": ["the", "like", "an", "where", "can", "there", "on", "that", "she", "play", "and", "you", "will", "see", "make", "a", "animal", "hit", "her"]}, {"sent": "and he can he can sit for entertain himself it seems like .", "words": ["can", "for", "like", "and", "it", "he", "sit"]}, {"sent": "would you like a snack ?", "words": ["a", "like", "you", "would"]}, {"sent": "and Id test it with my elbow like that .", "words": ["my", "with", "like", "and", "it", "that"]}, {"sent": "Id like to play with you guys .", "words": ["to", "play", "with", "like", "you"]}, {"sent": "bit like a bell .", "words": ["like", "a"]}, {"sent": "would you like a a a drink ?", "words": ["drink", "like", "you", "would", "a"]}, {"sent": "would you like some cereal ?", "words": ["like", "you", "would", "some", "cereal"]}, {"sent": "Ive never seen ones like that .", "words": ["that", "like"]}, {"sent": "like that .", "words": ["that", "like"]}, {"sent": "you know like I cook chicken every now and then but Ive been on the go so much lately that I um if I cook chicken itll just sit in the refrigerator for days .", "words": ["the", "refrigerator", "go", "like", "cook", "on", "chicken", "then", "that", "much", "for", "if", "so", "you", "and", "every", "in", "sit", "but"]}, {"sent": "and but now shes like enthralled by this bottle .", "words": ["like", "by", "and", "this", "bottle", "but"]}, {"sent": "what would you like to eat ?", "words": ["what", "to", "like", "you", "eat", "would"]}, {"sent": "aunt pat and daddy went to see mommy and the baby every day mike and daddy talked about what the babys name should be daddy said mommy had suggested naming her pat after aunt pat I like that said mike so do I said daddy so it was decided mike kept wondering what will the baby look like ?", "words": ["the", "what", "do", "to", "be", "so", "was", "like", "and", "that", "it", "every", "about", "look", "will", "see", "her"]}, {"sent": "like that .", "words": ["that", "like"]}, {"sent": "would you like to put the goat on this page ?", "words": ["the", "put", "to", "like", "you", "this", "would", "on"]}, {"sent": "they like to eat your food huh ?", "words": ["your", "to", "like", "eat", "they", "food"]}, {"sent": "yeah and she does not like that .", "words": ["does", "like", "and", "she", "that", "not"]}, {"sent": "like baby like the corn casserole that comes in the baby jars .", "words": ["the", "like", "in", "corn", "that"]}, {"sent": "what what do you like to eat at McDonalds ?", "words": ["what", "do", "to", "like", "you", "eat", "at"]}, {"sent": "hes like a baby and baby needs lots and lots of help .", "words": ["help", "of", "like", "and", "a"]}, {"sent": "Im glad that you like it .", "words": ["it", "that", "like", "you"]}, {"sent": "red like an apple green like peas .", "words": ["red", "peas", "like", "an", "green", "apple"]}, {"sent": "you like things that are nice and sweet dont you ?", "words": ["nice", "like", "you", "and", "dont", "are", "that"]}, {"sent": "does Thomas like Cornflakes ?", "words": ["does", "like"]}, {"sent": "so you can make like dashes .", "words": ["can", "so", "like", "you", "make"]}, {"sent": "Mommys glad you seem to like it .", "words": ["to", "it", "like", "you"]}, {"sent": "what would you like to do with them ?", "words": ["what", "do", "to", "with", "like", "them", "you", "would"]}, {"sent": "it looks like a horse though .", "words": ["horse", "it", "like", "a"]}, {"sent": "you like that idea .", "words": ["that", "like", "you"]}, {"sent": "I mean I was like tv shes not too bad at she usually only watches she watches an hour maybe an hour and a half a day if its like you know .", "words": ["if", "was", "like", "bad", "and", "you", "she", "tv", "at", "an", "too", "not", "a"]}, {"sent": "it it almost smells like popcorn .", "words": ["it", "like", "popcorn"]}, {"sent": "I like noise .", "words": ["like"]}, {"sent": "oh my God I bet you that shell be dark like Steve .", "words": ["my", "be", "like", "you", "dark", "that"]}, {"sent": "you just like to lick the icing off and leave the cake wouldnt you ?", "words": ["the", "off", "cake", "to", "like", "you", "and", "lick"]}, {"sent": "who makes noises like that ?", "words": ["that", "like", "who"]}, {"sent": "the stone I really like is this one .", "words": ["the", "is", "like", "this", "stone"]}, {"sent": "you know it would be easier if you held it like this and shook it .", "words": ["if", "be", "like", "you", "and", "this", "it", "would"]}, {"sent": "would you like to look at a book ?", "words": ["to", "like", "would", "you", "at", "look", "book", "a"]}, {"sent": "what dont you like about it ?", "words": ["what", "like", "you", "dont", "it", "about"]}, {"sent": "what do you like to eat ?", "words": ["what", "do", "to", "like", "you", "eat"]}, {"sent": "well Ill leave it like Ill leave it open so we know where were at and Ill just stand it down there look .", "words": ["where", "look", "we", "open", "so", "stand", "like", "and", "there", "it", "at", "down", "were"]}, {"sent": "what would you want what would you like to wash ?", "words": ["what", "to", "like", "you", "would", "wash"]}, {"sent": "Im sure you were there in five minutes like .", "words": ["like", "you", "there", "in", "were"]}, {"sent": "what do the dogs like to eat ?", "words": ["the", "what", "do", "to", "like", "eat"]}, {"sent": "its on like golf shot .", "words": ["on", "like"]}, {"sent": "look at that do you like Nemos boots ?", "words": ["do", "look", "like", "you", "boots", "at", "that"]}, {"sent": "I think she recognized it like she understands bee .", "words": ["think", "bee", "like", "it", "she"]}, {"sent": "does she like to turn the pages herself ?", "words": ["the", "does", "to", "like", "she"]}, {"sent": "ohdear like that ?", "words": ["that", "like"]}, {"sent": "hi I like your dress .", "words": ["dress", "your", "like"]}, {"sent": "that looks like a mouth .", "words": ["mouth", "that", "like", "a"]}, {"sent": "it looks to me like youre actually done eating because you dont look very hungry .", "words": ["to", "like", "hungry", "you", "dont", "it", "look", "me", "because"]}, {"sent": "try it like that .", "words": ["that", "it", "try", "like"]}, {"sent": "it was like bus wasnt it ?", "words": ["was", "it", "bus", "like"]}, {"sent": "do you think they might like to go in the house ?", "words": ["the", "do", "think", "go", "to", "like", "you", "they", "in", "house"]}, {"sent": "what does big bird look like ?", "words": ["what", "does", "bird", "like", "look", "big"]}, {"sent": "but see once you get it lit theres nothing like it .", "words": ["like", "you", "get", "it", "see", "but"]}, {"sent": "and a lot of kids know this books so sometimes theyre like ah youknow they they try to read because they knows whats gonna happen .", "words": ["of", "so", "to", "a lot", "like", "and", "this", "they", "read", "try", "try to", "because", "a"]}, {"sent": "xxx sat on the wall like that .", "words": ["on", "the", "that", "like"]}, {"sent": "which do you like better ?", "words": ["better", "do", "like", "you", "which"]}, {"sent": "like that .", "words": ["that", "like"]}, {"sent": "nursings what I like to do when we have a bath for two .", "words": ["what", "do", "for", "we", "to", "like", "have", "when", "a"]}, {"sent": "do you have a ball like this at home ?", "words": ["do", "like", "you", "have", "ball", "this", "at", "home", "a"]}, {"sent": "how would you like your hair today .", "words": ["how", "your", "like", "you", "would", "hair"]}, {"sent": "would you like me to help you .", "words": ["help", "to", "like", "you", "would", "me"]}, {"sent": "youre not sposta pull them like that .", "words": ["them", "like", "pull", "that", "not"]}, {"sent": "I like that .", "words": ["that", "like"]}, {"sent": "does he look like hes driving it ?", "words": ["does", "like", "it", "he", "look"]}, {"sent": "I guess you like that .", "words": ["that", "like", "you"]}, {"sent": "like she useta only be interested in things that made noise but now shell pick up like a spoon and be interested .", "words": ["up", "pick", "be", "like", "and", "in", "spoon", "that", "she", "but", "a"]}, {"sent": "peas she does not like .", "words": ["does", "peas", "like", "she", "not"]}, {"sent": "oh look at this I know you like this one .", "words": ["look", "like", "you", "this", "at"]}, {"sent": "things like this .", "words": ["this", "like"]}, {"sent": "like dirty beer yes put a cork in it like wine like when Mummy makes wine .", "words": ["put", "like", "in", "it", "dirty", "when", "a"]}, {"sent": "it does look like a trolley .", "words": ["does", "like", "it", "look", "a"]}, {"sent": "or squeeze it out like that .", "words": ["it", "that", "like", "out"]}, {"sent": "I like the way that feels .", "words": ["the", "that", "like"]}, {"sent": "but pretty much in every play thing that weve done here with this um longitudinal study whatever hes started with thats been like what weve played with most of the time and then at the end hes like moving to something else .", "words": ["the", "what", "of", "to", "play", "with", "like", "and", "here", "this", "in", "that", "pretty", "every", "at", "then", "much", "but"]}, {"sent": "would you like anything to eat perhaps a .", "words": ["to", "like", "you", "eat", "would", "a"]}, {"sent": "like um air fresheners and cans of oil and deicer .", "words": ["of", "like", "and"]}, {"sent": "youll like it .", "words": ["it", "like"]}, {"sent": "thats what you like to do ?", "words": ["what", "do", "to", "like", "you"]}, {"sent": "a a Swwws Mama can make that for you because I dont like eggs .", "words": ["can", "for", "like", "you", "dont", "make", "that", "because", "a"]}, {"sent": "ooh youre like a grownup .", "words": ["like", "a"]}, {"sent": "do you like it ?", "words": ["do", "it", "like", "you"]}, {"sent": "we brush Purdie dont we ?", "words": ["brush", "dont", "we"]}, {"sent": "dog yeah I dont think theres any kitty cats .", "words": ["think", "any", "dont", "kitty", "dog"]}, {"sent": "no dont look in there come here .", "words": ["here", "dont", "in", "there", "look"]}, {"sent": "dont know .", "words": ["dont"]}, {"sent": "I dont think she wants to go in the garage .", "words": ["the", "think", "go", "garage", "to", "dont", "in", "she"]}, {"sent": "dont put your sticker in there .", "words": ["put", "your", "dont", "in", "there"]}, {"sent": "you dont like him ?", "words": ["dont", "him", "like", "you"]}, {"sent": "you like to draw dont you ?", "words": ["to", "draw", "like", "you", "dont"]}, {"sent": "no dont do that .", "words": ["dont", "do", "that"]}, {"sent": "I dont know .", "words": ["dont"]}, {"sent": "if you dont want food we just leave it to one side dont we ?", "words": ["if", "we", "to", "you", "dont", "food", "it"]}, {"sent": "I dont want you lying on the floor just yet .", "words": ["on", "dont", "the", "you"]}, {"sent": "I dont know the rest .", "words": ["the", "dont"]}, {"sent": "dont .", "words": ["dont"]}, {"sent": "I dont think its my fault .", "words": ["dont", "think", "my"]}, {"sent": "I dont know if theyre gonna yeah it hasnt been open yet I dont know whats going on they probably dont have funding or xxx keep messing that place but .", "words": ["if", "open", "have", "dont", "they", "it", "on", "that", "but"]}, {"sent": "I dont know whats happening .", "words": ["dont"]}, {"sent": "you like hotdogs dont you ?", "words": ["dont", "like", "you"]}, {"sent": "I dont know what happened to to her arm .", "words": ["what", "to", "dont", "arm", "her"]}, {"sent": "hes saying dont to me .", "words": ["to", "dont", "me"]}, {"sent": "dont put that in .", "words": ["dont", "put", "that", "in"]}, {"sent": "well I dont know because its almost time for daddy to come in so we wanna be in here .", "words": ["for", "we", "to", "so", "be", "here", "dont", "in", "wanna", "because"]}, {"sent": "Charles got her on the sofa and told her dont be nervous he brought her a blanko blanket and cocoa and called the .", "words": ["the", "blanket", "be", "and", "dont", "he", "on", "sofa", "her", "a"]}, {"sent": "you dont dump your plate .", "words": ["your", "plate", "you", "dont", "dump"]}, {"sent": "though I dont know if you remember all of them .", "words": ["all", "if", "of", "them", "you", "dont"]}, {"sent": "dont do that to Meike .", "words": ["to", "dont", "do", "that"]}, {"sent": "you dont wanna play today .", "words": ["dont", "play", "wanna", "you"]}, {"sent": "they dont snap .", "words": ["dont", "they"]}, {"sent": "dont what am I fixing ?", "words": ["dont", "what", "am"]}, {"sent": "this is I dont know maybe outside of Caversham .", "words": ["is", "of", "outside", "dont", "this"]}, {"sent": "you dont want it on your lips do you ?", "words": ["do", "your", "you", "dont", "it", "on"]}, {"sent": "you dont know what happened when she got there .", "words": ["what", "you", "dont", "there", "when", "she"]}, {"sent": "I dont know .", "words": ["dont"]}, {"sent": "you dont want any garlic on your breath .", "words": ["your", "any", "you", "dont", "on"]}, {"sent": "dont dont rub it .", "words": ["dont", "it"]}, {"sent": "you dont wanna get this out ?", "words": ["you", "get", "dont", "this", "wanna", "out"]}, {"sent": "xxx to go down the stairs you dont needta take his hand to go down stairs .", "words": ["the", "go", "hand", "to", "you", "dont", "his", "stairs", "down", "take"]}, {"sent": "dont stop it .", "words": ["dont", "it", "stop"]}, {"sent": "need a vet for a horse dont you ?", "words": ["for", "need", "you", "dont", "horse", "a"]}, {"sent": "that kinda drives me crazy I dont really like that .", "words": ["dont", "me", "that", "like"]}, {"sent": "dont put it down my jumper mess pot .", "words": ["my", "put", "dont", "it", "down"]}, {"sent": "you dont want any more to go ?", "words": ["go", "to", "any", "you", "dont", "more"]}, {"sent": "why dont you just do what you normally do ?", "words": ["do", "what", "you", "dont", "why"]}, {"sent": "well dont forget to charge the battery this time will you ?", "words": ["the", "to", "you", "dont", "this", "will"]}, {"sent": "why dont you put his clothes away ?", "words": ["put", "you", "dont", "away", "his", "why"]}, {"sent": "you dont wanna feel that ?", "words": ["dont", "that", "wanna", "you"]}, {"sent": "make sure they dont fall down the stairs because that would hurt .", "words": ["the", "hurt", "fall", "dont", "they", "that", "would", "stairs", "down", "make", "because"]}, {"sent": "well a flashlight no I dont know a flashlight yes he said that that would help him hed feel more comfortable .", "words": ["help", "dont", "him", "more", "he", "would", "that", "a"]}, {"sent": "I dont hafta ask .", "words": ["dont"]}, {"sent": "you dont go on as much any more do you ?", "words": ["do", "go", "any", "you", "dont", "more", "on", "much"]}, {"sent": "I dont see no pennies .", "words": ["see", "dont"]}, {"sent": "Im sure you dont really but .", "words": ["dont", "but", "you"]}, {"sent": "oh we like kiwi dont we ?", "words": ["dont", "like", "we"]}, {"sent": "you dont have funnels and smoke down there .", "words": ["and", "have", "dont", "you", "there", "down"]}, {"sent": "because you dont love me .", "words": ["you", "dont", "love", "me", "because"]}, {"sent": "we dont do that do we ?", "words": ["dont", "do", "that", "we"]}, {"sent": "you dont wanna play that huh ?", "words": ["play", "you", "dont", "wanna", "that"]}, {"sent": "dont sniff then .", "words": ["dont", "then"]}, {"sent": "I dont think we bought it in Sainsburys actually Thomas .", "words": ["think", "we", "dont", "in", "it"]}, {"sent": "I dont wanna take them home .", "words": ["them", "dont", "wanna", "take", "home"]}, {"sent": "but I dont suppose he .", "words": ["dont", "he", "but"]}, {"sent": "it sounded like I dont think it would be Simonize .", "words": ["think", "be", "like", "dont", "it", "would"]}, {"sent": "no dont take one of me .", "words": ["of", "dont", "me", "take"]}, {"sent": "okay why dont you read this book to Mommy ?", "words": ["to", "you", "dont", "this", "read", "book", "why"]}, {"sent": "I dont know Purdie .", "words": ["dont"]}, {"sent": "yeah because we have a lot of animal toys and games and everything at home too but the toy food we dont .", "words": ["the", "we", "of", "a lot", "and", "have", "dont", "toy", "animal", "food", "but", "at", "home", "too", "because", "a"]}, {"sent": "oh I dont want any more .", "words": ["dont", "more", "any"]}, {"sent": "well dont post them then .", "words": ["dont", "then", "them"]}, {"sent": "I dont think they do .", "words": ["dont", "think", "do", "they"]}, {"sent": "oh no dont play with that tape recorder please okay ?", "words": ["play", "with", "tape", "dont", "that"]}, {"sent": "dont forget the trapezoid .", "words": ["the", "dont"]}, {"sent": "ah dont pull those .", "words": ["dont", "those", "pull"]}, {"sent": "oh no you dont hafta cry .", "words": ["dont", "cry", "you"]}, {"sent": "you dont want that ?", "words": ["dont", "that", "you"]}, {"sent": "you dont wanna take them out too many at a time .", "words": ["them", "you", "dont", "at", "wanna", "out", "too", "take", "a"]}, {"sent": "the doors dont open on that Thomas .", "words": ["the", "open", "dont", "on", "that"]}, {"sent": "dont mix them .", "words": ["dont", "them"]}, {"sent": "you dont like sprouts ?", "words": ["dont", "like", "you"]}, {"sent": "well I dont know for sure but Im just suggesting it might be there .", "words": ["for", "be", "dont", "there", "it", "but"]}, {"sent": "you dont wanna play with peekaboos ?", "words": ["play", "with", "you", "dont", "wanna"]}, {"sent": "I dont care .", "words": ["dont"]}, {"sent": "you dont like it that much huh ?", "words": ["like", "much", "you", "dont", "it", "that"]}, {"sent": "dont touch that .", "words": ["dont", "that", "touch"]}, {"sent": "if you dont talk Im going to turn the taperecorder off .", "words": ["the", "off", "if", "to", "you", "dont", "talk"]}, {"sent": "you dont touch anything .", "words": ["dont", "touch", "you"]}, {"sent": "I dont think the smell has had a chance to get out yet .", "words": ["the", "think", "to", "get", "dont", "out", "a"]}, {"sent": "I dont feel so good .", "words": ["dont", "so", "good"]}, {"sent": "so they dont need new names Im sorry .", "words": ["need", "so", "dont", "they", "new"]}, {"sent": "youve gotta play because if you dont play with puppet .", "words": ["if", "play", "with", "you", "dont", "because"]}, {"sent": "dont .", "words": ["dont"]}, {"sent": "no dont pull the ears .", "words": ["the", "dont", "pull"]}, {"sent": "you dont drive a bus or anything do you ?", "words": ["do", "you", "drive", "dont", "bus", "a"]}, {"sent": "you just dont know ?", "words": ["dont", "you"]}, {"sent": "so you dont hafta go unless they start hurting .", "words": ["go", "so", "you", "dont", "they"]}, {"sent": "dont put this over there please .", "words": ["put", "dont", "this", "there", "over"]}, {"sent": "please dont get many things .", "words": ["get", "dont"]}, {"sent": "I dont know did you look in the bag ?", "words": ["the", "you", "dont", "in", "did", "look"]}, {"sent": "I think this paints and these little pots are nice dont you ?", "words": ["these", "think", "nice", "little", "and", "this", "dont", "are", "you"]}, {"sent": "you dont want breakfast .", "words": ["dont", "you"]}, {"sent": "she just wants to have a look dont you ?", "words": ["to", "you", "have", "dont", "look", "she", "a"]}, {"sent": "because I think is the end of our sentence .", "words": ["the", "think", "is", "of", "our", "because"]}, {"sent": "hes mending our leaky windscreen .", "words": ["our"]}, {"sent": "can we take our hand outof our hair please ?", "words": ["can", "hand", "we", "our", "hair", "take"]}, {"sent": "we spent the day moving our stuff at the new house .", "words": ["the", "we", "our", "house", "at", "new"]}, {"sent": "they might all wanna join in with our game .", "words": ["all", "game", "with", "our", "they", "in", "wanna"]}, {"sent": "that man took our car didnt he ?", "words": ["he", "car", "that", "our"]}, {"sent": "not in our faces not in our faces .", "words": ["in", "not", "our"]}, {"sent": "he is girded with praise as our songs and prayers circle around him .", "words": ["is", "with", "our", "and", "him", "he", "around"]}, {"sent": "anything that came onto our farm xxx and sold .", "words": ["that", "and", "our"]}, {"sent": "I think Bob Higgens is gonna get our .", "words": ["get", "our", "think", "is"]}, {"sent": "our bus .", "words": ["bus", "our"]}, {"sent": "that would be our dessert .", "words": ["our", "be", "that", "would"]}, {"sent": "those are our main buss arent they ?", "words": ["they", "are", "those", "our"]}, {"sent": "what was he doing in our bed ?", "words": ["what", "was", "bed", "our", "in", "he"]}, {"sent": "yeah she really shes our girl .", "words": ["she", "our"]}, {"sent": "no were robots were gonna chop you in half with our sharp arms .", "words": ["with", "our", "you", "in", "were"]}, {"sent": "it is our only one .", "words": ["our", "it", "is"]}, {"sent": "do you remember when we went for our walk ?", "words": ["do", "for", "we", "you", "our", "walk", "when"]}, {"sent": "so were going in this way to our right .", "words": ["to", "so", "our", "this", "in", "were"]}, {"sent": "can um Ludovica see some of our pictures from um holiday ?", "words": ["can", "of", "our", "see", "some"]}, {"sent": "if you look at the ceiling up there in our house above the ceiling is our bedroom .", "words": ["the", "up", "if", "is", "you", "our", "there", "in", "at", "house", "look", "above", "bedroom"]}, {"sent": "and I noticed he found our junk food .", "words": ["our", "food", "he", "and"]}, {"sent": "oh lets put some furniture in our house shall we ?", "words": ["put", "we", "our", "in", "house", "some"]}, {"sent": "we hafta wash our brush dont we ?", "words": ["we", "brush", "our", "dont", "wash"]}, {"sent": "a bear and a raccoon got into our garbage ?", "words": ["into", "bear", "our", "and", "garbage", "a"]}, {"sent": "let us bang our head on it see how hard it is .", "words": ["how", "is", "head", "our", "us", "it", "hard", "on", "see"]}, {"sent": "we all have muscles and bones in our skin ?", "words": ["all", "we", "and", "have", "our", "in"]}, {"sent": "and our Kathryn had come down and taken the paints outof the kitchen and decided to paint the living room carpet with paints .", "words": ["the", "paint", "to", "living room", "with", "our", "and", "room", "kitchen", "down"]}, {"sent": "we have a snake in our yard dont we ?", "words": ["we", "our", "have", "dont", "in", "a"]}, {"sent": "keep your daughter away from our son she said .", "words": ["away", "your", "she", "our"]}, {"sent": "thats our bus coming now yeah thats our bus coming now .", "words": ["bus", "our"]}, {"sent": "what we need now is a nice family picture now that our new babys here .", "words": ["what", "need", "is", "we", "nice", "picture", "our", "here", "new", "that", "a"]}, {"sent": "its like our book Thomas and the big big bridge remember hes xxx .", "words": ["the", "like", "and", "our", "book", "big"]}, {"sent": "Im gonna draw a picture of our new house .", "words": ["of", "draw", "picture", "our", "house", "new", "a"]}, {"sent": "our kitchen is messy now .", "words": ["is", "kitchen", "our"]}, {"sent": "heres some kiwi should we some kiwi in our fruit salad put it in .", "words": ["put", "we", "our", "in", "it", "some"]}, {"sent": "mm we are missing pieces where did our pieces go oh theres couple more pieces over here .", "words": ["where", "we", "go", "our", "here", "more", "are", "did", "over"]}, {"sent": "I think thats all of our words that we say .", "words": ["all", "think", "we", "of", "say", "our", "that"]}, {"sent": "we get in our car and we drive outof Massachusetts and we drive up through Maine and go in our new house .", "words": ["up", "we", "go", "car", "our", "get", "and", "in", "drive", "house", "new"]}, {"sent": "I think we should start with his feet and work our way up .", "words": ["up", "think", "work", "we", "with", "and", "our", "his"]}, {"sent": "he hasnt come to our house darling .", "words": ["house", "to", "he", "our"]}, {"sent": "tell Nan what landed in our playground .", "words": ["what", "in", "our"]}, {"sent": "so if I see our farm animals xxx to the zoo arent they ?", "words": ["the", "zoo", "if", "so", "to", "our", "they", "see"]}, {"sent": "lets go wrap it up and put it in the trash and wash our hands .", "words": ["up", "the", "put", "go", "trash", "and", "our", "in", "it", "wash"]}, {"sent": "its called our solar system .", "words": ["our"]}, {"sent": "thing somethings gone wrong with our hoover hasnt it ?", "words": ["it", "with", "our"]}, {"sent": "Abe why dont we go get our coats on and go play in the snow ?", "words": ["the", "go", "we", "play", "snow", "our", "get", "dont", "and", "in", "on", "why"]}, {"sent": "where can we put our angels ?", "words": ["where", "can", "put", "we", "our"]}, {"sent": "so we can put cheese quavers in our cheese house Thomas .", "words": ["can", "put", "we", "so", "our", "in", "house", "cheese"]}, {"sent": "it says our long trip is over .", "words": ["is", "long", "our", "it", "over"]}, {"sent": "might hafta wash our hands after this .", "words": ["this", "wash", "our"]}, {"sent": "this is our last day okay .", "words": ["our", "this", "last", "is"]}, {"sent": "no ghosts in our house .", "words": ["in", "house", "our"]}, {"sent": "Cathy our dogs xxx serious problems .", "words": ["our"]}, {"sent": "well what was very interesting that happened to our house during that wind storm ?", "words": ["what", "was", "to", "our", "house", "that", "wind"]}, {"sent": "you have a look and it hasnt got an upstairs like our house .", "words": ["like", "and", "have", "you", "an", "it", "our", "house", "look", "a"]}, {"sent": "weve had such fun the babies yawned but now our mommies miss us .", "words": ["the", "us", "but", "our"]}, {"sent": "okay lets see here Boo shes going to get the coloring stuff well continue our interview .", "words": ["the", "to", "our", "here", "get", "see"]}, {"sent": "we had our lunch in this place .", "words": ["our", "this", "in", "we"]}, {"sent": "we needta wash some of our clothes today dont we ?", "words": ["we", "of", "our", "dont", "some", "wash"]}, {"sent": "we drink our milk we drink our milk .", "words": ["our", "milk", "drink", "we"]}, {"sent": "or shell walk around but then she come up to either me or Don and grab our legs and push us backwards ta sit us down on a stoop .", "words": ["up", "push", "to", "sit", "and", "our", "us", "around", "on", "me", "then", "down", "walk", "she", "but", "a"]}, {"sent": "what does mommy do when we do our kisses ?", "words": ["what", "do", "does", "we", "our", "when"]}, {"sent": "how about this is our cake .", "words": ["how", "cake", "is", "our", "this", "about"]}, {"sent": "alright come over here I need some help cleaning up this bookcase we gotta decide what were selling in our yard sale .", "words": ["up", "what", "help", "need", "we", "our", "here", "this", "in", "some", "over", "were"]}, {"sent": "we hear the horse go clop clop clop our pup goes fast asleep on top .", "words": ["the", "fast", "go", "we", "hear", "our", "asleep", "on", "horse"]}, {"sent": "we have our books .", "words": ["have", "our", "we"]}, {"sent": "Mister Mister Isabelle shouted thats our boat .", "words": ["boat", "our"]}, {"sent": "okay were just about to put our little pies in the oven .", "words": ["the", "put", "to", "our", "little", "in", "about", "were", "oven"]}, {"sent": "because then you could have added the teaspoon to our servings which would intensify the flavor .", "words": ["the", "to", "then", "you", "have", "our", "would", "could", "which", "because"]}, {"sent": "weve hauled some barges in our day .", "words": ["some", "in", "our"]}, {"sent": "what are our friends doing about that ?", "words": ["what", "our", "are", "about", "that"]}, {"sent": "do you think they were at our door ?", "words": ["do", "think", "you", "our", "they", "at", "were", "door"]}, {"sent": "well weve got three eggs havent we in our egg box ?", "words": ["box", "egg", "we", "our", "in"]}, {"sent": "atleast they do that at our school .", "words": ["do", "school", "our", "they", "at", "that"]}, {"sent": "should we take off our backpacks now ?", "words": ["off", "our", "take", "we"]}, {"sent": "oh were eating our breakfast nicely .", "words": ["were", "our"]}, {"sent": "at our house ?", "words": ["our", "house", "at"]}, {"sent": "they get away they take our hook we never take them home to cook .", "words": ["we", "to", "them", "our", "get", "they", "cook", "away", "take", "home"]}, {"sent": "recycle our water .", "words": ["water", "our"]}, {"sent": "no we cant put the books in our mouth .", "words": ["the", "put", "we", "mouth", "our", "in"]}, {"sent": "Imean our dog runs in and outof here .", "words": ["and", "here", "our", "in", "dog"]}, {"sent": "we put on our pajamas and hug all our mommas .", "words": ["all", "put", "we", "our", "and", "hug", "pajamas", "on"]}, {"sent": "but yeah even like in our house our food toys are in our in our basement .", "words": ["basement", "like", "our", "in", "food", "house", "are", "but"]}, {"sent": "and our space trip begins .", "words": ["our", "and"]}, {"sent": "but the big toys she plays at our house are actually her blocks .", "words": ["the", "her", "our", "are", "house", "at", "big", "she", "but"]}, {"sent": "did we see one of those on our trip ?", "words": ["those", "we", "of", "our", "did", "on", "see"]}, {"sent": "a little fox has been in our yard .", "words": ["our", "in", "little", "a"]}, {"sent": "so what if we leave our houses in forty five minutes .", "words": ["what", "if", "we", "so", "our", "in"]}, {"sent": "that brightens our pathways awhile .", "words": ["that", "our"]}, {"sent": "after we have breakfast we get to put our halloween costumes on .", "words": ["put", "we", "to", "our", "have", "get", "on"]}, {"sent": "is this our dinner ?", "words": ["our", "this", "is"]}, {"sent": "do yours like our .", "words": ["do", "like", "our"]}, {"sent": "and when we came home there was a big green truck outside our house delivering a parcel to some of the neighbors .", "words": ["the", "we", "of", "was", "outside", "to", "truck", "and", "our", "there", "big", "house", "green", "some", "when", "a", "home"]}, {"sent": "because we dont have any tins in our basket do we ?", "words": ["do", "we", "any", "our", "have", "dont", "in", "basket", "because"]}, {"sent": "hi youre here for our pajama party .", "words": ["here", "party", "for", "our"]}, {"sent": "pretend Danny xxx in our family .", "words": ["in", "pretend", "our"]}, {"sent": "our cat in a cupboard full of .", "words": ["full", "of", "our", "cat", "in", "a"]}, {"sent": "Dwww wasnt at our picnic yesterday .", "words": ["our", "at"]}, {"sent": "this is a road in our town .", "words": ["is", "our", "this", "in", "a"]}, {"sent": "and what else are they doing ?", "words": ["what", "are", "they", "and"]}, {"sent": "they love you dont they ?", "words": ["dont", "they", "love", "you"]}, {"sent": "they they could be jelly shapes couldnt they ?", "words": ["jelly", "they", "could", "be"]}, {"sent": "they had some tea uhn didnt they ?", "words": ["they", "some"]}, {"sent": "are they freshly picked from the field ?", "words": ["the", "they", "are"]}, {"sent": "what do they have up at the nursery ?", "words": ["up", "what", "do", "the", "have", "they", "at"]}, {"sent": "why are they crying ?", "words": ["they", "are", "why"]}, {"sent": "as they looked out at the rising sun Mufasa pointed to the light beams that stretched across the Pride lands .", "words": ["the", "light", "to", "sun", "they", "out", "at", "that"]}, {"sent": "the moon is high the sea is deep they rock and rock and rock to sleep .", "words": ["the", "moon", "is", "sleep", "to", "and", "they", "high", "rock"]}, {"sent": "they might they might fall .", "words": ["they", "fall"]}, {"sent": "they go splash .", "words": ["they", "splash", "go"]}, {"sent": "if they get cross with you they sting you .", "words": ["if", "with", "you", "get", "they"]}, {"sent": "where are they ?", "words": ["where", "they", "are"]}, {"sent": "they are sad arent they ?", "words": ["they", "sad", "are"]}, {"sent": "they make noise .", "words": ["they", "make"]}, {"sent": "they could they might .", "words": ["they", "could"]}, {"sent": "what are they ?", "words": ["what", "are", "they"]}, {"sent": "theyre all coming out to say hello are they ?", "words": ["all", "to", "say", "they", "are", "out"]}, {"sent": "what are they ?", "words": ["what", "are", "they"]}, {"sent": "are they for me ?", "words": ["they", "me", "are", "for"]}, {"sent": "or they could be .", "words": ["they", "could", "be"]}, {"sent": "are they fine ?", "words": ["they", "are", "fine"]}, {"sent": "what if they were hungry what if they were very hungry where would they go ?", "words": ["where", "what", "if", "go", "hungry", "they", "would", "were"]}, {"sent": "they are so lucky .", "words": ["they", "are", "so"]}, {"sent": "they bashed what ?", "words": ["they", "what"]}, {"sent": "I didnt take them out or shake them out they didnt fall out they were all out .", "words": ["all", "them", "shake", "fall", "they", "out", "take", "were"]}, {"sent": "what kinda juice are they drinking ?", "words": ["what", "juice", "are", "they"]}, {"sent": "they have havent they ?", "words": ["have", "they"]}, {"sent": "and down they fell .", "words": ["they", "down", "and"]}, {"sent": "but they bought him felt tips and pages and heres me .", "words": ["and", "they", "him", "me", "but"]}, {"sent": "can they have their dinner ?", "words": ["can", "they", "have", "their"]}, {"sent": "do they eat smarties ?", "words": ["eat", "do", "they"]}, {"sent": "are they wolves ?", "words": ["they", "are"]}, {"sent": "what did they do ?", "words": ["what", "do", "did", "they"]}, {"sent": "where are they going ?", "words": ["where", "they", "are"]}, {"sent": "say the turkeys they have not seen mother duck .", "words": ["the", "say", "have", "they", "duck", "not"]}, {"sent": "they only stick down once dont they ?", "words": ["dont", "they", "stick", "down"]}, {"sent": "are they going to scamp where they going ?", "words": ["to", "they", "are", "where"]}, {"sent": "and in the Winter when it was dark we useta see the lights flashing but recently theyve been coming quite early havent they ?", "words": ["the", "we", "was", "and", "they", "in", "it", "dark", "see", "when", "but"]}, {"sent": "well because they get lost dont they ?", "words": ["get", "they", "dont", "because"]}, {"sent": "who are they ?", "words": ["they", "are", "who"]}, {"sent": "I think what you mean is theyve got the orange lights on top of the lorry havent they ?", "words": ["the", "what", "think", "is", "of", "you", "they", "orange", "on"]}, {"sent": "are they four ?", "words": ["they", "are"]}, {"sent": "because they hafta be upside down dont they ?", "words": ["be", "dont", "they", "down", "because"]}, {"sent": "and they eat them .", "words": ["eat", "they", "them", "and"]}, {"sent": "they gave you medicine to put on it ?", "words": ["put", "to", "medicine", "you", "they", "it", "on"]}, {"sent": "oh they can fit under the gates .", "words": ["can", "the", "fit", "they", "under"]}, {"sent": "they get slightly ripped dont they ?", "words": ["get", "they", "dont"]}, {"sent": "and it was all then they were jumping in the mud so what happens ?", "words": ["the", "all", "what", "was", "so", "and", "they", "in", "it", "then", "were"]}, {"sent": "what are they pictures of ?", "words": ["of", "what", "are", "they"]}, {"sent": "hay hay is what they eat .", "words": ["eat", "what", "they", "is"]}, {"sent": "yes yes yes do you remember they were doing the ?", "words": ["the", "do", "you", "they", "were"]}, {"sent": "what do they have sometimes in the rocks there ?", "words": ["the", "what", "do", "have", "they", "in", "there"]}, {"sent": "they havent had any breakfast .", "words": ["they", "any"]}, {"sent": "theyre bushes rather than trees arent they ?", "words": ["they"]}, {"sent": "there they are watching the Rockettes .", "words": ["the", "there", "are", "they"]}, {"sent": "and what should they eat ?", "words": ["eat", "what", "they", "and"]}, {"sent": "are they friends ?", "words": ["they", "are"]}, {"sent": "let us see if they fit on your wrist .", "words": ["your", "if", "fit", "us", "they", "on", "see"]}, {"sent": "theyre two milk bottles are they ?", "words": ["milk", "they", "are"]}, {"sent": "they come every week and wave at us and we never know their names .", "words": ["their", "we", "and", "they", "us", "every", "at"]}, {"sent": "they do too .", "words": ["they", "do", "too"]}, {"sent": "ah are they all gonna play now .", "words": ["all", "they", "are", "play"]}, {"sent": "arent they ?", "words": ["they"]}, {"sent": "are they tired ?", "words": ["they", "are", "tired"]}, {"sent": "how do you think they go on the truck so they wont tip over .", "words": ["the", "do", "how", "think", "go", "so", "truck", "you", "they", "on", "over"]}, {"sent": "no little boys dont wear dresses do they ?", "words": ["dont", "do", "they", "little"]}, {"sent": "thats right they were .", "words": ["they", "were"]}, {"sent": "I think all our people are too big for this furniture arent they ?", "words": ["all", "think", "for", "our", "this", "they", "are", "too", "big"]}, {"sent": "I think they look very very nice but I think youve got the wrong shoes on them .", "words": ["the", "think", "nice", "them", "they", "look", "on", "but"]}, {"sent": "they painted the masks in bright colors and left them to dry .", "words": ["the", "to", "dry", "them", "and", "they", "in"]}, {"sent": "they gave it to Kathryn for Christmas .", "words": ["to", "they", "it", "for"]}, {"sent": "are they a bit blunt ?", "words": ["they", "are", "a"]}, {"sent": "little Del they have a diner where you can make a bunch of food .", "words": ["where", "can", "of", "little", "have", "they", "you", "food", "make", "a"]}, {"sent": "ya they have gone bye bye havent they ?", "words": ["have", "they"]}, {"sent": "and they spray water outof their trunk dont they ?", "words": ["their", "and", "they", "dont", "water"]}, {"sent": "but they didnt find much .", "words": ["they", "much", "but", "find"]}, {"sent": "they what ?", "words": ["they", "what"]}, {"sent": "where have they .", "words": ["where", "they", "have"]}, {"sent": "they do .", "words": ["they", "do"]}, {"sent": "so what are they doing ?", "words": ["so", "what", "are", "they"]}, {"sent": "or are they just throwing balls ?", "words": ["they", "are"]}, {"sent": "what do they do ?", "words": ["what", "do", "they"]}, {"sent": "and you know do you make the pizzas are they cooked inside the parlor ?", "words": ["the", "do", "inside", "you", "and", "they", "are", "make"]}, {"sent": "now they can all go together .", "words": ["can", "they", "all", "go"]}, {"sent": "those keys were hiding werent they ?", "words": ["they", "keys", "those", "were"]}, {"sent": "they opened a new restaurant in the garage .", "words": ["the", "garage", "they", "in", "new", "a"]}, {"sent": "Roman they need help .", "words": ["they", "help", "need"]}, {"sent": "are they gonna see deserts ?", "words": ["see", "they", "are"]}, {"sent": "they go rrr rrr rrr rrr and it makes butter .", "words": ["go", "and", "they", "it", "butter"]}, {"sent": "well to climb up the ladder and then they jump through it .", "words": ["up", "the", "to", "and", "they", "jump", "it", "ladder", "then", "climb"]}, {"sent": "they can choose from that .", "words": ["can", "they", "that"]}, {"sent": "they all fell off .", "words": ["all", "they", "off"]}, {"sent": "little boys do xxx dont they ?", "words": ["dont", "do", "they", "little"]}, {"sent": "what they watching ?", "words": ["what", "they"]}, {"sent": "the ducks sing that do they ?", "words": ["the", "do", "sing", "they", "that"]}, {"sent": "they ball .", "words": ["ball", "they"]}, {"sent": "are they going to swim ?", "words": ["to", "they", "are", "swim"]}, {"sent": "are they going to get on the train in a minute ?", "words": ["the", "train", "to", "get", "they", "in", "are", "on", "a"]}, {"sent": "what shape are they ?", "words": ["what", "are", "they"]}, {"sent": "they have snow around them .", "words": ["snow", "them", "have", "they", "around"]}, {"sent": "once itll go around here and then itll go back around the big one itll go twice .", "words": ["the", "back", "go", "and", "here", "around", "then", "big"]}, {"sent": "turn it around .", "words": ["it", "around"]}, {"sent": "quit running around .", "words": ["around"]}, {"sent": "Im not joking around .", "words": ["not", "around"]}, {"sent": "are you pulling the train around ?", "words": ["the", "train", "you", "are", "around"]}, {"sent": "start looking around .", "words": ["around"]}, {"sent": "the door opened and in comes one huge hippopotamus Id like a place for one please said the hippo to the rabbit behind the desk somewhere a fellow can really roll around I know just the place said the rabbit he led the hippo into the forest hotel around they went up and down till they came to a muddy river perfect said the hippo and the rabbit hippidy hopped back to his office .", "words": ["the", "can", "up", "for", "into", "back", "to", "behind", "like", "and", "they", "in", "he", "around", "his", "down", "a", "door"]}, {"sent": "yeh and move around .", "words": ["around", "and"]}, {"sent": "youre building a tunnel around me ?", "words": ["me", "around", "a"]}, {"sent": "oh youre sorry youre spinning around .", "words": ["around"]}, {"sent": "well look around .", "words": ["around", "look"]}, {"sent": "it was going around and around wasnt it ?", "words": ["was", "it", "and", "around"]}, {"sent": "well change your nappie and well let you splash around in some water if you like .", "words": ["water", "your", "if", "like", "you", "and", "in", "splash", "around", "some"]}, {"sent": "when he gets around the curve he can go faster .", "words": ["the", "can", "go", "he", "around", "when"]}, {"sent": "they climbing all around the bars huh ?", "words": ["all", "they", "the", "around"]}, {"sent": "froggy and the Mousie went walking all around .", "words": ["the", "all", "around", "and"]}, {"sent": "and she wore the shell containing Ariels voice around her neck .", "words": ["the", "and", "around", "her", "she"]}, {"sent": "are you still throwing my friends around ?", "words": ["around", "my", "are", "you"]}, {"sent": "well somebody in the store made that circle around that color honey .", "words": ["the", "store", "in", "around", "that"]}, {"sent": "turn around I cant hear you .", "words": ["hear", "you", "around"]}, {"sent": "why dont you go around to the back and see if you can push it open .", "words": ["the", "can", "push", "if", "back", "go", "open", "to", "you", "and", "dont", "it", "around", "see", "why"]}, {"sent": "well I think its around seven oclock .", "words": ["think", "around"]}, {"sent": "okay turn around oop .", "words": ["around"]}, {"sent": "oh coming around the mountain .", "words": ["the", "around"]}, {"sent": "alright come around .", "words": ["around"]}, {"sent": "turn it around so it fits .", "words": ["so", "it", "around"]}, {"sent": "its dangerous to walk around like that Aran .", "words": ["to", "like", "walk", "around", "that"]}, {"sent": "I cant get around it I hafta go .", "words": ["get", "go", "it", "around"]}, {"sent": "Kirsty looked around .", "words": ["around"]}, {"sent": "xxx to go around and around .", "words": ["to", "around", "and", "go"]}, {"sent": "shes bumping around a bit isnt she ?", "words": ["she", "a", "around"]}, {"sent": "you were hopping around saying what frogs say .", "words": ["what", "say", "you", "around", "were"]}, {"sent": "you think you can move around to the other side of the table so I can see your face ?", "words": ["can", "the", "think", "your", "table", "face", "of", "to", "so", "you", "other", "around", "see"]}, {"sent": "is it all that jumping around ?", "words": ["all", "is", "it", "around", "that"]}, {"sent": "turn around .", "words": ["around"]}, {"sent": "and uh She I dont know she sort a seems like shes losing her interest in books that she had a few months ago She used ta just pester continually with the book you know just follow you around the house but she seems shell even sit and read one on her own now .", "words": ["the", "with", "like", "and", "you", "dont", "in", "that", "house", "read", "on", "around", "book", "sit", "her", "she", "but", "a"]}, {"sent": "look at the bell around his neck .", "words": ["the", "around", "look", "at", "his"]}, {"sent": "Esther tells us that Roger Browns secretary tells us all we ever seem to do is sit around and drink tea .", "words": ["all", "do", "drink", "we", "is", "to", "and", "us", "around", "sit", "that"]}, {"sent": "when the moon rose and saw them warming themselves around the fire they had lit it shouted out an alarm .", "words": ["the", "moon", "them", "and", "they", "an", "it", "out", "around", "when"]}, {"sent": "he sniffs around for food when he is out and about .", "words": ["for", "is", "and", "food", "he", "out", "around", "about", "when"]}, {"sent": "and you shouldnt throw them around like that .", "words": ["throw", "them", "like", "you", "and", "around", "that"]}, {"sent": "Mia xxx walk around the thing .", "words": ["the", "walk", "around"]}, {"sent": "this is thats usually how she spends like a good I would say shell have a good pocket of maybe an hour in the afternoon or an hour in the morning hour in the afternoon although its usually in the afternoon when well go to my sisters house and she and the kids are running around and shell focus on like one like one of their toys and shell sit there and talk to herself and its usually to herself .", "words": ["the", "go", "of", "like", "an", "talk", "when", "their", "have", "there", "are", "on", "she", "how", "is", "to", "and", "good", "would", "a", "my", "say", "this", "in", "house", "around", "sit"]}, {"sent": "were going around tonight .", "words": ["were", "around"]}, {"sent": "and you can move em around the house .", "words": ["can", "the", "you", "and", "house", "around"]}, {"sent": "I could draw around it .", "words": ["it", "could", "draw", "around"]}, {"sent": "round and around and around round round .", "words": ["around", "and"]}, {"sent": "I cant put your nappie on when youre running around .", "words": ["put", "your", "around", "on", "when"]}, {"sent": "are you going to trace around it ?", "words": ["to", "you", "are", "it", "around"]}, {"sent": "zoom it around ?", "words": ["it", "around"]}, {"sent": "spin that around .", "words": ["that", "around"]}, {"sent": "I cant even get I around my foot .", "words": ["get", "my", "foot", "around"]}, {"sent": "okay lets turn around .", "words": ["around"]}, {"sent": "are you spinning around ?", "words": ["around", "are", "you"]}, {"sent": "wanna do ring around the rosy ?", "words": ["the", "do", "wanna", "around"]}, {"sent": "oh all the frogs are going around in a ring .", "words": ["the", "all", "in", "are", "around", "a"]}, {"sent": "if you bring me your big dolly Ill put this around the dolly .", "words": ["the", "put", "your", "if", "bring", "you", "this", "around", "me", "big"]}, {"sent": "another piece of tissue paper wave it all around .", "words": ["all", "tissue", "of", "paper", "another", "it", "around"]}, {"sent": "they do run around .", "words": ["they", "do", "run", "around"]}, {"sent": "its got wheels that go around and around and around .", "words": ["around", "that", "and", "go"]}, {"sent": "turn it around no thats not it look at the next there you are no oh here we are this one there look at it this one .", "words": ["the", "around", "we", "you", "here", "there", "this", "it", "at", "are", "look", "not"]}, {"sent": "turn them around .", "words": ["them", "around"]}, {"sent": "hold on with two hands please and turn around the right way .", "words": ["the", "hold", "with", "and", "around", "on"]}, {"sent": "I mean he said hed send a card around Christmas you know .", "words": ["around", "he", "you", "a"]}, {"sent": "no no dont fool around with that thing .", "words": ["dont", "that", "with", "around"]}, {"sent": "who ran around upstairs with Abi ?", "words": ["around", "with", "who"]}, {"sent": "around it .", "words": ["it", "around"]}, {"sent": "or she can run around .", "words": ["can", "run", "she", "around"]}, {"sent": "right right I mean they even sound I mean the topics they they talk about arent the same as truck drivers around oo thankyou .", "words": ["the", "around", "truck", "they", "same", "about", "talk"]}, {"sent": "turn it around .", "words": ["it", "around"]}, {"sent": "turn them around .", "words": ["them", "around"]}, {"sent": "okay you can hold it just try not to move around too much .", "words": ["can", "hold", "to", "much", "you", "it", "around", "too", "try", "not"]}, {"sent": "this one keeps going around and around .", "words": ["this", "and", "around"]}, {"sent": "and I wrapped it around the thing and stuck it and tied it in a bow the lot .", "words": ["the", "and", "in", "it", "stuck", "around", "a"]}, {"sent": "has he got some bees flying around him ?", "words": ["some", "him", "he", "around"]}, {"sent": "it just turns around .", "words": ["it", "around"]}, {"sent": "you could hardly walk around if youve got that many clothes on can you ?", "words": ["can", "if", "you", "that", "around", "on", "could", "walk"]}, {"sent": "she walked around .", "words": ["she", "around"]}, {"sent": "theres family around .", "words": ["around"]}, {"sent": "they just wanna sit around and sip cool drinks .", "words": ["and", "they", "wanna", "around", "sit"]}, {"sent": "eyes move around .", "words": ["around"]}, {"sent": "you look at the book with mummy and Ill make it go around for you .", "words": ["the", "around", "for", "go", "with", "you", "and", "at", "it", "look", "book", "make"]}, {"sent": "um so Ill call you back um somewhere around tenish .", "words": ["so", "around", "back", "you"]}, {"sent": "can you swing her around just a little bit ?", "words": ["can", "swing", "you", "little", "around", "her", "a"]}, {"sent": "now turn around .", "words": ["around"]}, {"sent": "xxx youknow it we dont like she take to toys like I have um little princess figurines but theyre princess dolls and she likes to put them in her her box and carry things around .", "words": ["put", "box", "we", "to", "them", "like", "her", "little", "have", "dont", "and", "it", "in", "carry", "around", "take", "she", "but"]}, {"sent": "and coming around now .", "words": ["around", "and"]}, {"sent": "we can move it around uhhuh .", "words": ["can", "it", "around", "we"]}, {"sent": "well you flip this and the train goes around and around .", "words": ["the", "train", "and", "you", "this", "around"]}, {"sent": "she was turning them around .", "words": ["was", "them", "she", "around"]}, {"sent": "I think like when she gets a little more kind of like tired like around naptime because of her routine then she kind of moves more toward books .", "words": ["think", "of", "her", "like", "little", "tired", "more", "around", "then", "when", "she", "because", "a"]}, {"sent": "he is walking around a field .", "words": ["around", "he", "a", "is"]}, {"sent": "look Im gonna make all these people run around here and then put them here .", "words": ["all", "these", "around", "put", "them", "and", "here", "run", "look", "then", "make"]}, {"sent": "what should we put around the table ?", "words": ["the", "what", "put", "table", "we", "around"]}, {"sent": "what about other piles of papers right around the computer ?", "words": ["the", "what", "of", "other", "about", "around"]}, {"sent": "around around around we go the ground is very far below .", "words": ["the", "go", "we", "is", "around"]}, {"sent": "running around and around .", "words": ["and", "around"]}, {"sent": "you worked around ?", "words": ["around", "you"]}, {"sent": "around here .", "words": ["here", "around"]}, {"sent": "better let her go to sleep then .", "words": ["better", "go", "sleep", "to", "then", "her"]}, {"sent": "oh its got like a wee spots to its lumps .", "words": ["to", "like", "a"]}, {"sent": "you turn left to go up to us at Jordanstown .", "words": ["up", "go", "to", "you", "us", "at"]}, {"sent": "when I went to work the other day .", "words": ["the", "work", "to", "other", "when"]}, {"sent": "we havent been to Macdonalds for a while .", "words": ["to", "for", "a", "we"]}, {"sent": "youre very kind to your dinosaurs .", "words": ["to", "your"]}, {"sent": "ask him if hell give it to you .", "words": ["if", "to", "you", "him", "it", "give"]}, {"sent": "that youve got to be watched when youre in the garden .", "words": ["the", "when", "to", "be", "in", "garden", "that"]}, {"sent": "Jason that tree is too small for you to climb .", "words": ["tree", "for", "is", "to", "you", "too", "climb", "that"]}, {"sent": "girls I want you to stop shouting .", "words": ["to", "stop", "you"]}, {"sent": "why what has Purdie done to you ?", "words": ["to", "what", "why", "you"]}, {"sent": "you want something to drink ?", "words": ["to", "drink", "you"]}, {"sent": "no this is the first time hes really been and hes youknow seems specifically drawn to them .", "words": ["the", "is", "to", "first", "them", "and", "this"]}, {"sent": "are you helping Mummy to build now ?", "words": ["to", "are", "build", "you"]}, {"sent": "you know when you go to the toddler group in HeatonMersey on a Tuesday ?", "words": ["the", "go", "to", "you", "in", "on", "when", "a"]}, {"sent": "want me to throw it ?", "words": ["to", "me", "it", "throw"]}, {"sent": "theyre going to go and live there .", "words": ["to", "there", "and", "go"]}, {"sent": "you remember how to do this ?", "words": ["do", "how", "to", "you", "this"]}, {"sent": "Ill just go and get a cloth to rinse you now .", "words": ["go", "to", "and", "get", "you", "a"]}, {"sent": "whos gonna to drink that milk ?", "words": ["milk", "to", "that", "drink"]}, {"sent": "are they going to go on the train to the cafe is that what you wanna do hm ?", "words": ["the", "what", "do", "go", "train", "to", "is", "you", "they", "are", "wanna", "on", "that"]}, {"sent": "which one do you want to hear .", "words": ["do", "to", "hear", "you", "want to", "which"]}, {"sent": "now we got one more leg to put in .", "words": ["put", "we", "to", "more", "in", "leg"]}, {"sent": "you want Mommy to hold the bear ?", "words": ["the", "hold", "to", "bear", "you"]}, {"sent": "now dont forget to eat this nice sausage roll .", "words": ["to", "nice", "eat", "dont", "this"]}, {"sent": "I dont want you to touch the grow core .", "words": ["the", "to", "touch", "you", "dont"]}, {"sent": "what happened to Thomass knee ?", "words": ["to", "what", "knee"]}, {"sent": "I hafta go to collect Adam from school .", "words": ["to", "school", "go"]}, {"sent": "whatre you going to whatre you making ?", "words": ["to", "you"]}, {"sent": "so then he came to pick you up from your school so that was kind of interesting .", "words": ["up", "your", "pick", "to", "so", "school", "was", "of", "you", "he", "then", "that"]}, {"sent": "wouldnt it be lovely to have somebody at home to come in and clean the house ?", "words": ["clean", "the", "to", "be", "and", "have", "in", "it", "house", "at", "home"]}, {"sent": "come to Daddy .", "words": ["to"]}, {"sent": "if you start throwing any of this stuff well hafta put the lid on and you wont be able to play with it for the rest of the time .", "words": ["the", "put", "for", "if", "of", "be", "to", "play", "any", "with", "you", "and", "this", "it", "on"]}, {"sent": "do you want Mummy to do it or is Nicole going to do it ?", "words": ["do", "is", "to", "you", "it"]}, {"sent": "no youre not gonna go to bed with it on .", "words": ["go", "to", "with", "bed", "it", "on", "not"]}, {"sent": "we go to the shop to buy magazines dont we ?", "words": ["the", "go", "we", "to", "dont", "buy"]}, {"sent": "what colors the ball going to be Dominic ?", "words": ["the", "what", "be", "to", "ball"]}, {"sent": "you got to find a b heres b over there .", "words": ["to", "you", "there", "find", "over", "a"]}, {"sent": "when are you going to learn to wipe yourself ?", "words": ["yourself", "to", "you", "are", "wipe", "when"]}, {"sent": "always ready to help Eddies answer to any problem is .", "words": ["to", "help", "any", "is"]}, {"sent": "so weve got to leave a gap .", "words": ["to", "so", "a"]}, {"sent": "okay Im going to go back downstairs .", "words": ["to", "back", "go"]}, {"sent": "do you know how to swim ?", "words": ["do", "how", "to", "swim", "you"]}, {"sent": "oh you like to see ah things fall ?", "words": ["to", "fall", "like", "you", "see"]}, {"sent": "one of the reasons Erin came back though is because she didnt get to hear the whole story of Shrek so I thought we could tell her it together even though shes never seen the movie .", "words": ["the", "back", "is", "of", "to", "so", "we", "hear", "get", "it", "story", "her", "she", "because", "could"]}, {"sent": "thats to put babys in .", "words": ["to", "put", "in"]}, {"sent": "he needs to be taken to the zoo .", "words": ["the", "zoo", "to", "be", "he"]}, {"sent": "to stay strong and healthy the brown horse eats all day .", "words": ["the", "all", "brown", "to", "and", "horse", "stay"]}, {"sent": "is there a bucket to dig with ?", "words": ["bucket", "is", "to", "with", "there", "a"]}, {"sent": "dance to the music .", "words": ["the", "to", "dance"]}, {"sent": "so perhaps well hafta come back to this one .", "words": ["to", "so", "back", "this"]}, {"sent": "trying to .", "words": ["to"]}, {"sent": "out to dinner .", "words": ["to", "out"]}, {"sent": "oh are your going to be busy tomorrow ?", "words": ["to", "are", "your", "be"]}, {"sent": "and then this evening when the students are all settled in Ive got to go to a cheese and wine Thomas .", "words": ["the", "all", "go", "to", "and", "this", "in", "are", "then", "cheese", "when", "a"]}, {"sent": "youve got to be shown what to do otherwise no .", "words": ["to", "what", "be", "do"]}, {"sent": "youll go straight to the school see with them .", "words": ["the", "go", "to", "school", "with", "them", "see"]}, {"sent": "xxx woujou like to draw a picture with your princess markers ?", "words": ["your", "to", "draw", "picture", "like", "with", "a"]}, {"sent": "do you wanna say hello to Uncle Steve ?", "words": ["do", "to", "say", "you", "wanna"]}, {"sent": "no I said your mothers listening to the radio .", "words": ["the", "to", "your", "radio"]}, {"sent": "you want the boys to go down the slide ?", "words": ["the", "slide", "go", "to", "you", "down"]}, {"sent": "you want me to move it over ?", "words": ["to", "you", "it", "me", "over"]}, {"sent": "no you just need to sleep to feel better .", "words": ["better", "need", "sleep", "to", "need to", "you"]}, {"sent": "is she going to find Elsa ?", "words": ["to", "she", "find", "is"]}, {"sent": "Thomas the bus driver is this a ticket to take me just one way to the sweetie shop ?", "words": ["the", "is", "to", "this", "me", "take", "bus", "a"]}, {"sent": "and I kept trying to pick you up .", "words": ["up", "pick", "to", "you", "and"]}, {"sent": "why dont you comb her hair and then Ill try to put a barrette on the top .", "words": ["the", "put", "try", "to", "comb", "you", "and", "dont", "on", "then", "hair", "her", "try to", "why", "a"]}, {"sent": "I bet its going to fall .", "words": ["to", "fall"]}, {"sent": "whats he got to do ?", "words": ["to", "do", "he"]}, {"sent": "and we went down to the sea on the tram didnt we ?", "words": ["the", "we", "to", "and", "on", "down"]}, {"sent": "there is bus stop right here and and it goes right to the campus .", "words": ["the", "is", "to", "stop", "and", "here", "there", "it", "bus"]}, {"sent": "Luna its going to be the last one are you ready ?", "words": ["the", "to", "last", "be", "you", "are"]}, {"sent": "Mummy to help you ?", "words": ["to", "help", "you"]}, {"sent": "do you wanna go to the parade on sunday ?", "words": ["the", "do", "go", "to", "you", "wanna", "on"]}, {"sent": "you forgot to smack that bottom .", "words": ["to", "bottom", "that", "you"]}, {"sent": "what are we going to do ?", "words": ["what", "do", "we", "to", "are"]}, {"sent": "just be careful not to get any food on the engines Thomas .", "words": ["the", "to", "be", "any", "get", "food", "careful", "on", "not"]}, {"sent": "I was like no GummyBears are not something to eat in the morning .", "words": ["the", "to", "was", "like", "eat", "in", "are", "not"]}, {"sent": "bit big to go under the bridge isnt he ?", "words": ["the", "go", "to", "under", "he", "big"]}, {"sent": "you want me to turn it around for you ?", "words": ["for", "to", "you", "it", "around", "me"]}, {"sent": "are you going to listen to her cough first ?", "words": ["listen", "to", "first", "you", "are", "her"]}, {"sent": "oh what has happened to the pussycat ?", "words": ["to", "what", "the"]}, {"sent": "so you want this to go like this .", "words": ["go", "to", "so", "like", "you", "this"]}, {"sent": "I would like to go skiing .", "words": ["go", "to", "like", "would"]}, {"sent": "its better just to put it in with your hands .", "words": ["better", "put", "your", "to", "with", "in", "it"]}, {"sent": "youre going to be a postman arent you ?", "words": ["to", "be", "you", "a"]}, {"sent": "do you think your pussy cat wants to go out ?", "words": ["do", "think", "your", "go", "to", "you", "cat", "out"]}, {"sent": "go to bed now LaLa .", "words": ["to", "bed", "go"]}, {"sent": "to that game for you .", "words": ["game", "for", "to", "you", "that"]}, {"sent": "wanna sit or do you want the cleaner to sit ?", "words": ["the", "do", "to", "you", "wanna", "sit"]}, {"sent": "whenever the students had free time they were permitted to go to the light bulb lab in the back of the classroom .", "words": ["the", "back", "go", "light", "to", "of", "they", "in", "were"]}, {"sent": "wait lets listen to Wanda then Michael .", "words": ["to", "wait", "then", "listen"]}, {"sent": "and Richards going to stay with us for a little while isnt he ?", "words": ["for", "to", "with", "and", "little", "us", "he", "stay", "a"]}, {"sent": "it has the cap on it to cover it .", "words": ["the", "to", "it", "cover", "on"]}, {"sent": "who else likes to oh .", "words": ["to", "who"]}, {"sent": "he is gonna say good bye to his piggie .", "words": ["is", "to", "say", "he", "good", "his"]}, {"sent": "shall we find some more animals to go in the middle ?", "words": ["the", "we", "go", "to", "more", "in", "find", "some"]}, {"sent": "oh now you see thats right that lighthouse looks like its got a little cottage or something attached to it .", "words": ["to", "like", "you", "little", "it", "see", "that", "a"]}, {"sent": "okay go to sleep .", "words": ["sleep", "to", "go"]}, {"sent": "ask him if hes okay .", "words": ["him", "if"]}, {"sent": "if your cars not there .", "words": ["there", "not", "your", "if"]}, {"sent": "you can leave that roll if you dont want it .", "words": ["can", "if", "you", "dont", "it", "that"]}, {"sent": "as youre growing up and getting too big for some of your toys it would be nice if you gave them to somebody else so they could enjoy them .", "words": ["of", "nice", "could", "be", "it", "some", "big", "up", "for", "if", "to", "so", "and", "you", "would", "too", "your", "them", "they"]}, {"sent": "if you follow what Mama does .", "words": ["what", "does", "if", "you"]}, {"sent": "let me see if I can find the right xxx number .", "words": ["can", "the", "if", "me", "find", "see"]}, {"sent": "we dont hafta if you dont want to .", "words": ["if", "we", "to", "you", "dont", "want to"]}, {"sent": "look around and see if there is something else .", "words": ["look", "if", "is", "and", "there", "around", "see"]}, {"sent": "so if Purdies going to waste it we might as well give it to him .", "words": ["if", "we", "to", "so", "him", "it", "give"]}, {"sent": "wouldnt it be quiet if there wasnt Thomas living here ?", "words": ["if", "be", "here", "there", "it", "quiet"]}, {"sent": "okay if you say so .", "words": ["so", "say", "if", "you"]}, {"sent": "if we have a bit of jelly at dinner time therell be enough left for tea wont there ?", "words": ["jelly", "for", "if", "we", "of", "be", "have", "there", "at", "a"]}, {"sent": "tell you what if shes gonna wear this lemme .", "words": ["what", "if", "this", "you"]}, {"sent": "if its red on the outside it will look like a brick house .", "words": ["red", "the", "if", "outside", "will", "like", "it", "house", "look", "on", "a"]}, {"sent": "if you dont want any more then we can pour the rest in here cant we ?", "words": ["can", "the", "if", "we", "pour", "any", "you", "here", "dont", "more", "in", "then"]}, {"sent": "does that make if feel better to have a bandaid on ?", "words": ["better", "does", "if", "to", "have", "that", "on", "make", "a"]}, {"sent": "well if the fireman has going back home now then he doesnt needta put his siren and his flashing lights on does he ?", "words": ["the", "does", "put", "if", "back", "and", "on", "he", "his", "then", "home"]}, {"sent": "see if all these people wanna go up here .", "words": ["up", "all", "these", "if", "go", "here", "wanna", "see"]}, {"sent": "yes if you like .", "words": ["if", "like", "you"]}, {"sent": "xxx I think Ill xxx in the floor if you go outside youre ruined .", "words": ["the", "think", "if", "go", "outside", "you", "in"]}, {"sent": "I cant find them if theyre not there .", "words": ["if", "them", "there", "find", "not"]}, {"sent": "you know what you can move this if you dont want it in the dining room .", "words": ["can", "what", "the", "if", "you", "this", "dont", "it", "in", "room"]}, {"sent": "she cant hold on if shes carrying a bag .", "words": ["hold", "if", "on", "she", "a"]}, {"sent": "and I think if they had gone away theyd have taken one of the cars with them so .", "words": ["the", "think", "if", "of", "so", "with", "them", "and", "have", "they", "away"]}, {"sent": "so carry on if you wanna break him .", "words": ["if", "so", "you", "carry", "him", "wanna", "on", "break"]}, {"sent": "see if you can find a cup .", "words": ["can", "cup", "if", "you", "find", "see", "a"]}, {"sent": "see if that fits look .", "words": ["see", "that", "if", "look"]}, {"sent": "and if everybody pays theres no problem .", "words": ["if", "and"]}, {"sent": "if you wanna make a real alligator you hafta make it on paper .", "words": ["if", "paper", "you", "it", "alligator", "wanna", "on", "make", "a"]}, {"sent": "perhaps if were going to hire the church hall we cant do a lot of things like that .", "words": ["the", "do", "if", "we", "of", "to", "church", "a lot", "like", "that", "were", "a"]}, {"sent": "if you just say no I dont know over and over again thats not very much data .", "words": ["if", "say", "much", "you", "and", "dont", "over", "not"]}, {"sent": "I was wondering if you might want a french fry .", "words": ["was", "if", "a", "you"]}, {"sent": "Thomas what happens if you get stuck in there ?", "words": ["what", "if", "you", "get", "there", "in", "stuck"]}, {"sent": "well if youre allowed to .", "words": ["to", "if"]}, {"sent": "shall we go and look in Jwwws room and see if weve got any clean clothes in there ?", "words": ["clean", "if", "go", "we", "any", "and", "there", "in", "room", "look", "see"]}, {"sent": "if thats alright then .", "words": ["then", "if"]}, {"sent": "if .", "words": ["if"]}, {"sent": "Im like she should stay home with a toddler for thirteen hours and see if she likes it .", "words": ["for", "if", "see", "with", "like", "and", "it", "stay", "she", "a", "home"]}, {"sent": "how about if you go shopping with the shopping basket and put things in the basket instead of in your mouth ?", "words": ["the", "how", "put", "your", "if", "go", "of", "mouth", "with", "you", "and", "in", "basket", "about"]}, {"sent": "well there isnt a mess yet but there will be if the lid comes off .", "words": ["the", "off", "if", "be", "there", "will", "but", "a"]}, {"sent": "what would you have done if that ape would have said .", "words": ["what", "if", "you", "have", "would", "that"]}, {"sent": "yeah but you can put it on your cheeks if you want .", "words": ["can", "put", "your", "if", "you", "it", "on", "but"]}, {"sent": "what do we do if its the same Becky ?", "words": ["the", "what", "do", "if", "we", "same"]}, {"sent": "if you lean forward in that chair youll go right over .", "words": ["if", "go", "chair", "you", "in", "over", "that"]}, {"sent": "do what you would do if you were at someones house .", "words": ["do", "what", "if", "you", "at", "house", "would", "were"]}, {"sent": "no well if they get put in the box by mistake .", "words": ["the", "put", "box", "if", "by", "get", "they", "in"]}, {"sent": "if youre messing about xxx swallow it you might choke .", "words": ["it", "if", "about", "you"]}, {"sent": "oh well Im not poorly but I might be if I dont go to the Dentist .", "words": ["the", "if", "go", "be", "to", "dont", "not", "but"]}, {"sent": "but you can wear it you can try it on if you want .", "words": ["can", "if", "you", "it", "on", "try", "but"]}, {"sent": "lets see if you can walk .", "words": ["can", "if", "you", "see", "walk"]}, {"sent": "if you want .", "words": ["if", "you"]}, {"sent": "but he but maybe if he had seen the book he might have been interested ?", "words": ["the", "if", "have", "he", "book", "but"]}, {"sent": "I was just seeing if you were awake or not .", "words": ["if", "was", "awake", "you", "were", "not"]}, {"sent": "if I could reach you right now .", "words": ["could", "if", "you"]}, {"sent": "well if you put it on .", "words": ["put", "if", "you", "it", "on"]}, {"sent": "well if youre playing with trucks Ill have a go at this jigsaw .", "words": ["if", "go", "with", "have", "this", "at", "a"]}, {"sent": "if I push it then the noise starts .", "words": ["the", "push", "if", "it", "then"]}, {"sent": "I Im not sure if it was the colors or .", "words": ["the", "if", "was", "it", "not"]}, {"sent": "if you try to stroke her tummy or her feathers of red .", "words": ["red", "try", "if", "of", "to", "you", "tummy", "her", "try to"]}, {"sent": "Jessica is in the office if youre looking for her honey .", "words": ["the", "for", "if", "is", "in", "her"]}, {"sent": "if you get a steel one .", "words": ["get", "if", "a", "you"]}, {"sent": "he can wear this if he wants .", "words": ["can", "this", "he", "if"]}, {"sent": "lets see if we can find it .", "words": ["can", "if", "we", "it", "find", "see"]}, {"sent": "xxx if he goes too fast then sometimes he can feel dizzy cant he ?", "words": ["can", "fast", "if", "he", "too", "then"]}, {"sent": "well hafta look in a minute and see if we can find it .", "words": ["can", "if", "we", "find", "and", "in", "it", "look", "see", "a"]}, {"sent": "if you had them both at one time they would melt .", "words": ["if", "them", "would", "you", "they", "at"]}, {"sent": "if you leave it standing .", "words": ["it", "if", "you"]}, {"sent": "if you put that bit does this wee bit go down here ?", "words": ["does", "put", "if", "go", "you", "here", "this", "down", "that"]}, {"sent": "now wait a second lemme see if its okay .", "words": ["see", "wait", "if", "a"]}, {"sent": "lets see if hes in there yep there he is .", "words": ["if", "is", "there", "in", "he", "see"]}, {"sent": "if you have you teeth brushed .", "words": ["have", "if", "you"]}, {"sent": "you can sit on here then Ill hold if for you and then we can write .", "words": ["can", "for", "hold", "if", "we", "then", "write", "you", "here", "and", "on", "sit"]}, {"sent": "dont know if Im gonna get up again .", "words": ["get", "dont", "up", "if"]}, {"sent": "find out see if there is a ball .", "words": ["if", "is", "ball", "there", "out", "find", "see", "a"]}, {"sent": "if it were thicker twine then it would stay up right I think .", "words": ["up", "think", "if", "it", "would", "stay", "then", "were"]}, {"sent": "cause if were going to put it together you hafta remember all right ?", "words": ["all", "put", "if", "to", "you", "it", "were"]}, {"sent": "playing with them even if I dont read it to him .", "words": ["if", "to", "with", "them", "dont", "him", "it", "read"]}, {"sent": "and that kinda helped because he would get all upset if she wrecked up his stuff and .", "words": ["up", "all", "if", "and", "get", "he", "would", "his", "that", "she", "because"]}, {"sent": "I make them sit at the kitchen table if they wanna do anything .", "words": ["the", "do", "table", "if", "them", "they", "kitchen", "at", "wanna", "sit", "make"]}, {"sent": "Im not sure if theyre blackberries or blackcurrants .", "words": ["if", "not"]}, {"sent": "if we dont tell anybody if nobody sees us .", "words": ["dont", "if", "us", "we"]}, {"sent": "if you turn it on the other side they attract .", "words": ["the", "if", "you", "other", "they", "it", "on"]}, {"sent": "could I use some of yours if I wanted to ?", "words": ["if", "of", "to", "some", "could"]}, {"sent": "yeah if you can find room .", "words": ["can", "if", "you", "room", "find"]}, {"sent": "and if she passes shell be leaving AuntieJanice and UncleStuart and going to University wont she ?", "words": ["if", "to", "be", "and", "she"]}, {"sent": "if you pull it Aran this will come easily to bits .", "words": ["if", "to", "pull", "you", "this", "it", "will"]}, {"sent": "if you wanna you can flatten the the xxx top up like this and put a face on here you know like that and glue it on and put hair on it .", "words": ["can", "the", "up", "put", "if", "face", "like", "you", "and", "this", "here", "it", "wanna", "on", "glue", "hair", "that", "a"]}, {"sent": "how about if we look for the piglets ?", "words": ["the", "how", "for", "if", "we", "about", "look"]}, {"sent": "what does that mean if its good ?", "words": ["what", "does", "if", "good", "that"]}, {"sent": "even if you have the green light because sometimes the car doesnt see that its sposta stop .", "words": ["the", "if", "light", "car", "stop", "you", "have", "green", "see", "that", "because"]}, {"sent": "if somebody sat on that theyd think the cushion was secure and theyd slip off and they might slip off with it .", "words": ["the", "off", "think", "if", "was", "with", "and", "they", "it", "on", "that"]}, {"sent": "look if you wanna put things on here right what well do right is put the house up there yeah ?", "words": ["the", "what", "put", "do", "up", "if", "is", "you", "here", "there", "house", "wanna", "look", "on"]}, {"sent": "well lets see if Laura worked .", "words": ["see", "if"]}, {"sent": "see if that room .", "words": ["see", "room", "that", "if"]}, {"sent": "you better blow on it Lara if you wanna leave it to cool .", "words": ["better", "if", "to", "you", "it", "wanna", "on", "blow"]}, {"sent": "theres room for you Matthew if you wanna go up .", "words": ["up", "for", "if", "go", "you", "room", "wanna"]}, {"sent": "I dont know if we ever talked about jackolanterns .", "words": ["dont", "if", "about", "we"]}, {"sent": "but if you take these off then you these bars are going to fall down .", "words": ["off", "these", "if", "to", "fall", "you", "are", "then", "down", "take", "but"]}, {"sent": "if theres no sickness toil or danger .", "words": ["if"]}, {"sent": "lets do the rest of it and see if it turns up .", "words": ["the", "do", "up", "if", "of", "and", "it", "see"]}, {"sent": "but what kind of animals were at the pumpkin patch ?", "words": ["the", "what", "of", "at", "pumpkin", "were", "but"]}, {"sent": "but whats the fish in ?", "words": ["the", "in", "but", "fish"]}, {"sent": "but our house is like that .", "words": ["is", "like", "our", "house", "that", "but"]}, {"sent": "well not exact same kind but different colors but very theres like we have um a brown board with uh he rudder blue screws but its practically the same .", "words": ["the", "we", "brown", "with", "like", "blue", "have", "same", "he", "not", "but", "a"]}, {"sent": "yeah but why is the little dog trying to pull the big dog ?", "words": ["the", "is", "to", "little", "pull", "but", "big", "dog", "why"]}, {"sent": "but only in your chair .", "words": ["chair", "in", "your", "but"]}, {"sent": "its my bedtime now but Im going to hide bye uh oh pew its a little .", "words": ["my", "hide", "to", "little", "but", "a"]}, {"sent": "but it hasnt got a mummie .", "words": ["it", "but", "a"]}, {"sent": "but I dont know if hes really into the whole dressing and undressing thing yet .", "words": ["the", "if", "into", "and", "dont", "but"]}, {"sent": "but shell just play with it .", "words": ["it", "play", "with", "but"]}, {"sent": "but I dont know .", "words": ["dont", "but"]}, {"sent": "oh its a train track but you need some building machines to repair it ?", "words": ["need", "train", "to", "you", "it", "some", "but", "a"]}, {"sent": "but Ive found it now .", "words": ["it", "but"]}, {"sent": "but you can bring your little green footstool here as well .", "words": ["can", "your", "bring", "you", "little", "here", "green", "but"]}, {"sent": "but theyre not Pillsburys a .", "words": ["not", "but", "a"]}, {"sent": "but little pigs is all there is and not a single bear .", "words": ["all", "is", "little", "and", "there", "bear", "not", "but", "a"]}, {"sent": "but Im not sure .", "words": ["not", "but"]}, {"sent": "okay so are were there any toys that he seemed interested last time but this time xxx .", "words": ["so", "last", "any", "there", "that", "are", "he", "this", "were", "but"]}, {"sent": "but maybe not very well .", "words": ["not", "but"]}, {"sent": "but not if you just throw it at me .", "words": ["if", "throw", "you", "it", "at", "me", "not", "but"]}, {"sent": "thats that letter goes there but whats that letter .", "words": ["there", "that", "but"]}, {"sent": "but I think its meant to be like that sweetheart .", "words": ["think", "to", "be", "like", "that", "but"]}, {"sent": "so thats kind they they have a farm part to it so um her brother knows about that cause he remembers that part but um .", "words": ["so", "to", "have", "they", "that", "it", "he", "about", "her", "but", "a"]}, {"sent": "its a bit of a fiddly job but you can try .", "words": ["can", "of", "you", "try", "but", "a"]}, {"sent": "but you like to pretend youre writing .", "words": ["to", "like", "you", "pretend", "but"]}, {"sent": "but not clapping .", "words": ["not", "but"]}, {"sent": "but when he sees kids he wants to touch their face .", "words": ["their", "face", "to", "touch", "he", "when", "but"]}, {"sent": "but I think she was more or less just average at talking .", "words": ["think", "was", "more", "at", "she", "but"]}, {"sent": "but today were going bye bye .", "words": ["were", "but"]}, {"sent": "but I must remember Ill leave it out there .", "words": ["there", "out", "it", "but"]}, {"sent": "but okay since you wanna make that statement right now .", "words": ["you", "make", "wanna", "that", "but"]}, {"sent": "oh but youre working on being able to swim the whole .", "words": ["the", "to", "swim", "on", "but"]}, {"sent": "I dont know what youre doing but youre mad .", "words": ["mad", "dont", "what", "but"]}, {"sent": "but look .", "words": ["but", "look"]}, {"sent": "no ooh but but but we gotta keep it on honey no no no no no .", "words": ["on", "it", "but", "we"]}, {"sent": "but I do think its quite fun .", "words": ["do", "think", "but"]}, {"sent": "but I dont want the birds eating all our fruit .", "words": ["the", "all", "our", "dont", "but"]}, {"sent": "but Z is the cat that will clean up that spot .", "words": ["the", "clean", "up", "is", "cat", "will", "that", "but"]}, {"sent": "but Gordon began to feel more and more feeble .", "words": ["to", "more", "but", "and"]}, {"sent": "but we saw the steam roller this morning didnt we ?", "words": ["the", "this", "but", "we"]}, {"sent": "alright but first we hafta put them into piles .", "words": ["put", "into", "we", "first", "them", "but"]}, {"sent": "I know it is wet and the sun isnt sunny but we can have lotsof good fun that is funny .", "words": ["the", "wet", "can", "is", "we", "sun", "and", "have", "it", "good", "that", "but"]}, {"sent": "chicks peep crickets cheep horses neigh donkeys spray but rubber duckies dont say quack .", "words": ["dont", "say", "but"]}, {"sent": "but sometimes when Mommy goes by herself Mommy uses the basket .", "words": ["the", "by", "basket", "when", "but"]}, {"sent": "I think that would have got me better quicker but never mind .", "words": ["better", "think", "have", "would", "me", "that", "but"]}, {"sent": "but lets see if we can do it because if we did one foot we can probably do the other one .", "words": ["can", "do", "the", "if", "we", "because", "other", "foot", "it", "did", "see", "but"]}, {"sent": "but I dont think its assoonas that .", "words": ["dont", "think", "that", "but"]}, {"sent": "yeah but he got a king .", "words": ["he", "but", "a"]}, {"sent": "we usually slice it but we havent today .", "words": ["it", "but", "we"]}, {"sent": "I know but its not your turn .", "words": ["your", "not", "but"]}, {"sent": "but you probably enjoyed it more on the train .", "words": ["the", "train", "you", "more", "it", "on", "but"]}, {"sent": "but what day ?", "words": ["what", "but"]}, {"sent": "but not too near the flowers because theyve got to stand in water .", "words": ["the", "water", "to", "stand", "in", "but", "too", "not", "because"]}, {"sent": "xxx but I dont know k .", "words": ["dont", "but"]}, {"sent": "but the farmers had moved away the barn was abandoned and the granary stood empty .", "words": ["the", "was", "and", "empty", "away", "but"]}, {"sent": "you werent very well but you werent that poorly .", "words": ["that", "but", "you"]}, {"sent": "she ne she very rarely plays with them if she does its only because Micah had it and then shell want it but then she loses interest drop it and the minute Micah picks it up again shell go and get it .", "words": ["the", "does", "up", "if", "go", "with", "them", "and", "get", "it", "drop", "but", "then", "she", "because"]}, {"sent": "another one we didnt do this morning but another favorite that we learnt from Julian .", "words": ["do", "we", "another", "this", "that", "but"]}, {"sent": "but youve got sausages .", "words": ["but"]}, {"sent": "but thats not Dominic .", "words": ["not", "but"]}, {"sent": "I mean she played with the food stuff last time but she was more interested last time in the stuffed animals I think .", "words": ["the", "think", "was", "last", "with", "food", "more", "in", "she", "but"]}, {"sent": "well Ive tried to get it out but it wont come .", "words": ["to", "get", "it", "out", "but"]}, {"sent": "here maybe we should dig it up a little bit oh but these might be little plants here .", "words": ["up", "these", "we", "be", "little", "here", "it", "but", "a"]}, {"sent": "I know but thats because you keep hitting it with your leg you hafta be careful honey .", "words": ["your", "be", "with", "you", "it", "leg", "but", "careful", "because"]}, {"sent": "but yeah he they have dolls at home .", "words": ["have", "they", "he", "at", "but", "home"]}, {"sent": "we should sing a song about carrots but I dont know one .", "words": ["we", "sing", "dont", "carrots", "about", "but", "a"]}, {"sent": "but the Briggs and make sure its that engine youknow .", "words": ["the", "and", "make", "that", "but"]}, {"sent": "but you dont needta Gra .", "words": ["dont", "but", "you"]}, {"sent": "it was very messy but it looked looked like it was good fun .", "words": ["was", "like", "it", "good", "but"]}, {"sent": "yeah but dont ask me ask ah ask Missus Blasingame .", "words": ["dont", "me", "but"]}, {"sent": "drilling I know but whats he drilling you told me what you thought it was before .", "words": ["what", "was", "you", "it", "he", "me", "but"]}, {"sent": "okay but the light is uh .", "words": ["the", "is", "but", "light"]}, {"sent": "but dont break it off .", "words": ["off", "dont", "it", "break", "but"]}, {"sent": "but I like Saint Thomas and I like uh Haiti .", "words": ["like", "but", "and"]}, {"sent": "oh j for jar good girl but Id j for jump .", "words": ["for", "jump", "good", "jar", "but"]}, {"sent": "but Purdies a tabby and Mollys quite different from Purdie .", "words": ["and", "but", "a"]}, {"sent": "but I think .", "words": ["think", "but"]}, {"sent": "I know but .", "words": ["but"]}, {"sent": "this is your sticker book but you ya already did them all .", "words": ["all", "your", "is", "them", "you", "this", "did", "book", "but"]}, {"sent": "but you know where Im going dont you ?", "words": ["where", "dont", "but", "you"]}, {"sent": "Mummy has just made a cup of tea but Ill hafta wait a few minutes for it to brew .", "words": ["cup", "wait", "for", "of", "to", "it", "but", "a"]}, {"sent": "but I put it back .", "words": ["put", "it", "back", "but"]}, {"sent": "but whats the baby .", "words": ["the", "but"]}, {"sent": "I can play a flute but I cant play a crown .", "words": ["can", "play", "but", "a"]}, {"sent": "yeah thats the house but wheres the kitchen ?", "words": ["the", "house", "kitchen", "but"]}, {"sent": "but Im trying to think if he actually has a doll .", "words": ["think", "if", "to", "doll", "he", "but", "a"]}, {"sent": "I dont know exactly what were gonna do but we hafta figure out another way for you to have a bathie .", "words": ["what", "do", "for", "we", "to", "you", "another", "dont", "have", "out", "were", "but", "a"]}, {"sent": "but do you know what I use that one for ?", "words": ["do", "what", "for", "you", "that", "but"]}, {"sent": "probably hell take mommy but not you .", "words": ["take", "not", "but", "you"]}, {"sent": "but I mean her vocabulary has always been marvelous .", "words": ["her", "but"]}, {"sent": "right but you dont want to bring up that its inappropriate for her to do that ?", "words": ["up", "do", "for", "to", "her", "bring", "you", "dont", "want to", "that", "but"]}, {"sent": "but did you play out at school ?", "words": ["school", "play", "you", "did", "out", "at", "but"]}, {"sent": "youll hafta just hafta see well look around a little bit but nothing scary .", "words": ["look", "little", "around", "see", "but", "a"]}, {"sent": "but I remember as a child you really got excited by them .", "words": ["them", "you", "by", "but", "a"]}, {"sent": "but look can you see that window over there ?", "words": ["can", "you", "there", "look", "over", "see", "window", "that", "but"]}, {"sent": "but you looked very sweet in it didnt you ?", "words": ["in", "it", "but", "you"]}, {"sent": "but we only did two verses .", "words": ["did", "but", "we"]}, {"sent": "I I know its a neck but whats this bone called here ?", "words": ["here", "this", "but", "a"]}, {"sent": "but I think normally I wouldve had a hard time with the babies .", "words": ["the", "think", "with", "hard", "but", "a"]}, {"sent": "but I needta do your shoelace .", "words": ["do", "your", "but"]}, {"sent": "not bringing you .", "words": ["not", "you"]}, {"sent": "Im not scared .", "words": ["not", "scared"]}, {"sent": "I might not .", "words": ["not"]}, {"sent": "yeah I mean a handful like the turkey and the peacock um there are a whole bunch in here that we dont have because theyre not the same collection .", "words": ["the", "we", "like", "and", "here", "there", "in", "are", "dont", "have", "same", "turkey", "that", "not", "because", "a"]}, {"sent": "because wed like the girl because were getting too tired of changing boys not girls .", "words": ["the", "of", "like", "tired", "too", "were", "not", "because"]}, {"sent": "oh no that is not all .", "words": ["all", "that", "not", "is"]}, {"sent": "Ive not gone swimming .", "words": ["not"]}, {"sent": "he did not write in your book .", "words": ["your", "write", "in", "he", "did", "book", "not"]}, {"sent": "well Ill try not to worry .", "words": ["to", "try", "not"]}, {"sent": "dollys not well ?", "words": ["not"]}, {"sent": "not always .", "words": ["not"]}, {"sent": "not sure really .", "words": ["not"]}, {"sent": "was that newspaper not for Daddy ?", "words": ["was", "for", "that", "not"]}, {"sent": "no thats not growing .", "words": ["not"]}, {"sent": "not yet .", "words": ["not"]}, {"sent": "not sposta spill it .", "words": ["spill", "it", "not"]}, {"sent": "not your feet .", "words": ["your", "not"]}, {"sent": "its not the same as that is it ?", "words": ["the", "is", "it", "same", "that", "not"]}, {"sent": "why not ?", "words": ["not", "why"]}, {"sent": "Im not putting these back very well am I ?", "words": ["these", "am", "not", "back"]}, {"sent": "youre not going to give him any but I bet he would .", "words": ["to", "any", "him", "he", "would", "give", "not", "but"]}, {"sent": "and Im not I told her if she hasta pass out balloons she only has twelve balloons .", "words": ["if", "not", "and", "out", "her", "she"]}, {"sent": "were not theyre not gonna take them out Peter .", "words": ["them", "out", "take", "not", "were"]}, {"sent": "if shes not going to she lives at .", "words": ["if", "to", "she", "at", "not"]}, {"sent": "thats not very nice .", "words": ["nice", "not"]}, {"sent": "its not enough roofs .", "words": ["not"]}, {"sent": "and thats not round at all is it ?", "words": ["all", "is", "and", "it", "at", "not"]}, {"sent": "hello youre the not dirty one .", "words": ["the", "dirty", "not"]}, {"sent": "because thats not nice .", "words": ["nice", "not", "because"]}, {"sent": "not real ?", "words": ["not"]}, {"sent": "not yet .", "words": ["not"]}, {"sent": "shes not responding though .", "words": ["not"]}, {"sent": "its a jigsaw not a train Thomas .", "words": ["train", "not", "a"]}, {"sent": "why not ?", "words": ["not", "why"]}, {"sent": "thats not the school bus .", "words": ["the", "school", "bus", "not"]}, {"sent": "Im not good at football .", "words": ["not", "good", "at"]}, {"sent": "not no not in the box in the bag somewhere in that bag .", "words": ["the", "box", "in", "that", "not"]}, {"sent": "why not ?", "words": ["not", "why"]}, {"sent": "hes not home alone is he ?", "words": ["he", "not", "is", "home"]}, {"sent": "not yet ?", "words": ["not"]}, {"sent": "Im not going to tell you but I know .", "words": ["to", "not", "but", "you"]}, {"sent": "weve not seen that program for a long time .", "words": ["for", "long", "that", "not", "a"]}, {"sent": "no not hi .", "words": ["not"]}, {"sent": "theres not theres one .", "words": ["not"]}, {"sent": "no youre not done .", "words": ["not"]}, {"sent": "youre not tipping them out .", "words": ["them", "not", "out"]}, {"sent": "not deux .", "words": ["not"]}, {"sent": "its not ab .", "words": ["not"]}, {"sent": "theyre not on the road any more are they ?", "words": ["the", "any", "they", "more", "are", "on", "not"]}, {"sent": "youre not .", "words": ["not"]}, {"sent": "no its not bolts .", "words": ["not"]}, {"sent": "hes not helping to pick his toys up .", "words": ["up", "pick", "to", "his", "not"]}, {"sent": "why not ?", "words": ["not", "why"]}, {"sent": "are they not miserable ?", "words": ["they", "are", "not"]}, {"sent": "yes what are we going to do grandpa carrying the spade thats not very likely Im afraid .", "words": ["the", "what", "do", "we", "to", "are", "not"]}, {"sent": "hes not asleep .", "words": ["asleep", "not"]}, {"sent": "I dont know what that is Id call that a piece of pork but Im not exactly sure .", "words": ["what", "is", "of", "dont", "that", "not", "but", "a"]}, {"sent": "Im not that cruel .", "words": ["that", "not"]}, {"sent": "and whos the little girl who says shes finished when shes not and the teachers get cross ?", "words": ["the", "little", "and", "get", "who", "when", "not"]}, {"sent": "its not a strawberry is it ?", "words": ["strawberry", "is", "it", "not", "a"]}, {"sent": "now its not ready yet Eleanor .", "words": ["not"]}, {"sent": "Im not sure what youre talking about now .", "words": ["what", "not", "about"]}, {"sent": "she might not .", "words": ["not", "she"]}, {"sent": "shes not whinging at all at the moment .", "words": ["the", "all", "not", "at"]}, {"sent": "theyre not gonna get torn when Fra is looking at it that is for sure .", "words": ["for", "is", "get", "that", "it", "at", "when", "not"]}, {"sent": "weve not trimmed it .", "words": ["it", "not"]}, {"sent": "youre not putting it in the dustbin .", "words": ["the", "in", "it", "not"]}, {"sent": "the suns shining but its not really very warm .", "words": ["the", "not", "but"]}, {"sent": "Im not gonna be your friend .", "words": ["be", "your", "not"]}, {"sent": "thats not for babies no thats not for babies .", "words": ["for", "not"]}, {"sent": "you not want the buttons done up ?", "words": ["the", "up", "not", "you"]}, {"sent": "hes not silly .", "words": ["not"]}, {"sent": "get that off you xxx youll not be wearing that .", "words": ["off", "be", "you", "get", "that", "not"]}, {"sent": "its not a hat is it ?", "words": ["is", "it", "hat", "not", "a"]}, {"sent": "no not at all .", "words": ["all", "not", "at"]}, {"sent": "is this book about a lion no its not .", "words": ["is", "this", "about", "book", "lion", "not", "a"]}, {"sent": "well said TopCat if I cant clang this alleys not the home for me .", "words": ["the", "for", "if", "this", "me", "not", "home"]}, {"sent": "right you know today Im not going to take you to Lwwws house .", "words": ["to", "you", "house", "take", "not"]}, {"sent": "is thats not a good picture of the fish isnt there a better one ?", "words": ["the", "better", "is", "of", "picture", "there", "good", "fish", "not", "a"]}, {"sent": "Im not sure if we did .", "words": ["if", "did", "not", "we"]}, {"sent": "not tomorrow pal .", "words": ["not"]}, {"sent": "not in your mouth Thomas .", "words": ["in", "mouth", "your", "not"]}, {"sent": "its not gonna fit in there .", "words": ["in", "fit", "not", "there"]}, {"sent": "not real .", "words": ["not"]}, {"sent": "were not having salad .", "words": ["were", "not"]}, {"sent": "the Hulks not for Halloween ?", "words": ["the", "for", "not"]}, {"sent": "thats Daddys job not Mommys .", "words": ["not"]}, {"sent": "you did not .", "words": ["not", "did", "you"]}, {"sent": "well why not ?", "words": ["not", "why"]}, {"sent": "oh its not gonna stay there is it ?", "words": ["is", "there", "it", "stay", "not"]}, {"sent": "youre not doing the shopping very well .", "words": ["the", "not"]}, {"sent": "its not far .", "words": ["not"]}, {"sent": "I hope not .", "words": ["not"]}, {"sent": "it might not have been .", "words": ["have", "it", "not"]}, {"sent": "theres not a horse in there .", "words": ["there", "in", "horse", "not", "a"]}, {"sent": "yeah sometimes you lose your motivation if youre not close enough to something huh ?", "words": ["your", "if", "to", "you", "close", "not"]}, {"sent": "thats not called play dough its called clay .", "words": ["play", "play dough", "not"]}, {"sent": "no no youre not gonna play with that .", "words": ["that", "play", "with", "not"]}, {"sent": "ones saying do not enter .", "words": ["do", "not"]}, {"sent": "not sure where the telephone is .", "words": ["where", "the", "is", "not", "telephone"]}, {"sent": "touch the green lily pad .", "words": ["the", "green", "touch"]}, {"sent": "dese are for you and Usher but theyre too hot to touch yet okay ?", "words": ["for", "to", "touch", "and", "you", "hot", "are", "too", "but"]}, {"sent": "Violet dont touch xxx .", "words": ["dont", "touch"]}, {"sent": "touch yes .", "words": ["touch"]}, {"sent": "dont touch her stuff .", "words": ["dont", "her", "touch"]}, {"sent": "no dont touch anything .", "words": ["dont", "touch"]}, {"sent": "youre not to touch the television .", "words": ["the", "to", "not", "touch"]}, {"sent": "those cant touch that .", "words": ["that", "those", "touch"]}, {"sent": "Mommy doesnt want you to touch .", "words": ["to", "touch", "you"]}, {"sent": "touch that bit .", "words": ["that", "touch"]}, {"sent": "I could I could touch my mothers shoulder without tippyingtoe .", "words": ["my", "could", "touch", "shoulder"]}, {"sent": "ya he can touch it .", "words": ["can", "it", "he", "touch"]}, {"sent": "dont touch the bowl because its hot .", "words": ["the", "bowl", "touch", "dont", "hot", "because"]}, {"sent": "dont touch it .", "words": ["dont", "it", "touch"]}, {"sent": "careful dont touch it .", "words": ["dont", "it", "touch", "careful"]}, {"sent": "yeah dont touch it now .", "words": ["dont", "it", "touch"]}, {"sent": "whoever touch it .", "words": ["it", "touch"]}, {"sent": "just look dont touch .", "words": ["dont", "touch", "look"]}, {"sent": "do you wanna touch it now ?", "words": ["do", "touch", "you", "it", "wanna"]}, {"sent": "and Fraser doesnt touch them does he ?", "words": ["does", "them", "touch", "and", "he"]}, {"sent": "and do it she wont let me touch them .", "words": ["do", "them", "touch", "and", "it", "me", "she"]}, {"sent": "okay Ill touch his tongue .", "words": ["his", "tongue", "touch"]}, {"sent": "because you mustnt touch .", "words": ["touch", "because", "you"]}, {"sent": "you touch mine .", "words": ["mine", "touch", "you"]}, {"sent": "if you touch those passports youll be in serious trouble .", "words": ["those", "if", "be", "touch", "you", "in"]}, {"sent": "dont touch those .", "words": ["dont", "those", "touch"]}, {"sent": "well Im picking up the part that didnt touch the floor .", "words": ["up", "the", "that", "touch"]}, {"sent": "now move away from here so Mama can touch this .", "words": ["can", "so", "touch", "here", "this", "away"]}, {"sent": "no you mustnt touch that chair .", "words": ["chair", "that", "touch", "you"]}, {"sent": "okay I wont touch it .", "words": ["it", "touch"]}, {"sent": "and you wont be allowed to touch it Thomas .", "words": ["to", "be", "touch", "you", "and", "it"]}, {"sent": "never touch her face .", "words": ["her", "face", "touch"]}, {"sent": "you mustnt touch her .", "words": ["her", "touch", "you"]}, {"sent": "touch your fingers like that .", "words": ["that", "your", "like", "touch"]}, {"sent": "you can touch them gently .", "words": ["can", "them", "touch", "you"]}, {"sent": "touch your nose .", "words": ["your", "touch", "nose"]}, {"sent": "dont touch it .", "words": ["dont", "it", "touch"]}, {"sent": "dont touch him will you ?", "words": ["touch", "you", "dont", "him", "will"]}, {"sent": "oh I think you made Pos head touch yours didnt you ?", "words": ["think", "head", "touch", "you"]}, {"sent": "no you may not touch all those wires no no no no .", "words": ["all", "those", "touch", "you", "not"]}, {"sent": "no touch .", "words": ["touch"]}, {"sent": "there its very important that you dont touch that pan .", "words": ["touch", "you", "dont", "there", "that"]}, {"sent": "but just dont touch that because youll break it .", "words": ["break", "touch", "dont", "it", "but", "that", "because"]}, {"sent": "Mummys pen ooh mustnt touch .", "words": ["touch", "pen"]}, {"sent": "you say it so dont hafta touch it .", "words": ["so", "say", "touch", "you", "dont", "it"]}, {"sent": "oh heres a touch book .", "words": ["book", "touch", "a"]}, {"sent": "dont touch it .", "words": ["dont", "it", "touch"]}, {"sent": "dont touch it with that .", "words": ["with", "touch", "dont", "it", "that"]}, {"sent": "say dont touch when its hot .", "words": ["say", "touch", "dont", "hot", "when"]}, {"sent": "its only a picture you can touch it if you like .", "words": ["can", "if", "picture", "like", "touch", "you", "it", "a"]}, {"sent": "yup and shes at home shes depending on how tired she is shes shell reach out and touch the pages .", "words": ["the", "how", "is", "touch", "and", "tired", "out", "at", "on", "she", "home"]}, {"sent": "so if you see a bee or a wasp outside you mustnt touch it .", "words": ["if", "so", "outside", "bee", "touch", "you", "it", "see", "a"]}, {"sent": "dont touch it now .", "words": ["dont", "it", "touch"]}, {"sent": "dont touch that .", "words": ["dont", "that", "touch"]}, {"sent": "Diandra he asked you not to touch them .", "words": ["to", "them", "touch", "you", "he", "not"]}, {"sent": "dont touch David .", "words": ["dont", "touch"]}, {"sent": "sweetie please dont touch .", "words": ["dont", "touch"]}, {"sent": "no dont touch them .", "words": ["dont", "them", "touch"]}, {"sent": "dont touch the actual pastry now Thomas please .", "words": ["the", "dont", "touch"]}, {"sent": "I said dont touch .", "words": ["dont", "touch"]}, {"sent": "no touch .", "words": ["touch"]}, {"sent": "I dont touch them and eat because those are tough .", "words": ["those", "them", "touch", "and", "eat", "dont", "are", "because"]}, {"sent": "no you dont touch Purdies food do you ?", "words": ["do", "touch", "you", "dont", "food"]}, {"sent": "no dont touch dont touch dont touch .", "words": ["dont", "touch"]}, {"sent": "dont touch because its very very hot .", "words": ["dont", "hot", "because", "touch"]}, {"sent": "dont touch that Sarah .", "words": ["dont", "that", "touch"]}, {"sent": "gonna touch it okay you wanna turn the page and touch it ?", "words": ["the", "touch", "you", "and", "it", "wanna"]}, {"sent": "yeah lets not touch the microphone too much .", "words": ["the", "touch", "too", "much", "not"]}, {"sent": "I have several like this one the touch and feel and then I have several versions of goodnight moon and some doctor Seuss books .", "words": ["the", "moon", "of", "like", "touch", "and", "have", "this", "some", "then"]}, {"sent": "you mustnt touch your nappie .", "words": ["your", "touch", "you"]}, {"sent": "touch it .", "words": ["it", "touch"]}, {"sent": "but mommys gonna fix this thing this isnt a thing for everyone to touch .", "words": ["for", "fix", "to", "touch", "this", "but", "a"]}, {"sent": "I dont think so but it may get hurt if you touch those things that are spinning around .", "words": ["think", "those", "if", "hurt", "so", "touch", "you", "get", "dont", "it", "are", "around", "that", "but"]}, {"sent": "yeah dont touch okay ?", "words": ["dont", "touch"]}, {"sent": "dont touch it .", "words": ["dont", "it", "touch"]}, {"sent": "dont touch anything .", "words": ["dont", "touch"]}, {"sent": "yes sothat you dont touch things .", "words": ["dont", "touch", "you"]}, {"sent": "dont touch Helens buttons .", "words": ["dont", "touch"]}, {"sent": "we mustnt touch the pan .", "words": ["the", "touch", "we"]}, {"sent": "touch it again .", "words": ["it", "touch"]}, {"sent": "turn the handle that way so that you cant touch it .", "words": ["the", "so", "touch", "you", "it", "that"]}, {"sent": "I willnt touch them .", "words": ["them", "touch"]}, {"sent": "dont touch that .", "words": ["dont", "that", "touch"]}, {"sent": "dont touch it though .", "words": ["dont", "it", "touch"]}, {"sent": "thats water but I dont want you to touch that okay ?", "words": ["to", "touch", "you", "dont", "water", "that", "but"]}, {"sent": "it goes all the way down so she can touch it doesnt it Lara ?", "words": ["the", "all", "can", "so", "touch", "it", "down", "she"]}, {"sent": "touch fur .", "words": ["touch"]}, {"sent": "no touch .", "words": ["touch"]}, {"sent": "mustnt touch though .", "words": ["touch"]}, {"sent": "dont touch them .", "words": ["dont", "them", "touch"]}, {"sent": "may I get touch it ?", "words": ["get", "it", "touch"]}, {"sent": "dont touch it .", "words": ["dont", "it", "touch"]}, {"sent": "dont touch .", "words": ["dont", "touch"]}, {"sent": "its gone where Fraser cant touch it .", "words": ["where", "it", "touch"]}, {"sent": "you think hes gonna touch the eggs ?", "words": ["the", "think", "touch", "you"]}, {"sent": "you mustnt touch the tape recorder .", "words": ["the", "tape", "touch", "you"]}, {"sent": "dont touch it though .", "words": ["dont", "it", "touch"]}, {"sent": "dont touch it .", "words": ["dont", "it", "touch"]}, {"sent": "you mustnt touch those .", "words": ["those", "touch", "you"]}, {"sent": "Mommy is very happy to see you eating those vanilla wafers .", "words": ["those", "happy", "is", "to", "vanilla", "you", "see"]}, {"sent": "that looks like those P and O um ferry train carriages we see on BurnageStation .", "words": ["those", "train", "we", "like", "and", "on", "see", "that"]}, {"sent": "okay oh you want him to wear those shoes ?", "words": ["to", "him", "those", "you"]}, {"sent": "those are nipples .", "words": ["are", "those"]}, {"sent": "dont pull those wires Sam .", "words": ["dont", "those", "pull"]}, {"sent": "what what are those ?", "words": ["what", "are", "those"]}, {"sent": "those Wiggles are very catchy catchy catchy .", "words": ["are", "those"]}, {"sent": "oh he doesnt like those cookies .", "words": ["he", "like", "those"]}, {"sent": "I saw those animals none of them sheep well that seemed better than falling asleep I stretched and whispered boy this is fun .", "words": ["better", "those", "is", "of", "none", "them", "and", "asleep", "this", "sheep", "that"]}, {"sent": "those are the babys eyes .", "words": ["the", "are", "those"]}, {"sent": "I actually bought some of those .", "words": ["of", "some", "those"]}, {"sent": "I gotta get him one of those .", "words": ["get", "of", "him", "those"]}, {"sent": "theyre getting very pink arent they those worms .", "words": ["they", "those"]}, {"sent": "I made those biscuits for you Abe .", "words": ["for", "those", "you"]}, {"sent": "those are the balls that the girls holding .", "words": ["the", "that", "are", "those"]}, {"sent": "xxx those are grapes .", "words": ["grapes", "are", "those"]}, {"sent": "those are magnets .", "words": ["are", "those"]}, {"sent": "the birds wearing a necklace made from those buttons .", "words": ["the", "necklace", "those", "a"]}, {"sent": "those little trees .", "words": ["those", "little"]}, {"sent": "I dont need those bottles .", "words": ["dont", "need", "those"]}, {"sent": "what about all those seats over there ?", "words": ["all", "what", "those", "there", "about", "over"]}, {"sent": "look at all those toys in there .", "words": ["all", "look", "those", "there", "in", "at"]}, {"sent": "what are those ?", "words": ["what", "are", "those"]}, {"sent": "those bridges that come up .", "words": ["up", "that", "those"]}, {"sent": "you stacked those up very good .", "words": ["up", "those", "good", "you"]}, {"sent": "Ces if you dont sit up Im taking those paper off of you .", "words": ["up", "off", "those", "if", "of", "paper", "you", "dont", "sit"]}, {"sent": "those are all the same colors .", "words": ["the", "all", "those", "are", "same"]}, {"sent": "I think those .", "words": ["think", "those"]}, {"sent": "so all those coats can go on the coat stand .", "words": ["can", "all", "the", "those", "go", "so", "stand", "on", "coat"]}, {"sent": "look we never saw one of those before did we ?", "words": ["those", "we", "of", "did", "look"]}, {"sent": "you leave those alone .", "words": ["those", "you"]}, {"sent": "put those in .", "words": ["in", "put", "those"]}, {"sent": "put those glass in the sink easy .", "words": ["the", "put", "those", "in", "sink", "glass"]}, {"sent": "you wanna take those out ?", "words": ["those", "you", "out", "wanna", "take"]}, {"sent": "lemme see those fingers .", "words": ["see", "those"]}, {"sent": "give her those back .", "words": ["give", "her", "back", "those"]}, {"sent": "are those her toes ?", "words": ["her", "are", "those"]}, {"sent": "not connecting with those animals are we honey .", "words": ["those", "we", "with", "are", "not"]}, {"sent": "those arent lions .", "words": ["those"]}, {"sent": "you have those and Ill have those .", "words": ["have", "those", "and", "you"]}, {"sent": "you liked those boys didnt you ?", "words": ["those", "you"]}, {"sent": "are those your ones ?", "words": ["your", "are", "those"]}, {"sent": "I dont know where those pieces are .", "words": ["where", "dont", "are", "those"]}, {"sent": "I bet that all those other pieces are xxx .", "words": ["all", "those", "other", "are", "that"]}, {"sent": "so with the books did he does he have any of those books at home or anything like that ?", "words": ["the", "does", "those", "of", "so", "with", "any", "like", "have", "he", "did", "at", "that", "home"]}, {"sent": "those are mine .", "words": ["mine", "are", "those"]}, {"sent": "those slugs eat Mummys plants Thomas .", "words": ["eat", "those"]}, {"sent": "and I knew that they werent midget gems because those are those little jelly sweets arent they that you like ?", "words": ["jelly", "those", "like", "and", "little", "they", "you", "are", "that", "because"]}, {"sent": "those are paints .", "words": ["are", "those"]}, {"sent": "they needta they needta go away really those .", "words": ["they", "away", "those", "go"]}, {"sent": "you know what those are ?", "words": ["what", "are", "those", "you"]}, {"sent": "I had one of those .", "words": ["of", "those"]}, {"sent": "at home shes playing more with um we have one of those FisherPrice tables .", "words": ["those", "we", "of", "with", "have", "more", "at", "home"]}, {"sent": "those are pigs .", "words": ["are", "those"]}, {"sent": "hey look at those painted houses .", "words": ["at", "those", "look"]}, {"sent": "what are those ?", "words": ["what", "are", "those"]}, {"sent": "you gonna build a tower with those blocks .", "words": ["those", "with", "you", "build", "a"]}, {"sent": "thats why Mummys put those sweetpeas down the side there so that they they can grow really big .", "words": ["the", "can", "put", "those", "so", "there", "they", "big", "down", "that", "why"]}, {"sent": "Mummy where did you find all those Mummy ?", "words": ["where", "all", "those", "you", "did", "find"]}, {"sent": "have they got one of those things that they put on the back ?", "words": ["the", "put", "those", "back", "of", "have", "they", "on", "that"]}, {"sent": "do you know our boys couldnt pronounce those when they were little ?", "words": ["do", "those", "our", "you", "they", "little", "when", "were"]}, {"sent": "what you call those again ?", "words": ["what", "those", "you"]}, {"sent": "do you wanna do one of those ?", "words": ["do", "those", "of", "you", "wanna"]}, {"sent": "the day that shes not looking at those and she starts doing it like that .", "words": ["the", "those", "like", "and", "she", "it", "at", "that", "not"]}, {"sent": "its all those things huh ?", "words": ["all", "those"]}, {"sent": "those are the headlights .", "words": ["the", "are", "those"]}, {"sent": "what are those ?", "words": ["what", "are", "those"]}, {"sent": "thats one of those things you wouldnt take when I pumped .", "words": ["those", "when", "of", "you", "take"]}, {"sent": "lets wipe those tears .", "words": ["wipe", "those"]}, {"sent": "will you move those please .", "words": ["will", "those", "you"]}, {"sent": "those the videos ?", "words": ["the", "those"]}, {"sent": "how are you going to go to the xxx in those trousers ?", "words": ["the", "how", "those", "go", "to", "you", "in", "are"]}, {"sent": "no those are wires .", "words": ["are", "those"]}, {"sent": "theyre going to be busy in that garage if all those vans vans and trucks are for repair arent they ?", "words": ["all", "for", "those", "if", "garage", "to", "be", "and", "they", "in", "are", "that"]}, {"sent": "you know how to sing and talk and yell and whisper you know all those things .", "words": ["all", "how", "those", "to", "and", "sing", "you", "talk"]}, {"sent": "theyve got alotof hay havent they those animals .", "words": ["they", "those"]}, {"sent": "what are all those blocks in the doll house .", "words": ["the", "all", "what", "those", "doll", "in", "are", "house"]}, {"sent": "and you posted all those letters .", "words": ["all", "those", "you", "and"]}, {"sent": "those are mice .", "words": ["are", "those"]}, {"sent": "those have thorns cupcake .", "words": ["have", "those"]}, {"sent": "oh those are pennys .", "words": ["are", "those"]}, {"sent": "how dya like those shoes ?", "words": ["how", "those", "like"]}, {"sent": "oh thats the cherry one I dont like those .", "words": ["the", "dont", "those", "like"]}, {"sent": "are those delicious toes ?", "words": ["are", "those"]}, {"sent": "look at all those letters .", "words": ["all", "at", "those", "look"]}, {"sent": "nobody likes to hear you play those .", "words": ["those", "to", "play", "you", "hear"]}, {"sent": "those are colors huh ?", "words": ["are", "those"]}, {"sent": "xxx knock over those graham crackers you gonna be cleaning up .", "words": ["up", "those", "be", "you", "knock", "over"]}, {"sent": "no those are his glasses .", "words": ["his", "are", "those", "glasses"]}, {"sent": "yeah should we put those in there ?", "words": ["put", "those", "we", "there", "in"]}, {"sent": "all kind of bunnys and bears and everything ah what are those ?", "words": ["all", "what", "those", "of", "and", "are"]}, {"sent": "what color flowers are those ?", "words": ["what", "are", "those"]}, {"sent": "those have sides .", "words": ["have", "those"]}, {"sent": "those babies sure wake up a lot .", "words": ["up", "those", "a lot", "wake", "a"]}, {"sent": "putting those pants back on .", "words": ["on", "back", "those", "pants"]}, {"sent": "yeah we have we do waffles and peas and he eats those I think he might recognize those .", "words": ["do", "think", "those", "we", "peas", "and", "have", "he"]}, {"sent": "well actually those bits dont go together .", "words": ["dont", "those", "go"]}, {"sent": "I get all those ?", "words": ["get", "all", "those"]}, {"sent": "where are the socks ?", "words": ["where", "the", "are"]}, {"sent": "because these arent old ones are they ?", "words": ["old", "these", "they", "are", "because"]}, {"sent": "are you sitting down ?", "words": ["down", "are", "you"]}, {"sent": "now what are we going to put those into ?", "words": ["what", "put", "those", "into", "we", "to", "are"]}, {"sent": "there we are .", "words": ["there", "are", "we"]}, {"sent": "what are we doing ?", "words": ["what", "are", "we"]}, {"sent": "are you juggling ?", "words": ["are", "you"]}, {"sent": "ah are you giving clown a big cuddle ?", "words": ["are", "big", "a", "you"]}, {"sent": "are you enjoying that ?", "words": ["are", "that", "you"]}, {"sent": "are you a good boy ?", "words": ["are", "good", "a", "you"]}, {"sent": "are you an old bumblebee ?", "words": ["old", "an", "are", "you"]}, {"sent": "what are you doing ?", "words": ["what", "are", "you"]}, {"sent": "are you a big girl ?", "words": ["are", "big", "a", "you"]}, {"sent": "are you going to be nice ?", "words": ["to", "be", "nice", "you", "are"]}, {"sent": "where are the other keys Jack ?", "words": ["where", "the", "other", "are", "keys"]}, {"sent": "those are the major ones .", "words": ["the", "are", "those"]}, {"sent": "those are your favorite .", "words": ["your", "are", "those"]}, {"sent": "are you playing with my hair ?", "words": ["my", "with", "you", "are", "hair"]}, {"sent": "well I dont think there are any big ones left now .", "words": ["think", "any", "dont", "there", "are", "big"]}, {"sent": "where are you going ?", "words": ["where", "are", "you"]}, {"sent": "well what are you doing with that basket .", "words": ["what", "with", "you", "are", "basket", "that"]}, {"sent": "one of these are broke too now .", "words": ["of", "these", "are", "too"]}, {"sent": "how about in real life are you are any of these yyy .", "words": ["these", "how", "of", "any", "you", "in", "are", "about"]}, {"sent": "oh think his feet are cold ?", "words": ["his", "think", "are", "cold"]}, {"sent": "theyre all hiding and you hafta guess which ones they are .", "words": ["all", "you", "and", "they", "are", "which"]}, {"sent": "there you are then .", "words": ["there", "then", "are", "you"]}, {"sent": "are they presents ?", "words": ["they", "are"]}, {"sent": "do you know what the dwarves names are ?", "words": ["the", "do", "what", "you", "are"]}, {"sent": "what are them ?", "words": ["what", "are", "them"]}, {"sent": "do you remember where we are going after Christmas ?", "words": ["where", "do", "we", "you", "are"]}, {"sent": "those are all Barbie things they are .", "words": ["all", "are", "those", "they"]}, {"sent": "are you cold oh well I dont want you to be cold .", "words": ["to", "be", "you", "dont", "are", "cold"]}, {"sent": "is that what youre trying to do are you gonna build with those ?", "words": ["what", "do", "those", "is", "to", "with", "you", "are", "build", "that"]}, {"sent": "are you ?", "words": ["are", "you"]}, {"sent": "what are them ?", "words": ["what", "are", "them"]}, {"sent": "here you are .", "words": ["here", "are", "you"]}, {"sent": "and what are they doing ?", "words": ["what", "are", "they", "and"]}, {"sent": "are you running around is that what youre doing ?", "words": ["what", "is", "you", "are", "around", "that"]}, {"sent": "where are you going ?", "words": ["where", "are", "you"]}, {"sent": "what are you gonna do ?", "words": ["what", "do", "are", "you"]}, {"sent": "you are such a big guy xxx xxx xxx .", "words": ["are", "big", "a", "you"]}, {"sent": "because then there are two fields .", "words": ["there", "then", "are", "because"]}, {"sent": "are you not a monkey ?", "words": ["monkey", "you", "are", "not", "a"]}, {"sent": "you are .", "words": ["are", "you"]}, {"sent": "thats right thats where they are .", "words": ["where", "they", "are"]}, {"sent": "you are gonna see who ?", "words": ["see", "who", "are", "you"]}, {"sent": "cutting my leg off are you ?", "words": ["off", "my", "you", "are", "leg"]}, {"sent": "are you glad ?", "words": ["are", "you"]}, {"sent": "are you going to sleep now ?", "words": ["sleep", "to", "are", "you"]}, {"sent": "steering wheels are round .", "words": ["are"]}, {"sent": "are you gonna help me tidy up ?", "words": ["up", "help", "you", "are", "me"]}, {"sent": "what colors are these ?", "words": ["what", "are", "these"]}, {"sent": "are they all different colors ?", "words": ["all", "they", "are"]}, {"sent": "what are you making ?", "words": ["what", "are", "you"]}, {"sent": "are you moving the balloons and playing PeepO with Purdie ?", "words": ["the", "with", "and", "you", "are"]}, {"sent": "are you my baby ?", "words": ["my", "are", "you"]}, {"sent": "what are those funny little white things ?", "words": ["what", "those", "little", "are", "white"]}, {"sent": "I know they are .", "words": ["they", "are"]}, {"sent": "an his hands his hands are probably have alotof trouble with his hands .", "words": ["with", "have", "an", "are", "his"]}, {"sent": "what are we gonna do downstairs ?", "words": ["what", "do", "are", "we"]}, {"sent": "there we are .", "words": ["there", "are", "we"]}, {"sent": "youre doing that bow red are you ?", "words": ["red", "are", "that", "you"]}, {"sent": "theses are different from .", "words": ["are"]}, {"sent": "oh so youre possibly a farmer now are you ?", "words": ["so", "are", "you", "a"]}, {"sent": "those are bigger cups .", "words": ["are", "those"]}, {"sent": "are you being a cheeky boy ?", "words": ["are", "a", "you"]}, {"sent": "what kind of animals are they ?", "words": ["of", "what", "are", "they"]}, {"sent": "there you are .", "words": ["there", "are", "you"]}, {"sent": "are you whistling while you work ?", "words": ["are", "work", "you"]}, {"sent": "are you sure ?", "words": ["are", "you"]}, {"sent": "but the bulls are dangerous arent they .", "words": ["the", "they", "are", "but"]}, {"sent": "are they ?", "words": ["they", "are"]}, {"sent": "you are .", "words": ["are", "you"]}, {"sent": "what are you gonna do ?", "words": ["what", "do", "are", "you"]}, {"sent": "these are sardines .", "words": ["these", "are"]}, {"sent": "but most of them are animals hes never seen .", "words": ["of", "are", "them", "but"]}, {"sent": "are we having fun .", "words": ["are", "we"]}, {"sent": "are they happy because the Christmas tree fell down ?", "words": ["the", "tree", "happy", "they", "are", "down", "because"]}, {"sent": "socks are dirty .", "words": ["are", "dirty"]}, {"sent": "are you going in the bath ?", "words": ["the", "in", "are", "you"]}, {"sent": "they are closed .", "words": ["they", "are"]}, {"sent": "them what are you ?", "words": ["what", "are", "them", "you"]}, {"sent": "what else are you gonna make me ?", "words": ["what", "you", "are", "me", "make"]}, {"sent": "you aint sliced in half are you ?", "words": ["in", "are", "you"]}, {"sent": "there are buffalo buffalo bisons and a great big bear with wings .", "words": ["with", "and", "bear", "there", "are", "big", "a"]}, {"sent": "those are the spacemen right ?", "words": ["the", "are", "those"]}, {"sent": "are you gonna sing a song ?", "words": ["sing", "are", "a", "you"]}, {"sent": "what are you doing behind the sofa Thomas ?", "words": ["the", "what", "behind", "you", "are", "sofa"]}, {"sent": "are you a sleepy puss ?", "words": ["sleepy", "are", "a", "you"]}, {"sent": "the people who made the paint are the ones who decided to paint the pictures on them .", "words": ["the", "paint", "to", "them", "are", "who", "on"]}, {"sent": "trees that lose their leaves in winter are called deciduous trees .", "words": ["in", "are", "that", "their"]}, {"sent": "passengers are getting on .", "words": ["on", "are"]}, {"sent": "what other cool things are in there ?", "words": ["what", "other", "there", "in", "are"]}, {"sent": "what are those ?", "words": ["what", "are", "those"]}, {"sent": "there we are put the jigsaws there .", "words": ["the", "put", "we", "there", "are"]}, {"sent": "are you going in the bath again ?", "words": ["the", "in", "are", "you"]}, {"sent": "what are you doing ?", "words": ["what", "are", "you"]}, {"sent": "where are you going ?", "words": ["where", "are", "you"]}, {"sent": "are they ?", "words": ["they", "are"]}, {"sent": "are you gonna give him the cake this time ?", "words": ["the", "cake", "you", "this", "him", "are", "give"]}, {"sent": "hm what should we give you ?", "words": ["what", "give", "you", "we"]}, {"sent": "give me some pants please .", "words": ["me", "give", "pants", "some"]}, {"sent": "give your toes give your toes a chance to breathe .", "words": ["to", "give", "your", "a"]}, {"sent": "alright how about if I give you the wagon ?", "words": ["the", "how", "if", "you", "about", "give"]}, {"sent": "you can give him my hair .", "words": ["can", "my", "you", "him", "give", "hair"]}, {"sent": "*VI1: uh Ill give you an Ill give you an estimate when I .", "words": ["give", "when", "an", "you"]}, {"sent": "Ill give you a spanking .", "words": ["give", "a", "you"]}, {"sent": "just give me one of yours then .", "words": ["of", "me", "give", "then"]}, {"sent": "you going to give back to Ursula now .", "words": ["to", "give", "back", "you"]}, {"sent": "Im gonna give you ten seconds to give me some bread otherwise Im not gonna shop here again .", "words": ["bread", "to", "you", "here", "me", "give", "not", "some"]}, {"sent": "are you gonna give her a bath ?", "words": ["you", "are", "give", "her", "a"]}, {"sent": "lets give some more to your baby .", "words": ["your", "to", "more", "some", "give"]}, {"sent": "can we give some the cup to the baby ?", "words": ["can", "the", "cup", "we", "to", "some", "give"]}, {"sent": "Im not picking it up until you give it to me properly .", "words": ["up", "to", "you", "it", "me", "give", "not"]}, {"sent": "well give him this because Purdies only going to waste it .", "words": ["to", "this", "him", "it", "give", "because"]}, {"sent": "were going to give her some juice .", "words": ["to", "her", "some", "give", "juice", "were"]}, {"sent": "the doggies coming to give him a kiss .", "words": ["the", "to", "him", "give", "kiss", "a"]}, {"sent": "give .", "words": ["give"]}, {"sent": "give them to Mummy because theyll stick on the carpet .", "words": ["the", "stick", "to", "them", "on", "give", "because"]}, {"sent": "did you give it your best effort asked his mother .", "words": ["your", "you", "it", "did", "his", "give"]}, {"sent": "give that to Mummy .", "words": ["to", "give", "that"]}, {"sent": "give me your hand .", "words": ["me", "give", "your", "hand"]}, {"sent": "give it to me .", "words": ["to", "give", "it", "me"]}, {"sent": "did you give your Mommy a present for Mothers day ?", "words": ["for", "your", "present", "you", "did", "give", "a"]}, {"sent": "you gonna give it to me thank you .", "words": ["to", "you", "it", "me", "give"]}, {"sent": "give dolly a drink ?", "words": ["give", "drink", "a"]}, {"sent": "give me the dogs cup .", "words": ["the", "me", "give", "cup"]}, {"sent": "I try and give you what looks like the good piece of toast and the same for Laura .", "words": ["the", "what", "for", "toast", "of", "like", "you", "and", "same", "good", "give", "try"]}, {"sent": "so Sarah said Gail doesnt give me any money .", "words": ["money", "so", "any", "me", "give"]}, {"sent": "ah give him to Clifford .", "words": ["to", "give", "him"]}, {"sent": "is he gonna give him some medicine and stuff huh ?", "words": ["is", "medicine", "and", "him", "he", "some", "give"]}, {"sent": "to give ?", "words": ["to", "give"]}, {"sent": "lets give Daddy some wheat germ .", "words": ["some", "give"]}, {"sent": "your Mommy would have a fit if she saw me give you Tab .", "words": ["your", "if", "fit", "you", "have", "would", "me", "give", "she", "a"]}, {"sent": "what can I give you ?", "words": ["can", "what", "give", "you"]}, {"sent": "youre going to give her some cups .", "words": ["to", "give", "her", "some"]}, {"sent": "what does the greengrocer give you ?", "words": ["the", "what", "does", "you", "give"]}, {"sent": "Jenny give me a kiss goodnight ?", "words": ["me", "give", "kiss", "a"]}, {"sent": "I want give this can you give this to Mama ?", "words": ["can", "to", "you", "this", "give"]}, {"sent": "you just give it a rub and give it back to me .", "words": ["back", "to", "and", "you", "it", "me", "give", "a"]}, {"sent": "watch Im gonna have give me the elephant .", "words": ["the", "watch", "elephant", "have", "me", "give"]}, {"sent": "I give her a bite of mine too .", "words": ["of", "mine", "too", "bite", "give", "her", "a"]}, {"sent": "and thats what they give you .", "words": ["what", "you", "and", "they", "give"]}, {"sent": "she wont have any to give to Robin and David .", "words": ["to", "any", "and", "have", "give", "she"]}, {"sent": "let me give him some eyes .", "words": ["me", "give", "him", "some"]}, {"sent": "are we gonna give them drinks ?", "words": ["give", "are", "them", "we"]}, {"sent": "Im not doing anything with panda unless you give it to me nicely .", "words": ["to", "with", "you", "it", "me", "give", "not"]}, {"sent": "shall we give them names ?", "words": ["give", "them", "we"]}, {"sent": "okay let me give you some tea ya ready ?", "words": ["me", "give", "some", "you"]}, {"sent": "you give that to me .", "words": ["to", "you", "me", "give", "that"]}, {"sent": "you gonna give that to Lisa ?", "words": ["to", "give", "that", "you"]}, {"sent": "are you gonna get her out to give her a wind ?", "words": ["to", "wind", "you", "get", "are", "out", "give", "her", "a"]}, {"sent": "a little pinch and then I give you a .", "words": ["then", "and", "little", "you", "give", "a"]}, {"sent": "what did Mrs Wood give you ?", "words": ["what", "give", "did", "you"]}, {"sent": "that will give you an even spread of tape .", "words": ["of", "tape", "you", "an", "will", "give", "that"]}, {"sent": "give it a scratch then ?", "words": ["give", "it", "then", "a"]}, {"sent": "xxx give me your hand and let me make xxx .", "words": ["your", "hand", "and", "me", "give", "make"]}, {"sent": "give it back .", "words": ["give", "it", "back"]}, {"sent": "we were sposta go to AuntieCwwws house to go and give her her birthday card but we didnt go did we ?", "words": ["go", "we", "to", "her", "and", "house", "did", "give", "were", "but"]}, {"sent": "give you a wipe .", "words": ["give", "wipe", "a", "you"]}, {"sent": "give me .", "words": ["me", "give"]}, {"sent": "and I give him two wee things of Disprol to take his temperature down .", "words": ["of", "to", "and", "him", "his", "give", "down", "take"]}, {"sent": "earlier on when I wanted to give you a hug on the rocking chair you wouldnt play .", "words": ["the", "to", "play", "rocking chair", "chair", "you", "hug", "on", "give", "when", "a"]}, {"sent": "can I give you a kiss ?", "words": ["can", "you", "give", "kiss", "a"]}, {"sent": "so whos gonna give the baby a bath ?", "words": ["the", "so", "give", "a"]}, {"sent": "then if its too noisy then Mummy will just give it a little mix on her own .", "words": ["give", "if", "noisy", "little", "on", "it", "too", "will", "then", "her", "a"]}, {"sent": "the other animals never give a moments thought to what is down there .", "words": ["the", "what", "is", "to", "other", "there", "give", "down", "a"]}, {"sent": "lets say I might hafta give your shoes back and get your old ones .", "words": ["old", "your", "back", "say", "and", "get", "give"]}, {"sent": "they give you it and my child needs an antibiotic .", "words": ["my", "and", "you", "they", "an", "it", "give"]}, {"sent": "lets give them a stroke .", "words": ["give", "them", "a"]}, {"sent": "how much did you give me ?", "words": ["how", "you", "did", "me", "give", "much"]}, {"sent": "sure will you give it back ?", "words": ["back", "you", "it", "will", "give"]}, {"sent": "give it to Mummy .", "words": ["to", "give", "it"]}, {"sent": "do want give me five ?", "words": ["do", "give", "me"]}, {"sent": "thats the thing give you a bit more peace of mind bit more freedom .", "words": ["the", "of", "you", "more", "give", "a"]}, {"sent": "give you a kiss later .", "words": ["give", "kiss", "a", "you"]}, {"sent": "no give him a minute .", "words": ["give", "him", "a"]}, {"sent": "give the baby a hug ?", "words": ["the", "give", "hug", "a"]}, {"sent": "are you going to give the baby a ride .", "words": ["the", "to", "you", "are", "give", "ride", "a"]}, {"sent": "Ill give you some smarties .", "words": ["some", "give", "you"]}, {"sent": "Ill give you will I give you a quick flash ?", "words": ["will", "give", "a", "you"]}, {"sent": "did Grandma give you Babybel cheese ?", "words": ["give", "cheese", "did", "you"]}, {"sent": "you tell me and Ill give it to you .", "words": ["to", "and", "you", "it", "me", "give"]}, {"sent": "oh will you give it xxx .", "words": ["will", "give", "it", "you"]}, {"sent": "and we had better give her a bath .", "words": ["better", "we", "and", "give", "her", "a"]}, {"sent": "give them to Cathy and she can open them herself .", "words": ["can", "open", "to", "them", "and", "give", "she"]}, {"sent": "what would you give Piggy from an Easter egg ?", "words": ["what", "egg", "you", "an", "would", "give"]}, {"sent": "Ill give you a pen .", "words": ["pen", "give", "a", "you"]}, {"sent": "give me the shampoo .", "words": ["the", "me", "give"]}, {"sent": "and who do you give it to ?", "words": ["do", "to", "you", "and", "it", "who", "give"]}, {"sent": "does it give you pretty fingers ?", "words": ["does", "you", "it", "pretty", "give"]}, {"sent": "give me your hand .", "words": ["me", "give", "your", "hand"]}, {"sent": "give me all the short ones .", "words": ["all", "me", "give", "the"]}, {"sent": "wait a minute give me your other arm so I can put your jersey on .", "words": ["can", "wait", "put", "your", "so", "other", "arm", "on", "me", "give", "a"]}, {"sent": "so in a little while Ill give you some .", "words": ["so", "little", "you", "in", "some", "give", "a"]}, {"sent": "should we give some some juice to the baby ?", "words": ["the", "we", "to", "some", "give", "juice"]}, {"sent": "here give me both of them so they could be in .", "words": ["of", "so", "be", "them", "here", "they", "in", "me", "give", "could"]}, {"sent": "give it to me .", "words": ["to", "give", "it", "me"]}, {"sent": "you can wear them for the party and then you must give them back .", "words": ["can", "the", "give", "for", "back", "them", "you", "party", "and", "then"]}, {"sent": "is that a good present ?", "words": ["present", "is", "good", "that", "a"]}, {"sent": "what makes you think its a present ?", "words": ["what", "think", "present", "you", "a"]}, {"sent": "its a birthday present .", "words": ["present", "a"]}, {"sent": "you wanna give him a present ?", "words": ["present", "you", "him", "wanna", "give", "a"]}, {"sent": "he has brought chipmunk a present a bunch of pansys .", "words": ["of", "he", "present", "a"]}, {"sent": "car for a birthday present .", "words": ["car", "for", "present", "a"]}, {"sent": "and this is the present .", "words": ["the", "present", "is", "and", "this"]}, {"sent": "I wanna give him a present .", "words": ["present", "him", "wanna", "give", "a"]}, {"sent": "here is the birthday present that Pierre and Maryse gave you .", "words": ["the", "present", "is", "and", "here", "you", "that"]}, {"sent": "and then when we left you got a little present do you remember what that was ?", "words": ["do", "what", "present", "we", "was", "you", "and", "little", "that", "then", "when", "a"]}, {"sent": "do you mean the lady that bought you the the present ?", "words": ["the", "do", "present", "you", "that"]}, {"sent": "well Thomas the bus driver could you please help me find the present for my little boy Thomas .", "words": ["the", "help", "for", "my", "present", "you", "little", "find", "me", "could", "bus"]}, {"sent": "what was the present you got today in the post ?", "words": ["the", "what", "present", "was", "you", "in"]}, {"sent": "a present .", "words": ["present", "a"]}, {"sent": "then were going to hafta take the other present away .", "words": ["the", "present", "to", "other", "away", "take", "then", "were"]}, {"sent": "for Michela and well bring her present and I thought we could go you know we could just go and it would be fun .", "words": ["for", "present", "we", "go", "be", "bring", "and", "you", "it", "would", "could", "her"]}, {"sent": "okay heres your present for you here you go .", "words": ["for", "your", "present", "go", "you", "here"]}, {"sent": "wheres my present ?", "words": ["my", "present"]}, {"sent": "and we put our present right there .", "words": ["put", "present", "we", "our", "and", "there"]}, {"sent": "a birthday present .", "words": ["present", "a"]}, {"sent": "to the big part of the present .", "words": ["the", "present", "of", "to", "big"]}, {"sent": "Lucy needs some help opening her present I think .", "words": ["help", "think", "present", "some", "her"]}, {"sent": "I got you a present .", "words": ["present", "a", "you"]}, {"sent": "oh you wanna open your present ?", "words": ["your", "present", "open", "you", "wanna"]}, {"sent": "it was a nice present for a while but .", "words": ["for", "present", "was", "nice", "it", "but", "a"]}, {"sent": "little present .", "words": ["present", "little"]}, {"sent": "this is your Christmas present to Grandma isnt it ?", "words": ["your", "present", "is", "to", "this", "it"]}, {"sent": "heres your present .", "words": ["your", "present"]}, {"sent": "yeah smurfs got a present hasnt he ?", "words": ["he", "present", "a"]}, {"sent": "holding a big present isnt she ?", "words": ["big", "she", "present", "a"]}, {"sent": "and Santa give me present .", "words": ["me", "give", "present", "and"]}, {"sent": "a present ?", "words": ["present", "a"]}, {"sent": "I have a present for you .", "words": ["for", "present", "you", "have", "a"]}, {"sent": "um a present .", "words": ["present", "a"]}, {"sent": "what did we give AuntyAwww for her present ?", "words": ["what", "for", "present", "we", "did", "give", "her"]}, {"sent": "that was his birthday present .", "words": ["his", "was", "that", "present"]}, {"sent": "and buy a birthday present okay ?", "words": ["present", "buy", "a", "and"]}, {"sent": "you do realize though Thomas when it comes to your birthday that youve already had your present from David and Sharon and from GrandmaBealby .", "words": ["do", "your", "present", "to", "you", "and", "that", "it", "when"]}, {"sent": "its an early birthday present or a belated birthday present .", "words": ["an", "present", "a"]}, {"sent": "and you get to bring a present home .", "words": ["present", "to", "bring", "you", "get", "and", "home", "a"]}, {"sent": "its a present .", "words": ["present", "a"]}, {"sent": "because its Jonathans present wrapped up nicely .", "words": ["up", "present", "because"]}, {"sent": "I think you should be giving AuntieAwwws baby a present .", "words": ["think", "present", "be", "you", "a"]}, {"sent": "well they already gave you a birthday present .", "words": ["they", "present", "a", "you"]}, {"sent": "it was Molly saying thank you for her birthday and Christmas present wasnt it ?", "words": ["for", "present", "was", "you", "and", "it", "her"]}, {"sent": "no thats for Jwwws birthday present because its Jwwws birthdayparty next week isnt it ?", "words": ["it", "for", "because", "present"]}, {"sent": "and a present .", "words": ["present", "a", "and"]}, {"sent": "its a pretend present .", "words": ["pretend", "present", "a"]}, {"sent": "thats a wee present for you .", "words": ["you", "for", "present", "a"]}, {"sent": "and this was a present from Pam .", "words": ["present", "was", "and", "this", "a"]}, {"sent": "is it a present ?", "words": ["present", "it", "a", "is"]}, {"sent": "he had a beautiful box covered in shiny paper and he didnt remember what he was sposta do with the present and Mrs Platford said give it to the baby .", "words": ["the", "what", "do", "box", "present", "was", "paper", "to", "with", "and", "in", "it", "he", "give", "a"]}, {"sent": "I choose Johns birthday present on my own .", "words": ["on", "my", "present"]}, {"sent": "what a surprise present ?", "words": ["what", "present", "a"]}, {"sent": "and they were a present from Dick and Wendy .", "words": ["present", "and", "they", "were", "a"]}, {"sent": "we couldnt find a birthday present for Jwww could we ?", "words": ["for", "present", "we", "find", "could", "a"]}, {"sent": "what have they bought for Johns present ?", "words": ["what", "for", "present", "have", "they"]}, {"sent": "were not too really thinking about that too much at the very present moment .", "words": ["the", "present", "much", "at", "about", "too", "that", "not", "were"]}, {"sent": "what what have we bought Purdie for a present ?", "words": ["what", "for", "present", "we", "have", "a"]}, {"sent": "its a present for me ?", "words": ["me", "for", "present", "a"]}, {"sent": "you want the present ?", "words": ["the", "present", "you"]}, {"sent": "thats Lindas present .", "words": ["present"]}, {"sent": "whats your favorite present ?", "words": ["your", "present"]}, {"sent": "this is a lovely present isnt it ?", "words": ["present", "is", "this", "it", "a"]}, {"sent": "or is it a present of money ?", "words": ["money", "present", "is", "of", "it", "a"]}, {"sent": "what a surprise present .", "words": ["what", "present", "a"]}, {"sent": "and I noticed that Jane had some actually because I was thinking it would make Jane a nice present .", "words": ["present", "was", "nice", "and", "it", "make", "would", "some", "that", "because", "a"]}, {"sent": "oh it was her birthday present .", "words": ["was", "it", "her", "present"]}, {"sent": "why do I get a present today ?", "words": ["do", "present", "get", "why", "a"]}, {"sent": "I have a little present for you .", "words": ["for", "present", "little", "have", "you", "a"]}, {"sent": "so well hafta go and buy her a present before Saturday .", "words": ["present", "go", "so", "and", "her", "buy", "a"]}, {"sent": "did you send your present to um Swww ?", "words": ["your", "present", "to", "you", "did"]}, {"sent": "that was a jolly good present wasnt it ?", "words": ["present", "was", "it", "good", "that", "a"]}, {"sent": "he bought her a present .", "words": ["her", "he", "present", "a"]}, {"sent": "will Santa will Santa give her a present ?", "words": ["present", "will", "give", "her", "a"]}, {"sent": "its a present isnt it ?", "words": ["it", "present", "a"]}, {"sent": "I know youre a present .", "words": ["present", "a"]}, {"sent": "oh is that a present for Nina ?", "words": ["for", "present", "is", "that", "a"]}, {"sent": "and so she brought this back as a present for you .", "words": ["for", "back", "present", "so", "and", "you", "this", "she", "a"]}, {"sent": "a present for Pooh .", "words": ["for", "present", "a"]}, {"sent": "wheres his present ?", "words": ["his", "present"]}, {"sent": "I have a little present for you for todays visit .", "words": ["for", "present", "little", "have", "you", "a"]}, {"sent": "but we have a little birthday present for Swww .", "words": ["for", "present", "we", "little", "have", "but", "a"]}, {"sent": "so we wanna take a little present round dont we ?", "words": ["present", "we", "so", "little", "dont", "wanna", "take", "a"]}, {"sent": "that was a Christmas present from Liz wasnt it ?", "words": ["present", "was", "it", "that", "a"]}, {"sent": "just a few lines to thank you for the very unexpected Christmas present waiting for me in church last Sunday .", "words": ["the", "for", "present", "to", "church", "last", "you", "in", "me", "a"]}, {"sent": "you wrapping it up like a Christmas present ?", "words": ["up", "present", "like", "you", "it", "a"]}, {"sent": "what a nice present .", "words": ["what", "nice", "present", "a"]}, {"sent": "that wasnt a Christmas present though was it ?", "words": ["present", "was", "it", "that", "a"]}, {"sent": "that was a nice present .", "words": ["present", "was", "nice", "that", "a"]}, {"sent": "what kind of a present ?", "words": ["of", "what", "present", "a"]}, {"sent": "its a christening present .", "words": ["present", "a"]}, {"sent": "present ?", "words": ["present"]}, {"sent": "a nice little present .", "words": ["present", "nice", "little", "a"]}, {"sent": "oh I would love a sleep present honey .", "words": ["present", "sleep", "love", "would", "a"]}, {"sent": "its a present ?", "words": ["present", "a"]}, {"sent": "well go into that big toy store to buy a present for Devon .", "words": ["for", "into", "present", "go", "store", "to", "toy", "that", "big", "buy", "a"]}, {"sent": "Ill get you a small present .", "words": ["get", "present", "a", "you"]}, {"sent": "did you say thank you for the present ?", "words": ["the", "for", "present", "say", "you", "did"]}, {"sent": "did you tell Daddy about your new trick ?", "words": ["your", "you", "did", "about", "new"]}, {"sent": "um hm they were finding a new place .", "words": ["they", "new", "were", "a"]}, {"sent": "did you get a new jigsawpuzzle Fraser ?", "words": ["you", "get", "did", "new", "a"]}, {"sent": "you wanna new one ?", "words": ["new", "wanna", "you"]}, {"sent": "was it a doll was it a new doll ?", "words": ["was", "doll", "it", "new", "a"]}, {"sent": "I thought you were going to take the three dinosaurs but now you left me a new one and now I have four .", "words": ["the", "to", "you", "and", "have", "but", "me", "new", "take", "were", "a"]}, {"sent": "because I kept showing him new things .", "words": ["him", "new", "because"]}, {"sent": "Ill get a new piece of paper and draw it on a new piece of paper .", "words": ["of", "paper", "draw", "and", "get", "it", "on", "new", "a"]}, {"sent": "okay now you can put a new wheel on it .", "words": ["can", "put", "you", "it", "on", "new", "a"]}, {"sent": "do they look like new ?", "words": ["do", "like", "they", "look", "new"]}, {"sent": "because its Nanas new carpet .", "words": ["new", "because"]}, {"sent": "are you trying to test that new tooth out here ?", "words": ["tooth", "to", "you", "here", "are", "out", "new", "that"]}, {"sent": "weve seen a new baby this week havent we ?", "words": ["this", "new", "we", "a"]}, {"sent": "Ill put these new clothes in your wardrobe .", "words": ["these", "put", "your", "in", "new"]}, {"sent": "its not really too new now is it ?", "words": ["is", "it", "too", "new", "not"]}, {"sent": "you could get Ive never bought a brand new one .", "words": ["you", "get", "could", "new", "a"]}, {"sent": "new books too .", "words": ["new", "too"]}, {"sent": "the bird flew up to its new home .", "words": ["the", "up", "to", "bird", "new", "home"]}, {"sent": "weve got new neighbors uhn havent we ?", "words": ["new", "we"]}, {"sent": "let me tell you a new word .", "words": ["me", "new", "a", "you"]}, {"sent": "put new wheels on ?", "words": ["on", "put", "new"]}, {"sent": "new puzzle .", "words": ["new", "puzzle"]}, {"sent": "Zoe tried on her new teeball uniform over and over and over .", "words": ["her", "and", "on", "new", "over"]}, {"sent": "were gonna get some new toys .", "words": ["get", "some", "new", "were"]}, {"sent": "is that a new one ?", "words": ["new", "that", "a", "is"]}, {"sent": "having new horses and new sheep .", "words": ["new", "sheep", "and"]}, {"sent": "out hatched a new Wendy .", "words": ["new", "out", "a"]}, {"sent": "remember you were gonna tell me all about your new school .", "words": ["all", "your", "school", "you", "about", "me", "new", "were"]}, {"sent": "she prob Im glad she probably forgot about that new toy .", "words": ["toy", "about", "new", "that", "she"]}, {"sent": "thats a new one .", "words": ["new", "a"]}, {"sent": "building a new house .", "words": ["new", "house", "a"]}, {"sent": "weve got a new postman havent we ?", "words": ["new", "we", "a"]}, {"sent": "look at this new toy .", "words": ["look", "this", "toy", "at", "new"]}, {"sent": "lets throw that away well get you a new one .", "words": ["throw", "you", "get", "away", "new", "that", "a"]}, {"sent": "gave you a whole set of brand new golf clubs .", "words": ["of", "new", "a", "you"]}, {"sent": "uh her father Maurice was busy working on a new invention youll win first prize at the fair tomorrow Belle predicted .", "words": ["the", "was", "first", "at", "on", "new", "her", "a"]}, {"sent": "is that a new one ?", "words": ["new", "that", "a", "is"]}, {"sent": "squamps this is Lilys new made up word squamps .", "words": ["up", "this", "new", "is"]}, {"sent": "Leo the Late Bloomer this is a new book isnt it ?", "words": ["the", "is", "this", "it", "book", "new", "a"]}, {"sent": "wow is that a new book ?", "words": ["is", "book", "new", "that", "a"]}, {"sent": "yeah Doras her new obsession pretty much .", "words": ["pretty", "new", "her", "much"]}, {"sent": "and um she actually just got a changing set for one of them to out on new diapers and things like that .", "words": ["for", "of", "to", "them", "like", "and", "out", "on", "new", "that", "she", "a"]}, {"sent": "look this is new .", "words": ["this", "new", "is", "look"]}, {"sent": "Roman how did you do on your new bike ?", "words": ["do", "how", "your", "you", "did", "on", "new"]}, {"sent": "do you mean new ?", "words": ["do", "new", "you"]}, {"sent": "do you wanna look at your new magazine that came in the mail with your toys ?", "words": ["the", "do", "your", "with", "you", "in", "at", "wanna", "look", "new", "that"]}, {"sent": "weve put a new piece of paper on .", "words": ["put", "of", "paper", "on", "new", "a"]}, {"sent": "isnt it new ?", "words": ["new", "it"]}, {"sent": "and then were gonna go soon to go look for the new Sleepy Bear whos lost .", "words": ["the", "for", "go", "to", "and", "look", "then", "new", "were"]}, {"sent": "and take them from the old house to the new house and now theyve got to take them all out and put it in the new house .", "words": ["the", "old", "all", "put", "to", "them", "and", "in", "it", "house", "out", "new", "take"]}, {"sent": "I think we need a new helmet Lara .", "words": ["think", "need", "we", "new", "a"]}, {"sent": "its not new .", "words": ["new", "not"]}, {"sent": "have you seen my new tractor Pat ?", "words": ["my", "you", "have", "tractor", "new"]}, {"sent": "hey heres a new picture of you .", "words": ["of", "picture", "you", "new", "a"]}, {"sent": "I think that Nina likes her new kangaroo .", "words": ["think", "new", "that", "her"]}, {"sent": "thats old water you have the new water right there .", "words": ["old", "the", "you", "have", "there", "water", "new"]}, {"sent": "for Tweedledum said Tweedledee had spoiled his nice new rattle .", "words": ["his", "new", "for", "nice"]}, {"sent": "AuntieMabel has got her new green ones on hasnt she ?", "words": ["green", "on", "new", "her", "she"]}, {"sent": "Dirk you cant compete with new people .", "words": ["new", "with", "you"]}, {"sent": "and theyre all brand spanking new not like our Toddlers .", "words": ["all", "like", "and", "our", "new", "not"]}, {"sent": "what did you do with your new friend Lauren ?", "words": ["what", "do", "your", "with", "you", "did", "new"]}, {"sent": "did you see the new cabinet William ?", "words": ["the", "you", "did", "see", "new"]}, {"sent": "thats kind of new and funny .", "words": ["of", "new", "and"]}, {"sent": "hafta buy you some new ones .", "words": ["some", "new", "buy", "you"]}, {"sent": "do you have a new ball .", "words": ["do", "you", "have", "ball", "new", "a"]}, {"sent": "is he a new doggie Jeremy ?", "words": ["new", "he", "a", "is"]}, {"sent": "you need the new train ?", "words": ["the", "need", "train", "you", "new"]}, {"sent": "and then you learned a new game too .", "words": ["game", "you", "and", "too", "then", "new", "a"]}, {"sent": "Eleanor is there a sandcastle in your new houses garden ?", "words": ["your", "is", "there", "in", "garden", "new", "a"]}, {"sent": "you wanna sit in your new chair ?", "words": ["your", "chair", "you", "in", "wanna", "sit", "new"]}, {"sent": "Mommy needs a new mop .", "words": ["new", "mop", "a"]}, {"sent": "to the very new Spa shop ?", "words": ["the", "to", "new"]}, {"sent": "oh so the new baby would just know how to do it huh ?", "words": ["the", "do", "how", "so", "to", "it", "would", "new"]}, {"sent": "Prudence sat on her new potty she sat and sat and sat and sat and sat and nothing came out .", "words": ["and", "potty", "out", "on", "new", "her", "she"]}, {"sent": "this time the roller has work to do it pushes and smooths the road is new .", "words": ["the", "do", "work", "is", "to", "and", "this", "it", "new"]}, {"sent": "well they do find new monkeys but what is this little one holding ?", "words": ["do", "what", "is", "little", "they", "this", "find", "new", "but"]}, {"sent": "I have a new idea .", "words": ["have", "new", "a"]}, {"sent": "have you got some new stories ?", "words": ["have", "some", "new", "you"]}, {"sent": "where is that new Dorothy ?", "words": ["where", "new", "that", "is"]}, {"sent": "some new tins .", "words": ["some", "new"]}, {"sent": "thats a new rhyme .", "words": ["new", "a"]}, {"sent": "but the Marmites a new jar and its very heavy .", "words": ["the", "and", "heavy", "new", "jar", "but", "a"]}, {"sent": "and Nana and Grandad also bought you Dobbin your new rocking horse .", "words": ["your", "you", "and", "horse", "new"]}, {"sent": "will you put her in a new bath ?", "words": ["put", "you", "in", "will", "new", "her", "a"]}, {"sent": "are you describing the new beaker that we bought you yesterday in Stockport ?", "words": ["the", "we", "you", "in", "are", "new", "that"]}, {"sent": "thats a new bag isnt it ?", "words": ["it", "new", "a"]}, {"sent": "Lara doesnt want a new carpet .", "words": ["new", "a"]}, {"sent": "its a new one .", "words": ["new", "a"]}, {"sent": "if you break it wed hafta go and buy a new one wouldnt we ?", "words": ["if", "go", "we", "you", "and", "it", "new", "break", "buy", "a"]}, {"sent": "because when you get a new baby you dont know it do you ?", "words": ["do", "you", "get", "dont", "it", "new", "when", "because", "a"]}, {"sent": "thats the other new one .", "words": ["the", "new", "other"]}, {"sent": "this is new is this your toy ?", "words": ["your", "is", "this", "toy", "new"]}, {"sent": "totally new .", "words": ["new"]}, {"sent": "oh it just needs a new belt .", "words": ["new", "it", "belt", "a"]}, {"sent": "your new what ?", "words": ["what", "new", "your"]}, {"sent": "but why do you like your new school ?", "words": ["do", "your", "school", "like", "you", "but", "new", "why"]}, {"sent": "your new favorite book .", "words": ["book", "new", "your"]}, {"sent": "for this new tree that was growing .", "words": ["tree", "for", "was", "this", "new", "that"]}, {"sent": "are they new xxx ?", "words": ["they", "new", "are"]}, {"sent": "remember Ben came round one day when it was new .", "words": ["was", "new", "it", "when"]}, {"sent": "that is not Grammy is car .", "words": ["car", "that", "not", "is"]}, {"sent": "car ?", "words": ["car"]}, {"sent": "when I travel I mostly go by car but have you ever been by train or plane or boat ?", "words": ["go", "train", "car", "boat", "by", "have", "you", "when", "but"]}, {"sent": "are you going to put it on the back of the car ?", "words": ["the", "put", "back", "of", "to", "car", "you", "are", "it", "on"]}, {"sent": "Pat in the car yeah .", "words": ["the", "car", "in"]}, {"sent": "because youve been playing with the trains today and the racing car havent you ?", "words": ["the", "car", "with", "and", "you", "because"]}, {"sent": "Billys car huh ?", "words": ["car"]}, {"sent": "and see wed bought a new car .", "words": ["car", "and", "see", "new", "a"]}, {"sent": "because because he doesnt have a car .", "words": ["car", "have", "he", "because", "a"]}, {"sent": "is panda going to drive the car ?", "words": ["the", "is", "to", "car", "drive"]}, {"sent": "he goes in the car .", "words": ["the", "car", "in", "he"]}, {"sent": "only one car can go by .", "words": ["can", "car", "by", "go"]}, {"sent": "you had it in the car didnt you ?", "words": ["the", "car", "you", "in", "it"]}, {"sent": "his car went will have .", "words": ["his", "car", "have", "will"]}, {"sent": "Marks driving off in his car .", "words": ["off", "his", "in", "car"]}, {"sent": "is he washing the car ?", "words": ["the", "car", "he", "is"]}, {"sent": "what kind of a car is this ?", "words": ["what", "is", "of", "car", "this", "a"]}, {"sent": "when I go I go and I drive my car to McDonalds .", "words": ["my", "go", "car", "to", "and", "drive", "when"]}, {"sent": "show me how to wash my car will you ?", "words": ["how", "my", "show", "to", "car", "you", "will", "me", "wash"]}, {"sent": "your car goes fast .", "words": ["car", "fast", "your"]}, {"sent": "heres our car .", "words": ["car", "our"]}, {"sent": "does your daddy have a car ?", "words": ["does", "your", "car", "have", "a"]}, {"sent": "thats the nurse revise reversing her car .", "words": ["the", "car", "her"]}, {"sent": "car is it two cars ?", "words": ["car", "it", "is"]}, {"sent": "this car .", "words": ["this", "car"]}, {"sent": "xxx nineteen seventy five car you win the car .", "words": ["the", "car", "you"]}, {"sent": "what was the name of that little boy getting into the car yesterday with his Mummy when I came to pick you up ?", "words": ["the", "what", "up", "into", "when", "of", "was", "car", "to", "with", "pick", "little", "you", "his", "that"]}, {"sent": "oh well perhaps hed better get in the car if its going to go under the bridge .", "words": ["the", "better", "if", "go", "car", "to", "get", "under", "in"]}, {"sent": "for this little car .", "words": ["this", "for", "car", "little"]}, {"sent": "oh its out in the car .", "words": ["the", "car", "in", "out"]}, {"sent": "its a broken car .", "words": ["car", "broken", "a"]}, {"sent": "police car was first ?", "words": ["was", "car", "first"]}, {"sent": "car do you need car ?", "words": ["car", "do", "need", "you"]}, {"sent": "he can take all the food back on his car .", "words": ["can", "all", "the", "back", "car", "his", "food", "he", "on", "take"]}, {"sent": "look at my car .", "words": ["car", "my", "at", "look"]}, {"sent": "Pwww is gone in her car .", "words": ["car", "in", "her", "is"]}, {"sent": "wheres the green car ?", "words": ["the", "car", "green"]}, {"sent": "Fraser with Jwww in the car .", "words": ["the", "car", "in", "with"]}, {"sent": "tell me what happens when we ride in the car .", "words": ["the", "what", "we", "car", "ride", "in", "me", "when"]}, {"sent": "over on the fence behind your daddys car .", "words": ["the", "your", "car", "behind", "on", "over"]}, {"sent": "shall we see if we can put a car on there ?", "words": ["can", "put", "if", "we", "car", "there", "on", "see", "a"]}, {"sent": "Mickeys driving a car by the looks of things isnt he ?", "words": ["the", "of", "car", "by", "he", "a"]}, {"sent": "can I have some gas in my bucket and then I can go back and fill up the car ?", "words": ["can", "up", "my", "the", "back", "bucket", "go", "car", "and", "have", "in", "some", "then"]}, {"sent": "a taxi is a car that comes to the house to take you somewhere .", "words": ["the", "is", "to", "car", "you", "house", "take", "that", "a"]}, {"sent": "um have you got some deicer for the car windscreen please ?", "words": ["the", "for", "car", "you", "have", "some"]}, {"sent": "and this is the car that the boys pulling .", "words": ["the", "is", "car", "and", "this", "that"]}, {"sent": "look bananas in the last car you bananas bananas thats where they are .", "words": ["the", "where", "car", "last", "you", "they", "in", "are", "look"]}, {"sent": "theres a racing car there .", "words": ["car", "there", "a"]}, {"sent": "thats another car .", "words": ["another", "car"]}, {"sent": "yeah just you have a car .", "words": ["have", "car", "a", "you"]}, {"sent": "put him in your car .", "words": ["put", "your", "car", "in", "him"]}, {"sent": "what does the car say ?", "words": ["the", "what", "does", "car", "say"]}, {"sent": "um whereas if hes in the car he makes he seems to make more noises and more babbling .", "words": ["the", "if", "car", "to", "and", "in", "more", "he", "make"]}, {"sent": "well does Peter need the car everyday ?", "words": ["the", "does", "need", "car"]}, {"sent": "yeah his is broken a dog outof a car .", "words": ["is", "car", "broken", "his", "dog", "a"]}, {"sent": "thats because its its a pretend car .", "words": ["car", "pretend", "because", "a"]}, {"sent": "well thats the driver of the car .", "words": ["the", "of", "car"]}, {"sent": "what colors daddys car ?", "words": ["what", "car"]}, {"sent": "you know how to drive a car ?", "words": ["how", "to", "car", "you", "drive", "a"]}, {"sent": "and then when youre in the car and you to sleep instead of your head going and falling all over the place .", "words": ["the", "all", "your", "sleep", "car", "to", "of", "head", "and", "you", "in", "over", "then", "when"]}, {"sent": "here go a car .", "words": ["here", "car", "a", "go"]}, {"sent": "what would you do if you got to ride in a race car ?", "words": ["what", "do", "if", "to", "car", "you", "in", "would", "ride", "a"]}, {"sent": "have you got the car to go in to the .", "words": ["the", "go", "to", "car", "you", "have", "in"]}, {"sent": "car ?", "words": ["car"]}, {"sent": "its a mini car ?", "words": ["car", "a"]}, {"sent": "and the car and everything else came with it .", "words": ["the", "car", "with", "and", "it"]}, {"sent": "car wash .", "words": ["car", "wash"]}, {"sent": "who do you think would win win the truck or the car ?", "words": ["the", "do", "think", "car", "truck", "you", "would", "who"]}, {"sent": "the only other car wreck I was in happened when I was on the way to a birthday party .", "words": ["the", "car", "was", "to", "other", "party", "in", "on", "when", "a"]}, {"sent": "were you in a car ?", "words": ["car", "you", "in", "were", "a"]}, {"sent": "a car ?", "words": ["car", "a"]}, {"sent": "why dont you go get the car ?", "words": ["the", "go", "car", "you", "get", "dont", "why"]}, {"sent": "just one car Ross ?", "words": ["car"]}, {"sent": "its a car with an l plate on it isnt it ?", "words": ["plate", "car", "with", "an", "it", "on", "a"]}, {"sent": "car .", "words": ["car"]}, {"sent": "car ?", "words": ["car"]}, {"sent": "wherere you gonna go in your car ?", "words": ["your", "go", "car", "you", "in"]}, {"sent": "well with your racing car .", "words": ["your", "car", "with"]}, {"sent": "then youre going to their car too .", "words": ["their", "to", "car", "too", "then"]}, {"sent": "theres a ball in the car .", "words": ["the", "car", "ball", "in", "a"]}, {"sent": "yeah Mummy went in the car to take Jwww to school didnt she ?", "words": ["the", "car", "to", "school", "in", "take", "she"]}, {"sent": "oh was I driving the car too quickly ?", "words": ["the", "was", "car", "too"]}, {"sent": "is that off the car ?", "words": ["off", "the", "is", "car", "that"]}, {"sent": "when its the yellow side up its the ride car and when its red side up its a racing car .", "words": ["the", "up", "red", "car", "ride", "and", "yellow", "when", "a"]}, {"sent": "its a car .", "words": ["car", "a"]}, {"sent": "you cant just arrest somebody when you dont know what theyve done and put them in a police car .", "words": ["what", "put", "car", "them", "and", "you", "dont", "in", "when", "a"]}, {"sent": "you putting the racing car back in the box ?", "words": ["the", "box", "back", "car", "you", "in"]}, {"sent": "rabbit I think he is in the car .", "words": ["the", "think", "is", "car", "in", "he"]}, {"sent": "how about a blue car ?", "words": ["how", "car", "blue", "about", "a"]}, {"sent": "nice blue police car .", "words": ["car", "nice", "blue"]}, {"sent": "well put the shall we put the racing car driver in the car as well ?", "words": ["the", "put", "we", "car", "in"]}, {"sent": "its another car .", "words": ["another", "car"]}, {"sent": "Bobbys fooling around with the back of Harrys car .", "words": ["the", "back", "of", "car", "with", "around"]}, {"sent": "car .", "words": ["car"]}, {"sent": "and this car has got green wheels .", "words": ["this", "green", "car", "and"]}, {"sent": "how do we hook what the car ?", "words": ["the", "do", "how", "what", "we", "car"]}, {"sent": "look at the little kittycat driving that car ?", "words": ["the", "look", "car", "little", "at", "that"]}, {"sent": "it was hiding under that car .", "words": ["was", "car", "under", "it", "that"]}, {"sent": "another car .", "words": ["another", "car"]}, {"sent": "yeah thats Mummys car isnt it ?", "words": ["car", "it"]}, {"sent": "so what cartoons did you watch ?", "words": ["what", "watch", "so", "you", "did"]}, {"sent": "think so ?", "words": ["so", "think"]}, {"sent": "so now you have a turtle puppet and a mouse .", "words": ["so", "you", "have", "and", "turtle", "mouse", "a"]}, {"sent": "so hell put them on the grill and then hell hell bring them to you .", "words": ["the", "put", "so", "to", "them", "bring", "and", "you", "on", "then"]}, {"sent": "so its its a furry animal and then just its appendages are .", "words": ["so", "and", "animal", "are", "then", "a"]}, {"sent": "so .", "words": ["so"]}, {"sent": "so that the tape recorder can pick up your voice ?", "words": ["the", "can", "up", "your", "pick", "so", "tape", "that"]}, {"sent": "eat so much .", "words": ["eat", "so", "much"]}, {"sent": "no I dont think so .", "words": ["dont", "think", "so"]}, {"sent": "so next we hafta do this bit here with all the times on .", "words": ["the", "do", "all", "we", "so", "with", "here", "this", "on"]}, {"sent": "Im not gonna be your friend now Asher so I .", "words": ["be", "so", "your", "not"]}, {"sent": "Im sitting so Ill get to play with it .", "words": ["so", "to", "play", "with", "get", "it"]}, {"sent": "we can turn them all over so you can see them .", "words": ["can", "all", "we", "so", "them", "you", "see", "over"]}, {"sent": "so xxx .", "words": ["so"]}, {"sent": "why is Jess so cross ?", "words": ["so", "why", "is"]}, {"sent": "snowy is so happy he hoots .", "words": ["so", "he", "happy", "is"]}, {"sent": "so ya want me to start ?", "words": ["to", "so", "me"]}, {"sent": "no so so just um .", "words": ["so"]}, {"sent": "and I was talkin and I saw that he had something but he was holding it so like far away like awkwardly and the expression on his face .", "words": ["the", "face", "was", "so", "like", "and", "his", "it", "he", "away", "on", "that", "but"]}, {"sent": "so I said I hadta bring them home didnt I ?", "words": ["so", "them", "bring", "home"]}, {"sent": "I think so .", "words": ["so", "think"]}, {"sent": "so people can take it home .", "words": ["can", "so", "it", "take", "home"]}, {"sent": "thats something so tall .", "words": ["so"]}, {"sent": "oh its so tasty .", "words": ["so"]}, {"sent": "you think so ?", "words": ["so", "think", "you"]}, {"sent": "so Scamps mommy and daddy were ?", "words": ["so", "were", "and"]}, {"sent": "bunny is so big .", "words": ["so", "bunny", "big", "is"]}, {"sent": "so .", "words": ["so"]}, {"sent": "you think so ?", "words": ["so", "think", "you"]}, {"sent": "its so dark .", "words": ["so", "dark"]}, {"sent": "I am so going .", "words": ["so", "am"]}, {"sent": "so .", "words": ["so"]}, {"sent": "youve just bought that so you can keep that cant you ?", "words": ["can", "so", "that", "you"]}, {"sent": "nothing else so colorful huh ?", "words": ["so"]}, {"sent": "so did you have fun seeing Jeannine today ?", "words": ["have", "so", "did", "you"]}, {"sent": "so it might be that Grandmas in hospital when were on holiday .", "words": ["when", "so", "be", "in", "it", "on", "that", "were"]}, {"sent": "and it was about ten oclock or so .", "words": ["was", "so", "and", "it", "about"]}, {"sent": "so she wont bother singing .", "words": ["so", "she"]}, {"sent": "so you dont drown huh case you fall in .", "words": ["so", "fall", "you", "dont", "in"]}, {"sent": "so what are you doing today Johnny ?", "words": ["so", "what", "are", "you"]}, {"sent": "and so shes .", "words": ["so", "and"]}, {"sent": "xxx lot of stock in so was busy .", "words": ["of", "so", "in", "was"]}, {"sent": "dont be so bossy .", "words": ["dont", "be", "so"]}, {"sent": "and he frightened the dog and the frog so bad that he ate the bug and he said .", "words": ["the", "bug", "so", "bad", "frog", "and", "he", "that", "dog"]}, {"sent": "so they were all sitting in the back oh my wait a minute hold on a second .", "words": ["the", "all", "my", "wait", "hold", "back", "so", "they", "in", "on", "were", "a"]}, {"sent": "dont get so excited .", "words": ["get", "dont", "so"]}, {"sent": "right and Bobby is having two cups and maybe even hell have three cups so its almost the same thing .", "words": ["the", "is", "so", "and", "have", "same"]}, {"sent": "so the mice had so the mouse had a knife huh to cut the cats down ?", "words": ["the", "cut", "so", "knife", "to", "down", "mouse", "a"]}, {"sent": "so has that one .", "words": ["so", "that"]}, {"sent": "and so he will immediately um approach them if if I can bring them to him .", "words": ["can", "if", "so", "to", "them", "bring", "and", "him", "he", "will"]}, {"sent": "so you gonna use the green one ?", "words": ["the", "so", "green", "you"]}, {"sent": "its not so bad .", "words": ["so", "not", "bad"]}, {"sent": "and so I was like that oh .", "words": ["was", "so", "like", "and", "that"]}, {"sent": "oh my gosh so exciting I cant believe it .", "words": ["so", "my", "it"]}, {"sent": "so xxx like the baby and the baby bottles yeah .", "words": ["the", "so", "like", "and"]}, {"sent": "youre so cute .", "words": ["cute", "so"]}, {"sent": "so who would be holding a lamb ?", "words": ["lamb", "so", "be", "would", "who", "a"]}, {"sent": "oh I didnt mean that way Im sorry I meant like this so that his head goes in this spot .", "words": ["so", "head", "like", "this", "in", "his", "that"]}, {"sent": "you mean Ive just got soaking wet and so has the house and you dont love me .", "words": ["the", "wet", "so", "and", "you", "dont", "house", "love", "me"]}, {"sent": "not so loud ThomasHenry .", "words": ["so", "not", "loud"]}, {"sent": "so Mommy can put him back on the mobile .", "words": ["can", "the", "put", "back", "so", "him", "on"]}, {"sent": "so hes xxx .", "words": ["so"]}, {"sent": "I dont think so .", "words": ["dont", "think", "so"]}, {"sent": "so what we can do is after lunch we can have a little walk out .", "words": ["can", "what", "do", "we", "is", "so", "little", "have", "out", "walk", "a"]}, {"sent": "well do you want your potty moving there so you can see the television ?", "words": ["can", "do", "the", "your", "so", "you", "there", "potty", "see"]}, {"sent": "I am so second fiddle Amanda .", "words": ["so", "am"]}, {"sent": "so he asked if his mother .", "words": ["his", "so", "he", "if"]}, {"sent": "haha so .", "words": ["so"]}, {"sent": "so he would he wouldnt recognize a kiwi in its whole state .", "words": ["so", "in", "he", "would", "a"]}, {"sent": "um so xxx you know how you go over the questions .", "words": ["the", "how", "go", "so", "you", "over"]}, {"sent": "why are you being so silly ?", "words": ["so", "are", "why", "you"]}, {"sent": "so do you not have the key for that ?", "words": ["the", "do", "for", "so", "you", "have", "that", "not"]}, {"sent": "we dont have the dog today so you can go ahead xxx go straight ahead .", "words": ["the", "can", "we", "go", "so", "you", "have", "dont", "dog"]}, {"sent": "youknow playing himself so .", "words": ["so"]}, {"sent": "so he gets the toaster the book and do you know what this is called ?", "words": ["the", "do", "what", "is", "so", "and", "you", "this", "he", "book"]}, {"sent": "so thats what its called .", "words": ["so", "what"]}, {"sent": "so what else did you do this morning ?", "words": ["what", "do", "so", "you", "this", "did"]}, {"sent": "I should have a spoon so I could eat my egg .", "words": ["my", "egg", "so", "have", "eat", "spoon", "could", "a"]}, {"sent": "how come youre yelling and screaming so much ?", "words": ["so", "how", "much", "and"]}, {"sent": "Im so sorry .", "words": ["so"]}, {"sent": "I think so too .", "words": ["so", "think", "too"]}, {"sent": "oh they look so nice on you .", "words": ["so", "nice", "you", "they", "look", "on"]}, {"sent": "so big so big .", "words": ["so", "big"]}, {"sent": "what were we following so we didnt miss anybody ?", "words": ["what", "so", "were", "we"]}, {"sent": "dont you think so ?", "words": ["dont", "think", "so", "you"]}, {"sent": "so theyre not falling out .", "words": ["so", "not", "out"]}, {"sent": "around the house and stuff so .", "words": ["the", "so", "and", "house", "around"]}, {"sent": "so Im glad we got involved in the program .", "words": ["the", "so", "in", "we"]}, {"sent": "is that so here we are .", "words": ["we", "is", "so", "here", "are", "that"]}, {"sent": "so it says write the word blue .", "words": ["the", "so", "write", "blue", "it"]}, {"sent": "its a great responsibility taking care of so many ducklings and they kept them very busy .", "words": ["of", "so", "them", "and", "they", "a"]}, {"sent": "well so am I .", "words": ["so", "am"]}, {"sent": "so that the snake will let the little boy go .", "words": ["the", "go", "so", "little", "will", "that"]}, {"sent": "think so ?", "words": ["so", "think"]}, {"sent": "okay so it has .", "words": ["so", "it"]}, {"sent": "and its so they can see over the top of the trees .", "words": ["can", "the", "of", "so", "and", "they", "see", "over"]}, {"sent": "out your mouth goes up into your sinuses and your nasal cavity and it just irritates it and inflames it so that .", "words": ["up", "your", "into", "so", "mouth", "and", "it", "out", "that"]}, {"sent": "no I dont think so .", "words": ["dont", "think", "so"]}, {"sent": "its gone now you better not throw that at me .", "words": ["better", "throw", "you", "at", "me", "that", "not"]}, {"sent": "I bet you this is better than that juice you were trying to give her .", "words": ["better", "is", "to", "her", "you", "this", "give", "juice", "that", "were"]}, {"sent": "itll make your mouth feel better wont it ?", "words": ["better", "your", "mouth", "it", "make"]}, {"sent": "they fit in better like that .", "words": ["better", "fit", "like", "they", "in", "that"]}, {"sent": "youd better give it to him .", "words": ["better", "to", "him", "it", "give"]}, {"sent": "thats better .", "words": ["better"]}, {"sent": "is that better ?", "words": ["better", "that", "is"]}, {"sent": "is that better ?", "words": ["better", "that", "is"]}, {"sent": "oh you better go lie down .", "words": ["go", "better", "down", "you"]}, {"sent": "all better .", "words": ["better", "all"]}, {"sent": "when you feel better .", "words": ["better", "when", "you"]}, {"sent": "does the oven make you feel better ?", "words": ["the", "does", "better", "you", "make", "oven"]}, {"sent": "is that better ?", "words": ["better", "that", "is"]}, {"sent": "which do you like better Sarah .", "words": ["better", "do", "like", "you", "which"]}, {"sent": "that better ?", "words": ["better", "that"]}, {"sent": "no I better hold it because youre busy doing other things there .", "words": ["better", "hold", "other", "there", "it", "because"]}, {"sent": "I think you better use glue now .", "words": ["better", "think", "glue", "you"]}, {"sent": "we better go home said Snuffy .", "words": ["go", "better", "home", "we"]}, {"sent": "youd better find it quickly or else Im going to ring for MisterPlod the policeman .", "words": ["the", "better", "for", "to", "it", "find"]}, {"sent": "better have a drink .", "words": ["have", "better", "drink", "a"]}, {"sent": "I promise itll be better if you eat the real thing .", "words": ["the", "better", "if", "be", "you", "eat"]}, {"sent": "so weve decided really its better to keep that inside and just on the pavement .", "words": ["the", "better", "so", "inside", "to", "and", "on", "that"]}, {"sent": "its lower how about that is that better ?", "words": ["better", "how", "is", "about", "that"]}, {"sent": "hey is it I think it goes better the other way .", "words": ["the", "better", "think", "is", "other", "it"]}, {"sent": "is that better ?", "words": ["better", "that", "is"]}, {"sent": "is that better than all the other ones ?", "words": ["the", "all", "better", "is", "other", "that"]}, {"sent": "thats better .", "words": ["better"]}, {"sent": "oh I dont think youd better have that .", "words": ["better", "think", "have", "dont", "that"]}, {"sent": "is it much better ?", "words": ["better", "it", "much", "is"]}, {"sent": "you better keep the diapers in one place .", "words": ["the", "better", "in", "you"]}, {"sent": "do you think wed better call the police ?", "words": ["the", "better", "do", "think", "you"]}, {"sent": "Daddy should have stirred it better .", "words": ["have", "better", "it"]}, {"sent": "thats better yeah .", "words": ["better"]}, {"sent": "they cut it better .", "words": ["better", "they", "it", "cut"]}, {"sent": "thats better .", "words": ["better"]}, {"sent": "theyre just shes better able to .", "words": ["to", "better"]}, {"sent": "better tell mommy to peel that .", "words": ["to", "better", "that"]}, {"sent": "you better block me .", "words": ["better", "me", "block", "you"]}, {"sent": "thats better .", "words": ["better"]}, {"sent": "okay are they better now ?", "words": ["better", "they", "are"]}, {"sent": "thats better though there is better than there .", "words": ["better", "is", "there"]}, {"sent": "you got a better idea ?", "words": ["better", "a", "you"]}, {"sent": "ah thats better theres lotsof room in here and its so cozy .", "words": ["better", "so", "and", "here", "in", "room"]}, {"sent": "thats better .", "words": ["better"]}, {"sent": "if youre having a problem you better get me .", "words": ["better", "if", "you", "get", "me", "a"]}, {"sent": "these still smell better than the other .", "words": ["better", "these", "other", "the"]}, {"sent": "well you better behave .", "words": ["better", "you"]}, {"sent": "which sounds better to you Sarah he jumping or hes jumping ?", "words": ["better", "to", "you", "he", "which"]}, {"sent": "youd better hurry because I think pandas gonna beat you .", "words": ["better", "think", "hurry", "you", "because"]}, {"sent": "better be careful .", "words": ["better", "be", "careful"]}, {"sent": "do you think it would be better if we put this in the box and the stickers out here ?", "words": ["the", "do", "think", "better", "put", "if", "box", "we", "be", "you", "and", "this", "in", "it", "here", "out", "would"]}, {"sent": "see which one you like better this time .", "words": ["better", "like", "you", "this", "see", "which"]}, {"sent": "thats much better now .", "words": ["better", "much"]}, {"sent": "oh thats so much better now it can be nice and hot .", "words": ["can", "better", "so", "be", "nice", "and", "hot", "it", "much"]}, {"sent": "you better stop throwing that ball .", "words": ["better", "stop", "you", "ball", "that"]}, {"sent": "well he better do it .", "words": ["better", "do", "it", "he"]}, {"sent": "now do you feel better for that now ?", "words": ["better", "do", "for", "you", "that"]}, {"sent": "well shall I open the blinds and then you can see a bit better ?", "words": ["the", "can", "better", "open", "you", "and", "see", "then", "a"]}, {"sent": "thats better ?", "words": ["better"]}, {"sent": "aye thats better .", "words": ["better"]}, {"sent": "we better put it down on the floor .", "words": ["the", "better", "put", "we", "it", "on", "down"]}, {"sent": "its better now .", "words": ["better"]}, {"sent": "wed be better to cut the picture and the story up .", "words": ["the", "better", "up", "cut", "be", "to", "picture", "and", "story"]}, {"sent": "do I have better hair now ?", "words": ["have", "do", "hair", "better"]}, {"sent": "I think Williams a much better pianist than the teddybear .", "words": ["the", "better", "think", "teddybear", "much", "a"]}, {"sent": "which do you like better ?", "words": ["better", "do", "like", "you", "which"]}, {"sent": "lemme lemme make it a little bit better .", "words": ["better", "little", "it", "make", "a"]}, {"sent": "sit her up better so she can hear the story .", "words": ["up", "better", "can", "the", "story", "so", "hear", "sit", "her", "she"]}, {"sent": "well I think I better get you dressed first .", "words": ["better", "think", "first", "you", "get"]}, {"sent": "you better drive the rest then .", "words": ["the", "better", "you", "drive", "then"]}, {"sent": "well I think the country will be much better off for it .", "words": ["the", "better", "think", "off", "for", "be", "it", "will", "much"]}, {"sent": "thats better .", "words": ["better"]}, {"sent": "you better leave that afro alone .", "words": ["better", "that", "you"]}, {"sent": "you hug Nomi feel better .", "words": ["hug", "better", "you"]}, {"sent": "that looks better already .", "words": ["better", "that"]}, {"sent": "you might be better at the gym now .", "words": ["the", "better", "be", "you", "at"]}, {"sent": "you can see better from back here .", "words": ["can", "better", "back", "you", "here", "see"]}, {"sent": "thats much better .", "words": ["better", "much"]}, {"sent": "thats better .", "words": ["better"]}, {"sent": "Michael feels better already .", "words": ["better"]}, {"sent": "Ill make him better .", "words": ["better", "him", "make"]}, {"sent": "oh its better now .", "words": ["better"]}, {"sent": "oh I dont think youd better take the top off .", "words": ["the", "better", "think", "off", "dont", "take"]}, {"sent": "does that feel better ?", "words": ["better", "does", "that"]}, {"sent": "better look for something blue then .", "words": ["better", "for", "blue", "look", "then"]}, {"sent": "hes all better .", "words": ["better", "all"]}, {"sent": "lets put our chair there thats much better .", "words": ["better", "put", "chair", "our", "there", "much"]}, {"sent": "thats better .", "words": ["better"]}, {"sent": "is that better ?", "words": ["better", "that", "is"]}, {"sent": "you better cut it .", "words": ["better", "it", "cut", "you"]}, {"sent": "thats a little better .", "words": ["better", "little", "a"]}, {"sent": "youre not getting the lollipop unless you do a little better than that .", "words": ["the", "do", "better", "you", "lollipop", "little", "that", "not", "a"]}, {"sent": "thats better .", "words": ["better"]}, {"sent": "that better ?", "words": ["better", "that"]}, {"sent": "they better .", "words": ["better", "they"]}, {"sent": "lots better .", "words": ["better"]}, {"sent": "hell hear better if you leave him alone you know .", "words": ["better", "if", "you", "hear", "him"]}, {"sent": "thats a better size .", "words": ["better", "a"]}, {"sent": "I think it had better be an engine shed because .", "words": ["better", "think", "be", "an", "it", "because"]}, {"sent": "its ever so much better now .", "words": ["better", "so", "much"]}, {"sent": "are you going to drive a car ?", "words": ["to", "car", "you", "drive", "are", "a"]}, {"sent": "does Billy drive in daddys blue van ?", "words": ["drive", "does", "in", "blue"]}, {"sent": "you dont have a fireman to drive that truck .", "words": ["to", "truck", "you", "have", "dont", "drive", "that", "a"]}, {"sent": "our house when we drive up but then I said whos house it is she would say Lilys .", "words": ["up", "we", "is", "say", "our", "drive", "it", "house", "would", "then", "when", "she", "but"]}, {"sent": "Ill drive the car .", "words": ["drive", "the", "car"]}, {"sent": "might be hard for you to drive on the carpet .", "words": ["the", "for", "to", "be", "you", "drive", "hard", "on"]}, {"sent": "are you going to drive it round the streets ?", "words": ["the", "to", "you", "drive", "are", "it"]}, {"sent": "but its too little to drive its just a toy car .", "words": ["to", "car", "little", "drive", "toy", "too", "but", "a"]}, {"sent": "will it drive under that bridge ?", "words": ["drive", "under", "it", "will", "that"]}, {"sent": "dont drive your tractor over your pieces .", "words": ["your", "drive", "dont", "tractor", "over"]}, {"sent": "is it trying to drive up the bank ?", "words": ["up", "the", "is", "to", "drive", "it"]}, {"sent": "are you saying drive off ?", "words": ["drive", "are", "off", "you"]}, {"sent": "are you gonna go and drive down the shops ?", "words": ["the", "go", "and", "drive", "you", "are", "down"]}, {"sent": "just when they reached the end of the drive grr roared the engine .", "words": ["the", "of", "drive", "they", "when"]}, {"sent": "so Ill not pull her tail or drive her away .", "words": ["so", "pull", "drive", "away", "her", "not"]}, {"sent": "did you drive the car ?", "words": ["the", "car", "you", "drive", "did"]}, {"sent": "can a baby drive the car ?", "words": ["can", "the", "car", "drive", "a"]}, {"sent": "drive the red car .", "words": ["drive", "the", "red", "car"]}, {"sent": "was he all set to drive off ?", "words": ["off", "all", "was", "to", "drive", "he"]}, {"sent": "you let the marbles drive .", "words": ["the", "drive", "you"]}, {"sent": "can you drive a tractor ?", "words": ["can", "you", "drive", "tractor", "a"]}, {"sent": "turn it this way if youre gonna drive .", "words": ["drive", "this", "it", "if"]}, {"sent": "make it drive through these .", "words": ["drive", "these", "it", "make"]}, {"sent": "can you drive this truck ?", "words": ["can", "truck", "you", "drive", "this"]}, {"sent": "and drive off in my car .", "words": ["off", "my", "car", "and", "drive", "in"]}, {"sent": "so you could drive .", "words": ["drive", "so", "could", "you"]}, {"sent": "well whos going to drive the train now theres no horse to drive it ?", "words": ["the", "train", "to", "drive", "it", "horse"]}, {"sent": "Sarah when are you gonna learn how to drive ?", "words": ["how", "to", "you", "drive", "are", "when"]}, {"sent": "see you stick stuff all in the drive and .", "words": ["the", "all", "stick", "you", "drive", "and", "in", "see"]}, {"sent": "you can drive your car in the kitchen .", "words": ["can", "the", "your", "car", "you", "drive", "in", "kitchen"]}, {"sent": "you gonna drive your car .", "words": ["drive", "car", "your", "you"]}, {"sent": "do you wanna watch him drive away in his new car ?", "words": ["do", "watch", "car", "you", "drive", "him", "away", "in", "wanna", "his", "new"]}, {"sent": "yep all the people drive under there .", "words": ["the", "all", "drive", "under", "there"]}, {"sent": "it says its gonna drive on your head .", "words": ["your", "head", "drive", "it", "on"]}, {"sent": "and what does he drive ?", "words": ["what", "does", "and", "drive", "he"]}, {"sent": "its what this drive in .", "words": ["drive", "what", "in", "this"]}, {"sent": "you want Dingo to drive into the tuba ?", "words": ["the", "into", "to", "you", "drive"]}, {"sent": "I think the postman is going to drive off in a few minutes in his red van .", "words": ["the", "off", "think", "red", "is", "to", "drive", "in", "his", "a"]}, {"sent": "because she hasta drive ?", "words": ["drive", "she", "because"]}, {"sent": "because shes not learnt to drive yet .", "words": ["drive", "to", "not", "because"]}, {"sent": "of course well drive everyone nuts but if we have a whole book then we can really .", "words": ["can", "if", "we", "of", "drive", "have", "book", "then", "but", "a"]}, {"sent": "and I havent been like I drive every so often and uh .", "words": ["so", "like", "and", "drive", "every"]}, {"sent": "okay this drive dinosaurs gonna drive back .", "words": ["drive", "this", "back"]}, {"sent": "do you remember I use to wear you in the sling in the car when we would drive xxx xxx xxx .", "words": ["the", "do", "we", "to", "car", "you", "drive", "in", "would", "when"]}, {"sent": "shall I drive ?", "words": ["drive"]}, {"sent": "oh look at this car do you drive this car ?", "words": ["do", "look", "car", "you", "drive", "this", "at"]}, {"sent": "whos gonna drive ?", "words": ["drive"]}, {"sent": "Im gonna drive the motorcycle boy .", "words": ["drive", "the", "motorcycle"]}, {"sent": "yep theyre gonna drive .", "words": ["drive"]}, {"sent": "pop me back up in the lift and Ill come and pick my car up and drive out .", "words": ["up", "the", "my", "pop", "back", "pick", "car", "and", "drive", "in", "out", "me"]}, {"sent": "needs somebody to drive it .", "words": ["drive", "to", "it"]}, {"sent": "oh I wanna drive my cars up in there .", "words": ["up", "my", "drive", "there", "in", "wanna"]}, {"sent": "the little animal doll characters then they have their vehicles and a train and shell sit and shell actually drive youknow like start to push them around and drive them then the ball that she plays with .", "words": ["the", "like", "their", "with", "have", "ball", "then", "that", "she", "push", "to", "doll", "little", "and", "drive", "a", "train", "them", "they", "animal", "around", "sit"]}, {"sent": "is he going for a drive in the car ?", "words": ["the", "for", "is", "car", "drive", "in", "he", "a"]}, {"sent": "can you get them to drive round ?", "words": ["can", "to", "them", "you", "get", "drive"]}, {"sent": "what do you like at the drive in ?", "words": ["the", "what", "do", "like", "you", "drive", "in", "at"]}, {"sent": "alright come on then drive it round here .", "words": ["drive", "here", "it", "on", "then"]}, {"sent": "you drive it right out and you put it in the parking lot and you walk back in please .", "words": ["the", "put", "back", "and", "drive", "you", "in", "it", "out", "walk"]}, {"sent": "you wanna drive my car ?", "words": ["my", "car", "you", "drive", "wanna"]}, {"sent": "help Daddy drive .", "words": ["drive", "help"]}, {"sent": "hes gonna drive okay .", "words": ["drive"]}, {"sent": "if we go home shall we drive in the car or shall we fly in the plane ?", "words": ["the", "if", "go", "we", "car", "drive", "in", "home"]}, {"sent": "you gonna drive ?", "words": ["drive", "you"]}, {"sent": "doesnt know how to drive at all .", "words": ["all", "how", "to", "drive", "at"]}, {"sent": "shall we find a little man to drive it ?", "words": ["we", "to", "little", "drive", "it", "find", "a"]}, {"sent": "and he came and brought a big bike where you hafta press this thing and drive it .", "words": ["where", "and", "you", "this", "drive", "it", "he", "big", "a"]}, {"sent": "youre gonna drive yourself are you ?", "words": ["drive", "yourself", "are", "you"]}, {"sent": "you mean Im lucky because I can drive on the road ?", "words": ["can", "the", "you", "drive", "on", "because"]}, {"sent": "whos gonna drive the car .", "words": ["drive", "the", "car"]}, {"sent": "the fist thing I do when I go to McDonalds is I drive to McDonalds I walk in the door and then what happens ?", "words": ["the", "do", "what", "go", "is", "to", "and", "drive", "walk", "in", "then", "when", "door"]}, {"sent": "wanna drive it ?", "words": ["drive", "it", "wanna"]}, {"sent": "look a drive through the country .", "words": ["drive", "the", "a", "look"]}, {"sent": "he could drive couldnt he ?", "words": ["drive", "could", "he"]}, {"sent": "whos going to drive it ?", "words": ["drive", "to", "it"]}, {"sent": "thank you for drive thank you so so much .", "words": ["for", "so", "you", "drive", "much"]}, {"sent": "drive the car I should say .", "words": ["drive", "the", "say", "car"]}, {"sent": "shall we build an arch for the tractor to drive through ?", "words": ["the", "for", "we", "to", "drive", "an", "build", "tractor"]}, {"sent": "trying to get up and get ready and drive over there .", "words": ["up", "to", "and", "get", "drive", "there", "over"]}, {"sent": "take it off the fireguard and drive it on the floor please .", "words": ["off", "the", "and", "drive", "it", "on", "take"]}, {"sent": "now drive very slow .", "words": ["drive", "slow"]}, {"sent": "its only cars that drive .", "words": ["drive", "that"]}, {"sent": "now you could drive .", "words": ["drive", "could", "you"]}, {"sent": "whos going to drive the car ?", "words": ["drive", "to", "car", "the"]}, {"sent": "you drive .", "words": ["drive", "you"]}, {"sent": "drive on the street Dingo .", "words": ["drive", "the", "on", "street"]}, {"sent": "you drive .", "words": ["drive", "you"]}, {"sent": "hes going to drive the train ?", "words": ["drive", "to", "train", "the"]}, {"sent": "may be the dog can drive the train .", "words": ["the", "can", "train", "be", "drive", "dog"]}, {"sent": "are you gonna go for a drive ?", "words": ["for", "go", "you", "drive", "are", "a"]}, {"sent": "dont dont drive that on the photograph darling because it will damage it .", "words": ["the", "will", "drive", "dont", "it", "on", "that", "because"]}, {"sent": "oh hes gonna drive a car ?", "words": ["drive", "car", "a"]}, {"sent": "you drive up this one Sally you drive up this one .", "words": ["drive", "this", "up", "you"]}, {"sent": "and a wheel to drive it with ?", "words": ["to", "with", "and", "drive", "it", "a"]}, {"sent": "who wants to drive ?", "words": ["drive", "to", "who"]}, {"sent": "to drive the truck .", "words": ["drive", "to", "truck", "the"]}, {"sent": "youre sposta drive it .", "words": ["drive", "it"]}, {"sent": "well is there a car for her to drive ?", "words": ["for", "is", "car", "to", "drive", "there", "her", "a"]}, {"sent": "didnt think one got dressed up to drive a tractor .", "words": ["up", "think", "to", "drive", "tractor", "a"]}, {"sent": "Abba has one in his car and you drive it too .", "words": ["car", "you", "and", "drive", "in", "it", "too", "his"]}, {"sent": "can you drive a tractor ?", "words": ["can", "you", "drive", "tractor", "a"]}, {"sent": "tried that last night huh ?", "words": ["last", "that"]}, {"sent": "I found one in the bread box last night xxx .", "words": ["the", "bread", "box", "last", "in"]}, {"sent": "did you get a shampoo last night ?", "words": ["last", "you", "get", "did", "a"]}, {"sent": "the last one .", "words": ["the", "last"]}, {"sent": "xxx the last bit .", "words": ["the", "last"]}, {"sent": "whatve we done then since we last saw Caroline .", "words": ["then", "last", "we"]}, {"sent": "Tonys going last .", "words": ["last"]}, {"sent": "can you put the last two pieces in ?", "words": ["can", "the", "put", "last", "you", "in"]}, {"sent": "Mommy had a weird dreams last night .", "words": ["last", "a"]}, {"sent": "yousee this last couple of weeks you know what he likes to be called ?", "words": ["what", "of", "to", "last", "be", "you", "this", "he"]}, {"sent": "so first of all what do you think has changed since last time ?", "words": ["what", "all", "do", "think", "of", "so", "last", "first", "you"]}, {"sent": "did you ask Ursula if shed like one before you took the last one ?", "words": ["the", "if", "last", "like", "you", "did"]}, {"sent": "but we went to stay with Nana last week .", "words": ["we", "to", "last", "with", "stay", "but"]}, {"sent": "this time last year Thomas was still having a sleep in the afternoon wasnt he ?", "words": ["the", "sleep", "was", "last", "this", "in", "he", "a"]}, {"sent": "she and Mike stayed up til one thirty or two last night discussing whether or not they would have children .", "words": ["up", "last", "would", "and", "have", "they", "she", "not"]}, {"sent": "you know did you know that Ryan useta work in the day care last year ?", "words": ["the", "work", "last", "you", "in", "did", "that"]}, {"sent": "yeah the last couple weeks have truly amazed me because shes adding i n g to her verbs like shell say Im jumping jumping not all the time but shes put is into a sentence now um or .", "words": ["the", "all", "i", "put", "into", "is", "to", "last", "say", "like", "have", "but", "me", "her", "not", "because", "a"]}, {"sent": "remember you were RainbowBrite last year ?", "words": ["last", "were", "you"]}, {"sent": "it was alright last night when I was in the house .", "words": ["the", "was", "last", "in", "it", "house", "when"]}, {"sent": "how did you take it apart last time ?", "words": ["how", "last", "you", "it", "did", "take"]}, {"sent": "well I think you ate the sausage last time .", "words": ["the", "think", "last", "you"]}, {"sent": "played with that one last week .", "words": ["last", "that", "with"]}, {"sent": "last one last one darling .", "words": ["last"]}, {"sent": "you woke up last night didnt you and cried because your elephant had fallen outof bed .", "words": ["up", "your", "last", "bed", "and", "you", "elephant", "because"]}, {"sent": "is that the last one ?", "words": ["the", "last", "that", "is"]}, {"sent": "yeah and Andy Andys been talking about it cause he couldnt do it last time so hes .", "words": ["do", "so", "last", "and", "it", "he", "about"]}, {"sent": "he wasnt able to well last time I was giving him stuff like here Kalin look at this .", "words": ["was", "to", "last", "like", "here", "this", "him", "at", "he", "look"]}, {"sent": "*IN2: and do you know what my last thing is in here ?", "words": ["do", "my", "what", "is", "last", "you", "and", "here", "in"]}, {"sent": "I wish I could find that last piece of jigsawpuzzle .", "words": ["of", "last", "wish", "find", "could", "that"]}, {"sent": "whats her last name ?", "words": ["last", "her"]}, {"sent": "well you told me last time they were gonna be pumas .", "words": ["be", "last", "you", "they", "me", "were"]}, {"sent": "what did the story say about sharing last night Adam ?", "words": ["the", "what", "last", "say", "did", "about", "story"]}, {"sent": "if you feel sick the very last thing you want is a sweetie .", "words": ["the", "if", "is", "last", "you", "sick", "a"]}, {"sent": "youre last box of toys .", "words": ["of", "last", "box"]}, {"sent": "we can put this last piece away and we can get your soft stuff out and you can make some shapes .", "words": ["can", "put", "soft", "your", "we", "last", "and", "get", "this", "you", "away", "out", "some", "make"]}, {"sent": "can I give you your last spoonful of yogurt ?", "words": ["can", "your", "of", "last", "yogurt", "you", "give"]}, {"sent": "she well I dont know what time she did it I took her out last night at ten .", "words": ["what", "last", "dont", "it", "did", "out", "at", "her", "she"]}, {"sent": "thats what you did last time .", "words": ["what", "last", "did", "you"]}, {"sent": "yeah mines stepped on the floor last night .", "words": ["on", "the", "last"]}, {"sent": "what about that nice one I got from the library last week ?", "words": ["the", "what", "last", "nice", "about", "that"]}, {"sent": "no well see you were ahead of me so you won and blue is the last one to get so you won .", "words": ["the", "is", "of", "so", "me", "last", "to", "you", "and", "blue", "get", "see", "were"]}, {"sent": "she had them last week .", "words": ["last", "them", "she"]}, {"sent": "I went for pizza last night too .", "words": ["last", "for", "pizza", "too"]}, {"sent": "the woman upstairs name is Dennis and the last name is Dennison .", "words": ["the", "last", "and", "is"]}, {"sent": "so anyway he called me last night .", "words": ["so", "me", "he", "last"]}, {"sent": "it doesnt get to six oclock until the very last page Joel .", "words": ["the", "to", "last", "get", "it"]}, {"sent": "and I think perhaps weve only seen one in the last few weeks .", "words": ["the", "think", "last", "and", "in"]}, {"sent": "so thats the last collection before Christmas .", "words": ["the", "so", "last"]}, {"sent": "so Ive been trying to think of some interesting things weve done since weve seen her in the last couple weeks .", "words": ["the", "think", "of", "to", "so", "last", "in", "some", "her"]}, {"sent": "and our Patrick was a a page boy last year .", "words": ["was", "last", "our", "and", "a"]}, {"sent": "I dont remember when I last did that .", "words": ["last", "dont", "that", "did", "when"]}, {"sent": "do you think youre gonna be able to get the last one on ?", "words": ["the", "do", "think", "to", "be", "last", "you", "get", "on"]}, {"sent": "you were very good at building those blocks last time .", "words": ["those", "last", "you", "good", "at", "were"]}, {"sent": "tried to fix it last night .", "words": ["to", "last", "it", "fix"]}, {"sent": "it wasnt last night .", "words": ["last", "it"]}, {"sent": "Nana bumped her head last week didnt she ?", "words": ["last", "head", "her", "she"]}, {"sent": "and I was cutting I was cutting back a lot of the dead things last night wasnt I ?", "words": ["the", "back", "of", "was", "last", "a lot", "and", "a"]}, {"sent": "I guess he hadnt been to Provence is the country in France but Toot went to Paris in France in the last book didnt he ?", "words": ["the", "is", "to", "last", "in", "he", "book", "but"]}, {"sent": "xxx let her do the last one .", "words": ["the", "do", "last", "her"]}, {"sent": "the last time I saw at the xxx .", "words": ["the", "last", "at"]}, {"sent": "did he do that last time or do you remember ?", "words": ["do", "last", "you", "he", "did", "that"]}, {"sent": "and I think the last one might be that one .", "words": ["the", "think", "be", "last", "and", "that"]}, {"sent": "and it says at last Thomas reached the seaside .", "words": ["the", "last", "and", "it", "at"]}, {"sent": "we have to stop looking at them last time okay no more .", "words": ["we", "to", "last", "stop", "them", "have", "more", "at"]}, {"sent": "no I washed alotof these glasses last night .", "words": ["these", "last", "glasses"]}, {"sent": "yeah she probably would have gone straight for just the food last time .", "words": ["the", "for", "last", "have", "food", "would", "she"]}, {"sent": "last week it was a rabbit we went through .", "words": ["we", "was", "last", "it", "a"]}, {"sent": "what did you do here last time when you were here ?", "words": ["what", "do", "last", "you", "here", "did", "when", "were"]}, {"sent": "okay whats your last name Kathryn ?", "words": ["last", "your"]}, {"sent": "plaster goes on last .", "words": ["on", "last"]}, {"sent": "you had yours done last week .", "words": ["last", "you"]}, {"sent": "remember what happened last night with Daddy ?", "words": ["what", "last", "with"]}, {"sent": "whatd we have last night ?", "words": ["have", "last", "we"]}, {"sent": "I had a lovely sleep last night .", "words": ["sleep", "last", "a"]}, {"sent": "is that your last little crumb ?", "words": ["your", "is", "last", "little", "that"]}, {"sent": "on the last day of September .", "words": ["on", "the", "last", "of"]}, {"sent": "last night .", "words": ["last"]}, {"sent": "theres a new book that Granny bought last week that we havent looked at .", "words": ["we", "last", "at", "book", "new", "that", "a"]}, {"sent": "did you see somebody on tv last night ?", "words": ["last", "you", "tv", "did", "on", "see"]}, {"sent": "last Saturday .", "words": ["last"]}, {"sent": "I think the last time we were here I read her a book .", "words": ["the", "think", "we", "last", "her", "here", "read", "book", "were", "a"]}, {"sent": "now I dont do you remember what um he primarily played with last time you were here ?", "words": ["do", "what", "last", "with", "you", "here", "dont", "he", "were"]}, {"sent": "um I dont think so I think she liked the keys more last time .", "words": ["the", "think", "so", "last", "dont", "more", "keys", "she"]}, {"sent": "who did you talk to on the telephone last night ?", "words": ["the", "to", "last", "you", "did", "talk", "who", "on", "telephone"]}, {"sent": "ah got it at last .", "words": ["last", "it", "at"]}, {"sent": "should I start it like I did the last time for you ?", "words": ["the", "for", "last", "like", "you", "it", "did"]}, {"sent": "its a funny O yep thats what daddy was telling you last night its a funny O it has this little tail on it .", "words": ["what", "was", "last", "you", "little", "this", "it", "on", "a"]}, {"sent": "do you remember seeing snow last year Thomas ?", "words": ["do", "last", "snow", "you"]}, {"sent": "horsie says something and tiger says the last part .", "words": ["the", "last", "tiger", "and"]}, {"sent": "youre doing what the post office lady did last night arent you ?", "words": ["the", "what", "last", "you", "did"]}, {"sent": "last chance .", "words": ["last"]}, {"sent": "hey Roman why dont you tell us about what you did last night ?", "words": ["what", "last", "you", "us", "dont", "did", "about", "why"]}, {"sent": "last night .", "words": ["last"]}, {"sent": "and last night we had a birthdaycake .", "words": ["last", "we", "a", "and"]}, {"sent": "do you remember we bought that little train set last Summer didnt we ?", "words": ["do", "we", "train", "last", "little", "you", "that"]}, {"sent": "both at Bethanys house last week last Thursday .", "words": ["last", "house", "at"]}, {"sent": "he has got bigger from last week .", "words": ["last", "he"]}, {"sent": "whens the last time you went to the doctor ?", "words": ["the", "to", "last", "you"]}, {"sent": "we went last night we saw them digging all that earth up didnt we ?", "words": ["up", "all", "we", "last", "them", "that"]}, {"sent": "because thats there where you were playing with it last time .", "words": ["where", "last", "with", "you", "there", "it", "were", "because"]}, {"sent": "is that for me too ?", "words": ["for", "is", "too", "me", "that"]}, {"sent": "hey you did too math great you got a green and a blue .", "words": ["you", "and", "blue", "did", "green", "too", "a"]}, {"sent": "because then well put milk in your bowl too .", "words": ["bowl", "put", "your", "in", "too", "milk", "then", "because"]}, {"sent": "oh Ill take the sticker off of Sallys head too .", "words": ["the", "off", "of", "head", "too", "take"]}, {"sent": "she wants me to bring the whole family too .", "words": ["the", "to", "bring", "too", "me", "she"]}, {"sent": "youve got new sneakers on too Sarah .", "words": ["on", "new", "too"]}, {"sent": "yes I like chips too .", "words": ["like", "too"]}, {"sent": "its not too bad a mess though Thomas .", "words": ["a", "not", "bad", "too"]}, {"sent": "no Griz is too big .", "words": ["too", "big", "is"]}, {"sent": "oh theres far too many on there .", "words": ["on", "there", "too"]}, {"sent": "but you dont wear it too much anymore right ?", "words": ["you", "dont", "it", "too", "much", "but"]}, {"sent": "too big Christopher to go in .", "words": ["go", "to", "in", "too", "big"]}, {"sent": "do you want the sea otter to eat the keys too ?", "words": ["the", "do", "to", "you", "eat", "too", "keys"]}, {"sent": "hes too heavy made me fall down .", "words": ["fall", "too", "me", "heavy", "down"]}, {"sent": "thats a dog too huh ?", "words": ["too", "dog", "a"]}, {"sent": "were sitting too close .", "words": ["close", "were", "too"]}, {"sent": "yeah you can cut with that too .", "words": ["can", "cut", "with", "you", "too", "that"]}, {"sent": "heres one too .", "words": ["too"]}, {"sent": "Fraser dont pull it too hard .", "words": ["pull", "dont", "it", "hard", "too"]}, {"sent": "this is too .", "words": ["this", "too", "is"]}, {"sent": "can I come too ?", "words": ["can", "too"]}, {"sent": "Fraser youre too young to play this .", "words": ["to", "play", "this", "too"]}, {"sent": "youre too busy talking and eating .", "words": ["and", "too"]}, {"sent": "you have little bunny slippers like that too huh ?", "words": ["bunny", "like", "little", "have", "you", "too", "that"]}, {"sent": "z is too small to see so dont try .", "words": ["is", "to", "so", "dont", "too", "see", "try"]}, {"sent": "now mommy has one too .", "words": ["too"]}, {"sent": "youre too big for those .", "words": ["for", "big", "those", "too"]}, {"sent": "but he throws them a bit too hard so we hafta beat him up .", "words": ["up", "we", "so", "them", "him", "he", "hard", "too", "but", "a"]}, {"sent": "suddenly all the spotlights have go on have gone on and the cameras too .", "words": ["the", "all", "go", "and", "have", "too", "on"]}, {"sent": "that doesnt sound too pleasant does it ?", "words": ["does", "it", "that", "too"]}, {"sent": "too sour ?", "words": ["too"]}, {"sent": "its nice to have coffee in these too .", "words": ["these", "to", "nice", "have", "in", "coffee", "too"]}, {"sent": "you can eat that too .", "words": ["can", "you", "eat", "too", "that"]}, {"sent": "so youre not doing anything too tough .", "words": ["so", "not", "too"]}, {"sent": "thats because weve got too much .", "words": ["much", "because", "too"]}, {"sent": "dont make too much noise please Thomas .", "words": ["dont", "make", "much", "too"]}, {"sent": "its too noisy .", "words": ["noisy", "too"]}, {"sent": "its Kamut too .", "words": ["too"]}, {"sent": "you did too .", "words": ["too", "did", "you"]}, {"sent": "itll itll ignore you too .", "words": ["too", "you"]}, {"sent": "are you serving these people too ?", "words": ["these", "too", "are", "you"]}, {"sent": "Im not too sure now .", "words": ["not", "too"]}, {"sent": "thats pretty exciting stuff up there too isnt it ?", "words": ["up", "there", "it", "pretty", "too"]}, {"sent": "dont be too long about it .", "words": ["be", "long", "dont", "it", "about", "too"]}, {"sent": "xxx you too .", "words": ["too", "you"]}, {"sent": "Im too busy .", "words": ["too"]}, {"sent": "you dont need the parcel its too dirty ?", "words": ["the", "need", "you", "dont", "dirty", "too"]}, {"sent": "you do too ?", "words": ["do", "too", "you"]}, {"sent": "theres a couple other books she likes too .", "words": ["too", "other", "she", "a"]}, {"sent": "is your hand too big ?", "words": ["your", "hand", "is", "too", "big"]}, {"sent": "and the cup too because Ive been working with her on her sippycup so she automatically went and put that in her mouth too so I think those she did youknow think they were like hers .", "words": ["the", "cup", "put", "think", "mouth", "like", "hers", "those", "with", "on", "were", "that", "she", "so", "and", "too", "because", "they", "in", "did", "her"]}, {"sent": "you remember what her daddys name is too ?", "words": ["what", "is", "you", "too", "her"]}, {"sent": "and your diapers probably feel pretty bogus when theyre wet too huh ?", "words": ["wet", "your", "and", "pretty", "too", "when"]}, {"sent": "and such a young baby too .", "words": ["too", "a", "and"]}, {"sent": "dont push too hard .", "words": ["dont", "push", "hard", "too"]}, {"sent": "yes thats a lady too .", "words": ["too", "a"]}, {"sent": "you gotta make sure they cant do that too .", "words": ["do", "you", "they", "that", "too", "make"]}, {"sent": "I can roll it too .", "words": ["can", "it", "too"]}, {"sent": "I think these are too small for you .", "words": ["these", "think", "for", "you", "are", "too"]}, {"sent": "I want a picture of you with him too .", "words": ["of", "picture", "with", "you", "him", "too", "a"]}, {"sent": "xxx mommys yawning too .", "words": ["too"]}, {"sent": "I have something else for you too look at .", "words": ["for", "you", "have", "at", "look", "too"]}, {"sent": "you can come too .", "words": ["can", "too", "you"]}, {"sent": "you put too much in your cup ?", "words": ["cup", "put", "your", "you", "in", "too", "much"]}, {"sent": "you made a tower too .", "words": ["too", "a", "you"]}, {"sent": "I an been too .", "words": ["an", "too"]}, {"sent": "here is more tools over here too .", "words": ["is", "here", "more", "too", "over"]}, {"sent": "this is for Daddy but why dont you draw a picture on this side too ?", "words": ["for", "is", "draw", "picture", "you", "this", "dont", "but", "too", "on", "why", "a"]}, {"sent": "youre gonna call the ball Gadiga too ?", "words": ["the", "ball", "too"]}, {"sent": "okay you dont hafta press it down too hard .", "words": ["you", "dont", "it", "hard", "too", "down"]}, {"sent": "dont eat too much Chester .", "words": ["eat", "dont", "much", "too"]}, {"sent": "youve got too much in it .", "words": ["in", "it", "much", "too"]}, {"sent": "theyre too heavy for you darling arent they ?", "words": ["for", "you", "they", "too", "heavy"]}, {"sent": "it was too long .", "words": ["was", "it", "long", "too"]}, {"sent": "too xxx too heavy ?", "words": ["heavy", "too"]}, {"sent": "its too hot oh okay thankyou .", "words": ["hot", "too"]}, {"sent": "is Dimitra at the table too ?", "words": ["the", "table", "is", "at", "too"]}, {"sent": "she can be destructful too sometimes but not too bad .", "words": ["can", "be", "not", "bad", "too", "she", "but"]}, {"sent": "we like to eat too .", "words": ["we", "to", "like", "eat", "too"]}, {"sent": "and a blue too .", "words": ["too", "blue", "a", "and"]}, {"sent": "they have some black and white tigers too .", "words": ["and", "have", "they", "black", "too", "some", "white"]}, {"sent": "mother possum carries her babys on her back too .", "words": ["on", "her", "back", "too"]}, {"sent": "till his whole body shook and pretty soon his hair shook too .", "words": ["and", "pretty", "too", "his", "hair"]}, {"sent": "its too big for Mommy .", "words": ["for", "big", "too"]}, {"sent": "pretend you hadta look at the other kids too and like sh she you were looking at the other kids and make sure she doesnt do anything to the other kids because shed shed do something to Ethan .", "words": ["the", "do", "too", "to", "like", "you", "other", "and", "at", "make", "look", "pretend", "were", "she", "because"]}, {"sent": "yeah lets try it on this one too .", "words": ["this", "it", "too", "on", "try"]}, {"sent": "now youre getting too smart .", "words": ["too"]}, {"sent": "shell get too .", "words": ["get", "too"]}, {"sent": "its too long Caroline isnt it ?", "words": ["it", "long", "too"]}, {"sent": "have some sweet potato too .", "words": ["have", "some", "too", "potato"]}, {"sent": "and you have some tofu there too .", "words": ["you", "have", "there", "and", "too", "some"]}, {"sent": "you better tell him to stop before he goes too far .", "words": ["better", "to", "stop", "you", "him", "he", "too"]}, {"sent": "Im too big .", "words": ["big", "too"]}, {"sent": "dont leave it on her for too long .", "words": ["for", "long", "dont", "it", "too", "on", "her"]}, {"sent": "so theres not too much mess when we come back .", "words": ["back", "we", "so", "much", "too", "when", "not"]}, {"sent": "aye her xxx is coming on too .", "words": ["on", "too", "her", "is"]}, {"sent": "so um she has more options now too .", "words": ["so", "more", "she", "too"]}, {"sent": "I am a robot too .", "words": ["too", "am", "a"]}, {"sent": "theyre too big .", "words": ["big", "too"]}, {"sent": "I know where it is too ?", "words": ["where", "too", "it", "is"]}, {"sent": "you know why the dragon felt bad ?", "words": ["the", "bad", "why", "you"]}, {"sent": "as they left a young man wearing an artists beret was taking his wet laundry out of a bag a very bad Corduroy had discovered .", "words": ["wet", "of", "was", "bad", "they", "an", "out", "his", "a"]}, {"sent": "too bad .", "words": ["bad", "too"]}, {"sent": "hes always in a bad mood .", "words": ["in", "bad", "a"]}, {"sent": "now you got a bad cold because he gave you a bad cold because its from here .", "words": ["bad", "you", "here", "he", "cold", "because", "a"]}, {"sent": "alright heres the bad person hes gonna come and .", "words": ["the", "bad", "and"]}, {"sent": "that was so bad too .", "words": ["was", "so", "bad", "too", "that"]}, {"sent": "is it do you think its in a bad way ?", "words": ["do", "think", "is", "bad", "you", "in", "it", "a"]}, {"sent": "its too bad we dont have a horse to give it to .", "words": ["we", "to", "bad", "have", "dont", "it", "too", "horse", "give", "a"]}, {"sent": "thank God I went on the highway because the highway wasnt bad but the other roads were really bad .", "words": ["the", "because", "bad", "other", "on", "were", "but"]}, {"sent": "youre gonna be a bad loser xxx you are uhn arent you ?", "words": ["be", "bad", "you", "are", "a"]}, {"sent": "yeah and Id get the sword from the bad guy and Id cut everybody .", "words": ["the", "cut", "bad", "and", "get"]}, {"sent": "Jeannines got a very bad headache .", "words": ["bad", "a"]}, {"sent": "xxx not bad .", "words": ["not", "bad"]}, {"sent": "its not bad .", "words": ["not", "bad"]}, {"sent": "fever and really bad .", "words": ["bad", "and"]}, {"sent": "he had a bad one all the way to his stomach huh ?", "words": ["the", "all", "to", "bad", "he", "his", "a"]}, {"sent": "great great Cathy not so bad at all .", "words": ["all", "so", "bad", "at", "not"]}, {"sent": "you bad girl .", "words": ["bad", "you"]}, {"sent": "big bad wolf .", "words": ["wolf", "big", "bad"]}, {"sent": "its not so bad out here now .", "words": ["so", "bad", "here", "out", "not"]}, {"sent": "was he a bad bear ?", "words": ["was", "bad", "bear", "he", "a"]}, {"sent": "they must have bad breath .", "words": ["have", "they", "bad"]}, {"sent": "come on and see how bad your God damned thing .", "words": ["how", "your", "bad", "and", "on", "see"]}, {"sent": "but he isnt too bad .", "words": ["he", "bad", "but", "too"]}, {"sent": "see The elephant and the bad baby ?", "words": ["the", "bad", "and", "elephant", "see"]}, {"sent": "with your bad cough ?", "words": ["your", "with", "bad"]}, {"sent": "oh thats no that bad .", "words": ["that", "bad"]}, {"sent": "which one is the bad one ?", "words": ["the", "which", "bad", "is"]}, {"sent": "the bad boys had stolen the balloon from the little boy .", "words": ["the", "balloon", "bad", "little"]}, {"sent": "somebody did something very bad .", "words": ["did", "bad"]}, {"sent": "feeling so bad about the whole thing Im like ugh thats just so yucky .", "words": ["the", "so", "yucky", "like", "bad", "about"]}, {"sent": "the rain and the big bad wolf .", "words": ["the", "wolf", "rain", "bad", "and", "big"]}, {"sent": "that sounds like a bad cold .", "words": ["like", "bad", "that", "cold", "a"]}, {"sent": "Id probably be just as bad .", "words": ["be", "bad"]}, {"sent": "it isnt that bad .", "words": ["it", "that", "bad"]}, {"sent": "thats for his bad finger .", "words": ["his", "finger", "for", "bad"]}, {"sent": "they were a bad idea those paints werent they ?", "words": ["those", "bad", "they", "were", "a"]}, {"sent": "what did he do that was bad ?", "words": ["what", "do", "was", "bad", "he", "did", "that"]}, {"sent": "was it a bad bump ?", "words": ["bump", "was", "bad", "it", "a"]}, {"sent": "thats too bad .", "words": ["bad", "too"]}, {"sent": "not so bad ?", "words": ["so", "not", "bad"]}, {"sent": "too bad they are made out of paper otherwise we could taste them .", "words": ["we", "of", "paper", "them", "taste", "bad", "they", "are", "out", "too", "could"]}, {"sent": "Ill be bad to ya .", "words": ["to", "be", "bad"]}, {"sent": "oh that ones not bad .", "words": ["that", "not", "bad"]}, {"sent": "hes not bad now .", "words": ["not", "bad"]}, {"sent": "I wouldnt do it again if it was as bad as that .", "words": ["do", "if", "was", "bad", "it", "that"]}, {"sent": "oh I dont want no big old bad thing after me .", "words": ["old", "bad", "dont", "me", "big"]}, {"sent": "Mummys bad too isnt she ?", "words": ["she", "bad", "too"]}, {"sent": "hes a bad boy ?", "words": ["bad", "a"]}, {"sent": "and if shes bad .", "words": ["if", "bad", "and"]}, {"sent": "thats a bad cough .", "words": ["bad", "a"]}, {"sent": "he is really really bad .", "words": ["he", "bad", "is"]}, {"sent": "I like bad monsters .", "words": ["like", "bad"]}, {"sent": "oh no olives then thats too bad .", "words": ["then", "bad", "too"]}, {"sent": "we have a bad little taperecorder its not talking to us .", "words": ["we", "to", "bad", "little", "have", "us", "not", "a"]}, {"sent": "you had bad dinners ?", "words": ["bad", "you"]}, {"sent": "what did they do that was bad ?", "words": ["what", "do", "was", "bad", "they", "did", "that"]}, {"sent": "itll be all bad .", "words": ["all", "be", "bad"]}, {"sent": "bad ?", "words": ["bad"]}, {"sent": "hes got a very bad cold .", "words": ["cold", "bad", "a"]}, {"sent": "that was a bad wreck .", "words": ["was", "that", "bad", "a"]}, {"sent": "oh bad .", "words": ["bad"]}, {"sent": "xxx but shes not as bad .", "words": ["not", "but", "bad"]}, {"sent": "not bad were just starting supper so you can have supper .", "words": ["can", "so", "bad", "you", "have", "were", "not"]}, {"sent": "it was very bad aye it was .", "words": ["was", "it", "bad"]}, {"sent": "thats bad news isnt it ?", "words": ["it", "bad"]}, {"sent": "put them in the books thats not a bad idea and then well know what page to read .", "words": ["the", "what", "put", "to", "them", "bad", "and", "in", "read", "then", "not", "a"]}, {"sent": "too bad .", "words": ["bad", "too"]}, {"sent": "when your mouth is bad these days xxx .", "words": ["these", "your", "is", "mouth", "bad", "when"]}, {"sent": "the big bad wolf the big bad wolf .", "words": ["the", "wolf", "big", "bad"]}, {"sent": "why are aeroplanes bad ?", "words": ["are", "bad", "why"]}, {"sent": "like between the two of them theyve their good days and their bad days .", "words": ["the", "their", "of", "them", "like", "bad", "and", "good"]}, {"sent": "bad tempered arent we ?", "words": ["bad", "we"]}, {"sent": "bad girl .", "words": ["bad"]}, {"sent": "not bad ?", "words": ["not", "bad"]}, {"sent": "things arent bad .", "words": ["bad"]}, {"sent": "youknow and uh Imean like if shes bad shes liable to shout out anything .", "words": ["if", "to", "like", "bad", "and", "out"]}, {"sent": "that was a bad idea .", "words": ["was", "that", "bad", "a"]}, {"sent": "he has a very bad temper you know he dont wanna listen .", "words": ["listen", "bad", "you", "dont", "he", "wanna", "a"]}, {"sent": "I dont think hes a bad guy no no no .", "words": ["dont", "think", "bad", "a"]}, {"sent": "lotsof candy can you chew it and swallow it please that was a bad idea .", "words": ["can", "was", "bad", "and", "you", "it", "candy", "that", "a"]}, {"sent": "have you got a bad cough ?", "words": ["have", "bad", "a", "you"]}, {"sent": "theyre bad ya know .", "words": ["bad"]}, {"sent": "youre not a bad girl .", "words": ["not", "bad", "a"]}, {"sent": "was it really bad ?", "words": ["was", "it", "bad"]}, {"sent": "now is he is good or is he bad ?", "words": ["he", "bad", "good", "is"]}, {"sent": "a bad one ?", "words": ["bad", "a"]}, {"sent": "the big bad wolf .", "words": ["the", "wolf", "big", "bad"]}, {"sent": "too bad hes eating it anyway .", "words": ["it", "bad", "too"]}, {"sent": "doesnt look so bad now .", "words": ["so", "bad", "look"]}, {"sent": "not bad Ross .", "words": ["not", "bad"]}, {"sent": "say a bad forest fire .", "words": ["say", "bad", "a"]}, {"sent": "um whos the bad guy in Spiderman ?", "words": ["the", "in", "bad"]}, {"sent": "hes a bad fella .", "words": ["bad", "a"]}, {"sent": "I think Im going to have a bad journey home .", "words": ["think", "to", "bad", "have", "home", "a"]}, {"sent": "oh thats not so bad .", "words": ["so", "not", "bad"]}, {"sent": "Ive got a bad cough today Stuart .", "words": ["bad", "a"]}, {"sent": "well its not too bad .", "words": ["not", "bad", "too"]}, {"sent": "every day huh ?", "words": ["every"]}, {"sent": "and now she likes her potty even more and uses it every time .", "words": ["and", "more", "it", "potty", "every", "her", "she"]}, {"sent": "well every Saturday wed be down .", "words": ["be", "down", "every"]}, {"sent": "and I try every day to read to her and we .", "words": ["we", "to", "her", "and", "read", "every", "try"]}, {"sent": "I I leave him out I I leave him out here every day its its fine thanks .", "words": ["fine", "here", "him", "every", "out"]}, {"sent": "you seen the same people coming every morning .", "words": ["the", "same", "every", "you"]}, {"sent": "and I would see her every day .", "words": ["and", "every", "would", "see", "her"]}, {"sent": "every time .", "words": ["every"]}, {"sent": "*EX1: xxx keeps growing every day .", "words": ["every"]}, {"sent": "every day when John gets sleepy he .", "words": ["he", "sleepy", "when", "every"]}, {"sent": "does he give you a wee wave every now and again ?", "words": ["does", "you", "and", "he", "every", "give", "a"]}, {"sent": "every day when Mary gets hungry .", "words": ["hungry", "when", "every"]}, {"sent": "I did every one before now lets see if I can do every other one .", "words": ["can", "do", "if", "other", "every", "did", "see"]}, {"sent": "Ill read you every book in your room if you want .", "words": ["your", "if", "you", "in", "room", "read", "every", "book"]}, {"sent": "every time I get out of the car he wants to feel the tires .", "words": ["the", "of", "car", "to", "get", "he", "every", "out"]}, {"sent": "okay youre going to turn into a orange if you keep eating a orange every half hour .", "words": ["if", "into", "to", "you", "orange", "every", "a"]}, {"sent": "because after nearly every mealtime the train and the trailer hafta be washed dont they ?", "words": ["the", "train", "be", "and", "dont", "they", "every", "because"]}, {"sent": "but as from next Wednesday every morning .", "words": ["every", "but"]}, {"sent": "every one ?", "words": ["every"]}, {"sent": "and the same with every mealtime .", "words": ["the", "with", "and", "same", "every"]}, {"sent": "every bite gets bigger mouth and a bigger mouth and a bigger mouth .", "words": ["mouth", "and", "every", "bite", "a"]}, {"sent": "youll hafta go every day .", "words": ["every", "go"]}, {"sent": "well Renee goes to bed every night just like Nathaniel does .", "words": ["does", "to", "bed", "like", "every"]}, {"sent": "so a little you know an aquarium that every morning he hasnt done it lately but for months there it was turtle every time we walked into his room cause he wanted to see the aquarium .", "words": ["the", "into", "an", "there", "it", "that", "for", "so", "was", "to", "you", "little", "every", "turtle", "see", "a", "we", "room", "he", "his", "but"]}, {"sent": "I say hes gonna need a haircut every month .", "words": ["need", "say", "every", "a"]}, {"sent": "do we get one every time we go to the grocery ?", "words": ["the", "do", "go", "we", "to", "get", "every"]}, {"sent": "and do you remember every morning when the postman came you went rushing to the door didnt you ?", "words": ["the", "do", "to", "you", "and", "every", "when", "door"]}, {"sent": "every day when John gets sleepy .", "words": ["sleepy", "when", "every"]}, {"sent": "she gave him things she told him secrets and sang lullabies to him every night .", "words": ["to", "and", "him", "every", "she"]}, {"sent": "there is an exception to every rule isnt there ?", "words": ["is", "to", "there", "an", "every"]}, {"sent": "every once in a while hell ground beef sometimes youknow a hamburger other times he can care less .", "words": ["can", "hamburger", "other", "in", "he", "every", "a"]}, {"sent": "you bring it out every now and again .", "words": ["bring", "you", "and", "it", "every", "out"]}, {"sent": "and we know every inch of the way from .", "words": ["the", "we", "of", "and", "every"]}, {"sent": "were gonna hafta clean every single bin .", "words": ["clean", "were", "every"]}, {"sent": "every go to there quickly .", "words": ["to", "every", "there", "go"]}, {"sent": "wi every things being recorded .", "words": ["every"]}, {"sent": "and on every barn is a weathervane of course .", "words": ["is", "of", "and", "every", "on", "a"]}, {"sent": "every day when John gets sleepy .", "words": ["sleepy", "when", "every"]}, {"sent": "um like every other .", "words": ["every", "like", "other"]}, {"sent": "every day .", "words": ["every"]}, {"sent": "every sort of sweet imaginable .", "words": ["of", "every"]}, {"sent": "every time I go what ?", "words": ["what", "every", "go"]}, {"sent": "so every day we hear .", "words": ["so", "hear", "every", "we"]}, {"sent": "xxx went the cough every now and then .", "words": ["the", "then", "every", "and"]}, {"sent": "but its never the same every day ?", "words": ["the", "same", "every", "but"]}, {"sent": "every time we talk it moves .", "words": ["it", "every", "talk", "we"]}, {"sent": "and my grandfather said its high time that you know of the terribly horrible thing that Zukes do in every Zuke house and in every Zuke town .", "words": ["the", "do", "my", "of", "you", "and", "in", "house", "every", "high", "that"]}, {"sent": "every night .", "words": ["every"]}, {"sent": "every where a moo moo .", "words": ["where", "every", "a"]}, {"sent": "every day when John gets sleepy .", "words": ["sleepy", "when", "every"]}, {"sent": "we have that at home she reads it every night with her dad .", "words": ["we", "with", "her", "have", "it", "every", "at", "that", "she", "home"]}, {"sent": "and every now and again you hafta go to the Optician like like AuntySal .", "words": ["the", "go", "to", "like", "you", "and", "every"]}, {"sent": "every day .", "words": ["every"]}, {"sent": "yeah shes played with food every time .", "words": ["food", "with", "every"]}, {"sent": "and every time you have a meal we hafta wipe down the tray on your highchair dont we ?", "words": ["the", "your", "we", "down", "you", "have", "and", "dont", "every", "on", "wipe", "tray", "a"]}, {"sent": "why does ah why does every David hafta do it ?", "words": ["does", "do", "it", "every", "why"]}, {"sent": "every day when Mary gets .", "words": ["when", "every"]}, {"sent": "this tank is getting lower every week .", "words": ["this", "every", "is"]}, {"sent": "Im taking a slight bit from every body to give to Gabriel .", "words": ["to", "give", "every", "a"]}, {"sent": "well which means every body eats it which means you know somebody likes like .", "words": ["like", "you", "it", "every", "which"]}, {"sent": "every day when Mary gets hungry .", "words": ["hungry", "when", "every"]}, {"sent": "every night I was in bed xxx nine .", "words": ["was", "in", "every", "bed"]}, {"sent": "every single meal time these days you say you feel a little bit sick .", "words": ["these", "say", "you", "little", "sick", "every", "a"]}, {"sent": "if you only give him two every day .", "words": ["if", "you", "him", "every", "give"]}, {"sent": "he reads with daddy every night before bed .", "words": ["bed", "he", "every", "with"]}, {"sent": "she wasnt interested I said okay she wanted dressup she wanted every princess dressup thing there is .", "words": ["there", "every", "she", "is"]}, {"sent": "what about dolls does he every play with dolls ?", "words": ["what", "does", "play", "with", "he", "every", "about"]}, {"sent": "I do this every day Thomas .", "words": ["do", "every", "this"]}, {"sent": "maybe once every two weeks .", "words": ["every"]}, {"sent": "do you have a ponytail every day when you go to school ?", "words": ["do", "go", "to", "school", "you", "have", "every", "when", "a"]}, {"sent": "every mermaid and merman were in bed fast asleep .", "words": ["fast", "bed", "and", "asleep", "in", "every", "were"]}, {"sent": "he gets knocked down every time he goes through there .", "words": ["there", "down", "he", "every"]}, {"sent": "just because it would because he wouldnt focus on any one thing he would just dive into every separate thing .", "words": ["into", "any", "it", "he", "every", "would", "on", "because"]}, {"sent": "youve got to have a clean brush before you start every time .", "words": ["clean", "brush", "to", "you", "have", "every", "a"]}, {"sent": "and nearly every day .", "words": ["every", "and"]}, {"sent": "every time he has a day off all youre doing is dishes .", "words": ["off", "all", "is", "he", "every", "a"]}, {"sent": "every now and then the refrigerator turns itself on and off .", "words": ["the", "off", "refrigerator", "and", "every", "on", "then"]}, {"sent": "and I asked him every single day Jeannine .", "words": ["him", "every", "and"]}, {"sent": "you get two dollars every sataday and you usually go to the movies right ?", "words": ["the", "go", "to", "and", "get", "you", "every"]}, {"sent": "every girl in the kingdom must be there .", "words": ["the", "be", "there", "in", "every"]}, {"sent": "and every once in a while Peter the horse would come along and say hello to her .", "words": ["the", "to", "say", "and", "in", "every", "would", "horse", "her", "a"]}, {"sent": "right so really every time I hear her sneeze I give her a tablet .", "words": ["so", "hear", "every", "give", "her", "a"]}, {"sent": "we useta see Jake every week didnt we ?", "words": ["see", "every", "we"]}, {"sent": "every platform on every station .", "words": ["on", "every"]}, {"sent": "she gets juice every once in a while and she certainly knows the word juice but I dont think she would recognize that as juice .", "words": ["the", "think", "and", "dont", "in", "every", "would", "juice", "that", "she", "but", "a"]}, {"sent": "oh this is what we say every night before we go to bed .", "words": ["what", "we", "is", "go", "to", "say", "bed", "this", "every"]}, {"sent": "do you every have a dream where youre outside ?", "words": ["where", "do", "outside", "you", "have", "every", "a"]}, {"sent": "every strawberry has a little pip .", "words": ["little", "every", "a", "strawberry"]}, {"sent": "xxx every body xxx .", "words": ["every"]}, {"sent": "every one minute you put a towel up its gone .", "words": ["up", "put", "you", "towel", "every", "a"]}, {"sent": "rather than just every day doing the same .", "words": ["the", "same", "every"]}, {"sent": "you got crayon on every floor in the house .", "words": ["the", "crayon", "you", "in", "house", "every", "on"]}, {"sent": "I hafta make editorial comments every now and then .", "words": ["then", "make", "every", "and"]}, {"sent": "he has three bottles every day .", "words": ["he", "every"]}, {"sent": "you did and I have it for you its right inside lemme lemme go get it for you now because I forget every time .", "words": ["for", "go", "inside", "and", "have", "you", "get", "it", "did", "every", "because"]}, {"sent": "and she knows nearly every nursery rhyme .", "words": ["every", "she", "and"]}, {"sent": "in between every child every xxx .", "words": ["in", "every"]}, {"sent": "so every week the same mothers go with the same kids .", "words": ["the", "go", "so", "with", "same", "every"]}, {"sent": "every day when Mary gets hungry .", "words": ["hungry", "when", "every"]}, {"sent": "that doesnt mind coming put every five minutes when my chip pans on fire .", "words": ["put", "my", "when", "every", "on", "that"]}, {"sent": "and put it on your head sit up for a minute I hafta go get it .", "words": ["up", "put", "for", "your", "go", "head", "and", "get", "it", "on", "sit", "a"]}, {"sent": "I may go this weekend though me and Denise .", "words": ["this", "me", "and", "go"]}, {"sent": "it doesnt go down there .", "words": ["down", "it", "there", "go"]}, {"sent": "I think theyre all packed away in these boxes to go to your new house .", "words": ["all", "think", "these", "your", "go", "to", "in", "away", "house", "new"]}, {"sent": "here you go .", "words": ["here", "go", "you"]}, {"sent": "go on and another one .", "words": ["on", "another", "and", "go"]}, {"sent": "there you go .", "words": ["go", "there", "you"]}, {"sent": "you go to the window and tell me whats happening .", "words": ["the", "go", "to", "you", "and", "me", "window"]}, {"sent": "when we go to the donut store what kind of donuts do you get ?", "words": ["the", "donut", "what", "do", "go", "we", "to", "store", "of", "you", "get", "when"]}, {"sent": "shall we go and put some petrol in it ?", "words": ["put", "go", "we", "and", "in", "it", "some"]}, {"sent": "lets go back outside and play at the pool .", "words": ["the", "back", "go", "outside", "play", "pool", "and", "at"]}, {"sent": "come on lets go for a wee wee .", "words": ["on", "for", "a", "go"]}, {"sent": "er because crisps and egg dont go .", "words": ["egg", "go", "and", "dont", "because"]}, {"sent": "you dont push Eleanor off the stool if you cant go to the party .", "words": ["off", "the", "push", "if", "go", "to", "you", "party", "dont"]}, {"sent": "does it go there ?", "words": ["does", "it", "there", "go"]}, {"sent": "okay here we go .", "words": ["here", "go", "we"]}, {"sent": "okay here we go .", "words": ["here", "go", "we"]}, {"sent": "willnt go on there will it ?", "words": ["go", "will", "there", "it", "on"]}, {"sent": "does it not go over his feet ?", "words": ["does", "go", "it", "his", "over", "not"]}, {"sent": "and did greatgrandma did greatgrandma go ?", "words": ["go", "did", "and"]}, {"sent": "why cant he go downstairs ?", "words": ["he", "why", "go"]}, {"sent": "me go home ?", "words": ["me", "home", "go"]}, {"sent": "no go up the stairs ?", "words": ["up", "the", "stairs", "go"]}, {"sent": "Caroline have a go .", "words": ["have", "go", "a"]}, {"sent": "Ill go get Jem .", "words": ["get", "go"]}, {"sent": "let him go just let him go .", "words": ["him", "go"]}, {"sent": "go peepee .", "words": ["go"]}, {"sent": "alright Ill go out now and ?", "words": ["and", "out", "go"]}, {"sent": "you member where they go ?", "words": ["where", "they", "go", "you"]}, {"sent": "when we go out and play what do you make in it ?", "words": ["what", "do", "go", "we", "play", "and", "you", "in", "make", "it", "out", "when"]}, {"sent": "do you wanna go to sleep .", "words": ["do", "go", "sleep", "to", "you", "wanna"]}, {"sent": "go and get the little chair then we can put the bits on the chair .", "words": ["the", "can", "put", "go", "we", "chair", "little", "get", "and", "on", "then"]}, {"sent": "but she wouldnt go and next thing okay let me go find the things that go on the pizza or let me go take it to the stove .", "words": ["the", "go", "to", "pizza", "and", "it", "stove", "find", "on", "me", "take", "that", "she", "but"]}, {"sent": "okay lets go sit back over here .", "words": ["back", "go", "here", "sit", "over"]}, {"sent": "FriarsBush I would like her to go to .", "words": ["go", "to", "like", "would", "her"]}, {"sent": "the movie should be over about three or two it it depends on what time we go .", "words": ["the", "what", "we", "go", "be", "it", "about", "on", "over"]}, {"sent": "you go open the door so we can get out .", "words": ["the", "can", "go", "open", "so", "we", "you", "get", "out", "door"]}, {"sent": "lets go .", "words": ["go"]}, {"sent": "go break them .", "words": ["them", "break", "go"]}, {"sent": "there you go .", "words": ["go", "there", "you"]}, {"sent": "shall we go to the doctors ?", "words": ["go", "to", "the", "we"]}, {"sent": "there ya go .", "words": ["there", "go"]}, {"sent": "where do you go ?", "words": ["where", "do", "go", "you"]}, {"sent": "Id better go for Adam .", "words": ["better", "for", "go"]}, {"sent": "does he go in ?", "words": ["does", "in", "he", "go"]}, {"sent": "to go into Frasers tummy .", "words": ["to", "tummy", "into", "go"]}, {"sent": "you go when its ?", "words": ["go", "when", "you"]}, {"sent": "there we go .", "words": ["go", "there", "we"]}, {"sent": "go .", "words": ["go"]}, {"sent": "do you wanna go downstairs now ?", "words": ["go", "do", "wanna", "you"]}, {"sent": "you hafta go to school so put your clothes on otherwise MissusCwww will say oh has she come to has she come to school with her pyjamas on ?", "words": ["put", "your", "go", "to", "school", "so", "will", "say", "with", "you", "on", "her", "she"]}, {"sent": "did you go on the snow ?", "words": ["the", "go", "snow", "you", "did", "on"]}, {"sent": "wants to go out of the window .", "words": ["the", "go", "of", "to", "out", "window"]}, {"sent": "she said to the kittens now go outside but be careful .", "words": ["the", "go", "to", "outside", "be", "careful", "she", "but"]}, {"sent": "ya hafta go .", "words": ["go"]}, {"sent": "lets go sit over there then .", "words": ["go", "then", "there", "sit", "over"]}, {"sent": "you go play and Mommy is gonna hang up these coats .", "words": ["up", "these", "go", "is", "play", "and", "you"]}, {"sent": "and then were gonna go .", "words": ["go", "then", "were", "and"]}, {"sent": "but you go out for a few moments .", "words": ["for", "go", "you", "out", "but", "a"]}, {"sent": "there ya go .", "words": ["there", "go"]}, {"sent": "you go in there .", "words": ["go", "there", "in", "you"]}, {"sent": "go on then .", "words": ["on", "then", "go"]}, {"sent": "go and get your not sword saw .", "words": ["your", "go", "and", "get", "not"]}, {"sent": "here go .", "words": ["here", "go"]}, {"sent": "go aah .", "words": ["go"]}, {"sent": "shall we go and throw it in the bin ?", "words": ["the", "throw", "go", "we", "and", "in", "it"]}, {"sent": "here we go .", "words": ["here", "go", "we"]}, {"sent": "and when they go up in the sky when they get let when you let go of them where do they go ?", "words": ["up", "the", "where", "do", "go", "sky", "of", "them", "and", "get", "they", "in", "you", "when"]}, {"sent": "does daddy go on the schoolbus ?", "words": ["on", "does", "the", "go"]}, {"sent": "okay go and sit down .", "words": ["sit", "down", "and", "go"]}, {"sent": "I dont wanna go .", "words": ["dont", "wanna", "go"]}, {"sent": "go ahead honey .", "words": ["go"]}, {"sent": "there we go whoops wheres that other hand there we go .", "words": ["go", "we", "hand", "other", "there", "that"]}, {"sent": "yeah but then they could not go .", "words": ["go", "they", "then", "not", "but", "could"]}, {"sent": "wonder if we can go .", "words": ["can", "go", "if", "we"]}, {"sent": "and then people like you and me go in and buy them .", "words": ["go", "them", "like", "you", "and", "in", "me", "then", "buy"]}, {"sent": "there you go .", "words": ["go", "there", "you"]}, {"sent": "whered it go ?", "words": ["it", "go"]}, {"sent": "do you wanna go for a sleep ?", "words": ["do", "for", "go", "sleep", "you", "wanna", "a"]}, {"sent": "and then you go .", "words": ["go", "then", "you", "and"]}, {"sent": "and did we go on lots of rides ?", "words": ["go", "we", "of", "and", "did", "on"]}, {"sent": "you not wanna go and play over there ?", "words": ["go", "play", "and", "you", "there", "wanna", "over", "not"]}, {"sent": "we hafta go to the store and buy breakfast food because Im all outof food .", "words": ["the", "all", "go", "we", "to", "store", "and", "food", "buy", "because"]}, {"sent": "go on .", "words": ["on", "go"]}, {"sent": "well go on the bus .", "words": ["on", "the", "bus", "go"]}, {"sent": "I hafta go to the store .", "words": ["the", "to", "store", "go"]}, {"sent": "how old do ya think you hafta be before you get to go play golf on the big golf course ?", "words": ["old", "do", "how", "think", "the", "go", "be", "to", "play", "you", "get", "on", "big"]}, {"sent": "it didnt go off on its own though that tractor did it ?", "words": ["off", "go", "it", "tractor", "did", "on", "that"]}, {"sent": "here you go xxx .", "words": ["here", "go", "you"]}, {"sent": "go scare her .", "words": ["her", "go"]}, {"sent": "well go on then .", "words": ["on", "then", "go"]}, {"sent": "go down oops you lost there you go .", "words": ["there", "down", "you", "go"]}, {"sent": "daddy lets go upstairs and tell a story .", "words": ["story", "a", "and", "go"]}, {"sent": "let it go for a second .", "words": ["it", "a", "for", "go"]}, {"sent": "you wanna put some shorts on because we gonna go outside and take that stuff to the garage .", "words": ["the", "put", "shorts", "we", "go", "outside", "to", "garage", "you", "and", "that", "wanna", "on", "some", "take", "because"]}, {"sent": "there ya go all of your friends .", "words": ["all", "your", "go", "of", "there"]}, {"sent": "and youd go .", "words": ["go", "and"]}, {"sent": "there we go .", "words": ["go", "there", "we"]}, {"sent": "its much better because thats the one thats meant to go on it .", "words": ["the", "better", "go", "to", "it", "on", "much", "because"]}, {"sent": "xxx stick her in the closet .", "words": ["the", "stick", "closet", "in", "her"]}, {"sent": "but I give her like an extra metre and a half of material .", "words": ["of", "like", "and", "an", "give", "her", "but", "a"]}, {"sent": "looks like shes even had her fringe curled .", "words": ["her", "like"]}, {"sent": "because I think Hwww she needs her nappie changed doesnt she ?", "words": ["think", "her", "she", "because"]}, {"sent": "wanna put her in here ?", "words": ["put", "here", "in", "wanna", "her"]}, {"sent": "but her hers is her hair thick like ?", "words": ["is", "like", "hers", "hair", "her", "but"]}, {"sent": "oh you gave her a hug .", "words": ["hug", "her", "a", "you"]}, {"sent": "lets put her in the stocking ?", "words": ["the", "in", "put", "her"]}, {"sent": "whatd you buy her ?", "words": ["her", "buy", "you"]}, {"sent": "then Sue would be my mother and Id be her little girl .", "words": ["my", "be", "and", "little", "would", "then", "her"]}, {"sent": "go ahead bring it to her .", "words": ["go", "to", "bring", "it", "her"]}, {"sent": "in the morning after she gets up and moves the cat and brushes her teeth and combs her ears and moves the cat Olivia gets dressed .", "words": ["the", "up", "and", "cat", "in", "her", "she"]}, {"sent": "she loves to run about in her bare feet .", "words": ["to", "run", "in", "about", "her", "she"]}, {"sent": "when she comes in well feed her then .", "words": ["her", "in", "feed", "then", "when", "she"]}, {"sent": "you dropped her a little bit ?", "words": ["little", "her", "a", "you"]}, {"sent": "oh Thomas look at this page whats happened to her now ?", "words": ["look", "to", "this", "at", "her"]}, {"sent": "Eleanor can ride her bike herself .", "words": ["can", "her", "ride"]}, {"sent": "Mummys just taking her glasses off to wash because she cant see .", "words": ["off", "to", "wash", "glasses", "see", "her", "she", "because"]}, {"sent": "she babysits you and her Mommy or ?", "words": ["her", "she", "and", "you"]}, {"sent": "you can play with Amy and her helmet .", "words": ["can", "play", "with", "and", "you", "her"]}, {"sent": "tell her rockabye baby .", "words": ["her"]}, {"sent": "thread her foot through .", "words": ["foot", "her"]}, {"sent": "because if she had her way they would go outside a lot .", "words": ["if", "go", "a lot", "outside", "they", "would", "her", "she", "because", "a"]}, {"sent": "you want her to be blind ?", "words": ["to", "be", "her", "you"]}, {"sent": "well get her when we go out to singing .", "words": ["when", "go", "we", "to", "get", "out", "her"]}, {"sent": "my mummie bought her it .", "words": ["my", "it", "her"]}, {"sent": "lets give her one more little splash .", "words": ["little", "more", "splash", "give", "her"]}, {"sent": "and would you like to give them to her .", "words": ["to", "them", "like", "you", "and", "would", "give", "her"]}, {"sent": "so weve got her these little slippers and weve also got her a little sweater havent we ?", "words": ["these", "sweater", "we", "so", "and", "little", "her", "a"]}, {"sent": "Im going to get her bottle .", "words": ["get", "to", "bottle", "her"]}, {"sent": "give her one of her toys please .", "words": ["of", "give", "her"]}, {"sent": "Ill get my aunt Ann and my aunt Joyce thats her sister in law and her sister and my daddys sister aunt Eva and her daughter and son .", "words": ["my", "and", "get", "in", "her"]}, {"sent": "who comes with her sometimes ?", "words": ["her", "with", "who"]}, {"sent": "no her name is not penny .", "words": ["penny", "her", "not", "is"]}, {"sent": "yep shes wearing her dress .", "words": ["dress", "her"]}, {"sent": "a bib because shes dribbling on her her clothes .", "words": ["bib", "on", "her", "because", "a"]}, {"sent": "all I hafta do is say to her .", "words": ["all", "do", "is", "to", "say", "her"]}, {"sent": "and if I dont hurt her shell do me no harm .", "words": ["do", "if", "hurt", "and", "dont", "me", "her"]}, {"sent": "see her going in ?", "words": ["see", "in", "her"]}, {"sent": "Bill took him up at four like four oclock xxx he could stay with her .", "words": ["up", "with", "like", "him", "he", "at", "stay", "could", "her"]}, {"sent": "did you give her that .", "words": ["her", "you", "did", "give", "that"]}, {"sent": "let her see .", "words": ["see", "her"]}, {"sent": "but she has not got her Wellingtons on again .", "words": ["she", "on", "her", "not", "but"]}, {"sent": "and and her daddys name is Raoul .", "words": ["her", "is", "and"]}, {"sent": "okay keep it open while Im putting her in okay we keep that open there .", "words": ["we", "open", "there", "in", "it", "that", "her"]}, {"sent": "you will see her again .", "words": ["will", "see", "her", "you"]}, {"sent": "and then she also wrote had written down Samara wanted Thomas to sit next to her .", "words": ["to", "sit", "and", "then", "down", "her", "she"]}, {"sent": "and she was sitting in her pram as happy as Larry .", "words": ["happy", "was", "and", "in", "her", "she"]}, {"sent": "dont pass it to her .", "words": ["to", "dont", "it", "her"]}, {"sent": "xxx take her temperature .", "words": ["her", "take"]}, {"sent": "see her eyes and her nose and her mouth ?", "words": ["mouth", "and", "see", "her", "nose"]}, {"sent": "the cans I think are very new to her .", "words": ["the", "think", "to", "are", "new", "her"]}, {"sent": "because Nicola has her friends there to play .", "words": ["to", "play", "there", "her", "because"]}, {"sent": "tell her to comeon ?", "words": ["to", "her"]}, {"sent": "with her bigger dolls whatever they came in .", "words": ["they", "her", "with", "in"]}, {"sent": "youre telling Purdie theres some food for her ?", "words": ["some", "food", "for", "her"]}, {"sent": "now how could I help that girl get her hair washed ?", "words": ["how", "help", "her", "get", "could", "hair", "that"]}, {"sent": "you tell her about that .", "words": ["that", "her", "about", "you"]}, {"sent": "and look at her hair .", "words": ["and", "at", "look", "hair", "her"]}, {"sent": "and it said Lala played with her enormous ball .", "words": ["with", "and", "ball", "it", "her"]}, {"sent": "my mother and father in law they keep her in the afternoon .", "words": ["the", "my", "and", "they", "in", "her"]}, {"sent": "do you think there are any foods here that shes now um that are familiar to her either by look or name ?", "words": ["do", "think", "to", "any", "her", "you", "here", "there", "by", "are", "look", "that"]}, {"sent": "you wanna change her clothes ?", "words": ["her", "wanna", "you"]}, {"sent": "is that her nappie ?", "words": ["her", "that", "is"]}, {"sent": "who lost her messages .", "words": ["her", "who"]}, {"sent": "but she wont fit well let her be on the outside and visit them .", "words": ["the", "be", "fit", "outside", "them", "and", "on", "her", "she", "but"]}, {"sent": "but thats she wants you to at her and thats a form of attention .", "words": ["of", "to", "you", "and", "at", "her", "she", "but", "a"]}, {"sent": "shall I hold Morag and you can brush her hair .", "words": ["can", "hold", "brush", "you", "and", "hair", "her"]}, {"sent": "well she bit you because you youve not been nice to her .", "words": ["to", "nice", "you", "she", "her", "not", "because"]}, {"sent": "because then the next one you give her after twelve will be four oclock .", "words": ["the", "be", "then", "you", "will", "give", "her", "because"]}, {"sent": "dont talk to her .", "words": ["to", "dont", "her", "talk"]}, {"sent": "you can talk to her .", "words": ["can", "to", "you", "talk", "her"]}, {"sent": "go ahead and change her ?", "words": ["her", "and", "go"]}, {"sent": "now turn your hand so that they go on her feet .", "words": ["your", "hand", "go", "so", "her", "they", "on", "that"]}, {"sent": "heres her shoe .", "words": ["her", "shoe"]}, {"sent": "theres a check for her which I will tell her about on the fridge for her to get on Friday .", "words": ["the", "for", "to", "which", "get", "on", "about", "will", "her", "a"]}, {"sent": "oh did she bang her head ?", "words": ["head", "her", "did", "she"]}, {"sent": "are you chasing her ?", "words": ["are", "her", "you"]}, {"sent": "the baba was baba crying because she missed her Mummy ?", "words": ["the", "was", "her", "she", "because"]}, {"sent": "youre going to take her ?", "words": ["to", "her", "take"]}, {"sent": "what do you think her name is ?", "words": ["what", "do", "think", "is", "you", "her"]}, {"sent": "but yeah babydolls Id say is probably her favorite activity .", "words": ["her", "say", "but", "is"]}, {"sent": "watch he can lean her .", "words": ["can", "her", "watch", "he"]}, {"sent": "and in her other hand that doesnt have the flower .", "words": ["the", "hand", "flower", "other", "and", "have", "in", "that", "her"]}, {"sent": "tell her to catch it .", "words": ["to", "catch", "her", "it"]}, {"sent": "is that what you said to her ?", "words": ["what", "is", "to", "her", "you", "that"]}, {"sent": "whatll I do with her ?", "words": ["do", "her", "with"]}, {"sent": "gonna get her pants out ?", "words": ["get", "out", "her", "pants"]}, {"sent": "here well have her sit on your hat .", "words": ["your", "here", "have", "hat", "on", "sit", "her"]}, {"sent": "Chantilly washing her ?", "words": ["her"]}, {"sent": "doesnt let us help her feed her .", "words": ["us", "help", "her", "feed"]}, {"sent": "thats her son thats her little boy cept now hes grown up and he lives in Africa .", "words": ["up", "and", "little", "in", "he", "her"]}, {"sent": "hold her babys hand .", "words": ["her", "hold", "hand"]}, {"sent": "and your man turned round to her .", "words": ["to", "her", "your", "and"]}, {"sent": "and you go and see her sometimes and go and play with her trains dont you ?", "words": ["go", "play", "with", "you", "and", "dont", "see", "her"]}, {"sent": "well usually I would put her her headband on last sweetie after you get her dress on .", "words": ["put", "dress", "last", "you", "get", "would", "on", "her"]}, {"sent": "shes still got her ribbon in her hair hasnt she ?", "words": ["in", "hair", "her", "she"]}, {"sent": "be very gentle with her Thomas .", "words": ["be", "gentle", "with", "her"]}, {"sent": "got her .", "words": ["her"]}, {"sent": "xxx hafta send these to her when its her birthday .", "words": ["to", "these", "her", "when"]}, {"sent": "everything turned back to normal .", "words": ["to", "back"]}, {"sent": "when Daddy gets back with the car John .", "words": ["the", "back", "car", "with", "when"]}, {"sent": "because your Mumll be back and we said that wed make her a latte didnt we ?", "words": ["your", "back", "we", "be", "her", "and", "make", "that", "because", "a"]}, {"sent": "we are gonna go back outside .", "words": ["back", "go", "we", "outside", "are"]}, {"sent": "no come back over here .", "words": ["here", "over", "back"]}, {"sent": "you are going very fast come back come back .", "words": ["fast", "are", "back", "you"]}, {"sent": "theres some here in the back that we know .", "words": ["the", "back", "we", "here", "in", "some", "that"]}, {"sent": "a sore back ?", "words": ["back", "a"]}, {"sent": "the money came back outof the cash register .", "words": ["the", "back", "money"]}, {"sent": "push this back for me please .", "words": ["for", "push", "back", "this", "me"]}, {"sent": "and she went up to the toilet and didnt wanna put it back on again .", "words": ["up", "the", "put", "back", "to", "and", "it", "wanna", "on", "she"]}, {"sent": "do you want me to put tape back around it ?", "words": ["do", "put", "back", "to", "tape", "you", "it", "around", "me"]}, {"sent": "because he wants to take him back to the village .", "words": ["the", "back", "to", "him", "he", "take", "because"]}, {"sent": "back thats right back there we are .", "words": ["there", "are", "back", "we"]}, {"sent": "little yellow ducklings swim across the pond and back following their mommy duck splish splash quack .", "words": ["the", "their", "back", "swim", "and", "little", "splash", "duck", "yellow"]}, {"sent": "put this put the screwdriver back please .", "words": ["the", "this", "put", "back"]}, {"sent": "see at the back ?", "words": ["the", "see", "back", "at"]}, {"sent": "okay put him back in .", "words": ["in", "put", "him", "back"]}, {"sent": "Santas got to go back in the box and go back to sleep .", "words": ["the", "box", "back", "go", "sleep", "to", "and", "in"]}, {"sent": "oh this house gets back to Marianne right ?", "words": ["to", "this", "house", "back"]}, {"sent": "whats it whats it say on the back ?", "words": ["the", "back", "say", "it", "on"]}, {"sent": "come on come on back in here .", "words": ["on", "in", "here", "back"]}, {"sent": "here hold this Ill be right back f .", "words": ["hold", "back", "be", "here", "this"]}, {"sent": "are you going back to bed ?", "words": ["back", "to", "bed", "you", "are"]}, {"sent": "having a ride in the back of the tractor .", "words": ["the", "back", "of", "in", "tractor", "ride", "a"]}, {"sent": "and when were taking sweets out of the houses we can put the chocolates back in the jar cant we ?", "words": ["the", "can", "put", "back", "we", "of", "jar", "and", "in", "out", "when", "were"]}, {"sent": "can you put it back in ?", "words": ["can", "put", "back", "you", "in", "it"]}, {"sent": "Tigerella caught it and she threw it back .", "words": ["it", "she", "back", "and"]}, {"sent": "can ya put them back ?", "words": ["can", "put", "them", "back"]}, {"sent": "theyre back on wow .", "words": ["on", "back"]}, {"sent": "youre putting them back in the box are you ?", "words": ["the", "box", "back", "them", "you", "in", "are"]}, {"sent": "youre rubbing your back along the wall .", "words": ["the", "your", "back"]}, {"sent": "well put that back okay ?", "words": ["put", "that", "back"]}, {"sent": "colors back that is right .", "words": ["that", "back", "is"]}, {"sent": "wanna do some writing on the back ?", "words": ["the", "do", "back", "wanna", "on", "some"]}, {"sent": "ahhah there are other tiny bits put them back in .", "words": ["tiny", "put", "back", "them", "other", "there", "in", "are"]}, {"sent": "if I push it back towards you .", "words": ["push", "if", "back", "you", "it"]}, {"sent": "are you on Erins back right now oh my gosh you are up high where are you flying ?", "words": ["up", "where", "my", "back", "you", "are", "high", "on"]}, {"sent": "so it shoots it back the other way .", "words": ["the", "back", "so", "other", "it"]}, {"sent": "shall we sit him back on ?", "words": ["back", "we", "him", "on", "sit"]}, {"sent": "okay now go get those xxx come back xxx Ronald .", "words": ["get", "those", "back", "go"]}, {"sent": "but you can wait until I come back up cant you ?", "words": ["can", "up", "wait", "back", "you", "but"]}, {"sent": "okay now get back here .", "words": ["get", "here", "back"]}, {"sent": "its back on television again isnt it ?", "words": ["on", "it", "back"]}, {"sent": "gonna put your highchair back ?", "words": ["put", "your", "back"]}, {"sent": "give me back my Heidi so I can tidy her hair .", "words": ["can", "my", "back", "so", "me", "give", "hair", "her"]}, {"sent": "she wanted me to go back next month .", "words": ["back", "go", "to", "me", "she"]}, {"sent": "xxx you need to back up .", "words": ["up", "need", "back", "need to", "to", "you"]}, {"sent": "and then reaching around and pulling it back .", "words": ["back", "and", "it", "around", "then"]}, {"sent": "lets scoot the chair back .", "words": ["the", "back", "chair"]}, {"sent": "put you see the back on it just like that .", "words": ["the", "put", "back", "like", "you", "it", "on", "see", "that"]}, {"sent": "heres the lawn mower coming back .", "words": ["the", "lawn mower", "back"]}, {"sent": "alright well go back to the penny .", "words": ["the", "back", "go", "to", "penny"]}, {"sent": "well hafta get it back off them then willnt we ?", "words": ["off", "back", "we", "them", "get", "it", "then"]}, {"sent": "are you coming back for Chuckie ?", "words": ["are", "for", "back", "you"]}, {"sent": "put him back down ?", "words": ["put", "down", "him", "back"]}, {"sent": "we were taking the rubbish through the back gate werent we ?", "words": ["the", "were", "back", "we"]}, {"sent": "can you send it back ?", "words": ["can", "it", "back", "you"]}, {"sent": "as the beast embraced Belle Gaston stabbed him in the back and the beast let out a terrible roar .", "words": ["the", "back", "and", "in", "him", "out", "a"]}, {"sent": "I need you to go back there and play now .", "words": ["need", "back", "go", "to", "play", "you", "and", "there"]}, {"sent": "and then well go put her back in .", "words": ["put", "back", "go", "and", "in", "then", "her"]}, {"sent": "theres a lit there is actually some sugar on that chair at the back Thomas I can see .", "words": ["the", "can", "back", "is", "see", "chair", "there", "at", "on", "some", "that", "a"]}, {"sent": "well this has got to go back first Thomas .", "words": ["back", "go", "to", "first", "this"]}, {"sent": "and its coming back on Monday .", "words": ["on", "back", "and"]}, {"sent": "push it back .", "words": ["it", "push", "back"]}, {"sent": "Teletubbies have come back havent they ?", "words": ["have", "they", "back"]}, {"sent": "I actually came back and says to him .", "words": ["to", "him", "back", "and"]}, {"sent": "pick him up and put him back in ?", "words": ["up", "put", "back", "pick", "and", "him", "in"]}, {"sent": "you gonna try to put the pumpkins back up ?", "words": ["the", "up", "put", "back", "to", "you", "try", "try to"]}, {"sent": "can you do something on that back for me ?", "words": ["can", "do", "for", "back", "you", "on", "me", "that"]}, {"sent": "do you wanna go in the back for a minute ?", "words": ["the", "do", "for", "back", "go", "you", "in", "wanna", "a"]}, {"sent": "put it back .", "words": ["put", "it", "back"]}, {"sent": "do you need some money back ?", "words": ["money", "do", "need", "back", "you", "some"]}, {"sent": "xxx back home .", "words": ["back", "home"]}, {"sent": "put the seesaw back .", "words": ["the", "put", "back"]}, {"sent": "you hafta push it back in there .", "words": ["push", "back", "you", "there", "in", "it"]}, {"sent": "can you fix them back on ?", "words": ["can", "back", "fix", "them", "you", "on"]}, {"sent": "put it back in there .", "words": ["put", "back", "there", "in", "it"]}, {"sent": "and just take the chances coming back on the subway .", "words": ["the", "back", "and", "on", "take"]}, {"sent": "you want it back ?", "words": ["it", "back", "you"]}, {"sent": "hes got eyes on the back and eyes on the front .", "words": ["on", "the", "back", "and"]}, {"sent": "get xxx the back of the line .", "words": ["get", "the", "of", "back"]}, {"sent": "and a cupcake goes back in the stove .", "words": ["the", "back", "and", "in", "stove", "a"]}, {"sent": "so we put the people in the ambulance in the back of there .", "words": ["the", "put", "back", "we", "of", "so", "there", "in"]}, {"sent": "put our molds back in here .", "words": ["put", "back", "our", "here", "in"]}, {"sent": "will you play back half of it ?", "words": ["back", "of", "play", "you", "it", "will"]}, {"sent": "can you put them back on ?", "words": ["can", "put", "back", "them", "you", "on"]}, {"sent": "I can make it go all the way down here and all the way back here before you even seen it .", "words": ["can", "all", "the", "back", "go", "and", "here", "you", "it", "down", "make"]}, {"sent": "alright Ill put William back on .", "words": ["on", "put", "back"]}, {"sent": "youll hafta creep in and bring them back then .", "words": ["back", "them", "bring", "and", "in", "then"]}, {"sent": "and then we can take them all back to the shed together .", "words": ["can", "all", "the", "back", "we", "to", "them", "and", "then", "take"]}, {"sent": "it was things youknow things youd throw back into the linen basket .", "words": ["the", "throw", "back", "into", "was", "it", "basket"]}, {"sent": "are you coming back now ?", "words": ["are", "back", "you"]}, {"sent": "Daddy will be back in a minute .", "words": ["back", "be", "in", "will", "a"]}, {"sent": "shall we put that back in the bag then ?", "words": ["the", "put", "back", "we", "in", "then", "that"]}, {"sent": "can I have mine back please ?", "words": ["can", "mine", "have", "back"]}, {"sent": "can you bring it back ?", "words": ["can", "back", "bring", "you", "it"]}, {"sent": "shall we put them back now ?", "words": ["put", "them", "back", "we"]}, {"sent": "well when I go back there Ill get you one from there or you can get it yourself .", "words": ["can", "yourself", "back", "go", "you", "get", "there", "it", "when"]}, {"sent": "does that look nice ?", "words": ["does", "nice", "that", "look"]}, {"sent": "it lights up nice and bright .", "words": ["up", "it", "and", "nice"]}, {"sent": "thats nice .", "words": ["nice"]}, {"sent": "but youre a nice big bear .", "words": ["nice", "bear", "big", "but", "a"]}, {"sent": "nice hit William .", "words": ["nice", "hit"]}, {"sent": "hes a nice mouse .", "words": ["nice", "mouse", "a"]}, {"sent": "this such a nice kiawe tree here .", "words": ["tree", "nice", "here", "this", "a"]}, {"sent": "thats not nice .", "words": ["nice", "not"]}, {"sent": "you dont have brown eyes Ill make you nice blue eyes .", "words": ["brown", "nice", "you", "have", "dont", "blue", "make"]}, {"sent": "we bought some nice ThomasTheTankEngine books .", "words": ["some", "nice", "we"]}, {"sent": "thats not nice is it Thomas ?", "words": ["it", "nice", "not", "is"]}, {"sent": "oh theyre nice .", "words": ["nice"]}, {"sent": "is it nice ?", "words": ["nice", "it", "is"]}, {"sent": "thats nice .", "words": ["nice"]}, {"sent": "Mummy smells nice does she ?", "words": ["does", "nice", "she"]}, {"sent": "Jac be nice .", "words": ["be", "nice"]}, {"sent": "um nice breakfast .", "words": ["nice"]}, {"sent": "ooh does that make a nice sound ?", "words": ["does", "nice", "that", "make", "a"]}, {"sent": "nice baby xxx .", "words": ["nice"]}, {"sent": "oh a nice hug .", "words": ["hug", "nice", "a"]}, {"sent": "he patted the cow on her nose and went back to the house thats great isnt that nice ?", "words": ["the", "back", "to", "nice", "and", "that", "he", "house", "on", "her", "cow", "nose"]}, {"sent": "doesnt it look like it would be a nice blanket ?", "words": ["look", "be", "nice", "like", "it", "would", "blanket", "a"]}, {"sent": "that might be nice .", "words": ["be", "nice", "that"]}, {"sent": "were going to get your baby some nice presents .", "words": ["your", "to", "nice", "get", "some", "were"]}, {"sent": "oh a nice refreshing Coke ?", "words": ["nice", "a"]}, {"sent": "you sit there and Ill do a nice Sammy snake with your new pencils .", "words": ["do", "your", "nice", "with", "and", "you", "there", "sit", "new", "a"]}, {"sent": "but theyve got nice frames .", "words": ["nice", "but"]}, {"sent": "look at the nice snowman .", "words": ["the", "look", "nice", "snowman", "at"]}, {"sent": "thats nice .", "words": ["nice"]}, {"sent": "thats nice .", "words": ["nice"]}, {"sent": "nice ?", "words": ["nice"]}, {"sent": "have a nice xxx .", "words": ["have", "nice", "a"]}, {"sent": "is she nice ?", "words": ["nice", "she", "is"]}, {"sent": "does the water taste nice ?", "words": ["the", "does", "nice", "taste", "water"]}, {"sent": "thats nice .", "words": ["nice"]}, {"sent": "we can if its nice and hot .", "words": ["can", "if", "we", "nice", "and", "hot"]}, {"sent": "this is really a nice toy .", "words": ["is", "nice", "this", "toy", "a"]}, {"sent": "no now that is not nice .", "words": ["nice", "that", "not", "is"]}, {"sent": "youve done his nice eyes havent you ?", "words": ["his", "nice", "you"]}, {"sent": "ooh that sounds nice .", "words": ["nice", "that"]}, {"sent": "very nice .", "words": ["nice"]}, {"sent": "is it nice ?", "words": ["nice", "it", "is"]}, {"sent": "the spider is pretty friendly hes nice .", "words": ["the", "nice", "pretty", "is"]}, {"sent": "its a book of stories where theyre all nice .", "words": ["where", "all", "of", "nice", "book", "a"]}, {"sent": "what a nice baby .", "words": ["what", "nice", "a"]}, {"sent": "is it nice ?", "words": ["nice", "it", "is"]}, {"sent": "do you know where theres a nice forest he asked .", "words": ["where", "do", "nice", "you", "he", "a"]}, {"sent": "theyre nice and safe now .", "words": ["nice", "and"]}, {"sent": "thats really nice .", "words": ["nice"]}, {"sent": "oh thats nice .", "words": ["nice"]}, {"sent": "what happened to that nice boy who was eating cherries a minute ago ?", "words": ["what", "to", "was", "nice", "who", "that", "a"]}, {"sent": "thatll be nice uhn wont it ?", "words": ["be", "nice", "it"]}, {"sent": "oh thats nice isnt it ?", "words": ["it", "nice"]}, {"sent": "a nice weekend .", "words": ["nice", "a"]}, {"sent": "did you get some nice presents for your birthday ?", "words": ["for", "your", "nice", "you", "get", "did", "some"]}, {"sent": "very nice .", "words": ["nice"]}, {"sent": "oh and there is nice big crayons .", "words": ["is", "nice", "and", "there", "big"]}, {"sent": "oh nice .", "words": ["nice"]}, {"sent": "oh be nice Lee .", "words": ["be", "nice"]}, {"sent": "how nice .", "words": ["how", "nice"]}, {"sent": "what if hes nice ?", "words": ["what", "nice", "if"]}, {"sent": "your nice blue ring .", "words": ["nice", "your", "blue"]}, {"sent": "thats nice .", "words": ["nice"]}, {"sent": "oh thats not nice .", "words": ["nice", "not"]}, {"sent": "so theyd both come in from the cold walk theyd both had a nice warm bath and then what did they do ?", "words": ["the", "what", "do", "so", "nice", "and", "they", "in", "did", "then", "walk", "cold", "a"]}, {"sent": "thats a nice name isnt it ?", "words": ["it", "nice", "a"]}, {"sent": "yeah nice baby .", "words": ["nice"]}, {"sent": "that tastes nice .", "words": ["nice", "that"]}, {"sent": "thats nice at that age .", "words": ["nice", "that", "at"]}, {"sent": "not being very nice .", "words": ["nice", "not"]}, {"sent": "thats not very nice .", "words": ["nice", "not"]}, {"sent": "thats not nice .", "words": ["nice", "not"]}, {"sent": "ooh nice baby .", "words": ["nice"]}, {"sent": "yeah thats the present for Grandma it look very nice .", "words": ["the", "for", "present", "nice", "it", "look"]}, {"sent": "its a nice day .", "words": ["nice", "a"]}, {"sent": "oh nice youre giving the baby some pizza .", "words": ["the", "some", "nice", "pizza"]}, {"sent": "thatll be nice .", "words": ["be", "nice"]}, {"sent": "thats a nice fish isnt it ?", "words": ["it", "nice", "fish", "a"]}, {"sent": "oh hes nice .", "words": ["nice"]}, {"sent": "I oh everybodys in there is everybody nice and warm in there ?", "words": ["is", "nice", "and", "there", "in"]}, {"sent": "nice clean teethies .", "words": ["clean", "nice"]}, {"sent": "how many times do you think you can be nice to Purdie in a day ?", "words": ["can", "do", "how", "think", "be", "to", "nice", "you", "in", "a"]}, {"sent": "did you have a nice nights sleep ?", "words": ["sleep", "nice", "you", "have", "did", "a"]}, {"sent": "oh I think theyre nice .", "words": ["think", "nice"]}, {"sent": "there you go sit there nice .", "words": ["go", "nice", "you", "there", "sit"]}, {"sent": "xxx your big beds nice isnt it ?", "words": ["it", "big", "nice", "your"]}, {"sent": "theyre going to have breakfast in the garden because its very nice .", "words": ["the", "to", "nice", "have", "in", "garden", "because"]}, {"sent": "does that taste nice ?", "words": ["does", "nice", "that", "taste"]}, {"sent": "okay have a nice day dear .", "words": ["have", "nice", "a"]}, {"sent": "its not nice to throw Xavier .", "words": ["to", "throw", "nice", "not"]}, {"sent": "thats not nice .", "words": ["nice", "not"]}, {"sent": "oh youre a nice boy .", "words": ["nice", "a"]}, {"sent": "well hes nice too .", "words": ["nice", "too"]}, {"sent": "thats not nice .", "words": ["nice", "not"]}, {"sent": "are your hands nice and clean ?", "words": ["clean", "your", "nice", "and", "are"]}, {"sent": "have you had a nice morning ?", "words": ["have", "nice", "a", "you"]}, {"sent": "thats not nice .", "words": ["nice", "not"]}, {"sent": "thats nice .", "words": ["nice"]}, {"sent": "and the only way we can stop it aching is for the dentist to do something to your mouth and it isnt very nice .", "words": ["the", "can", "do", "for", "your", "we", "is", "to", "mouth", "stop", "nice", "and", "it"]}, {"sent": "oh look .", "words": ["look"]}, {"sent": "do you look read the book .", "words": ["the", "do", "you", "read", "look", "book"]}, {"sent": "shall we have a look at it ?", "words": ["we", "have", "at", "it", "look", "a"]}, {"sent": "look at poor wee teddy .", "words": ["poor", "at", "look"]}, {"sent": "oh look at him .", "words": ["him", "at", "look"]}, {"sent": "look at the zebra .", "words": ["the", "zebra", "at", "look"]}, {"sent": "look there he is falling over .", "words": ["is", "there", "he", "look", "over"]}, {"sent": "these animals dont look as if they belong to the same set .", "words": ["the", "these", "if", "to", "dont", "they", "same", "look"]}, {"sent": "lets have a look oh its got purply bits as well .", "words": ["have", "look", "a"]}, {"sent": "look .", "words": ["look"]}, {"sent": "look .", "words": ["look"]}, {"sent": "its like NewYorkcity look with all the big towers .", "words": ["the", "all", "with", "like", "look", "big"]}, {"sent": "you have a look at all the numbers .", "words": ["the", "all", "you", "have", "at", "look", "a"]}, {"sent": "look at me look at me .", "words": ["me", "at", "look"]}, {"sent": "look .", "words": ["look"]}, {"sent": "no its not a brack bush but I it will look black because were gonna use the black paint wont it ?", "words": ["the", "paint", "because", "black", "it", "look", "will", "were", "not", "but", "a"]}, {"sent": "look at the colors youve got in here .", "words": ["the", "look", "here", "in", "at"]}, {"sent": "can you see this chair Fraser look .", "words": ["can", "chair", "you", "this", "look", "see"]}, {"sent": "look this bears saying my name is PandaBear .", "words": ["this", "my", "is", "look"]}, {"sent": "look it .", "words": ["it", "look"]}, {"sent": "look the door opens .", "words": ["the", "door", "look"]}, {"sent": "look .", "words": ["look"]}, {"sent": "hey look look .", "words": ["look"]}, {"sent": "ah whos this look what theyre doing .", "words": ["this", "what", "look"]}, {"sent": "and look and when doctor fright .", "words": ["look", "when", "and"]}, {"sent": "goodness sake because you look really really different .", "words": ["look", "because", "you"]}, {"sent": "what does Chris like to look at in the book ?", "words": ["the", "what", "does", "to", "like", "in", "at", "look", "book"]}, {"sent": "look at the flames there .", "words": ["the", "there", "at", "look"]}, {"sent": "oh now look who has come .", "words": ["who", "look"]}, {"sent": "okay you can look at it later with me .", "words": ["can", "with", "you", "it", "at", "look", "me"]}, {"sent": "uhs look Mia you have more pairs than mommy does .", "words": ["does", "you", "have", "more", "look"]}, {"sent": "look .", "words": ["look"]}, {"sent": "look how I put it in like that .", "words": ["how", "put", "like", "in", "it", "look", "that"]}, {"sent": "look .", "words": ["look"]}, {"sent": "oh look should put the tree together ?", "words": ["the", "tree", "put", "look"]}, {"sent": "dont you think Daddy would look a bit funny walking outside with spiders on his feet ?", "words": ["think", "outside", "with", "would", "you", "his", "dont", "look", "on", "a"]}, {"sent": "youve only got four look .", "words": ["look"]}, {"sent": "were finding them all now look .", "words": ["all", "look", "were", "them"]}, {"sent": "oh no look .", "words": ["look"]}, {"sent": "and look .", "words": ["look", "and"]}, {"sent": "oh look whos on the fence .", "words": ["on", "the", "look"]}, {"sent": "well he does look a little like Pepper .", "words": ["does", "like", "little", "he", "look", "a"]}, {"sent": "look at you go .", "words": ["go", "at", "you", "look"]}, {"sent": "do you wanna look for the bunny ?", "words": ["the", "do", "for", "bunny", "you", "wanna", "look"]}, {"sent": "look at those tatty shirts .", "words": ["at", "those", "look"]}, {"sent": "they look like theyre getting out dust sheets dont they ?", "words": ["like", "dont", "they", "out", "look"]}, {"sent": "look yeah you know what that is ?", "words": ["what", "is", "you", "look", "that"]}, {"sent": "look whos this ?", "words": ["this", "look"]}, {"sent": "look .", "words": ["look"]}, {"sent": "hell still turn it hold it upside down and look at it .", "words": ["hold", "and", "it", "at", "look", "down"]}, {"sent": "because the yellow ones go like this look .", "words": ["the", "go", "like", "this", "look", "yellow", "because"]}, {"sent": "hows he look ?", "words": ["he", "look"]}, {"sent": "can you look all over for it ?", "words": ["can", "all", "for", "you", "it", "look", "over"]}, {"sent": "Nicholas look .", "words": ["look"]}, {"sent": "here look .", "words": ["here", "look"]}, {"sent": "look there arent any there arent any big ones .", "words": ["there", "big", "any", "look"]}, {"sent": "look .", "words": ["look"]}, {"sent": "look at that light .", "words": ["light", "at", "that", "look"]}, {"sent": "look at that .", "words": ["at", "that", "look"]}, {"sent": "you hafta look xxx .", "words": ["look", "you"]}, {"sent": "Bey look .", "words": ["look"]}, {"sent": "look theres xxx .", "words": ["look"]}, {"sent": "stand and have a look .", "words": ["stand", "and", "have", "look", "a"]}, {"sent": "ooh look at that .", "words": ["at", "that", "look"]}, {"sent": "Francis look at all these yeah .", "words": ["all", "at", "these", "look"]}, {"sent": "look .", "words": ["look"]}, {"sent": "you wanna look at them ?", "words": ["them", "you", "at", "wanna", "look"]}, {"sent": "now look what you did said the fish to the cat .", "words": ["the", "what", "to", "you", "cat", "did", "look", "fish"]}, {"sent": "but look at .", "words": ["at", "but", "look"]}, {"sent": "look what Daddys brought you .", "words": ["what", "you", "look"]}, {"sent": "play with the babies look they got a bottle you can feed them .", "words": ["the", "can", "feed", "play", "with", "them", "you", "they", "look", "bottle", "a"]}, {"sent": "thats a k look at that thats .", "words": ["that", "look", "at", "a"]}, {"sent": "look up is he there ?", "words": ["up", "is", "there", "he", "look"]}, {"sent": "look heres how you eat you go .", "words": ["how", "go", "you", "eat", "look"]}, {"sent": "I still think they look small .", "words": ["they", "think", "look"]}, {"sent": "do you wanna look ?", "words": ["do", "look", "wanna", "you"]}, {"sent": "Joey come on come over here and let me look at your ah .", "words": ["your", "and", "here", "at", "look", "on", "me", "over"]}, {"sent": "and Grandma and Granddad come to look after Thomas .", "words": ["to", "look", "and"]}, {"sent": "look .", "words": ["look"]}, {"sent": "look .", "words": ["look"]}, {"sent": "look lets find the cement truck back here you see ?", "words": ["the", "look", "back", "truck", "you", "here", "find", "see"]}, {"sent": "look Theo .", "words": ["look"]}, {"sent": "oh look .", "words": ["look"]}, {"sent": "look .", "words": ["look"]}, {"sent": "look at those cars .", "words": ["at", "those", "look"]}, {"sent": "oh look at this .", "words": ["this", "at", "look"]}, {"sent": "look .", "words": ["look"]}, {"sent": "xxx look what planet are we on ?", "words": ["what", "we", "are", "look", "on"]}, {"sent": "look inside there .", "words": ["there", "inside", "look"]}, {"sent": "look .", "words": ["look"]}, {"sent": "look at the bricks .", "words": ["the", "at", "look"]}, {"sent": "look at that bubble .", "words": ["at", "that", "look"]}, {"sent": "this way this way look this way .", "words": ["this", "look"]}, {"sent": "look at them .", "words": ["at", "them", "look"]}, {"sent": "look theyre wee teddy bears too .", "words": ["too", "look"]}, {"sent": "this ones for them look .", "words": ["this", "for", "them", "look"]}, {"sent": "they look pretty good .", "words": ["they", "pretty", "good", "look"]}, {"sent": "Ive got a bruise there look .", "words": ["there", "look", "a"]}, {"sent": "you stick it in your finger look .", "words": ["stick", "your", "you", "finger", "in", "it", "look"]}, {"sent": "look .", "words": ["look"]}, {"sent": "I did earlier it was cold .", "words": ["was", "it", "did", "cold"]}, {"sent": "and dolly wont get cold in bed will she ?", "words": ["bed", "and", "get", "in", "she", "will", "cold"]}, {"sent": "an easy thing to be when you are cold and lonely .", "words": ["to", "be", "you", "and", "an", "are", "when", "cold"]}, {"sent": "okay now we put in one cup of cold .", "words": ["cup", "put", "we", "of", "in", "cold"]}, {"sent": "cold fruit juice warm cereal with cream pancakes with butter and maple syrup .", "words": ["with", "and", "butter", "juice", "cereal", "cold"]}, {"sent": "my hands feel cold Thomas .", "words": ["my", "cold"]}, {"sent": "yes that cold jug of tea .", "words": ["of", "that", "cold"]}, {"sent": "yeah would your feet be really cold ?", "words": ["be", "your", "cold", "would"]}, {"sent": "because you was cold uhn wasnt you ?", "words": ["was", "cold", "because", "you"]}, {"sent": "ooh Im cold cold in here .", "words": ["here", "in", "cold"]}, {"sent": "lets shut the door to keep the cold out .", "words": ["the", "to", "out", "cold", "door"]}, {"sent": "so I could see if it would be a warm or cold day today .", "words": ["if", "so", "be", "it", "would", "see", "could", "cold", "a"]}, {"sent": "its cold .", "words": ["cold"]}, {"sent": "very warm very warm oh now your cold thats very cold thats freezing cold .", "words": ["your", "cold"]}, {"sent": "oh yeah because Im catching a cold and I want uh .", "words": ["and", "cold", "because", "a"]}, {"sent": "um once and then I brought him back again they said it was cold brought it back again and I said I am in such sleep deprivation from youknow keeping away and it could happen during the night too and whats the sound of not breathing its yyy youknow .", "words": ["the", "back", "sleep", "was", "of", "not", "and", "they", "him", "it", "in", "am", "away", "too", "then", "cold", "could"]}, {"sent": "she wont be cold now .", "words": ["be", "she", "cold"]}, {"sent": "Im still coughing from that cold I had .", "words": ["that", "cold"]}, {"sent": "it really got cold .", "words": ["it", "cold"]}, {"sent": "and do you think its very cold under ground or warm ?", "words": ["do", "think", "you", "and", "under", "cold"]}, {"sent": "theyre cold .", "words": ["cold"]}, {"sent": "cold medicine ?", "words": ["medicine", "cold"]}, {"sent": "well some people eat them cold .", "words": ["eat", "some", "them", "cold"]}, {"sent": "xxx I want cold water .", "words": ["cold", "water"]}, {"sent": "you want a sip of cold coffee ?", "words": ["of", "you", "coffee", "cold", "a"]}, {"sent": "it was even too cold for Mummy .", "words": ["for", "was", "it", "too", "cold"]}, {"sent": "well all be cold getting sodas .", "words": ["all", "be", "cold"]}, {"sent": "its not very cold I dont think you need it .", "words": ["think", "need", "cold", "you", "dont", "it", "not"]}, {"sent": "its cold isnt it ?", "words": ["it", "cold"]}, {"sent": "it is quite cold today .", "words": ["it", "cold", "is"]}, {"sent": "its raining and its cold .", "words": ["cold", "and"]}, {"sent": "shell get cold wont she ?", "words": ["get", "cold", "she"]}, {"sent": "shed get a bit cold sleeping without any clothes on wouldnt she ?", "words": ["any", "get", "she", "on", "cold", "a"]}, {"sent": "its too cold today actually .", "words": ["cold", "too"]}, {"sent": "its cold today .", "words": ["cold"]}, {"sent": "I dont think its cold outside .", "words": ["dont", "think", "outside", "cold"]}, {"sent": "its too cold .", "words": ["cold", "too"]}, {"sent": "youve got a cold havent you ?", "words": ["cold", "you", "a"]}, {"sent": "this is so you wont be cold .", "words": ["is", "be", "so", "you", "this", "cold"]}, {"sent": "are they cold ?", "words": ["they", "are", "cold"]}, {"sent": "because your daddy Daddy was blue it was that cold outside .", "words": ["your", "was", "outside", "blue", "it", "that", "cold", "because"]}, {"sent": "cause Im gonna let you put the cold water in .", "words": ["the", "put", "you", "in", "water", "cold"]}, {"sent": "yes Nomi but I wanna get you dry so you dont get cold .", "words": ["so", "dry", "you", "get", "dont", "wanna", "cold", "but"]}, {"sent": "keep on catching a cold .", "words": ["on", "cold", "a"]}, {"sent": "she she made me feel really cold last night when she got on Mummy .", "words": ["cold", "last", "on", "me", "when", "she"]}, {"sent": "Im going to wash these now in some cold water .", "words": ["these", "to", "in", "water", "some", "wash", "cold"]}, {"sent": "gosh no wonder you are feeling cold .", "words": ["are", "cold", "you"]}, {"sent": "shes had real bad cold .", "words": ["cold", "bad"]}, {"sent": "maybe that will keep it cold .", "words": ["will", "it", "that", "cold"]}, {"sent": "yeah because the next load thats got to be cold .", "words": ["the", "to", "be", "cold", "because"]}, {"sent": "want another sip of cold coffee .", "words": ["another", "of", "coffee", "cold"]}, {"sent": "are you feet cold ?", "words": ["are", "cold", "you"]}, {"sent": "does it feel very cold darling ?", "words": ["does", "it", "cold"]}, {"sent": "is it cold ?", "words": ["it", "cold", "is"]}, {"sent": "are her hands cold ?", "words": ["are", "her", "cold"]}, {"sent": "she must be cold .", "words": ["be", "she", "cold"]}, {"sent": "yes there is containers with cold stuff in them .", "words": ["is", "with", "them", "there", "in", "cold"]}, {"sent": "machine wash cold cold cold .", "words": ["wash", "cold"]}, {"sent": "you want cold cereal or hot cereal ?", "words": ["hot", "cereal", "cold", "you"]}, {"sent": "its cold .", "words": ["cold"]}, {"sent": "it is cold .", "words": ["it", "cold", "is"]}, {"sent": "feels like its cold out there .", "words": ["there", "out", "like", "cold"]}, {"sent": "are you cold ?", "words": ["are", "cold", "you"]}, {"sent": "have you got a cold ?", "words": ["have", "cold", "a", "you"]}, {"sent": "my lips were chapped from the cold .", "words": ["the", "my", "were", "cold"]}, {"sent": "you know what if you dont eat foods gonna be cold and you gonna be mad .", "words": ["what", "if", "be", "mad", "you", "eat", "dont", "and", "cold"]}, {"sent": "your hands cold .", "words": ["your", "cold"]}, {"sent": "Dadas got a cold .", "words": ["cold", "a"]}, {"sent": "youre hands are cold .", "words": ["are", "cold"]}, {"sent": "unless its sposta be cold .", "words": ["be", "cold"]}, {"sent": "oh its cold on your hands .", "words": ["on", "your", "cold"]}, {"sent": "I dont want you to pour cold water into the bath xxx .", "words": ["the", "into", "to", "pour", "you", "dont", "water", "cold"]}, {"sent": "cold juice .", "words": ["juice", "cold"]}, {"sent": "this is really for when the weather goes very cold this is .", "words": ["the", "for", "is", "this", "when", "cold"]}, {"sent": "its very cold out .", "words": ["cold", "out"]}, {"sent": "its cold .", "words": ["cold"]}, {"sent": "its too cold outside .", "words": ["outside", "cold", "too"]}, {"sent": "its a little cold .", "words": ["cold", "little", "a"]}, {"sent": "thats cold .", "words": ["cold"]}, {"sent": "plain cold milk .", "words": ["milk", "cold"]}, {"sent": "what is annoying Elizabeth Elephant under that nice cold shower ?", "words": ["what", "is", "shower", "nice", "under", "that", "cold"]}, {"sent": "och yes well if shed had a cold .", "words": ["if", "cold", "a"]}, {"sent": "I gather sun rays for the cold dark winter days .", "words": ["the", "for", "sun", "dark", "cold"]}, {"sent": "oh is he cold ?", "words": ["he", "cold", "is"]}, {"sent": "Mummys gonna drink some of her coffee if its not cold .", "words": ["if", "drink", "cold", "of", "coffee", "some", "her", "not"]}, {"sent": "very cold .", "words": ["cold"]}, {"sent": "cold .", "words": ["cold"]}, {"sent": "well it was cold out wasnt it ?", "words": ["was", "it", "cold", "out"]}, {"sent": "oh is it so cold ?", "words": ["so", "it", "cold", "is"]}, {"sent": "BobtheBuilder had a cold didnt he ?", "words": ["he", "cold", "a"]}, {"sent": "only for five minutes because its too cold to play today .", "words": ["for", "to", "play", "too", "cold", "because"]}, {"sent": "whenever shes cold you should put her hat back on .", "words": ["put", "back", "you", "on", "hat", "her", "cold"]}, {"sent": "peas porridge cold .", "words": ["peas", "cold"]}, {"sent": "I know it was really cold last week .", "words": ["was", "last", "it", "cold"]}, {"sent": "are her feet cold ?", "words": ["are", "her", "cold"]}, {"sent": "Mommy is cold .", "words": ["cold", "is"]}, {"sent": "yeah its getting chilly out getting cold .", "words": ["cold", "out"]}, {"sent": "would you like some cold water ?", "words": ["water", "like", "you", "would", "some", "cold"]}, {"sent": "brother bear woke woke up one cold frosty morning .", "words": ["up", "cold", "bear"]}, {"sent": "brother bear woke up one cold frosty morning .", "words": ["up", "cold", "bear"]}, {"sent": "Thomas dont be naughty I dont want your feet in my face thank you .", "words": ["my", "your", "face", "be", "you", "dont", "in", "naughty"]}, {"sent": "did you tell him not to be naughty again ?", "words": ["to", "be", "naughty", "you", "him", "did", "not"]}, {"sent": "you being naughty yesterday werent you ?", "words": ["naughty", "you"]}, {"sent": "do I look as if Ive been a naughty lady .", "words": ["do", "if", "look", "naughty", "a"]}, {"sent": "naughty dog .", "words": ["dog", "naughty"]}, {"sent": "both of them because theyve both been naughty uhn havent they hm ?", "words": ["of", "them", "they", "naughty", "because"]}, {"sent": "is baby naughty aswell ?", "words": ["naughty", "is"]}, {"sent": "that was very naughty .", "words": ["was", "that", "naughty"]}, {"sent": "is Jwww naughty ?", "words": ["naughty", "is"]}, {"sent": "Henrys been a naughty boy hes chewed and chewed my favorite toy .", "words": ["my", "and", "toy", "naughty", "a"]}, {"sent": "is tiger naughty ?", "words": ["naughty", "tiger", "is"]}, {"sent": "next time youre being a naughty boy Ill shout yes please .", "words": ["naughty", "a"]}, {"sent": "but youre getting naughty because youre taking them from behind .", "words": ["them", "behind", "but", "naughty", "because"]}, {"sent": "and then tiger says no tiger is not naughty .", "words": ["is", "naughty", "and", "tiger", "then", "not"]}, {"sent": "because its naughty .", "words": ["naughty", "because"]}, {"sent": "naughty Fraser and naughty Jwww .", "words": ["naughty", "and"]}, {"sent": "shes naughty isnt she ?", "words": ["naughty", "she"]}, {"sent": "N Nemos very naughty .", "words": ["naughty"]}, {"sent": "and naughty ?", "words": ["naughty", "and"]}, {"sent": "that is naughty .", "words": ["that", "naughty", "is"]}, {"sent": "are you a good penguin or a naughty penguin ?", "words": ["penguin", "you", "are", "good", "naughty", "a"]}, {"sent": "stop being naughty .", "words": ["stop", "naughty"]}, {"sent": "naughty Mummy ?", "words": ["naughty"]}, {"sent": "does he do good things or bad things or naughty things ?", "words": ["does", "do", "bad", "he", "good", "naughty"]}, {"sent": "lotsof naughty boys ?", "words": ["naughty"]}, {"sent": "youre always naughty arent you ?", "words": ["naughty", "you"]}, {"sent": "and n naughty Nick .", "words": ["naughty", "and"]}, {"sent": "naughty Mummy .", "words": ["naughty"]}, {"sent": "you naughty butterfly .", "words": ["butterfly", "naughty", "you"]}, {"sent": "naughty naughty cow .", "words": ["naughty", "cow"]}, {"sent": "thats naughty .", "words": ["naughty"]}, {"sent": "its naughty .", "words": ["naughty"]}, {"sent": "and theyre sometimes theyre very naughty youknow .", "words": ["naughty", "and"]}, {"sent": "Ruths never naughty .", "words": ["naughty"]}, {"sent": "yeah thats for being naughty .", "words": ["for", "naughty"]}, {"sent": "is it a naughty horse ?", "words": ["is", "it", "horse", "naughty", "a"]}, {"sent": "well what happens when youre naughty ?", "words": ["what", "when", "naughty"]}, {"sent": "Ill say naughty Daddy for drawing on my tablecloth .", "words": ["my", "for", "say", "on", "naughty"]}, {"sent": "is that camel naughty Fraser ?", "words": ["that", "naughty", "is"]}, {"sent": "was it naughty Fraser ?", "words": ["was", "it", "naughty"]}, {"sent": "you didnt say that was naughty .", "words": ["was", "say", "you", "that", "naughty"]}, {"sent": "because I spank you when youre naughty ?", "words": ["when", "naughty", "because", "you"]}, {"sent": "because they hafta they hafta charge high prices to cover up for all the the people who are naughty and dont pay for things .", "words": ["up", "all", "the", "for", "to", "and", "they", "dont", "are", "cover", "high", "who", "naughty", "because"]}, {"sent": "theyre not taking turns with the clown theyre being naughty see theyre both pulling at the clown and whats this little boy doing ?", "words": ["the", "with", "naughty", "and", "little", "this", "at", "see", "not"]}, {"sent": "theres a naughty ginger pussy cat that comes here and upsets Purdie .", "words": ["and", "here", "cat", "that", "naughty", "a"]}, {"sent": "well Jwww was being naughty wasnt he ?", "words": ["was", "he", "naughty"]}, {"sent": "am I being naughty ?", "words": ["am", "naughty"]}, {"sent": "that if you blow your whistle naughty boys will come running out saying hello Im naughty ?", "words": ["your", "if", "naughty", "you", "out", "will", "that", "blow"]}, {"sent": "Fraser youre being naughty .", "words": ["naughty"]}, {"sent": "are those naughty teeth coming through again ?", "words": ["are", "those", "naughty"]}, {"sent": "thats naughty .", "words": ["naughty"]}, {"sent": "thats me being a bit naughty .", "words": ["me", "naughty", "a"]}, {"sent": "thats naughty Conor .", "words": ["naughty"]}, {"sent": "you think hes naughty ?", "words": ["think", "naughty", "you"]}, {"sent": "arent you naughty ?", "words": ["naughty", "you"]}, {"sent": "he was being naughty then wasnt he ?", "words": ["was", "then", "he", "naughty"]}, {"sent": "what do you think Ive done to be naughty Thomas ?", "words": ["what", "do", "think", "to", "be", "you", "naughty"]}, {"sent": "thats naughty now .", "words": ["naughty"]}, {"sent": "has she ever been naughty ?", "words": ["she", "naughty"]}, {"sent": "yeah she is naughty .", "words": ["she", "naughty", "is"]}, {"sent": "well you were a very naughty girl .", "words": ["were", "naughty", "a", "you"]}, {"sent": "that was naughty of him wasnt it ?", "words": ["of", "was", "him", "it", "that", "naughty"]}, {"sent": "thats naughty .", "words": ["naughty"]}, {"sent": "oh hes naughty that Jack isnt he ?", "words": ["he", "that", "naughty"]}, {"sent": "thats naughty .", "words": ["naughty"]}, {"sent": "am I naughty Mummy ?", "words": ["am", "naughty"]}, {"sent": "you naughty girl .", "words": ["naughty", "you"]}, {"sent": "oh eat it quick before somebody naughty gets to it .", "words": ["eat", "to", "it", "naughty"]}, {"sent": "thats naughty .", "words": ["naughty"]}, {"sent": "you always get naughty when Amys having her bottle .", "words": ["when", "you", "get", "bottle", "her", "naughty"]}, {"sent": "thats naughty .", "words": ["naughty"]}, {"sent": "do I look as if Ive been naughty ?", "words": ["do", "if", "naughty", "look"]}, {"sent": "thats naughty .", "words": ["naughty"]}, {"sent": "theyre naughty those noisy boys arent they .", "words": ["they", "those", "naughty", "noisy"]}, {"sent": "are you naughty boys ?", "words": ["are", "naughty", "you"]}, {"sent": "daddys naughty .", "words": ["naughty"]}, {"sent": "thats a bit naughty .", "words": ["naughty", "a"]}, {"sent": "naughty naughty is that what hes saying ?", "words": ["what", "that", "naughty", "is"]}, {"sent": "naughty .", "words": ["naughty"]}, {"sent": "why is it naughty ?", "words": ["it", "naughty", "why", "is"]}, {"sent": "if theyre naughty then they hafta go into um .", "words": ["if", "into", "go", "they", "then", "naughty"]}, {"sent": "youre getting really naughty you are Harry .", "words": ["are", "naughty", "you"]}, {"sent": "sometimes when girls are very naughty theyre very naughty when they go the park .", "words": ["the", "go", "they", "are", "park", "when", "naughty"]}, {"sent": "is she a naughty naughty girl ?", "words": ["naughty", "she", "a", "is"]}, {"sent": "is it four naughty men in the police station or four naughty policemen ?", "words": ["the", "is", "in", "it", "naughty"]}, {"sent": "get off the train naughty baby .", "words": ["the", "off", "train", "get", "naughty"]}, {"sent": "Cwwws naughty ?", "words": ["naughty"]}, {"sent": "is Po being naughty ?", "words": ["naughty", "is"]}, {"sent": "but youve not been naughty .", "words": ["not", "but", "naughty"]}, {"sent": "whos the naughty engine ?", "words": ["the", "naughty"]}, {"sent": "naughty .", "words": ["naughty"]}, {"sent": "is she naughty ?", "words": ["she", "naughty", "is"]}, {"sent": "hes too naughty to be let out .", "words": ["to", "be", "out", "too", "naughty"]}, {"sent": "naughty spider .", "words": ["naughty"]}, {"sent": "or do they come in because youve got someone naughty ?", "words": ["do", "they", "in", "naughty", "because"]}, {"sent": "oh youre being naughty again .", "words": ["naughty"]}, {"sent": "because he was doing something naughty with the bushes .", "words": ["the", "was", "with", "he", "naughty", "because"]}, {"sent": "cause hes been naughty .", "words": ["naughty"]}, {"sent": "dont be so naughty otherwise shell always remember you as a naughty person .", "words": ["be", "so", "you", "dont", "naughty", "a"]}, {"sent": "naughty key .", "words": ["naughty"]}, {"sent": "Thomas you hafta be quick because look the trains coming and its going to fall off .", "words": ["the", "off", "be", "to", "fall", "you", "and", "look", "because"]}, {"sent": "did it fall over ?", "words": ["fall", "it", "over", "did"]}, {"sent": "what did he fall ?", "words": ["what", "fall", "he", "did"]}, {"sent": "see thatll fall over .", "words": ["see", "over", "fall"]}, {"sent": "what how high was the couch was it a long fall ?", "words": ["the", "what", "how", "was", "couch", "long", "fall", "it", "high", "a"]}, {"sent": "you might fall .", "words": ["fall", "you"]}, {"sent": "I didnt fall over .", "words": ["over", "fall"]}, {"sent": "Fitzie fall down .", "words": ["down", "fall"]}, {"sent": "xxx push the chair closer so you wont fall .", "words": ["the", "push", "so", "fall", "chair", "you"]}, {"sent": "that did fall down .", "words": ["fall", "down", "that", "did"]}, {"sent": "whos going to fall ?", "words": ["to", "fall"]}, {"sent": "all fall down .", "words": ["all", "down", "fall"]}, {"sent": "it did fall down .", "words": ["down", "fall", "it", "did"]}, {"sent": "hes going to fall down if youre not careful .", "words": ["if", "to", "fall", "careful", "down", "not"]}, {"sent": "yeah xxx fall down .", "words": ["down", "fall"]}, {"sent": "steady because youll fall Thomas .", "words": ["fall", "because"]}, {"sent": "why did it fall apart ?", "words": ["fall", "it", "did", "why"]}, {"sent": "and youre gonna fall back .", "words": ["fall", "back", "and"]}, {"sent": "mind it doesnt fall off .", "words": ["off", "it", "fall"]}, {"sent": "ha fall down .", "words": ["down", "fall"]}, {"sent": "I told you he might fall off .", "words": ["off", "he", "fall", "you"]}, {"sent": "fall off the house .", "words": ["off", "the", "house", "fall"]}, {"sent": "xxx fall .", "words": ["fall"]}, {"sent": "dont fall over .", "words": ["dont", "over", "fall"]}, {"sent": "youll make them fall on your toes .", "words": ["your", "them", "fall", "on", "make"]}, {"sent": "just be careful you dont fall .", "words": ["be", "fall", "you", "dont", "careful"]}, {"sent": "do you think hes going to fall in ?", "words": ["do", "think", "to", "fall", "you", "in"]}, {"sent": "we all fall down .", "words": ["all", "down", "fall", "we"]}, {"sent": "how did you fall ?", "words": ["fall", "how", "did", "you"]}, {"sent": "youre gonna fall Laura .", "words": ["fall"]}, {"sent": "yeah the camera will fall down if you touch it so dont touch it okay ?", "words": ["the", "camera", "if", "so", "fall", "touch", "you", "dont", "it", "will", "down"]}, {"sent": "did you fall ?", "words": ["fall", "did", "you"]}, {"sent": "I thought it was gonna fall down .", "words": ["was", "down", "it", "fall"]}, {"sent": "youll fall dont you want a new bed ?", "words": ["fall", "bed", "you", "dont", "new", "a"]}, {"sent": "careful it might fall off again .", "words": ["off", "it", "fall", "careful"]}, {"sent": "did she fall down ?", "words": ["fall", "down", "did", "she"]}, {"sent": "I was trying to get out and and I went and fall fall down .", "words": ["to", "was", "fall", "and", "get", "out", "down"]}, {"sent": "mind you dont fall .", "words": ["dont", "fall", "you"]}, {"sent": "we all fall .", "words": ["all", "fall", "we"]}, {"sent": "you wanted it to fall down ?", "words": ["to", "fall", "you", "it", "down"]}, {"sent": "youll fall darling .", "words": ["fall"]}, {"sent": "if they fall outof the tree now .", "words": ["the", "tree", "if", "fall", "they"]}, {"sent": "its so you dont fall out of bed .", "words": ["of", "so", "fall", "bed", "you", "dont", "out"]}, {"sent": "it fall ?", "words": ["it", "fall"]}, {"sent": "I say your head will fall off .", "words": ["off", "your", "head", "say", "fall", "will"]}, {"sent": "come on youll fall .", "words": ["on", "fall"]}, {"sent": "because youll fall and youll hurt yourself .", "words": ["yourself", "hurt", "fall", "and", "because"]}, {"sent": "it says copy the NooNoo and all fall down .", "words": ["the", "all", "fall", "and", "it", "down"]}, {"sent": "no he doesnt fall down yet Nomi hes climbing very carefully .", "words": ["down", "he", "fall"]}, {"sent": "the horse is climbing on the doggie but he fall did he fall off ?", "words": ["the", "off", "is", "fall", "on", "he", "did", "horse", "but"]}, {"sent": "dont fall .", "words": ["dont", "fall"]}, {"sent": "why do we fall a ?", "words": ["do", "we", "fall", "why", "a"]}, {"sent": "youll fall out of the window .", "words": ["the", "of", "fall", "out", "window"]}, {"sent": "youll fall .", "words": ["fall"]}, {"sent": "I can see that I hope he doesnt fall out .", "words": ["can", "fall", "he", "out", "see", "that"]}, {"sent": "did it fall ?", "words": ["fall", "it", "did"]}, {"sent": "they both fall .", "words": ["they", "fall"]}, {"sent": "fall .", "words": ["fall"]}, {"sent": "it all fall down darling .", "words": ["all", "down", "it", "fall"]}, {"sent": "how did you come to fall off the seesaw xxx the horse ?", "words": ["off", "the", "how", "to", "fall", "you", "did", "horse"]}, {"sent": "careful you dont fall again .", "words": ["dont", "fall", "you", "careful"]}, {"sent": "because if you fall youll hurt yourself .", "words": ["yourself", "if", "hurt", "fall", "you", "because"]}, {"sent": "Ill tell you what well stand her up there so she wont fall over .", "words": ["up", "what", "stand", "so", "fall", "you", "there", "over", "her", "she"]}, {"sent": "why did you fall on top of me ?", "words": ["of", "fall", "you", "did", "on", "me", "why"]}, {"sent": "if they fall outof the tree now .", "words": ["the", "tree", "if", "fall", "they"]}, {"sent": "and make him fall off the wall .", "words": ["off", "the", "fall", "and", "him", "make"]}, {"sent": "it will fall ?", "words": ["will", "it", "fall"]}, {"sent": "could fall down .", "words": ["could", "down", "fall"]}, {"sent": "they fall down on Beccas toes ?", "words": ["on", "they", "down", "fall"]}, {"sent": "they fall uh fell apart .", "words": ["they", "fall"]}, {"sent": "Jon sit down before you fall .", "words": ["sit", "down", "fall", "you"]}, {"sent": "they might fall in a bucket .", "words": ["bucket", "fall", "they", "in", "a"]}, {"sent": "you said all fall down ?", "words": ["all", "down", "fall", "you"]}, {"sent": "all fall down .", "words": ["all", "down", "fall"]}, {"sent": "fall down .", "words": ["down", "fall"]}, {"sent": "how did it fall on you ?", "words": ["how", "fall", "you", "it", "did", "on"]}, {"sent": "do not fall down .", "words": ["do", "fall", "down", "not"]}, {"sent": "you think shell fall asleep soon ?", "words": ["asleep", "think", "fall", "you"]}, {"sent": "it wasnt going to fall down .", "words": ["to", "down", "it", "fall"]}, {"sent": "youre going to fall off your chair .", "words": ["off", "your", "to", "fall", "chair"]}, {"sent": "youre gonna fall .", "words": ["fall"]}, {"sent": "if you try pushing bits on the sticky out bits it will fall apart .", "words": ["the", "if", "sticky", "will", "fall", "you", "it", "out", "on", "try"]}, {"sent": "Von has magical shoes they fall off every three minutes he just looks at them and they fall off .", "words": ["off", "them", "fall", "and", "they", "he", "every", "at"]}, {"sent": "so the thing doesnt fall on you .", "words": ["the", "so", "fall", "you", "on"]}, {"sent": "did you fall ?", "words": ["fall", "did", "you"]}, {"sent": "LaLa looks like hes about to fall or shes about to fall .", "words": ["to", "fall", "like", "about"]}, {"sent": "where did you fall ?", "words": ["where", "fall", "did", "you"]}, {"sent": "you needta be careful standing up there you might fall out .", "words": ["up", "be", "fall", "you", "there", "out", "careful"]}, {"sent": "I think it is gonna fall .", "words": ["think", "it", "fall", "is"]}, {"sent": "and if it comes out the other end then theyre going to fall into the water .", "words": ["the", "if", "into", "to", "fall", "and", "other", "it", "out", "water", "then"]}, {"sent": "if you try to take one from the other places the whole thing will fall apart .", "words": ["the", "try", "if", "to", "fall", "you", "other", "will", "take", "try to"]}, {"sent": "you dont help them fall out and that grinding hurts your teeth .", "words": ["help", "your", "them", "fall", "you", "and", "dont", "out", "that"]}, {"sent": "oop theyre starting to fall asleep Ethan shh .", "words": ["to", "fall", "asleep"]}, {"sent": "when these fall out when your first teeth fall out .", "words": ["these", "your", "first", "fall", "out", "when"]}, {"sent": "did it fall down ?", "words": ["down", "fall", "it", "did"]}, {"sent": "HumptyDumpty had a great fall .", "words": ["fall", "a"]}, {"sent": "I dont wanna fall over thankyou Warren .", "words": ["dont", "over", "fall", "wanna"]}, {"sent": "after he saws it off hes gonna fall .", "words": ["off", "it", "he", "fall"]}, {"sent": "did that fall down ?", "words": ["fall", "down", "that", "did"]}, {"sent": "youll fall off .", "words": ["off", "fall"]}, {"sent": "see whose who can do it the quietest .", "words": ["can", "do", "the", "it", "who", "see"]}, {"sent": "who do you see there ?", "words": ["do", "you", "there", "who", "see"]}, {"sent": "who brings the letters ?", "words": ["the", "who"]}, {"sent": "who else do you wanna come to your party ?", "words": ["do", "your", "to", "you", "party", "wanna", "who"]}, {"sent": "who is who is in there ?", "words": ["there", "in", "is", "who"]}, {"sent": "and somebody who likes feather cushions can have them rather than wasting them because theyre in good condition arent they ?", "words": ["can", "them", "and", "have", "they", "in", "good", "who", "because"]}, {"sent": "can you not see who it is ?", "words": ["can", "is", "you", "it", "who", "see", "not"]}, {"sent": "who did they met ?", "words": ["they", "did", "who"]}, {"sent": "Sarah who sings that song ?", "words": ["that", "who"]}, {"sent": "who is it ?", "words": ["it", "is", "who"]}, {"sent": "who else ?", "words": ["who"]}, {"sent": "for who ?", "words": ["for", "who"]}, {"sent": "who do we know with guitars .", "words": ["do", "with", "we", "who"]}, {"sent": "thats who he is .", "words": ["he", "is", "who"]}, {"sent": "who came to see Carl ?", "words": ["see", "to", "who"]}, {"sent": "so who is he ?", "words": ["so", "he", "is", "who"]}, {"sent": "who else has glasses ?", "words": ["glasses", "who"]}, {"sent": "who eats that ?", "words": ["that", "who"]}, {"sent": "and who is this ?", "words": ["who", "this", "is", "and"]}, {"sent": "who shall Mummy bring in to the doctor .", "words": ["the", "to", "bring", "in", "who"]}, {"sent": "they were all dressed in blue with long pointed heads the dwarfs seemed very happy to see Dorothy who now spoke to their leader .", "words": ["the", "all", "their", "happy", "to", "with", "long", "blue", "they", "in", "who", "see", "were"]}, {"sent": "who wants chips ?", "words": ["who"]}, {"sent": "who does ?", "words": ["does", "who"]}, {"sent": "who ?", "words": ["who"]}, {"sent": "the lady who works there or the mummy who comes for coffee ?", "words": ["the", "for", "there", "coffee", "who"]}, {"sent": "I dont know who that is .", "words": ["dont", "that", "is", "who"]}, {"sent": "who was it who was awake though when they shouldve been asleep ?", "words": ["was", "awake", "they", "asleep", "it", "who", "when"]}, {"sent": "who lives in that house there ?", "words": ["there", "in", "house", "who", "that"]}, {"sent": "who is that ?", "words": ["that", "is", "who"]}, {"sent": "and I have a little girl called Kathryn who you met .", "words": ["little", "have", "and", "you", "who", "a"]}, {"sent": "who dont you get each peach pear plum and we can read it together .", "words": ["can", "each", "we", "you", "get", "dont", "and", "it", "read", "who"]}, {"sent": "who ?", "words": ["who"]}, {"sent": "who are you writing a letter to ?", "words": ["to", "you", "are", "who", "a"]}, {"sent": "who is he ?", "words": ["he", "is", "who"]}, {"sent": "who is the Weener .", "words": ["the", "is", "who"]}, {"sent": "who was driving ?", "words": ["was", "who"]}, {"sent": "who laid the eggs though ?", "words": ["the", "who"]}, {"sent": "and look they come out and who has the cat ?", "words": ["the", "look", "and", "they", "cat", "out", "who"]}, {"sent": "who taught you about syllables ?", "words": ["about", "you", "who"]}, {"sent": "whats who doing ?", "words": ["who"]}, {"sent": "who ?", "words": ["who"]}, {"sent": "whos froggy baby who who who ?", "words": ["who"]}, {"sent": "and who ate them up ?", "words": ["up", "who", "them", "and"]}, {"sent": "who ?", "words": ["who"]}, {"sent": "who ?", "words": ["who"]}, {"sent": "and who are you ?", "words": ["who", "are", "you", "and"]}, {"sent": "who got you that ?", "words": ["that", "you", "who"]}, {"sent": "and who who were you playing with ?", "words": ["with", "you", "and", "who", "were"]}, {"sent": "who did that on purpose ?", "words": ["on", "that", "did", "who"]}, {"sent": "and who was playing with you this morning ?", "words": ["was", "with", "you", "and", "this", "who"]}, {"sent": "who who bit your toe ?", "words": ["toe", "your", "who"]}, {"sent": "who brought you the doll that you like so well ?", "words": ["the", "so", "like", "doll", "you", "who", "that"]}, {"sent": "who whats his name ?", "words": ["his", "who"]}, {"sent": "who ?", "words": ["who"]}, {"sent": "who stays who stays with you when Daddy goes to work ?", "words": ["work", "to", "with", "you", "who", "when"]}, {"sent": "NewYork who lives in NewYork ?", "words": ["in", "who"]}, {"sent": "who from ?", "words": ["who"]}, {"sent": "who what ?", "words": ["what", "who"]}, {"sent": "somebody has upset you but you dont know who ?", "words": ["dont", "who", "but", "you"]}, {"sent": "do you know who it is thats naughty ?", "words": ["do", "is", "you", "it", "who", "naughty"]}, {"sent": "who do you want first ?", "words": ["do", "first", "you", "who"]}, {"sent": "to who ?", "words": ["to", "who"]}, {"sent": "anybody who wants to see the Doctor can go and see him .", "words": ["the", "can", "go", "to", "and", "him", "who", "see"]}, {"sent": "who is under the hat huh ?", "words": ["the", "is", "under", "who", "hat"]}, {"sent": "who you calling ?", "words": ["you", "who"]}, {"sent": "who else will you take with you ?", "words": ["with", "you", "who", "will", "take"]}, {"sent": "now you know who the family are dont you ?", "words": ["the", "you", "dont", "are", "who"]}, {"sent": "who ?", "words": ["who"]}, {"sent": "who are you hm ?", "words": ["are", "you", "who"]}, {"sent": "who do you gotta give it to ?", "words": ["do", "to", "you", "it", "who", "give"]}, {"sent": "weve got to sign a card for the nice little girl who is going to be two .", "words": ["the", "for", "is", "to", "be", "nice", "little", "who", "a"]}, {"sent": "then you tell me who it is .", "words": ["is", "you", "it", "who", "me", "then"]}, {"sent": "hes the one who who makes tape .", "words": ["the", "tape", "who"]}, {"sent": "who is this person ?", "words": ["this", "is", "who"]}, {"sent": "who do you think that is ?", "words": ["do", "think", "is", "you", "who", "that"]}, {"sent": "who can find shapes ?", "words": ["can", "find", "who"]}, {"sent": "who do you live with ?", "words": ["do", "with", "you", "who"]}, {"sent": "well who does he want ?", "words": ["does", "he", "who"]}, {"sent": "who did ?", "words": ["did", "who"]}, {"sent": "who did those two pages ?", "words": ["those", "did", "who"]}, {"sent": "Im not gonna read it but who is it ?", "words": ["is", "it", "read", "who", "not", "but"]}, {"sent": "who is it ?", "words": ["it", "is", "who"]}, {"sent": "one elephant right who had a .", "words": ["who", "a", "elephant"]}, {"sent": "who eat .", "words": ["eat", "who"]}, {"sent": "what matters is who does the nicest drawing .", "words": ["the", "what", "does", "is", "who"]}, {"sent": "push who ?", "words": ["push", "who"]}, {"sent": "tell Rose who we know who has a snake .", "words": ["we", "a", "who"]}, {"sent": "who are those ?", "words": ["are", "those", "who"]}, {"sent": "who do you think that is ?", "words": ["do", "think", "is", "you", "who", "that"]}, {"sent": "who is it ?", "words": ["it", "is", "who"]}, {"sent": "who gave back the spoons ?", "words": ["the", "back", "who"]}, {"sent": "who wasnt there ?", "words": ["there", "who"]}, {"sent": "and Daddy and Mommy and Naima thats who will be here tomorrow .", "words": ["be", "and", "here", "who", "will"]}, {"sent": "okay now who else go to swim ?", "words": ["go", "to", "swim", "who"]}, {"sent": "and who ?", "words": ["who", "and"]}, {"sent": "and who is Sophie ?", "words": ["who", "is", "and"]}, {"sent": "who ?", "words": ["who"]}, {"sent": "I dont know who it is .", "words": ["dont", "it", "is", "who"]}, {"sent": "so who are you up with Tanya ?", "words": ["up", "so", "with", "you", "are", "who"]}, {"sent": "Daddy and who is he with ?", "words": ["is", "with", "and", "he", "who"]}, {"sent": "would you like to go outside ?", "words": ["go", "to", "outside", "like", "you", "would"]}, {"sent": "I was going to say there is a bee outside .", "words": ["is", "to", "was", "bee", "say", "outside", "there", "a"]}, {"sent": "oh look whos outside now .", "words": ["outside", "look"]}, {"sent": "out bugs are outside .", "words": ["outside", "are", "out"]}, {"sent": "outside of this apartment building and then they go .", "words": ["go", "of", "outside", "and", "this", "they", "then"]}, {"sent": "outside ?", "words": ["outside"]}, {"sent": "and when its dark outside .", "words": ["outside", "dark", "when", "and"]}, {"sent": "well oh I dont think we really needta take her to the Doctor but Im glad shes outside .", "words": ["the", "think", "we", "to", "outside", "her", "dont", "take", "but"]}, {"sent": "oh I could hear Purdie crying but shes outside .", "words": ["outside", "could", "but", "hear"]}, {"sent": "when you come in from outside we take your shoes off .", "words": ["off", "your", "we", "outside", "you", "in", "take", "when"]}, {"sent": "outside okay .", "words": ["outside"]}, {"sent": "lets go outside now ?", "words": ["outside", "go"]}, {"sent": "because its a lovely day outside isnt it ?", "words": ["outside", "it", "because", "a"]}, {"sent": "can you see those little pink flowers in the bush outside at the front ?", "words": ["can", "the", "those", "outside", "little", "you", "in", "at", "see"]}, {"sent": "before we go outside Momma hasta feed the pets okay ?", "words": ["the", "go", "we", "outside", "feed"]}, {"sent": "Purdie wants to go back outside doesnt she ?", "words": ["back", "go", "to", "outside", "she"]}, {"sent": "you can go outside when daddy comes back .", "words": ["can", "back", "go", "outside", "you", "when"]}, {"sent": "do you see something outside ?", "words": ["see", "do", "outside", "you"]}, {"sent": "were not gonna go outside right now .", "words": ["outside", "were", "not", "go"]}, {"sent": "bring him on the outside so hes between the two of us .", "words": ["the", "of", "outside", "so", "bring", "us", "him", "on"]}, {"sent": "did you see a puppy outside ?", "words": ["outside", "you", "puppy", "did", "see", "a"]}, {"sent": "shall we hang these clothes up outside do you think Fraser ?", "words": ["up", "these", "do", "think", "we", "outside", "you"]}, {"sent": "outside we have the shed with Daddys lawnmower in it and the bicycles .", "words": ["the", "we", "outside", "with", "and", "have", "in", "it"]}, {"sent": "why dont you put it on the chair and when we leave the room you can take it with you and could put it in the wastebasket outside .", "words": ["the", "can", "put", "we", "outside", "with", "chair", "you", "and", "dont", "in", "it", "room", "on", "take", "when", "why", "could"]}, {"sent": "what should Erin be able to see thats outside ?", "words": ["what", "to", "be", "outside", "see"]}, {"sent": "okay we should get your shoes on incase you decide to go outside .", "words": ["your", "we", "go", "to", "outside", "you", "get", "on"]}, {"sent": "we dont play outside when its dark .", "words": ["we", "outside", "play", "dont", "dark", "when"]}, {"sent": "there goes a car outside .", "words": ["there", "outside", "car", "a"]}, {"sent": "it is wet outside .", "words": ["wet", "it", "outside", "is"]}, {"sent": "Daddy is gonna go outside .", "words": ["go", "outside", "is"]}, {"sent": "both are outside really arent they ?", "words": ["outside", "are", "they"]}, {"sent": "shes quite happy outside .", "words": ["outside", "happy"]}, {"sent": "he is going outside to play with his friends .", "words": ["is", "to", "outside", "play", "with", "he", "his"]}, {"sent": "Imean theres a park like beside the Loughshore outside this wee pub so there was .", "words": ["the", "outside", "beside", "so", "was", "like", "this", "there", "park", "a"]}, {"sent": "but thats broken outside .", "words": ["outside", "but", "broken"]}, {"sent": "lets count the people outside the bus .", "words": ["the", "outside", "bus"]}, {"sent": "its a shame were not outside isnt it ?", "words": ["outside", "it", "were", "not", "a"]}, {"sent": "oh Molly sometimes crosses the road and then goes to sleep underneath the black taxi that is parked outside doesnt she ?", "words": ["the", "is", "sleep", "to", "outside", "and", "black", "then", "that", "she"]}, {"sent": "were going to open the door and shake the cloth outside .", "words": ["the", "open", "to", "outside", "shake", "and", "were", "door"]}, {"sent": "cold outside yes .", "words": ["outside", "cold"]}, {"sent": "outside of Sues door .", "words": ["of", "outside", "door"]}, {"sent": "other bit outside ?", "words": ["outside", "other"]}, {"sent": "theres the blue truck outside .", "words": ["the", "outside", "truck", "blue"]}, {"sent": "its nice outside .", "words": ["outside", "nice"]}, {"sent": "either that or theyre poorly because Sonia and Rods cars were both outside werent they ?", "words": ["outside", "and", "they", "that", "were", "because"]}, {"sent": "outside ?", "words": ["outside"]}, {"sent": "that outside ?", "words": ["outside", "that"]}, {"sent": "well stick your turtleneck outside it .", "words": ["outside", "stick", "it", "your"]}, {"sent": "look Brians outside .", "words": ["outside", "look"]}, {"sent": "Cami must be outside too .", "words": ["be", "outside", "too"]}, {"sent": "theres lots of clouds outside today .", "words": ["of", "outside"]}, {"sent": "but we have put that outside because that makes a clicketyclicketyclick sound when youre using it .", "words": ["put", "when", "we", "outside", "because", "have", "it", "that", "but", "a"]}, {"sent": "were gonna go outside now .", "words": ["outside", "were", "go"]}, {"sent": "their friend Owl is waiting for the children to come outside .", "words": ["the", "for", "their", "is", "to", "outside"]}, {"sent": "youll probably get filthy dirty outside .", "words": ["get", "outside", "dirty"]}, {"sent": "you were outside werent you ?", "words": ["outside", "were", "you"]}, {"sent": "we cant go outside now poppet .", "words": ["go", "outside", "we"]}, {"sent": "you cant go outside while the barbecues on okay ?", "words": ["the", "go", "outside", "you", "on"]}, {"sent": "was somebody outside ?", "words": ["was", "outside"]}, {"sent": "yeah what kind of things do you do when youre outside ?", "words": ["what", "do", "of", "outside", "you", "when"]}, {"sent": "hes actually outside .", "words": ["outside"]}, {"sent": "because when you look outside normally you can see a pussycat walking around the garden cant you ?", "words": ["can", "the", "around", "outside", "you", "garden", "look", "see", "when", "because", "a"]}, {"sent": "oh its brightening up a bit now Thomas outside .", "words": ["up", "outside", "a"]}, {"sent": "oh there is a bird outside ?", "words": ["is", "outside", "bird", "there", "a"]}, {"sent": "Eleanor whats the weather like outside today ?", "words": ["the", "outside", "like"]}, {"sent": "its a snail like you have outside .", "words": ["outside", "like", "you", "have", "a"]}, {"sent": "whats outside ?", "words": ["outside"]}, {"sent": "you hear the cars outside ?", "words": ["the", "outside", "hear", "you"]}, {"sent": "she has gone outside .", "words": ["outside", "she"]}, {"sent": "what else does she do outside ?", "words": ["what", "do", "does", "outside", "she"]}, {"sent": "we can not go outside .", "words": ["can", "go", "we", "outside", "not"]}, {"sent": "weve got a compost bin outside havent we ?", "words": ["outside", "we", "a"]}, {"sent": "is it really sunny outside ?", "words": ["outside", "it", "is"]}, {"sent": "she bundled him up and put him outside on his motorcycle Felixs mama listened at the window but he was not making his motorcycle noises .", "words": ["up", "motorcycle", "put", "the", "outside", "was", "not", "and", "his", "him", "he", "at", "on", "window", "she", "but"]}, {"sent": "then you can take a nap and go outside .", "words": ["can", "go", "outside", "you", "and", "then", "take", "a"]}, {"sent": "go outside and play with Corney .", "words": ["go", "outside", "play", "with", "and"]}, {"sent": "and then we can go outside in in twenty minutes .", "words": ["can", "go", "we", "outside", "and", "in", "then"]}, {"sent": "what were you doing when you were outside this morning ?", "words": ["what", "when", "outside", "you", "this", "were"]}, {"sent": "you useta run outside and try and knock all the heads off the flowers and leaves off the trees .", "words": ["the", "all", "off", "outside", "and", "you", "run", "knock", "try"]}, {"sent": "quite a queue now outside isnt there ?", "words": ["outside", "there", "a"]}, {"sent": "hey whos coming outside ?", "words": ["outside"]}, {"sent": "it has got fish swimming round the outside hasnt it ?", "words": ["the", "outside", "it", "fish"]}, {"sent": "whats he doing outside ?", "words": ["outside", "he"]}, {"sent": "no well not on the inside see this keys on the inside you have a key on the outside .", "words": ["the", "outside", "inside", "you", "have", "this", "on", "see", "keys", "not", "a"]}, {"sent": "Lucy how would you like to go outside with Elise ?", "words": ["how", "go", "to", "outside", "with", "like", "you", "would"]}, {"sent": "she has just nipped outside .", "words": ["outside", "she"]}, {"sent": "its built its a web outside the crack .", "words": ["the", "outside", "a"]}, {"sent": "inside the hotel or outside ?", "words": ["the", "outside", "inside"]}, {"sent": "its on the outside of the glass isnt it ?", "words": ["the", "of", "outside", "it", "glass", "on"]}, {"sent": "outside in the rain ?", "words": ["the", "outside", "in", "rain"]}, {"sent": "outside .", "words": ["outside"]}, {"sent": "yes its light outside isnt it ?", "words": ["light", "outside", "it"]}, {"sent": "peering outside she sipped tea from her cup and eagerly watched for the sun to come .", "words": ["cup", "the", "for", "outside", "sun", "to", "and", "her", "she"]}, {"sent": "if I see you jumping outside again Ill smack you .", "words": ["see", "outside", "if", "you"]}, {"sent": "got butterflies to put on the the outside of your house if you wanna .", "words": ["the", "put", "your", "if", "of", "to", "outside", "you", "house", "wanna", "on"]}, {"sent": "the ambulance is outside of the hospital .", "words": ["the", "outside", "of", "is"]}, {"sent": "did you go outside ?", "words": ["go", "outside", "did", "you"]}, {"sent": "the girls can eat outside .", "words": ["the", "outside", "can", "eat"]}, {"sent": "so I suppose Ive just put them outside to get a bit of air .", "words": ["put", "of", "outside", "so", "to", "them", "get", "a"]}, {"sent": "*CH1: Swww is asleep .", "words": ["asleep", "is"]}, {"sent": "hes not asleep .", "words": ["asleep", "not"]}, {"sent": "it says owls on the night shift because the owl stays awake at night when everyone else is asleep .", "words": ["the", "is", "awake", "owl", "asleep", "it", "at", "on", "when", "because"]}, {"sent": "Frederick looks like hes asleep .", "words": ["asleep", "like"]}, {"sent": "oh is she asleep again ?", "words": ["asleep", "she", "is"]}, {"sent": "hm babys asleep .", "words": ["asleep"]}, {"sent": "you know when youre asleep ?", "words": ["asleep", "when", "you"]}, {"sent": "are you going fast asleep ?", "words": ["asleep", "fast", "are", "you"]}, {"sent": "because theyre all asleep .", "words": ["all", "asleep", "because"]}, {"sent": "then count them and youll fall asleep .", "words": ["them", "fall", "and", "asleep", "then"]}, {"sent": "Dwwws asleep downstairs .", "words": ["asleep"]}, {"sent": "is she asleep in there ?", "words": ["is", "asleep", "in", "there", "she"]}, {"sent": "everyones fast asleep .", "words": ["asleep", "fast"]}, {"sent": "is your baby asleep ?", "words": ["asleep", "your", "is"]}, {"sent": "asleep .", "words": ["asleep"]}, {"sent": "fast asleep .", "words": ["asleep", "fast"]}, {"sent": "is she asleep ?", "words": ["asleep", "she", "is"]}, {"sent": "shes asleep in the lounge isnt she ?", "words": ["the", "asleep", "in", "she"]}, {"sent": "but she usually falls asleep for a little while on the plane .", "words": ["the", "for", "little", "asleep", "on", "she", "but", "a"]}, {"sent": "so dad sat at the wheel again come Miffy in you leap such a lovely day she said and then she fell asleep .", "words": ["the", "so", "you", "and", "asleep", "in", "at", "then", "she", "a"]}, {"sent": "are you asleep ?", "words": ["asleep", "are", "you"]}, {"sent": "asleep .", "words": ["asleep"]}, {"sent": "is she asleep now ?", "words": ["asleep", "she", "is"]}, {"sent": "and fell asleep on the ground .", "words": ["on", "asleep", "the", "and"]}, {"sent": "youre sposta be asleep Nina ?", "words": ["be", "asleep"]}, {"sent": "Amys asleep .", "words": ["asleep"]}, {"sent": "the stars asleep ?", "words": ["the", "asleep"]}, {"sent": "whats this asleep ?", "words": ["this", "asleep"]}, {"sent": "who is asleep ?", "words": ["asleep", "is", "who"]}, {"sent": "is she asleep ?", "words": ["asleep", "she", "is"]}, {"sent": "of course Mummy tidied up while you were asleep .", "words": ["up", "of", "you", "asleep", "were"]}, {"sent": "asleep ?", "words": ["asleep"]}, {"sent": "well the story made Biff feel quite sleepy so Mummy took her back to bed and she fell asleep right away .", "words": ["the", "back", "so", "to", "bed", "and", "asleep", "away", "story", "sleepy", "her", "she"]}, {"sent": "Percy and Toby were still asleep Ill give them silly stickinthemuds a surprise chuckled Thomas .", "words": ["them", "and", "asleep", "give", "were", "a"]}, {"sent": "its asleep yeah .", "words": ["asleep"]}, {"sent": "and this tiny bear whos asleep in bed .", "words": ["tiny", "bed", "bear", "and", "this", "asleep", "in"]}, {"sent": "hes not asleep Becky .", "words": ["asleep", "not"]}, {"sent": "is there anyone their picture taken while you were asleep ?", "words": ["their", "is", "picture", "you", "there", "asleep", "were"]}, {"sent": "are you asleep ?", "words": ["asleep", "are", "you"]}, {"sent": "and asleep in the purple shadows theres a yellow .", "words": ["the", "and", "asleep", "in", "yellow", "a"]}, {"sent": "dogs asleep .", "words": ["asleep"]}, {"sent": "are those men still asleep in that truck there ?", "words": ["those", "truck", "asleep", "in", "are", "there", "that"]}, {"sent": "Jess is already asleep .", "words": ["asleep", "is"]}, {"sent": "I wore my wool pajamas but when I feel asleep .", "words": ["my", "asleep", "pajamas", "when", "but"]}, {"sent": "gosh hes asleep train isnt he ?", "words": ["asleep", "he", "train"]}, {"sent": "I think Caroline would love to be asleep .", "words": ["think", "to", "be", "asleep", "love", "would"]}, {"sent": "asleep ?", "words": ["asleep"]}, {"sent": "asleep .", "words": ["asleep"]}, {"sent": "last night as she was falling asleep she was saying like .", "words": ["was", "last", "like", "asleep", "she"]}, {"sent": "and you were still asleep and I cuddled you up with the quilt .", "words": ["up", "the", "with", "you", "and", "asleep", "were"]}, {"sent": "okay you fell asleep .", "words": ["asleep", "you"]}, {"sent": "and she was asleep .", "words": ["was", "she", "asleep", "and"]}, {"sent": "shes asleep .", "words": ["asleep"]}, {"sent": "oh they were all asleep ?", "words": ["all", "they", "were", "asleep"]}, {"sent": "I fell asleep one night .", "words": ["asleep"]}, {"sent": "hes asleep .", "words": ["asleep"]}, {"sent": "are you the pussy cat asleep on the road ?", "words": ["the", "you", "cat", "asleep", "are", "on"]}, {"sent": "fast asleep .", "words": ["asleep", "fast"]}, {"sent": "is she asleep now ?", "words": ["asleep", "she", "is"]}, {"sent": "here Ill pretend that Im asleep and then you pretend .", "words": ["then", "and", "here", "asleep", "you", "pretend", "that"]}, {"sent": "is dolly still asleep ?", "words": ["asleep", "is"]}, {"sent": "ah hes asleep .", "words": ["asleep"]}, {"sent": "she didnt go to bed until it must have been half past twelve twelve before she fell asleep last night and she was up at seven this morning .", "words": ["up", "go", "to", "last", "was", "bed", "and", "have", "asleep", "this", "it", "at", "she"]}, {"sent": "xxx still asleep in bed ?", "words": ["asleep", "in", "bed"]}, {"sent": "they looked around and soon discovered SnowWhite asleep in bed .", "words": ["bed", "and", "they", "asleep", "in", "around"]}, {"sent": "Daddys gonna stay asleep this morning .", "words": ["stay", "asleep", "this"]}, {"sent": "fell asleep ?", "words": ["asleep"]}, {"sent": "you still do lie in bed when youre asleep .", "words": ["do", "bed", "you", "asleep", "in", "when"]}, {"sent": "oh is PostmanPat asleep ?", "words": ["asleep", "is"]}, {"sent": "and you were fast asleep and didnt hear us .", "words": ["fast", "you", "and", "asleep", "hear", "us", "were"]}, {"sent": "you dont shout Mama in the night because Mama was fast asleep .", "words": ["the", "fast", "was", "you", "dont", "in", "asleep", "because"]}, {"sent": "is Eleanor asleep ?", "words": ["asleep", "is"]}, {"sent": "shush theyre asleep arent they .", "words": ["asleep", "they"]}, {"sent": "your hands asleep as well ?", "words": ["asleep", "your"]}, {"sent": "shes not asleep .", "words": ["asleep", "not"]}, {"sent": "or Mummy falls asleep .", "words": ["asleep"]}, {"sent": "thats where we left him asleep .", "words": ["where", "asleep", "him", "we"]}, {"sent": "she is asleep yeah .", "words": ["asleep", "she", "is"]}, {"sent": "the ladys asleep .", "words": ["the", "asleep"]}, {"sent": "oh is that dolly asleep Liz ?", "words": ["asleep", "that", "is"]}, {"sent": "hes under the haystack fast asleep .", "words": ["the", "under", "fast", "asleep"]}, {"sent": "babys asleep .", "words": ["asleep"]}, {"sent": "asleep all day ?", "words": ["all", "asleep"]}, {"sent": "PostmanPats asleep .", "words": ["asleep"]}, {"sent": "is your baby asleep ?", "words": ["asleep", "your", "is"]}, {"sent": "shes fast asleep is she ?", "words": ["asleep", "fast", "she", "is"]}, {"sent": "shush baby asleep .", "words": ["asleep"]}, {"sent": "you were fast asleep when they were doing that .", "words": ["fast", "when", "you", "asleep", "they", "that", "were"]}, {"sent": "found the small girl asleep by the gates .", "words": ["the", "asleep", "by"]}, {"sent": "look shes got her eyes shut and shes very quiet so she must be asleep .", "words": ["so", "be", "and", "asleep", "quiet", "look", "her", "she"]}, {"sent": "youre not fast asleep .", "words": ["asleep", "fast", "not"]}, {"sent": "and his cats asleep thats .", "words": ["his", "asleep", "and"]}, {"sent": "youve been asleep since November said Frog .", "words": ["asleep"]}, {"sent": "I bet you cant guess whos fast asleep on the suitcase upstairs in the attic .", "words": ["the", "fast", "you", "asleep", "in", "on"]}, {"sent": "shes nicely asleep isnt she ?", "words": ["asleep", "she"]}, {"sent": "going Nathaniel I think Nathaniels asleep isnt he ?", "words": ["asleep", "think", "he"]}, {"sent": "shes fast asleep isnt she ?", "words": ["asleep", "fast", "she"]}, {"sent": "is she fast asleep ?", "words": ["asleep", "fast", "she", "is"]}, {"sent": "are you asleep in a tree ?", "words": ["tree", "you", "asleep", "in", "are", "a"]}, {"sent": "on the love seat Honey .", "words": ["on", "the", "love"]}, {"sent": "oh Im sorry love .", "words": ["love"]}, {"sent": "I love you a hundred times .", "words": ["a", "love", "you"]}, {"sent": "I love it .", "words": ["it", "love"]}, {"sent": "I dont think youve got any pockets in those trousers love .", "words": ["think", "those", "any", "dont", "in", "love"]}, {"sent": "I dont love what ?", "words": ["dont", "what", "love"]}, {"sent": "with lots of love .", "words": ["of", "with", "love"]}, {"sent": "I love when you ask that kind of thing right when Im trying to get outof here .", "words": ["when", "of", "to", "you", "get", "here", "love", "that"]}, {"sent": "what love ?", "words": ["what", "love"]}, {"sent": "yeah you needta love her and make her happy .", "words": ["happy", "her", "and", "you", "love", "make"]}, {"sent": "oh never mind love .", "words": ["love"]}, {"sent": "I love strawberry pie .", "words": ["love", "strawberry"]}, {"sent": "dont love .", "words": ["dont", "love"]}, {"sent": "heehee love the baby huh you love the baby kisses ?", "words": ["the", "love", "you"]}, {"sent": "who do you love ?", "words": ["do", "love", "you", "who"]}, {"sent": "huh I love bananas .", "words": ["love"]}, {"sent": "you dont know anything youre just so tired love arent you ?", "words": ["so", "tired", "you", "dont", "love"]}, {"sent": "I love you didnt nobody love me .", "words": ["me", "love", "you"]}, {"sent": "Im in love .", "words": ["in", "love"]}, {"sent": "I would really love a biscuit .", "words": ["a", "love", "would"]}, {"sent": "dont break the books love .", "words": ["the", "dont", "break", "love"]}, {"sent": "haha two reasons to love that one .", "words": ["to", "that", "love"]}, {"sent": "you dont love me so youre not talking .", "words": ["so", "you", "dont", "love", "me", "not"]}, {"sent": "I love you .", "words": ["love", "you"]}, {"sent": "love you you know .", "words": ["love", "you"]}, {"sent": "and Tellytubbies love each other .", "words": ["other", "each", "love", "and"]}, {"sent": "I said Pippo would love a ripe juicy plum .", "words": ["a", "love", "would"]}, {"sent": "we love chinese food .", "words": ["food", "love", "we"]}, {"sent": "great is the measure of our fathers love .", "words": ["the", "is", "of", "our", "love"]}, {"sent": "xxx love .", "words": ["love"]}, {"sent": "I know love .", "words": ["love"]}, {"sent": "we would love to help in the dogs birthday parade .", "words": ["the", "help", "we", "to", "in", "love", "would"]}, {"sent": "I love you .", "words": ["love", "you"]}, {"sent": "weve runt outof butter my love .", "words": ["my", "butter", "love"]}, {"sent": "come on love because you said yourself that if you go out in the wind your cheeks get sore .", "words": ["the", "yourself", "your", "if", "go", "you", "get", "in", "love", "out", "on", "that", "because", "wind"]}, {"sent": "I love little pussy .", "words": ["love", "little"]}, {"sent": "love a lot .", "words": ["a lot", "love", "a"]}, {"sent": "I just say you go in and do your best love and just .", "words": ["do", "your", "go", "say", "and", "you", "in", "love"]}, {"sent": "whatre you saying love ?", "words": ["love", "you"]}, {"sent": "you love your Grandma ?", "words": ["your", "love", "you"]}, {"sent": "and I could love you too .", "words": ["you", "and", "love", "too", "could"]}, {"sent": "I love being off so I do .", "words": ["off", "so", "do", "love"]}, {"sent": "you love her ?", "words": ["her", "love", "you"]}, {"sent": "what do you want love ?", "words": ["what", "do", "love", "you"]}, {"sent": "so you say love mum and dad that says because thats from my mummie isnt it ?", "words": ["my", "so", "say", "and", "you", "it", "love", "that", "because"]}, {"sent": "aw we love dogs .", "words": ["love", "we"]}, {"sent": "yes love .", "words": ["love"]}, {"sent": "sorry love .", "words": ["love"]}, {"sent": "you know I love your sunglasses .", "words": ["your", "love", "you"]}, {"sent": "thank you my love ?", "words": ["my", "love", "you"]}, {"sent": "Thomas and Mummy and Purdie love each other .", "words": ["other", "each", "love", "and"]}, {"sent": "hed love it .", "words": ["it", "love"]}, {"sent": "I love little pussy .", "words": ["love", "little"]}, {"sent": "you love pasta cheese and custard .", "words": ["and", "cheese", "love", "you"]}, {"sent": "I would love a drink .", "words": ["a", "drink", "love", "would"]}, {"sent": "let me look love .", "words": ["me", "love", "look"]}, {"sent": "thats right love .", "words": ["love"]}, {"sent": "I love Mama .", "words": ["love"]}, {"sent": "I love you I hate you I love you xxx .", "words": ["hate", "love", "you"]}, {"sent": "its too late now my love .", "words": ["my", "love", "too"]}, {"sent": "aah Teletubbies love each other very much .", "words": ["much", "each", "love", "other"]}, {"sent": "you giving them a big hug and telling them you love them ?", "words": ["them", "you", "and", "love", "hug", "big", "a"]}, {"sent": "love you .", "words": ["love", "you"]}, {"sent": "I love you .", "words": ["love", "you"]}, {"sent": "love me love my bear .", "words": ["me", "my", "love", "bear"]}, {"sent": "just be very careful with them love .", "words": ["be", "them", "with", "love", "careful"]}, {"sent": "I love you sweetie .", "words": ["love", "you"]}, {"sent": "do you love Twww ?", "words": ["do", "love", "you"]}, {"sent": "you love to eat them .", "words": ["to", "them", "you", "eat", "love"]}, {"sent": "because I Id love her her hair to be long in a way .", "words": ["to", "be", "long", "in", "love", "hair", "her", "because", "a"]}, {"sent": "because if you love somebody and we lose them and you dont feel sad that would be terrible .", "words": ["if", "we", "be", "them", "and", "you", "dont", "love", "would", "sad", "that", "because"]}, {"sent": "because I love it too .", "words": ["love", "it", "because", "too"]}, {"sent": "I love you .", "words": ["love", "you"]}, {"sent": "oh you love the salty cheese dont you ?", "words": ["the", "you", "dont", "love", "cheese"]}, {"sent": "give her a love .", "words": ["give", "her", "love", "a"]}, {"sent": "farmers love to wake up early roosters love to crow .", "words": ["up", "to", "wake", "love"]}, {"sent": "xxx I love to kiss your little neck .", "words": ["your", "to", "little", "love", "kiss"]}, {"sent": "make yourself comfortable love .", "words": ["yourself", "make", "love"]}, {"sent": "well thankyou Id love to touch it .", "words": ["to", "it", "touch", "love"]}, {"sent": "I can love you .", "words": ["can", "love", "you"]}, {"sent": "I love you you read this story beautifully .", "words": ["you", "this", "read", "love", "story"]}, {"sent": "love you have a nice day .", "words": ["nice", "you", "have", "love", "a"]}, {"sent": "you love those spoons .", "words": ["those", "love", "you"]}, {"sent": "oh Id love to play .", "words": ["to", "play", "love"]}, {"sent": "I dont have grapes love .", "words": ["have", "dont", "grapes", "love"]}, {"sent": "I love him .", "words": ["him", "love"]}, {"sent": "I love you .", "words": ["love", "you"]}, {"sent": "and fell in love with it .", "words": ["with", "and", "in", "it", "love"]}, {"sent": "you love strawberry icecream .", "words": ["strawberry", "love", "you"]}, {"sent": "whats the matter love ?", "words": ["the", "love"]}, {"sent": "guess how much I love you ?", "words": ["how", "much", "love", "you"]}, {"sent": "you love Bella ?", "words": ["love", "you"]}, {"sent": "I love chickennuggets too .", "words": ["love", "too"]}, {"sent": "do you needta wee love ?", "words": ["do", "love", "you"]}, {"sent": "hooray hooray I love it today .", "words": ["it", "love"]}, {"sent": "shut the door my love .", "words": ["the", "my", "love", "door"]}, {"sent": "I love Ellas nose .", "words": ["love", "nose"]}, {"sent": "I love you too Abe .", "words": ["too", "love", "you"]}, {"sent": "Im not in love ?", "words": ["in", "not", "love"]}, {"sent": "yes Id love to see what color the Mummys car key is .", "words": ["the", "what", "is", "to", "car", "love", "see"]}, {"sent": "but she saw that her Mummy and Daddy had an ice cream and they gave her a tiny lick and she wanted some more .", "words": ["tiny", "ice cream", "and", "they", "that", "an", "more", "some", "ice", "lick", "her", "she", "but", "a"]}, {"sent": "it have all the tiny hundreds and thousands on top all different colors .", "words": ["tiny", "all", "the", "and", "have", "it", "on"]}, {"sent": "the little tiny mitten .", "words": ["the", "tiny", "little"]}, {"sent": "if you would like to bee the little tiny tick .", "words": ["the", "tiny", "if", "to", "bee", "like", "you", "little", "would"]}, {"sent": "its got a tiny tiny bit of snow on it .", "words": ["tiny", "of", "snow", "it", "on", "a"]}, {"sent": "theres theres a tiny light .", "words": ["tiny", "light", "a"]}, {"sent": "tiny tiny one at the end .", "words": ["tiny", "the", "at"]}, {"sent": "they were very tiny seeds .", "words": ["tiny", "they", "were"]}, {"sent": "teeny tiny cheese ?", "words": ["tiny", "cheese"]}, {"sent": "the little tiny one with the hat on .", "words": ["the", "tiny", "with", "little", "on", "hat"]}, {"sent": "tiny tiger thinks the peacock is afraid of her .", "words": ["tiny", "the", "is", "of", "tiger", "her"]}, {"sent": "tiny tiger ?", "words": ["tiny", "tiger"]}, {"sent": "look at how tiny .", "words": ["tiny", "how", "at", "look"]}, {"sent": "one Sunday morning the warm sun came up and pop out of the egg came a tiny and very .", "words": ["the", "up", "tiny", "pop", "egg", "of", "sun", "and", "out", "a"]}, {"sent": "I wonder if you have a tiny uh a very small pebble .", "words": ["tiny", "if", "you", "have", "a"]}, {"sent": "theyre only tiny .", "words": ["tiny"]}, {"sent": "its like a video and on the thing he looks round was a tiny little camera .", "words": ["the", "tiny", "camera", "was", "like", "and", "little", "he", "on", "a"]}, {"sent": "these are little tiny Hammerhead babies see their little hammers little tiny hammers ?", "words": ["tiny", "these", "their", "little", "are", "see"]}, {"sent": "and that is a very tiny is that where we live ?", "words": ["tiny", "where", "is", "we", "and", "that", "a"]}, {"sent": "that little tiny speck you think youre gonna pick it up .", "words": ["tiny", "up", "think", "pick", "you", "little", "it", "that"]}, {"sent": "och theyre lovely when you get a wee nurse of them when theyre tiny .", "words": ["tiny", "of", "them", "you", "get", "when", "a"]}, {"sent": "you mean that tiny tiny little scratch ?", "words": ["tiny", "that", "little", "you"]}, {"sent": "look a tiny baby .", "words": ["tiny", "a", "look"]}, {"sent": "and so moisture can get inside the wood and change the shape of the piece a tiny tiny tiny bit .", "words": ["can", "the", "tiny", "of", "so", "inside", "and", "get", "a"]}, {"sent": "a snail is tiny tiny .", "words": ["tiny", "is", "a"]}, {"sent": "but it was tiny .", "words": ["tiny", "was", "it", "but"]}, {"sent": "Snow white found seven tiny bowls seven tiny chairs and seven tiny beds .", "words": ["tiny", "white", "and"]}, {"sent": "tiny tiger .", "words": ["tiny", "tiger"]}, {"sent": "but little tiny hands little tiny hands .", "words": ["tiny", "but", "little"]}, {"sent": "tiny are they just little babies ?", "words": ["tiny", "they", "are", "little"]}, {"sent": "tiny tiny tiny tiny poorly .", "words": ["tiny"]}, {"sent": "and we only useta give you a tiny bit then .", "words": ["tiny", "we", "then", "you", "and", "give", "a"]}, {"sent": "Ive just got a tiny scratch on my arm .", "words": ["tiny", "my", "arm", "on", "a"]}, {"sent": "where do you want the very tiny one ?", "words": ["where", "do", "the", "tiny", "you"]}, {"sent": "the grand duke went to every home in the kingdom looking for the maiden whose foot fit the tiny glass shoe .", "words": ["the", "tiny", "for", "to", "fit", "shoe", "in", "foot", "every", "glass", "home"]}, {"sent": "a real little tiny baby .", "words": ["tiny", "little", "a"]}, {"sent": "yeah and you were tiny tiny tiny and then you got bigger and bigger and bigger and come out .", "words": ["tiny", "you", "and", "out", "then", "were"]}, {"sent": "must be a very tiny dog Thomas .", "words": ["tiny", "be", "dog", "a"]}, {"sent": "and I put a tiny tiny splash of tea in and the rest is milk .", "words": ["tiny", "the", "put", "is", "of", "and", "in", "splash", "milk", "a"]}, {"sent": "theres a tiny bubble .", "words": ["tiny", "a"]}, {"sent": "tiny cars for a race .", "words": ["tiny", "for", "a"]}, {"sent": "tiny little monkey .", "words": ["tiny", "monkey", "little"]}, {"sent": "these little girls wear these tiny shoes .", "words": ["tiny", "these", "little"]}, {"sent": "you are youre tiny tiny tiny tiny teeny tiny baby .", "words": ["tiny", "are", "you"]}, {"sent": "one warm day from a tiny egg a little cricket was born .", "words": ["tiny", "egg", "was", "little", "a"]}, {"sent": "what about when you were tiny ?", "words": ["tiny", "what", "when", "you", "about", "were"]}, {"sent": "she was a teenie tiny baby .", "words": ["tiny", "was", "she", "a"]}, {"sent": "tiny isnt it ?", "words": ["tiny", "it"]}, {"sent": "little tiny baby .", "words": ["tiny", "little"]}, {"sent": "a very tiny one .", "words": ["tiny", "a"]}, {"sent": "and then we have those tiny packets that Grandma brings you .", "words": ["tiny", "those", "we", "and", "have", "you", "then", "that"]}, {"sent": "yes you you come outof a tiny hole .", "words": ["tiny", "a", "you"]}, {"sent": "a tiny tiny screw .", "words": ["tiny", "a"]}, {"sent": "you took the little tiny one ?", "words": ["the", "tiny", "little", "you"]}, {"sent": "oh thats a tiny one .", "words": ["tiny", "a"]}, {"sent": "Snow White found seven tiny bowls seven tiny chairs and seven tiny beds .", "words": ["tiny", "and"]}, {"sent": "oh wait theres a tiny bit of pink flower .", "words": ["tiny", "wait", "of", "flower", "a"]}, {"sent": "thats a little bit tiny .", "words": ["tiny", "little", "a"]}, {"sent": "will you just drink this tiny drop of milk ?", "words": ["tiny", "milk", "drink", "of", "you", "this", "drop", "will"]}, {"sent": "all there is is one tiny bite .", "words": ["tiny", "all", "is", "there", "bite"]}, {"sent": "right theres a tiny xxx on this piece of paper .", "words": ["tiny", "of", "paper", "this", "on", "a"]}, {"sent": "tiny pig is counting .", "words": ["tiny", "is", "pig"]}, {"sent": "just tiny seedlings look .", "words": ["tiny", "look"]}, {"sent": "thats a tiny mousie .", "words": ["tiny", "a"]}, {"sent": "I think some of the tiny houses only allowed us to put two in .", "words": ["tiny", "the", "think", "put", "of", "to", "us", "in", "some"]}, {"sent": "another spider a tiny tiny spider .", "words": ["another", "tiny", "a"]}, {"sent": "very tiny pocket is it ?", "words": ["tiny", "it", "is"]}, {"sent": "oh it does need a tiny thing uhn doesnt it ?", "words": ["tiny", "does", "need", "it", "a"]}, {"sent": "just think it we had a tiny candle inside there that would light up our way at night wouldnt it ?", "words": ["tiny", "up", "think", "we", "light", "inside", "our", "there", "it", "at", "would", "that", "a"]}, {"sent": "and this lil tiny purple one ?", "words": ["tiny", "this", "and"]}, {"sent": "its hard to pull that little tiny handle .", "words": ["tiny", "to", "little", "pull", "hard", "that"]}, {"sent": "awfully tiny arms .", "words": ["tiny"]}, {"sent": "little tiny tie .", "words": ["tiny", "little"]}, {"sent": "hes awful tiny isnt he ?", "words": ["tiny", "he"]}, {"sent": "it is very tiny isnt it ?", "words": ["tiny", "it", "is"]}, {"sent": "oh are you a little tiny baby ?", "words": ["tiny", "little", "you", "are", "a"]}, {"sent": "they are very tiny .", "words": ["tiny", "they", "are"]}, {"sent": "just a tiny little bit .", "words": ["tiny", "little", "a"]}, {"sent": "aw look a little teeny tiny .", "words": ["tiny", "little", "a", "look"]}, {"sent": "theres no name and isnt it a shame its so tiny .", "words": ["tiny", "so", "and", "it", "a"]}, {"sent": "theyre tiny finger nails arent they John ?", "words": ["tiny", "finger", "they"]}, {"sent": "its difficult to get back that little tiny one screw .", "words": ["tiny", "back", "to", "little", "get", "that"]}, {"sent": "tiny little feet .", "words": ["tiny", "little"]}, {"sent": "beautiful candles and journals and um all these little tiny ceramic things and .", "words": ["tiny", "all", "these", "little", "and"]}, {"sent": "and how about right in the middle put a little nose a little tiny circle nose .", "words": ["the", "tiny", "how", "put", "nose", "and", "little", "in", "about", "a"]}, {"sent": "I know a little tiny mouse carrying a banana .", "words": ["tiny", "little", "banana", "mouse", "a"]}, {"sent": "we went on a big coach to Gibralter but to go up to the top of the rock we went on a tiny mini bus .", "words": ["up", "the", "tiny", "we", "go", "to", "of", "on", "rock", "big", "bus", "but", "a"]}, {"sent": "uhhuh you have a tiny one and a big one .", "words": ["tiny", "and", "have", "you", "big", "a"]}, {"sent": "its only a tiny xxx .", "words": ["tiny", "a"]}, {"sent": "or youre my baby ever since you were teenie and tiny .", "words": ["tiny", "my", "and", "you", "were"]}, {"sent": "theres all ti sort of a tiny hippopotamus and a tiny rhinoceros too but I give buns to the elephant when I went down to the zoo .", "words": ["tiny", "all", "the", "zoo", "of", "to", "and", "elephant", "too", "give", "down", "when", "but", "a"]}, {"sent": "um I would say maybe toys a tiny bit more .", "words": ["tiny", "say", "more", "would", "a"]}, {"sent": "little tiny crisp .", "words": ["tiny", "little"]}, {"sent": "it actually is raining a little tiny bit .", "words": ["tiny", "is", "little", "it", "a"]}, {"sent": "when you useta be really tiny at night we useta sing .", "words": ["tiny", "we", "be", "you", "sing", "at", "when"]}, {"sent": "a tiny bug shes a little ant and and she is a girl .", "words": ["tiny", "bug", "is", "little", "and", "ant", "she", "a"]}, {"sent": "its a tiny crumb Thomas .", "words": ["tiny", "a"]}, {"sent": "are you a big girl or a tiny baby ?", "words": ["tiny", "you", "are", "big", "a"]}, {"sent": "its that tiny .", "words": ["tiny", "that"]}, {"sent": "are you done with your juice ?", "words": ["your", "with", "you", "are", "juice"]}, {"sent": "you drank some juice at playgroup yesterday didnt you ?", "words": ["some", "juice", "at", "you"]}, {"sent": "more juice .", "words": ["more", "juice"]}, {"sent": "and youre having some juice .", "words": ["some", "juice", "and"]}, {"sent": "more juice .", "words": ["more", "juice"]}, {"sent": "you can shake grapefruits sometimes and you can hear the juice .", "words": ["can", "the", "shake", "hear", "and", "you", "juice"]}, {"sent": "it looks like um a bottle of juice .", "words": ["of", "like", "it", "bottle", "juice", "a"]}, {"sent": "youve dropped your juice .", "words": ["juice", "your"]}, {"sent": "nice juice ?", "words": ["nice", "juice"]}, {"sent": "thats it you dont bash it because if you do all the juice will flavor .", "words": ["the", "do", "all", "if", "you", "dont", "it", "will", "juice", "because"]}, {"sent": "that good juice ?", "words": ["juice", "that", "good"]}, {"sent": "please juice .", "words": ["juice"]}, {"sent": "juice a straw .", "words": ["juice", "a"]}, {"sent": "now your orange juice is ready .", "words": ["juice", "orange", "your", "is"]}, {"sent": "go get you some juice .", "words": ["go", "you", "get", "some", "juice"]}, {"sent": "apple juice ?", "words": ["apple", "juice"]}, {"sent": "babys gonna drink the juice .", "words": ["the", "juice", "drink"]}, {"sent": "juice .", "words": ["juice"]}, {"sent": "he has spilt his juice hasnt he ?", "words": ["his", "juice", "he"]}, {"sent": "see theyre pouring the juice and see how they made a mess .", "words": ["the", "how", "and", "they", "see", "juice", "a"]}, {"sent": "drink some juice ?", "words": ["some", "juice", "drink"]}, {"sent": "the price for certain things such as juice and milk .", "words": ["the", "for", "and", "milk", "juice"]}, {"sent": "want some prune juice ?", "words": ["some", "juice"]}, {"sent": "looks like theres orange juice in that pot .", "words": ["like", "in", "orange", "juice", "that"]}, {"sent": "you think its apple juice ?", "words": ["apple", "think", "juice", "you"]}, {"sent": "ow roa rou roa rou roa rou ra juice .", "words": ["juice"]}, {"sent": "would you like a drink of juice ?", "words": ["drink", "of", "like", "you", "would", "juice", "a"]}, {"sent": "I either have fruit juice yoghurt or I dont bother .", "words": ["have", "dont", "juice"]}, {"sent": "would you like a drink of juice ?", "words": ["drink", "of", "like", "you", "would", "juice", "a"]}, {"sent": "whats she eating with her juice in the bowl ?", "words": ["the", "bowl", "with", "in", "juice", "her", "she"]}, {"sent": "oh theres your juice cup .", "words": ["cup", "juice", "your"]}, {"sent": "theres some apple juice .", "words": ["apple", "some", "juice"]}, {"sent": "juice .", "words": ["juice"]}, {"sent": "thats apple juice .", "words": ["apple", "juice"]}, {"sent": "juice ?", "words": ["juice"]}, {"sent": "magic juice .", "words": ["juice"]}, {"sent": "lets put the juice in there too .", "words": ["the", "put", "there", "in", "too", "juice"]}, {"sent": "youd like apple juice .", "words": ["apple", "juice", "like"]}, {"sent": "its juice .", "words": ["juice"]}, {"sent": "heres your juice .", "words": ["juice", "your"]}, {"sent": "and you had some orange juice .", "words": ["you", "and", "orange", "some", "juice"]}, {"sent": "where has juice gone ?", "words": ["where", "juice"]}, {"sent": "is that juice cold ?", "words": ["juice", "that", "cold", "is"]}, {"sent": "you dont want any juice ?", "words": ["dont", "juice", "any", "you"]}, {"sent": "you want me to drink the juice ?", "words": ["the", "drink", "to", "you", "me", "juice"]}, {"sent": "here is the juice .", "words": ["here", "the", "juice", "is"]}, {"sent": "juice .", "words": ["juice"]}, {"sent": "Im drinking my orange juice .", "words": ["my", "orange", "juice"]}, {"sent": "watch your juice .", "words": ["juice", "watch", "your"]}, {"sent": "apple juice .", "words": ["apple", "juice"]}, {"sent": "yes juice .", "words": ["juice"]}, {"sent": "Mia you get juice ?", "words": ["get", "juice", "you"]}, {"sent": "xxx get the apple juice yeah heres the apple juice .", "words": ["get", "the", "apple", "juice"]}, {"sent": "want some juice ?", "words": ["some", "juice"]}, {"sent": "you want juice juice ?", "words": ["juice", "you"]}, {"sent": "no thats apple juice .", "words": ["apple", "juice"]}, {"sent": "what you doing with your juice ?", "words": ["what", "your", "with", "you", "juice"]}, {"sent": "good juice ?", "words": ["juice", "good"]}, {"sent": "juice .", "words": ["juice"]}, {"sent": "here why dont I pour you some juice ?", "words": ["pour", "you", "here", "dont", "some", "juice", "why"]}, {"sent": "more juice .", "words": ["more", "juice"]}, {"sent": "Dipsy has still got water running down his head xxx too much juice .", "words": ["water", "juice", "head", "too", "his", "down", "much"]}, {"sent": "thats our juice .", "words": ["juice", "our"]}, {"sent": "juice .", "words": ["juice"]}, {"sent": "got some juice hasnt she ?", "words": ["some", "juice", "she"]}, {"sent": "you not want juice you want a botbot ?", "words": ["juice", "not", "a", "you"]}, {"sent": "what are you hiding in your juice Thomas ?", "words": ["what", "your", "you", "in", "are", "juice"]}, {"sent": "Ill bring you the apple juice .", "words": ["the", "bring", "you", "apple", "juice"]}, {"sent": "sweety were gonna eat in a minute you eat drink juice now quick youll get filled up very quickly can you wait a few minutes ?", "words": ["up", "can", "wait", "drink", "you", "eat", "get", "in", "juice", "were", "a"]}, {"sent": "are you gonna drink your juice ?", "words": ["your", "drink", "you", "are", "juice"]}, {"sent": "vegetable juice .", "words": ["juice"]}, {"sent": "thats blackcurrant juice .", "words": ["juice"]}, {"sent": "whats whats hiding behind your juice Tommy Thomas ?", "words": ["your", "juice", "behind"]}, {"sent": "you want some juice ?", "words": ["some", "juice", "you"]}, {"sent": "Peter you want milk or juice ?", "words": ["milk", "juice", "you"]}, {"sent": "is it good orange juice ?", "words": ["is", "it", "orange", "good", "juice"]}, {"sent": "no thats not juice darling .", "words": ["juice", "not"]}, {"sent": "maybe some orange juice ?", "words": ["some", "orange", "juice"]}, {"sent": "now drink your juice .", "words": ["juice", "your", "drink"]}, {"sent": "an orange squeezed out orange juice .", "words": ["an", "orange", "juice", "out"]}, {"sent": "oh youre not gonna get any juice out that buddy Im sorry .", "words": ["any", "get", "out", "juice", "that", "not"]}, {"sent": "want apple juice ?", "words": ["apple", "juice"]}, {"sent": "and I guess those little um apple juice things in the little sippycup .", "words": ["the", "those", "little", "and", "in", "apple", "juice"]}, {"sent": "opening a carton of orange juice .", "words": ["of", "orange", "juice", "a"]}, {"sent": "well thats peach you can make it into juice .", "words": ["can", "into", "you", "it", "juice", "make"]}, {"sent": "drink your drink your juice and eat your cookies .", "words": ["your", "drink", "and", "eat", "juice"]}, {"sent": "juice ?", "words": ["juice"]}, {"sent": "dont try and give Purdie and juice though will you ?", "words": ["and", "you", "dont", "will", "give", "juice", "try"]}, {"sent": "arent you gonna make me some juice ?", "words": ["you", "me", "juice", "make", "some"]}, {"sent": "is your grape juice allgone ?", "words": ["juice", "your", "is"]}, {"sent": "we should go in and have a tortilla and some juice .", "words": ["go", "we", "and", "have", "in", "some", "juice", "a"]}, {"sent": "you just want the apple juice and peanutbutter huh .", "words": ["the", "and", "you", "apple", "juice"]}, {"sent": "water in your juice ?", "words": ["in", "juice", "your", "water"]}, {"sent": "you have grape juice .", "words": ["have", "juice", "you"]}, {"sent": "are you going to bring me some juice ?", "words": ["to", "bring", "you", "are", "me", "juice", "some"]}, {"sent": "want some orange juice ?", "words": ["some", "orange", "juice"]}, {"sent": "juice .", "words": ["juice"]}, {"sent": "would you like some juice ?", "words": ["like", "you", "would", "some", "juice"]}, {"sent": "you still have a lot of juice .", "words": ["of", "a lot", "you", "have", "juice", "a"]}, {"sent": "apple juice wasnt it ?", "words": ["apple", "juice", "it"]}, {"sent": "you have your hair cut at the barbers dont you ?", "words": ["the", "your", "cut", "you", "have", "dont", "at", "hair"]}, {"sent": "I was going to use the hair dryer but then I thought it would wake you up so I didnt bother .", "words": ["the", "up", "to", "was", "so", "you", "it", "would", "wake", "then", "hair", "but", "dryer"]}, {"sent": "well would you like to try and put your hair that way tomorrow ?", "words": ["put", "your", "to", "like", "you", "and", "that", "would", "hair", "try"]}, {"sent": "who helped Momma fix curl ya hair ?", "words": ["fix", "hair", "who"]}, {"sent": "in Mummys hair and your hair ?", "words": ["in", "hair", "your", "and"]}, {"sent": "and Mommy grab your hair .", "words": ["hair", "your", "and"]}, {"sent": "Abe I hafta brush your hair and brush your teeth .", "words": ["brush", "hair", "your", "and"]}, {"sent": "you didnt want a hair on you ?", "words": ["on", "hair", "a", "you"]}, {"sent": "xxx blonde hair and blue eyes .", "words": ["blue", "hair", "and"]}, {"sent": "oh yeah that piece of hair .", "words": ["of", "hair", "that"]}, {"sent": "you going to pull Markys hair ?", "words": ["to", "hair", "pull", "you"]}, {"sent": "look at his hair .", "words": ["his", "hair", "at", "look"]}, {"sent": "yeah youve got some hair too .", "words": ["some", "hair", "too"]}, {"sent": "no its pretend hair .", "words": ["pretend", "hair"]}, {"sent": "you cant set her hair in rollers .", "words": ["in", "hair", "her", "you"]}, {"sent": "are you brushing your hair ?", "words": ["hair", "are", "your", "you"]}, {"sent": "he doesnt have much hair does he ?", "words": ["does", "have", "he", "hair", "much"]}, {"sent": "some red hair .", "words": ["red", "some", "hair"]}, {"sent": "and a bit of hair on the head .", "words": ["the", "of", "head", "and", "on", "hair", "a"]}, {"sent": "this is the way we dry our hair off dry our hair off dry our hair off .", "words": ["the", "off", "we", "is", "dry", "our", "this", "hair"]}, {"sent": "and do you put her hair into a you pull her hair back ?", "words": ["do", "put", "into", "back", "you", "and", "pull", "hair", "her", "a"]}, {"sent": "you like the way your hair is combed ?", "words": ["the", "your", "is", "like", "you", "hair"]}, {"sent": "dont pull her hair .", "words": ["dont", "hair", "her", "pull"]}, {"sent": "yeah you have shmutz in your hair .", "words": ["your", "you", "have", "in", "hair"]}, {"sent": "is that Daddys hair dryer ?", "words": ["dryer", "hair", "that", "is"]}, {"sent": "I know this bumble bee has such interesting hair .", "words": ["this", "hair", "bee"]}, {"sent": "a little hair .", "words": ["hair", "little", "a"]}, {"sent": "youre gonna play with my hair are you ?", "words": ["my", "play", "with", "you", "are", "hair"]}, {"sent": "well because shes got long hair .", "words": ["long", "hair", "because"]}, {"sent": "dont you take care of you dollys hair ?", "words": ["of", "you", "dont", "hair", "take"]}, {"sent": "lemme comb your hair .", "words": ["your", "hair", "comb"]}, {"sent": "dont mess her hair up now .", "words": ["up", "dont", "hair", "her"]}, {"sent": "thats the hair dryer right ?", "words": ["the", "hair", "dryer"]}, {"sent": "I think the fly went into your hair .", "words": ["the", "think", "your", "into", "hair"]}, {"sent": "theres a hair on your hand ?", "words": ["your", "hand", "on", "hair", "a"]}, {"sent": "didnt come into your bedroom did I and cut your hair last night ?", "words": ["your", "into", "cut", "last", "and", "did", "hair", "bedroom"]}, {"sent": "Mummy has got to go and have her hair cut this weekend .", "words": ["cut", "go", "to", "and", "have", "this", "hair", "her"]}, {"sent": "is it Purdies hair ?", "words": ["hair", "it", "is"]}, {"sent": "thats his hair .", "words": ["his", "hair"]}, {"sent": "have you even see anybody with orchard hair ?", "words": ["with", "you", "have", "see", "hair"]}, {"sent": "what colors Carolines hair ?", "words": ["what", "hair"]}, {"sent": "come here and let me comb your hair and Ill let you play and play and play .", "words": ["your", "play", "comb", "and", "here", "you", "me", "hair"]}, {"sent": "hair .", "words": ["hair"]}, {"sent": "how are we gonna help our friend get her hair washed ?", "words": ["how", "help", "we", "our", "get", "are", "hair", "her"]}, {"sent": "that is Daddys hair .", "words": ["hair", "that", "is"]}, {"sent": "ach isnt her hair lovely ?", "words": ["hair", "her"]}, {"sent": "thats hair gel .", "words": ["hair"]}, {"sent": "ofcourse she picked it up by the hair .", "words": ["up", "the", "by", "it", "hair", "she"]}, {"sent": "wait what we dont put food in her hair .", "words": ["what", "wait", "put", "we", "dont", "food", "in", "hair", "her"]}, {"sent": "so the hair wont make them itch .", "words": ["the", "so", "them", "hair", "make"]}, {"sent": "I just thought they would you know especially with the hair .", "words": ["the", "with", "you", "they", "would", "hair"]}, {"sent": "hey can I fix your hair while you are fixing the horses ?", "words": ["can", "the", "your", "fix", "you", "are", "hair"]}, {"sent": "I wanna comb comb my dolls hair .", "words": ["my", "hair", "comb", "wanna"]}, {"sent": "his crazy hair .", "words": ["his", "hair"]}, {"sent": "she had more hair and was darker .", "words": ["was", "and", "more", "hair", "she"]}, {"sent": "more hair okay .", "words": ["more", "hair"]}, {"sent": "that one says the wooly hair has the woolly mammoth has long shaggy hair and is just thirteen feet tall .", "words": ["the", "is", "long", "and", "hair", "that"]}, {"sent": "how do you get a hair cut ?", "words": ["do", "how", "cut", "you", "get", "hair", "a"]}, {"sent": "look you got butter in your hair or something today .", "words": ["your", "you", "in", "butter", "look", "hair"]}, {"sent": "oh dont put your fork in my hair .", "words": ["put", "my", "your", "dont", "in", "fork", "hair"]}, {"sent": "she always had the hair .", "words": ["the", "hair", "she"]}, {"sent": "what spiky hair ?", "words": ["what", "hair"]}, {"sent": "who gave you the hair band ?", "words": ["the", "hair", "you", "who"]}, {"sent": "do you wanna comb the dolls hair ?", "words": ["the", "do", "comb", "you", "wanna", "hair"]}, {"sent": "hes got some cool hair huh ?", "words": ["some", "hair"]}, {"sent": "because my mother even even has red hair .", "words": ["red", "my", "hair", "because"]}, {"sent": "look at Morags hair .", "words": ["hair", "at", "look"]}, {"sent": "because I remember that was the day that Mummy put nail varnish remover on her hair instead of conditioner .", "words": ["the", "put", "of", "was", "nail", "her", "on", "hair", "that", "because"]}, {"sent": "yes you pulled a girls hair .", "words": ["hair", "a", "you"]}, {"sent": "his hair .", "words": ["his", "hair"]}, {"sent": "oh I think its probably a piece of mommys hair .", "words": ["of", "think", "hair", "a"]}, {"sent": "youll pull his hair out .", "words": ["his", "hair", "out", "pull"]}, {"sent": "wheres your hair grip gone ?", "words": ["hair", "your"]}, {"sent": "for your hair .", "words": ["hair", "for", "your"]}, {"sent": "wanna give her some hair maybe ?", "words": ["wanna", "some", "give", "hair", "her"]}, {"sent": "thats her hair .", "words": ["hair", "her"]}, {"sent": "you were sitting on there and you were holding onto that bit of hair there werent you ?", "words": ["of", "and", "you", "there", "that", "on", "hair", "were"]}, {"sent": "I cant do your hair like that .", "words": ["do", "your", "like", "hair", "that"]}, {"sent": "is she gonna have orange hair ?", "words": ["is", "have", "orange", "hair", "she"]}, {"sent": "and what color hair has GranddadBeably got ?", "words": ["what", "hair", "and"]}, {"sent": "no we mustnt pull dollys hair must we ?", "words": ["hair", "pull", "we"]}, {"sent": "we could use a hair .", "words": ["could", "hair", "a", "we"]}, {"sent": "theres his hair .", "words": ["his", "hair"]}, {"sent": "I dont think it would go on your hair very well .", "words": ["think", "your", "go", "dont", "it", "would", "on", "hair"]}, {"sent": "you carry her by the hair .", "words": ["the", "by", "carry", "you", "hair", "her"]}, {"sent": "it couldnt hurt your hair .", "words": ["hurt", "hair", "it", "your"]}, {"sent": "do you think she wants her hair in a tail ?", "words": ["do", "think", "you", "in", "hair", "her", "she", "a"]}, {"sent": "someday Ill have brown hair again .", "words": ["have", "hair", "brown"]}, {"sent": "thats your hair .", "words": ["hair", "your"]}, {"sent": "on her hair ?", "words": ["on", "hair", "her"]}, {"sent": "oh gonna comb your hair ?", "words": ["your", "hair", "comb"]}, {"sent": "Tom has blue hair .", "words": ["hair", "blue"]}, {"sent": "look how dirty her hair is .", "words": ["how", "is", "dirty", "look", "hair", "her"]}, {"sent": "and they have some little bit of hair on their body the way bats do .", "words": ["the", "do", "their", "of", "little", "have", "they", "and", "on", "some", "hair"]}, {"sent": "youre getting bread in your hair sweetie .", "words": ["in", "bread", "hair", "your"]}, {"sent": "so I can see your what your hair looks like .", "words": ["can", "what", "your", "so", "like", "see", "hair"]}, {"sent": "this is fun I like this fun little hair .", "words": ["is", "like", "little", "this", "hair"]}, {"sent": "can you say Daddys hair ?", "words": ["can", "hair", "say", "you"]}, {"sent": "there oh Nathaniel is such a good boy got his hair all washed .", "words": ["all", "is", "there", "good", "his", "hair", "a"]}, {"sent": "I gonna wash hair now .", "words": ["hair", "wash"]}, {"sent": "thats right were not were not ready yet .", "words": ["were", "not"]}, {"sent": "were not doing this then ?", "words": ["this", "then", "were", "not"]}, {"sent": "what are you saying to whoever you were talking to ?", "words": ["what", "to", "you", "are", "were"]}, {"sent": "now they were both very good girls .", "words": ["they", "were", "good"]}, {"sent": "I will take you when were all done okay ?", "words": ["all", "when", "you", "will", "take", "were"]}, {"sent": "no were not playing dominos were playing I forget what the name of this is called but we kind of .", "words": ["the", "what", "is", "of", "we", "this", "were", "not", "but"]}, {"sent": "and were going to mix these tayberries .", "words": ["to", "were", "these", "and"]}, {"sent": "she keeps asking me what they were what the monsters were like .", "words": ["the", "what", "like", "they", "me", "were", "she"]}, {"sent": "were not into fruit .", "words": ["were", "not", "into"]}, {"sent": "there were lotsof dinosaur children in Bolands neighborhood .", "words": ["there", "in", "were"]}, {"sent": "do you remember that book that we were looking at yesterday ?", "words": ["do", "we", "you", "at", "book", "that", "were"]}, {"sent": "I always thought when people were like yeah well well hafta knock the tile out and redo it Im like yeah but why is it cracking because youre going to put new tile down its gonna happen again .", "words": ["the", "why", "put", "is", "to", "down", "like", "because", "and", "knock", "it", "out", "but", "new", "when", "were"]}, {"sent": "because I was afraid you were hurt .", "words": ["hurt", "was", "you", "were", "because"]}, {"sent": "the jackolantern had many mouths and there were some smiling and some frowning .", "words": ["the", "and", "there", "some", "were"]}, {"sent": "were gonna need some protective eye covering so you dont throw things right at me .", "words": ["need", "throw", "so", "me", "eye", "you", "dont", "at", "some", "were"]}, {"sent": "you you were pretending to be Grandma .", "words": ["to", "be", "were", "you"]}, {"sent": "and you were on the slide .", "words": ["the", "slide", "you", "and", "on", "were"]}, {"sent": "so that when you were cycling along .", "words": ["when", "so", "you", "that", "were"]}, {"sent": "instead were having crate up here .", "words": ["up", "here", "were"]}, {"sent": "were being filmed right now .", "words": ["were"]}, {"sent": "you were being a right sillybilly with that .", "words": ["with", "you", "that", "were", "a"]}, {"sent": "no were not .", "words": ["were", "not"]}, {"sent": "no were not gonna watch tv right now .", "words": ["tv", "watch", "were", "not"]}, {"sent": "we were over we were over last when was it just when Adam was about two .", "words": ["when", "we", "was", "last", "it", "about", "over", "were"]}, {"sent": "thought you were helping me ?", "words": ["me", "were", "you"]}, {"sent": "what were you doing over at Nanas last night ?", "words": ["what", "last", "you", "at", "over", "were"]}, {"sent": "hes saying if we feed you were all going to starve because you need so much more food than us .", "words": ["all", "need", "if", "we", "to", "so", "much", "you", "us", "more", "food", "feed", "were", "because"]}, {"sent": "xxx like you were doing .", "words": ["were", "like", "you"]}, {"sent": "no were not gonna glue .", "words": ["glue", "were", "not"]}, {"sent": "but you would share it with her if she were here wouldnt you ?", "words": ["were", "share", "if", "with", "you", "here", "it", "would", "her", "she", "but"]}, {"sent": "you were fighting ?", "words": ["were", "you"]}, {"sent": "were still playing for a little while .", "words": ["for", "were", "little", "a"]}, {"sent": "were gonna cook it .", "words": ["cook", "it", "were"]}, {"sent": "you know where were going today ?", "words": ["where", "were", "you"]}, {"sent": "and those hot dogs were great .", "words": ["hot", "those", "were", "and"]}, {"sent": "do you know what were gonna hafta do ?", "words": ["do", "what", "were", "you"]}, {"sent": "you were werent you ?", "words": ["were", "you"]}, {"sent": "you were asleep for one .", "words": ["asleep", "for", "were", "you"]}, {"sent": "my eyelids were heavy when there were four .", "words": ["my", "when", "there", "heavy", "were"]}, {"sent": "oh were not quite all done are we ?", "words": ["all", "we", "are", "were", "not"]}, {"sent": "they were having a fallout werent they ?", "words": ["they", "were", "a"]}, {"sent": "there were some today .", "words": ["there", "were", "some"]}, {"sent": "were going to see these animals when we go to the circus .", "words": ["the", "these", "when", "we", "go", "to", "see", "were"]}, {"sent": "you werent listening were you probably ?", "words": ["were", "you"]}, {"sent": "Tisha you were here yesterday right ?", "words": ["here", "were", "you"]}, {"sent": "or were you pretending ?", "words": ["were", "you"]}, {"sent": "were not teaching her that yet but were just letting her sort of play with it .", "words": ["of", "play", "her", "with", "it", "but", "that", "not", "were"]}, {"sent": "were drawing Sheba .", "words": ["were"]}, {"sent": "okay yeah because I noticed you were intro you were showing him a lot of different foods in the pile .", "words": ["the", "of", "a lot", "you", "him", "in", "were", "because", "a"]}, {"sent": "xxx who were you pointing to when you said that ?", "words": ["when", "to", "you", "that", "who", "were"]}, {"sent": "you were hiding .", "words": ["were", "you"]}, {"sent": "were making you some friends .", "words": ["some", "were", "you"]}, {"sent": "Arlene was ironing like I told you they were in her room she was in her room ironing and Kerry and Derwood were in there talking .", "words": ["was", "her", "like", "you", "and", "they", "in", "room", "there", "were", "she"]}, {"sent": "this is what you really were doing .", "words": ["what", "is", "you", "this", "were"]}, {"sent": "were not going anywhere yet but later on were going to go to the shops .", "words": ["the", "go", "to", "on", "were", "not", "but"]}, {"sent": "no were gonna eat dinner with it on .", "words": ["with", "eat", "it", "on", "were"]}, {"sent": "were poor .", "words": ["were", "poor"]}, {"sent": "were you ?", "words": ["were", "you"]}, {"sent": "theres a big nest that we noticed yesterday and there were two birds flying towards it .", "words": ["we", "and", "there", "big", "it", "that", "were", "a"]}, {"sent": "Lily were not having another treat now okay ?", "words": ["another", "were", "not"]}, {"sent": "well no we cant play it back now were in the middle of doing it .", "words": ["the", "back", "we", "of", "play", "in", "it", "were"]}, {"sent": "so it might be one of the new students that you didnt know when you were there .", "words": ["the", "when", "of", "so", "be", "you", "there", "it", "new", "that", "were"]}, {"sent": "because you were only three at the beginning of April .", "words": ["the", "of", "you", "at", "were", "because"]}, {"sent": "it says its brake blocks were on fire .", "words": ["on", "it", "were"]}, {"sent": "because they were already on the table .", "words": ["the", "table", "they", "on", "were", "because"]}, {"sent": "yeah were gonna turn the sprinkler on .", "words": ["the", "on", "were", "sprinkler"]}, {"sent": "Travis and Spud were there .", "words": ["there", "were", "and"]}, {"sent": "yep because were going to go to a new gym class .", "words": ["go", "to", "new", "were", "because", "a"]}, {"sent": "were not going to have a bath now darling .", "words": ["to", "have", "were", "not", "a"]}, {"sent": "no you werent in the mood for a birthdayparty last night were you ?", "words": ["the", "for", "last", "you", "in", "were", "a"]}, {"sent": "were you not playing with a jigsawpuzzle ?", "words": ["with", "you", "were", "not", "a"]}, {"sent": "what were we reading ?", "words": ["what", "were", "we"]}, {"sent": "right were gonna keep a wee eye on him .", "words": ["eye", "him", "on", "were", "a"]}, {"sent": "no because were gonna have tea soon .", "words": ["have", "were", "because"]}, {"sent": "yes you were .", "words": ["were", "you"]}, {"sent": "and after breakfast weve got to decide what were both wearing .", "words": ["to", "what", "were", "and"]}, {"sent": "Imean is he familiar with the names you were using to refer to them ?", "words": ["the", "is", "to", "with", "them", "you", "he", "were"]}, {"sent": "hi Jen you know what were being taped right now .", "words": ["what", "were", "you"]}, {"sent": "you mean when you were just out ?", "words": ["out", "when", "were", "you"]}, {"sent": "you were fighting over Betty ?", "words": ["over", "were", "you"]}, {"sent": "were nice happy and full now .", "words": ["full", "happy", "nice", "and", "were"]}, {"sent": "because were were working .", "words": ["were", "because"]}, {"sent": "you you were going to get the policeman .", "words": ["the", "to", "you", "get", "were"]}, {"sent": "thats why you were in his way .", "words": ["you", "in", "his", "were", "why"]}, {"sent": "were the flowers that Daddy gave Mummy .", "words": ["the", "that", "were"]}, {"sent": "the only thing is when you were in the shop you said it and it sounded like you said the other thing and I think we got a funny look from one of the people in the shop .", "words": ["the", "think", "when", "is", "we", "of", "like", "you", "and", "other", "in", "it", "look", "were", "a"]}, {"sent": "ew no were just pretending okay ?", "words": ["were"]}, {"sent": "but were getting down there on a few things arent we ?", "words": ["we", "there", "on", "down", "were", "but", "a"]}, {"sent": "were planting .", "words": ["were"]}, {"sent": "well were finished Nan has something new .", "words": ["new", "were"]}, {"sent": "now were gonna go for a ride .", "words": ["for", "go", "ride", "were", "a"]}, {"sent": "were there some gaps still ?", "words": ["there", "were", "some"]}, {"sent": "I thought you were saving it .", "words": ["it", "were", "you"]}, {"sent": "gonna go in the xxx were gonna go in the car were gonna go in the truck actually the van .", "words": ["the", "go", "car", "truck", "in", "were"]}, {"sent": "what kind of animals were at the zoo ?", "words": ["the", "what", "zoo", "of", "at", "were"]}, {"sent": "no one knows how many of them were killed .", "words": ["of", "how", "them", "were"]}, {"sent": "you were on the mountain ?", "words": ["on", "the", "were", "you"]}, {"sent": "you wanted some lunch while we were here you wanted some bologna .", "words": ["we", "you", "here", "some", "were"]}, {"sent": "were little compared to you .", "words": ["to", "were", "you", "little"]}, {"sent": "thats a car that can fly .", "words": ["can", "car", "that", "a"]}, {"sent": "okay I can use green .", "words": ["can", "green"]}, {"sent": "can you make a brown line ?", "words": ["can", "brown", "you", "make", "a"]}, {"sent": "yes you can .", "words": ["can", "you"]}, {"sent": "can I have that pen ?", "words": ["can", "have", "that", "pen"]}, {"sent": "can you remind me I would like to stop at TraderJoes and I need to look and see if there is one between New York and here in Connecticut .", "words": ["can", "look", "need", "if", "is", "see", "to", "need to", "stop", "like", "you", "and", "there", "here", "at", "in", "would", "me"]}, {"sent": "later you can see it .", "words": ["can", "see", "it", "you"]}, {"sent": "oh nono you can have this one .", "words": ["can", "this", "have", "you"]}, {"sent": "can you catch my hair ?", "words": ["can", "my", "catch", "you", "hair"]}, {"sent": "alright can .", "words": ["can"]}, {"sent": "ask him if be can come tomorrow ?", "words": ["can", "be", "him", "if"]}, {"sent": "look through and see if you can find anymore cars .", "words": ["can", "if", "find", "you", "and", "look", "see"]}, {"sent": "we just hafta do the best we can .", "words": ["the", "do", "can", "we"]}, {"sent": "can I have a drink ?", "words": ["can", "have", "drink", "a"]}, {"sent": "can go there .", "words": ["can", "there", "go"]}, {"sent": "can Ive a bite ?", "words": ["can", "bite", "a"]}, {"sent": "can you sing ring around the rosie ?", "words": ["can", "the", "you", "sing", "around"]}, {"sent": "can you stir it up ?", "words": ["can", "it", "up", "you"]}, {"sent": "Im just throwing the ball I can sit down .", "words": ["the", "can", "ball", "sit", "down"]}, {"sent": "I can see him uhn can you ?", "words": ["can", "see", "him", "you"]}, {"sent": "you cant start saying youre hungry can you ?", "words": ["can", "hungry", "you"]}, {"sent": "can you do ?", "words": ["can", "do", "you"]}, {"sent": "can you read a book ?", "words": ["can", "you", "read", "book", "a"]}, {"sent": "and then you can pull this .", "words": ["can", "pull", "you", "and", "this", "then"]}, {"sent": "no we can put the birthday card on the wall or we can put it on your bookshelf .", "words": ["can", "the", "put", "your", "we", "it", "on"]}, {"sent": "look how many cars can fit on the car transporter .", "words": ["can", "the", "how", "car", "fit", "look", "on"]}, {"sent": "well hafta have another trip next week somewhere to buy some see if we can find anything else .", "words": ["can", "if", "we", "see", "to", "have", "another", "find", "some", "buy"]}, {"sent": "can you sit up and bounce it back ?", "words": ["can", "up", "back", "and", "you", "it", "sit"]}, {"sent": "of course you can .", "words": ["of", "can", "you"]}, {"sent": "are they talking about all the animals they can see ?", "words": ["the", "all", "can", "they", "are", "about", "see"]}, {"sent": "can you remember ?", "words": ["can", "you"]}, {"sent": "can you can you tell a funny story too ?", "words": ["can", "you", "too", "story", "a"]}, {"sent": "we can play with any of these thing cant we or we can play with your other toys .", "words": ["can", "these", "your", "we", "of", "play", "any", "with", "other"]}, {"sent": "maybe you can fly that one to um .", "words": ["can", "to", "that", "you"]}, {"sent": "they can ride the tractor .", "words": ["can", "the", "they", "tractor", "ride"]}, {"sent": "cops can certainly do it xxx .", "words": ["can", "do", "it"]}, {"sent": "you can stay at Jackies house .", "words": ["can", "you", "house", "at", "stay"]}, {"sent": "can I have some milk and sugar in my tea ?", "words": ["can", "my", "and", "have", "in", "milk", "some"]}, {"sent": "you can have some .", "words": ["can", "some", "have", "you"]}, {"sent": "now look they can walk upstairs too .", "words": ["can", "look", "they", "too", "walk"]}, {"sent": "you know I wanna take you home and you can be my Little computer friend okay ?", "words": ["can", "my", "be", "and", "you", "wanna", "take", "home"]}, {"sent": "you can watch a little bit later on .", "words": ["can", "watch", "little", "you", "on", "a"]}, {"sent": "can you go like that ?", "words": ["can", "go", "like", "you", "that"]}, {"sent": "can you count them ?", "words": ["can", "them", "you"]}, {"sent": "can you be the storyteller now .", "words": ["can", "be", "the", "you"]}, {"sent": "waiting waiting for me so you can nick more of my food .", "words": ["can", "my", "for", "of", "so", "you", "more", "food", "me"]}, {"sent": "can you say mast ?", "words": ["can", "say", "you"]}, {"sent": "just because I say youre not bouncing on the settee doesnt mean you can bounce on me .", "words": ["the", "can", "say", "you", "on", "me", "not", "because"]}, {"sent": "I can jump high .", "words": ["can", "jump", "high"]}, {"sent": "ask horsie if you can one .", "words": ["can", "if", "you"]}, {"sent": "you can recognize the sign for Tesco cant you ?", "words": ["can", "the", "for", "you"]}, {"sent": "can you see it .", "words": ["can", "see", "it", "you"]}, {"sent": "they got beds they can lay on .", "words": ["can", "they", "on"]}, {"sent": "said we can make them .", "words": ["can", "them", "make", "we"]}, {"sent": "May the eighth and Laura wants to talk so she can hear her voice .", "words": ["the", "can", "to", "so", "and", "hear", "talk", "her", "she"]}, {"sent": "can you do that ?", "words": ["can", "do", "that", "you"]}, {"sent": "and then you can close the box okay ?", "words": ["can", "the", "box", "then", "you", "and", "close"]}, {"sent": "so that the trains traveling on this track can go through because the bridge is too short so the drawbridge has to lift up .", "words": ["the", "can", "up", "go", "is", "so", "to", "this", "too", "on", "that", "because"]}, {"sent": "I can understand why you said it .", "words": ["can", "it", "why", "you"]}, {"sent": "you can go to the store in a little while if youre a good girl .", "words": ["can", "the", "if", "go", "to", "store", "you", "little", "in", "good", "a"]}, {"sent": "you can see Sues door quite clearly .", "words": ["can", "see", "door", "you"]}, {"sent": "can you see ?", "words": ["can", "see", "you"]}, {"sent": "if youre over there you can get it .", "words": ["can", "if", "you", "get", "there", "it", "over"]}, {"sent": "it can be washed .", "words": ["can", "be", "it"]}, {"sent": "okay you can have it .", "words": ["can", "have", "it", "you"]}, {"sent": "can he ride on that Duplo horse ?", "words": ["can", "ride", "horse", "he", "on", "that"]}, {"sent": "can I buy food at the store ?", "words": ["can", "the", "store", "food", "at", "buy"]}, {"sent": "how how can you tell Im not on the phone ?", "words": ["can", "the", "how", "you", "on", "not"]}, {"sent": "what can you see now ?", "words": ["can", "what", "see", "you"]}, {"sent": "you can get any size you want .", "words": ["can", "get", "any", "you"]}, {"sent": "can I use this ?", "words": ["can", "this"]}, {"sent": "yes you can .", "words": ["can", "you"]}, {"sent": "can you say hi honey ?", "words": ["can", "say", "you"]}, {"sent": "nobody else can see it so no one else knows whats happening .", "words": ["can", "see", "it", "so"]}, {"sent": "oh its the shaker song it can make a funny sound .", "words": ["the", "can", "it", "make", "a"]}, {"sent": "lets see if we can get a man .", "words": ["can", "if", "we", "get", "see", "a"]}, {"sent": "can I move this out ?", "words": ["can", "this", "out"]}, {"sent": "well you can .", "words": ["can", "you"]}, {"sent": "they can stand up there cant they ?", "words": ["can", "up", "stand", "they", "there"]}, {"sent": "can you see ?", "words": ["can", "see", "you"]}, {"sent": "these ones you can just put wherever you want .", "words": ["can", "these", "put", "you"]}, {"sent": "can mommy draw too ?", "words": ["can", "draw", "too"]}, {"sent": "can you get the baby for mommy ?", "words": ["can", "the", "for", "you", "get"]}, {"sent": "can you sit and eat breakfast and then Ill change you .", "words": ["can", "then", "and", "eat", "you", "sit"]}, {"sent": "along came a crocodile as cheeky as can be .", "words": ["can", "be", "a"]}, {"sent": "do ya think you can put it back together again ?", "words": ["can", "do", "think", "put", "back", "you", "it"]}, {"sent": "okay let me look and see if I can find if he has a name .", "words": ["can", "if", "me", "find", "and", "he", "look", "see", "a"]}, {"sent": "yes perhaps he can if he can run that fast .", "words": ["can", "fast", "if", "run", "he", "that"]}, {"sent": "can you stretch right up ?", "words": ["can", "up", "you"]}, {"sent": "and then we can put the other buttons back on .", "words": ["can", "the", "put", "back", "we", "and", "other", "on", "then"]}, {"sent": "xxx so we can .", "words": ["can", "so", "we"]}, {"sent": "Ill get you some paper towels and you can wipe up the floor .", "words": ["can", "up", "the", "paper", "and", "get", "you", "some", "wipe"]}, {"sent": "can you pour me some juice ?", "words": ["can", "pour", "you", "me", "juice", "some"]}, {"sent": "well you can play with that one .", "words": ["can", "play", "with", "you", "that"]}, {"sent": "can we have the radio on ?", "words": ["can", "the", "radio", "we", "have", "on"]}, {"sent": "can you see it ?", "words": ["can", "see", "it", "you"]}, {"sent": "hes a special dog he can change the color of his spots .", "words": ["can", "the", "of", "he", "his", "dog", "a"]}, {"sent": "can you say a word thats really hard ?", "words": ["can", "say", "you", "hard", "a"]}, {"sent": "soon you can go to swimming lessons .", "words": ["can", "to", "go", "you"]}, {"sent": "if you wanna give it to her then give it to her on the roof .", "words": ["the", "if", "to", "then", "you", "it", "roof", "wanna", "on", "give", "her"]}, {"sent": "shall we put them on the roof here ?", "words": ["the", "put", "we", "them", "here", "roof", "on"]}, {"sent": "look look on the roof .", "words": ["on", "the", "roof", "look"]}, {"sent": "can you put the roof on .", "words": ["can", "the", "put", "you", "roof", "on"]}, {"sent": "dont take the roof off the red car now .", "words": ["the", "off", "red", "car", "dont", "roof", "take"]}, {"sent": "yyy roof ?", "words": ["roof"]}, {"sent": "got to put a roof on have we ?", "words": ["put", "we", "to", "have", "roof", "on", "a"]}, {"sent": "the lid the roof doesnt open on it though because roofs dont generally open on cars .", "words": ["the", "open", "dont", "it", "roof", "on", "because"]}, {"sent": "the fish was stuck in some chocolate buttons on the roof ?", "words": ["the", "was", "chocolate", "in", "stuck", "roof", "fish", "on", "some"]}, {"sent": "now when youve put the roof back on the blue bus .", "words": ["the", "put", "back", "blue", "roof", "on", "when", "bus"]}, {"sent": "I think those are sposta be the tiles on the roof .", "words": ["the", "think", "those", "be", "are", "roof", "on"]}, {"sent": "and the other day when I was in the a magpie knocked something out of the gutter and it landed with a big thud and on the roof and thats what it was .", "words": ["the", "what", "of", "was", "with", "other", "and", "in", "it", "big", "roof", "out", "on", "when", "a"]}, {"sent": "has the roof come off the blue bus ?", "words": ["the", "off", "blue", "roof", "bus"]}, {"sent": "theres Tobys roof there .", "words": ["there", "roof"]}, {"sent": "shall we do draw some tiles on the roof ?", "words": ["the", "do", "we", "draw", "roof", "on", "some"]}, {"sent": "right at the very top of the roof .", "words": ["the", "of", "roof", "at"]}, {"sent": "on the roof maybe oh he steals .", "words": ["on", "the", "he", "roof"]}, {"sent": "pretend roof .", "words": ["pretend", "roof"]}, {"sent": "well just put the roof on then .", "words": ["the", "put", "roof", "on", "then"]}, {"sent": "dad can you get the kite off the roof for us please ?", "words": ["can", "the", "off", "for", "you", "get", "us", "roof"]}, {"sent": "and therere many things up on top of the roof here .", "words": ["up", "the", "of", "and", "here", "roof", "on"]}, {"sent": "right are you gonna put the roof on now ?", "words": ["the", "put", "you", "are", "roof", "on"]}, {"sent": "thats the roof .", "words": ["the", "roof"]}, {"sent": "on the roof ?", "words": ["on", "the", "roof"]}, {"sent": "put that roof back on .", "words": ["put", "back", "roof", "on", "that"]}, {"sent": "a toilet on a roof ?", "words": ["on", "roof", "a"]}, {"sent": "theyre probably outside on the roof .", "words": ["on", "outside", "roof", "the"]}, {"sent": "I dont think its quite as serious as his roof coming off but .", "words": ["off", "think", "dont", "roof", "his", "but"]}, {"sent": "whats the boy doing way up on the roof ?", "words": ["the", "on", "up", "roof"]}, {"sent": "called get out now the roof is coming down .", "words": ["the", "is", "get", "roof", "out", "down"]}, {"sent": "because hes made a roof look .", "words": ["look", "roof", "because", "a"]}, {"sent": "roof roof roof .", "words": ["roof"]}, {"sent": "you stop at the roof ?", "words": ["the", "stop", "you", "roof", "at"]}, {"sent": "has the roof fallen off the bus ?", "words": ["the", "bus", "roof", "off"]}, {"sent": "can you see on the roof ?", "words": ["can", "the", "you", "roof", "on", "see"]}, {"sent": "spin the rooster on the roof .", "words": ["the", "rooster", "on", "roof"]}, {"sent": "oh the roof of the cabs falling off .", "words": ["the", "of", "roof", "off"]}, {"sent": "well youve pulled half the roof off aswell .", "words": ["the", "roof", "off"]}, {"sent": "that looks a very big roof .", "words": ["big", "that", "roof", "a"]}, {"sent": "I think thats just the ce um not the ceiling the roof .", "words": ["the", "think", "not", "roof"]}, {"sent": "but if you take that tape off then the top will fall apart it wont be like a little roof anymore see what I mean ?", "words": ["off", "the", "what", "fall", "tape", "like", "be", "it", "roof", "then", "take", "that", "if", "you", "little", "will", "see", "a", "but"]}, {"sent": "its the roof of one of these .", "words": ["the", "of", "these", "roof"]}, {"sent": "on the roof ?", "words": ["on", "the", "roof"]}, {"sent": "roof roof .", "words": ["roof"]}, {"sent": "shall I put the roof on ?", "words": ["the", "put", "on", "roof"]}, {"sent": "is that the roof and theres a tree over the roof .", "words": ["the", "tree", "is", "and", "roof", "over", "that", "a"]}, {"sent": "xxx oh dear oh dear our chicken coop isnt doesnt have a roof .", "words": ["our", "have", "roof", "chicken", "a"]}, {"sent": "to the roof ?", "words": ["the", "to", "roof"]}, {"sent": "oh hes fell off the roof .", "words": ["off", "the", "roof"]}, {"sent": "not roof on the top of the house .", "words": ["the", "of", "house", "roof", "on", "not"]}, {"sent": "forgot the roof .", "words": ["the", "roof"]}, {"sent": "a purple roof ?", "words": ["roof", "a"]}, {"sent": "the roof thing ?", "words": ["the", "roof"]}, {"sent": "I dont think it must be a different toy where the the roof comes up .", "words": ["where", "the", "think", "up", "be", "dont", "toy", "it", "roof", "a"]}, {"sent": "just put the roof on .", "words": ["the", "put", "on", "roof"]}, {"sent": "its got a roof .", "words": ["roof", "a"]}, {"sent": "here the roof for that .", "words": ["the", "for", "here", "roof", "that"]}, {"sent": "well this is a roof .", "words": ["this", "roof", "a", "is"]}, {"sent": "mhm could go like that could be a roof .", "words": ["go", "be", "like", "roof", "could", "that", "a"]}, {"sent": "my doggies up on the roof and I cant get him down .", "words": ["up", "the", "my", "and", "get", "him", "roof", "on", "down"]}, {"sent": "ahhah so that looks like this is the roof right ?", "words": ["the", "is", "so", "like", "this", "roof", "that"]}, {"sent": "the washing machine on the roof ?", "words": ["the", "on", "roof", "washing machine"]}, {"sent": "on the roof .", "words": ["on", "the", "roof"]}, {"sent": "now you could fix up the house again and put the roof on .", "words": ["up", "the", "put", "fix", "you", "and", "house", "roof", "on", "could"]}, {"sent": "put the roof on the car .", "words": ["the", "put", "car", "roof", "on"]}, {"sent": "he says roof roof roof .", "words": ["he", "roof"]}, {"sent": "look that matches the roof .", "words": ["the", "that", "roof", "look"]}, {"sent": "then well make a roof in a minute for it .", "words": ["for", "in", "it", "roof", "then", "make", "a"]}, {"sent": "the chimney goes on the roof .", "words": ["the", "on", "roof"]}, {"sent": "oh you cant see the roof .", "words": ["the", "see", "roof", "you"]}, {"sent": "green roof ?", "words": ["roof", "green"]}, {"sent": "put a roof .", "words": ["put", "roof", "a"]}, {"sent": "whats happening to the church roof at the moment ?", "words": ["the", "to", "church", "roof", "at"]}, {"sent": "you dont have a roof do you ?", "words": ["do", "you", "have", "dont", "roof", "a"]}, {"sent": "roof a little roof .", "words": ["roof", "little", "a"]}, {"sent": "yeah on that big house there up on the roof .", "words": ["up", "the", "there", "that", "house", "roof", "on", "big"]}, {"sent": "the door fell off then the roof off .", "words": ["the", "off", "roof", "then", "door"]}, {"sent": "knocked the roof off .", "words": ["the", "roof", "off"]}, {"sent": "because if theres no roof on the hotel and it rains well get wet .", "words": ["the", "wet", "if", "and", "get", "it", "roof", "on", "because"]}, {"sent": "sat on the roof .", "words": ["on", "the", "roof"]}, {"sent": "Im riding the train roof roof roof roof roof .", "words": ["the", "roof", "train"]}, {"sent": "its probably something on the uh roof .", "words": ["on", "the", "roof"]}, {"sent": "its the roof .", "words": ["the", "roof"]}, {"sent": "you see I build those ones well because the roof has come off one and the other one has stayed intact .", "words": ["the", "off", "those", "you", "and", "other", "build", "roof", "see", "because"]}, {"sent": "lets just put it upside down again and put the roof on .", "words": ["the", "put", "and", "it", "roof", "on", "down"]}, {"sent": "perhaps nobodyll notice that the roof isnt isnt closed .", "words": ["the", "that", "roof"]}, {"sent": "you dont put cases on the roof of a house uhn do you ?", "words": ["the", "do", "put", "of", "you", "dont", "house", "roof", "on", "a"]}, {"sent": "whats that on the roof ?", "words": ["on", "the", "that", "roof"]}, {"sent": "there wont be any more space for the roof .", "words": ["the", "for", "be", "any", "there", "more", "roof"]}, {"sent": "what color is this honey what color is the roof ?", "words": ["the", "what", "is", "this", "roof"]}, {"sent": "that must be the roof .", "words": ["the", "be", "that", "roof"]}, {"sent": "roof .", "words": ["roof"]}, {"sent": "wheres the other bit of the roof ?", "words": ["the", "of", "roof", "other"]}, {"sent": "how could you sleep higher than the roof ?", "words": ["the", "how", "sleep", "you", "roof", "could"]}, {"sent": "yes here is the house and there is the roof and the chimney .", "words": ["the", "is", "and", "here", "there", "house", "roof"]}, {"sent": "where is the big black roof ?", "words": ["where", "the", "is", "black", "roof", "big"]}, {"sent": "is on the roof ?", "words": ["on", "the", "roof", "is"]}, {"sent": "thats the way the roof goes on .", "words": ["the", "on", "roof"]}, {"sent": "on the roof .", "words": ["on", "the", "roof"]}, {"sent": "there we go its a roof xxx .", "words": ["go", "we", "there", "roof", "a"]}, {"sent": "I dont think thats a very good way to build a house .", "words": ["think", "to", "dont", "build", "house", "good", "a"]}, {"sent": "are you going to build something else then with those ?", "words": ["those", "to", "with", "you", "are", "build", "then"]}, {"sent": "we could make little bricks as well and build a house couldnt we ?", "words": ["we", "and", "little", "build", "house", "could", "make", "a"]}, {"sent": "do you think we should build a different house ?", "words": ["do", "think", "we", "you", "build", "house", "a"]}, {"sent": "Kalie you can build something tall with that .", "words": ["can", "with", "you", "build", "that"]}, {"sent": "have you got some bricks to build it ?", "words": ["to", "you", "have", "it", "build", "some"]}, {"sent": "what are you going to build ?", "words": ["what", "to", "you", "are", "build"]}, {"sent": "well you wanna build a bicycle for the gorilla too ?", "words": ["the", "for", "bicycle", "you", "build", "wanna", "too", "a"]}, {"sent": "youre gonna build him standing up okay but it looks like he needs another leg .", "words": ["up", "like", "another", "him", "it", "build", "he", "leg", "but"]}, {"sent": "so you get the lego out and lets build a wall .", "words": ["the", "so", "you", "get", "and", "build", "out", "a"]}, {"sent": "you pick him up and build him up together again .", "words": ["up", "pick", "and", "you", "him", "build"]}, {"sent": "Ill build another one .", "words": ["another", "build"]}, {"sent": "shall we build some up ?", "words": ["up", "some", "build", "we"]}, {"sent": "what are you gonna build with the legos you gonna build you know what .", "words": ["the", "what", "with", "you", "are", "build"]}, {"sent": "lets build a tower .", "words": ["build", "a"]}, {"sent": "we can build a village on the back of this so things stay up better .", "words": ["can", "the", "up", "better", "back", "we", "of", "so", "this", "build", "on", "stay", "a"]}, {"sent": "oh well Im gonna build something with the little ones okay ?", "words": ["the", "build", "with", "little"]}, {"sent": "Mommy build for Henry .", "words": ["for", "build"]}, {"sent": "I bet you cant build .", "words": ["build", "you"]}, {"sent": "now now lets build it tall .", "words": ["it", "build"]}, {"sent": "well you do what you want and I think Ill build my own building .", "words": ["do", "what", "think", "my", "and", "you", "build"]}, {"sent": "shall we build it ?", "words": ["it", "build", "we"]}, {"sent": "on the way shell see a tree in your back yard shell want to build a tree house ah .", "words": ["the", "tree", "your", "back", "to", "in", "build", "house", "want to", "on", "see", "a"]}, {"sent": "lets build something .", "words": ["build"]}, {"sent": "you wanna build a sandwich ?", "words": ["you", "build", "wanna", "sandwich", "a"]}, {"sent": "xxx can build .", "words": ["can", "build"]}, {"sent": "mm try to build that ?", "words": ["try", "to", "build", "that", "try to"]}, {"sent": "well are you going to build anything else ?", "words": ["to", "are", "build", "you"]}, {"sent": "Mommy build for Henry .", "words": ["for", "build"]}, {"sent": "shall we build a tower ?", "words": ["build", "a", "we"]}, {"sent": "can I build you a house ?", "words": ["can", "you", "house", "build", "a"]}, {"sent": "well there were some steps build onto the lorry werent there ?", "words": ["the", "there", "build", "some", "were"]}, {"sent": "shall I build something ?", "words": ["build"]}, {"sent": "build another house .", "words": ["another", "house", "build"]}, {"sent": "lets build .", "words": ["build"]}, {"sent": "arent you gonna build me a bridge or something ?", "words": ["me", "build", "a", "you"]}, {"sent": "right well weve got to build a house up havent we first ?", "words": ["up", "we", "to", "first", "house", "build", "a"]}, {"sent": "why dont you build that big thing that you build ?", "words": ["you", "dont", "big", "build", "that", "why"]}, {"sent": "youre going to build a bridge with the book ?", "words": ["the", "to", "with", "build", "book", "a"]}, {"sent": "whatre you gonna build ?", "words": ["build", "you"]}, {"sent": "you can build stuff with them .", "words": ["can", "them", "with", "you", "build"]}, {"sent": "alright you needta build some more .", "words": ["some", "more", "build", "you"]}, {"sent": "dyou wanna move over that way so I can build this for you ?", "words": ["can", "for", "so", "you", "this", "build", "wanna", "over", "that"]}, {"sent": "I think we better build a wall between the tiger and the horsie .", "words": ["the", "better", "think", "we", "and", "build", "tiger", "a"]}, {"sent": "now what are you gonna build for her ?", "words": ["what", "for", "you", "are", "build", "her"]}, {"sent": "lets build these bricks up .", "words": ["up", "these", "build"]}, {"sent": "you build the house and mummyll just watch you .", "words": ["the", "watch", "and", "you", "house", "build"]}, {"sent": "did you hafta build it up ?", "words": ["up", "you", "it", "build", "did"]}, {"sent": "theyll build their hand ?", "words": ["build", "their", "hand"]}, {"sent": "shall we build a road for it ?", "words": ["for", "we", "it", "build", "a"]}, {"sent": "Daddy is gonna build a tower .", "words": ["build", "a", "is"]}, {"sent": "cant I build a lion house ?", "words": ["house", "lion", "build", "a"]}, {"sent": "mommys gonna build a .", "words": ["build", "a"]}, {"sent": "weve got to build some more bricks up oh .", "words": ["up", "to", "more", "build", "some"]}, {"sent": "you wanna build what ?", "words": ["what", "build", "wanna", "you"]}, {"sent": "lets build up the wall a bit more so they cant get out .", "words": ["up", "the", "so", "get", "they", "more", "build", "out", "a"]}, {"sent": "shall we build something else then ?", "words": ["then", "build", "we"]}, {"sent": "well build some blocks on here uhhuh .", "words": ["on", "some", "here", "build"]}, {"sent": "Im trying to build the house so you can put things down the chimney .", "words": ["the", "can", "put", "to", "so", "you", "house", "build", "down"]}, {"sent": "Im gonna build a tower .", "words": ["build", "a"]}, {"sent": "whatre you going to build ?", "words": ["to", "build", "you"]}, {"sent": "you build a really tall tower .", "words": ["build", "a", "you"]}, {"sent": "build a sandwich .", "words": ["build", "sandwich", "a"]}, {"sent": "you build it quite tall and tall .", "words": ["it", "build", "and", "you"]}, {"sent": "Im going to get all these bricks and build an aquarium .", "words": ["all", "these", "to", "and", "get", "an", "build"]}, {"sent": "you think you can build a tower with Mommy ?", "words": ["can", "think", "with", "you", "build", "a"]}, {"sent": "build houses ?", "words": ["build"]}, {"sent": "what would you like to build ?", "words": ["what", "to", "like", "you", "build", "would"]}, {"sent": "well will we build it up and put the trains on it ?", "words": ["up", "the", "put", "we", "and", "on", "it", "build", "will"]}, {"sent": "build it again .", "words": ["it", "build"]}, {"sent": "Nicole build a house ?", "words": ["house", "build", "a"]}, {"sent": "whatll we build ?", "words": ["build", "we"]}, {"sent": "in which case I dont needta build up Pingus igloo do I ?", "words": ["up", "do", "dont", "in", "build", "which"]}, {"sent": "alright you ready to build your train ?", "words": ["your", "train", "to", "you", "build"]}, {"sent": "what did Mommy build ?", "words": ["what", "build", "did"]}, {"sent": "Mommy build for Henry .", "words": ["for", "build"]}, {"sent": "xxx try to build a big one .", "words": ["try", "to", "build", "big", "try to", "a"]}, {"sent": "maybe we should try to build a tower with this kind of blocks these dont f oh youre making them all fall down can you make these stand up ?", "words": ["of", "fall", "can", "with", "down", "try to", "up", "these", "all", "to", "you", "dont", "make", "try", "a", "we", "stand", "them", "this", "build"]}, {"sent": "lets build them up in here .", "words": ["up", "them", "here", "in", "build"]}, {"sent": "did he build it like that ?", "words": ["like", "it", "build", "he", "did", "that"]}, {"sent": "are we going to build Pingus igloo ?", "words": ["to", "are", "build", "we"]}, {"sent": "are you gonna build me something ?", "words": ["me", "are", "build", "you"]}, {"sent": "well Ill build it higher .", "words": ["it", "build"]}, {"sent": "wanna try to build a patio ?", "words": ["to", "build", "wanna", "try", "try to", "a"]}, {"sent": "I cant build .", "words": ["build"]}, {"sent": "build them up a bit higher if you wanna make a bridge .", "words": ["up", "if", "them", "you", "build", "wanna", "make", "a"]}, {"sent": "she can build bridges .", "words": ["can", "build", "she"]}, {"sent": "I think Im going to build a mountain .", "words": ["to", "think", "build", "a"]}, {"sent": "you hafta build it .", "words": ["it", "build", "you"]}, {"sent": "thought you wanted mummie to build a bridge .", "words": ["to", "build", "a", "you"]}, {"sent": "what shall we build then ?", "words": ["what", "then", "build", "we"]}, {"sent": "I thought you were gonna build something for me .", "words": ["for", "you", "build", "me", "were"]}, {"sent": "you build up these bricks for me or shall we have some different bricks ?", "words": ["up", "these", "for", "we", "you", "have", "build", "me", "some"]}, {"sent": "Ive only got one day to build this up .", "words": ["up", "to", "build", "this"]}, {"sent": "will that keep all the horses and cows in there if we build this fence like that ?", "words": ["the", "all", "if", "we", "like", "and", "there", "in", "this", "build", "will", "that"]}, {"sent": "these ones build better these .", "words": ["better", "these", "build"]}, {"sent": "what are you going to build that ?", "words": ["what", "to", "you", "are", "build", "that"]}, {"sent": "shall we build it ?", "words": ["it", "build", "we"]}, {"sent": "Mummys trying to build a bridge .", "words": ["to", "build", "a"]}, {"sent": "it just goes and then the whole wall falls to bits and then they can build a new .", "words": ["the", "can", "to", "and", "they", "it", "build", "then", "new", "a"]}, {"sent": "theres a worm at the bottom of the garden .", "words": ["the", "bottom", "of", "garden", "at", "a"]}, {"sent": "xxx bottom piece .", "words": ["bottom"]}, {"sent": "are you doing the bottom bit now ?", "words": ["the", "are", "bottom", "you"]}, {"sent": "its on the bottom of your face ?", "words": ["the", "bottom", "your", "face", "of", "on"]}, {"sent": "oh yes you like to bite the bottom off the cone and then all the icecream melts and runs down your arm doesnt it ?", "words": ["the", "off", "all", "bottom", "your", "to", "like", "you", "and", "it", "arm", "bite", "then", "down"]}, {"sent": "here Ill throw in some berries and you gotta catch them from the bottom of the pool .", "words": ["the", "bottom", "throw", "of", "catch", "them", "pool", "and", "here", "you", "in", "some"]}, {"sent": "lower teeth extra teeth bottom teeth teeny teeth very clean teeth .", "words": ["clean", "bottom"]}, {"sent": "if I went on it my foot would touch the bottom of the water slide ?", "words": ["the", "water", "my", "bottom", "slide", "if", "of", "touch", "foot", "it", "would", "on"]}, {"sent": "bottom Thomas .", "words": ["bottom"]}, {"sent": "on the bottom of her feet .", "words": ["the", "bottom", "of", "on", "her"]}, {"sent": "that one has got a big lump of Blu tack on the bottom .", "words": ["the", "bottom", "of", "that", "on", "big", "a"]}, {"sent": "maybe if we put that one down on the bottom .", "words": ["the", "put", "bottom", "if", "we", "on", "down", "that"]}, {"sent": "I hemmed the bottom .", "words": ["the", "bottom"]}, {"sent": "xxx bottom .", "words": ["bottom"]}, {"sent": "pieces on the bottom .", "words": ["on", "the", "bottom"]}, {"sent": "steam or smoke coming out of the bottom of the space rocket doesnt it ?", "words": ["the", "bottom", "of", "it", "out"]}, {"sent": "well I think thats a little bit of a label on the bottom of the beaker .", "words": ["the", "think", "bottom", "of", "little", "on", "a"]}, {"sent": "hey let me see whats on your bottom look this says slugger where did you get this ?", "words": ["where", "bottom", "your", "me", "you", "get", "this", "did", "look", "on", "see"]}, {"sent": "you have food on the bottom of your shoes .", "words": ["the", "bottom", "your", "of", "you", "have", "food", "on"]}, {"sent": "xxx at the bottom .", "words": ["the", "bottom", "at"]}, {"sent": "the big one down the bottom ?", "words": ["the", "down", "big", "bottom"]}, {"sent": "bottom .", "words": ["bottom"]}, {"sent": "this is the skirting board at the bottom of the wall .", "words": ["the", "bottom", "is", "of", "this", "at"]}, {"sent": "youre not getting a wet bottom down there are you ?", "words": ["wet", "bottom", "you", "there", "are", "down", "not", "a"]}, {"sent": "on the bottom of the church ?", "words": ["the", "bottom", "of", "church", "on"]}, {"sent": "theres a worm at the bottom of the garden .", "words": ["the", "bottom", "of", "garden", "at", "a"]}, {"sent": "bottom .", "words": ["bottom"]}, {"sent": "put that at the bottom of the pile .", "words": ["the", "put", "bottom", "of", "at", "that"]}, {"sent": "babys sitting on Annes bottom .", "words": ["on", "bottom"]}, {"sent": "is there a top and a bottom to this Jane ?", "words": ["bottom", "is", "to", "and", "there", "this", "a"]}, {"sent": "and do you know what find at the bottom of the rainbow ?", "words": ["the", "do", "what", "bottom", "of", "you", "and", "at", "find"]}, {"sent": "seven goes at the bottom .", "words": ["the", "bottom", "at"]}, {"sent": "lets see now heres the bottom of the house .", "words": ["the", "bottom", "of", "house", "see"]}, {"sent": "you know we sign birthday cards we put kiss kiss kiss on the bottom ?", "words": ["the", "put", "bottom", "we", "you", "on", "kiss"]}, {"sent": "Im trying to scrape the bottom of the jar so that youve got enough on there that you can taste it .", "words": ["the", "can", "bottom", "of", "to", "so", "taste", "you", "there", "that", "it", "on", "jar"]}, {"sent": "wash your bottom .", "words": ["your", "bottom", "wash"]}, {"sent": "your bottom is a little bit stinky Nathaniel .", "words": ["bottom", "your", "is", "little", "a"]}, {"sent": "lets get the cheese out from under your bottom though and ice cream and hotdog .", "words": ["the", "ice cream", "bottom", "your", "and", "get", "under", "out", "ice", "cheese"]}, {"sent": "its the bottom part of a house .", "words": ["the", "bottom", "of", "house", "a"]}, {"sent": "just at the bottom of the page please .", "words": ["the", "of", "bottom", "at"]}, {"sent": "thats on the bottom too .", "words": ["on", "the", "bottom", "too"]}, {"sent": "how can my bottom be smelling ?", "words": ["can", "how", "bottom", "my", "be"]}, {"sent": "so books would probably be at the bottom .", "words": ["the", "bottom", "so", "be", "would", "at"]}, {"sent": "get your bottom so clean .", "words": ["clean", "bottom", "your", "so", "get"]}, {"sent": "uhhuh see all that poop on the bottom ?", "words": ["the", "all", "bottom", "on", "see", "that"]}, {"sent": "lets clear a space for your bottom to sit down and do it then .", "words": ["do", "for", "your", "bottom", "to", "then", "and", "it", "sit", "down", "a"]}, {"sent": "do you remember what goes here near the cats bottom ?", "words": ["the", "do", "what", "bottom", "you", "here"]}, {"sent": "it has fallen off at the bottom .", "words": ["off", "the", "bottom", "it", "at"]}, {"sent": "lets have a little look at your bottom .", "words": ["bottom", "your", "little", "have", "at", "look", "a"]}, {"sent": "right down the bottom .", "words": ["the", "down", "bottom"]}, {"sent": "its like the bottom of yours .", "words": ["the", "of", "bottom", "like"]}, {"sent": "you hafta hold onto the bottom Nathaniel .", "words": ["the", "bottom", "hold", "you"]}, {"sent": "your bottom would get a little bit hot .", "words": ["bottom", "your", "little", "get", "hot", "would", "a"]}, {"sent": "and you wrote a message for daddy on the bottom .", "words": ["the", "for", "bottom", "you", "and", "on", "a"]}, {"sent": "goes to the bottom .", "words": ["the", "to", "bottom"]}, {"sent": "did Mummy smack your bottom ?", "words": ["bottom", "your", "did"]}, {"sent": "he has got wheels on the bottom .", "words": ["on", "the", "bottom", "he"]}, {"sent": "thats their hive at the bottom and theyve made lots of honey so everybody wants to eat their honey .", "words": ["the", "bottom", "their", "of", "so", "to", "and", "eat", "at"]}, {"sent": "and you want a plaster on your bottom and you want the fire engine to do it ?", "words": ["the", "do", "bottom", "your", "to", "you", "and", "it", "on", "a"]}, {"sent": "oh theyve got all the guys in the bottom of the boat huh ?", "words": ["the", "all", "bottom", "of", "in", "boat"]}, {"sent": "shall I spank her bottom ?", "words": ["bottom", "her"]}, {"sent": "the the very bottom ?", "words": ["the", "bottom"]}, {"sent": "nice clean bottom .", "words": ["clean", "nice", "bottom"]}, {"sent": "and the t curls at the bottom .", "words": ["the", "at", "bottom", "and"]}, {"sent": "theyre gonna smack your bottom .", "words": ["bottom", "your"]}, {"sent": "thats on the bottom .", "words": ["on", "the", "bottom"]}, {"sent": "oh thats the bottom of your moneybox .", "words": ["the", "of", "bottom", "your"]}, {"sent": "the bottoms open aswell and theyll all fall out the bottom .", "words": ["the", "all", "bottom", "open", "fall", "and", "out"]}, {"sent": "Ill smack your bottom if you make a hole in there .", "words": ["bottom", "your", "if", "you", "there", "in", "make", "a"]}, {"sent": "it look almost like a jellyfish but its sitting at the bottom of the ground .", "words": ["the", "bottom", "of", "like", "it", "at", "look", "but", "a"]}, {"sent": "thats the arms on the bottom ?", "words": ["the", "on", "bottom"]}, {"sent": "and whats the bottom shape ?", "words": ["the", "bottom", "and"]}, {"sent": "youve got m for mousie on the bottom .", "words": ["on", "the", "for", "bottom"]}, {"sent": "almost except you hafta start at the bottom and then go up and down and up .", "words": ["the", "up", "bottom", "go", "and", "you", "at", "then", "down"]}, {"sent": "hes nibbled your bottom again .", "words": ["bottom", "your"]}, {"sent": "because of my fat bottom on them .", "words": ["my", "bottom", "of", "them", "on", "because"]}, {"sent": "well its the dogs bottom actually .", "words": ["the", "bottom"]}, {"sent": "Simba is at the bottom of the tree .", "words": ["the", "tree", "bottom", "is", "of", "at"]}, {"sent": "it says to Daddy with love from and at the bottom it says Aran .", "words": ["the", "bottom", "to", "with", "and", "it", "love", "at"]}, {"sent": "you mean the bottom ?", "words": ["the", "bottom", "you"]}, {"sent": "and then all the way down to the bottom .", "words": ["the", "all", "bottom", "to", "and", "then", "down"]}, {"sent": "well wheels normally go at the bottom dont they pet ?", "words": ["the", "bottom", "go", "dont", "they", "at"]}, {"sent": "thats the top half and thats the bottom half .", "words": ["the", "bottom", "and"]}, {"sent": "at the bottom ?", "words": ["the", "bottom", "at"]}, {"sent": "well put some holes in the bottom .", "words": ["the", "put", "bottom", "in", "some"]}, {"sent": "what is that down in the bottom of the glass ?", "words": ["the", "what", "bottom", "is", "of", "in", "glass", "down", "that"]}, {"sent": "your leg and bottom look a little bit sore .", "words": ["bottom", "your", "and", "little", "leg", "look", "a"]}, {"sent": "you might end up sinking to the bottom of the bath .", "words": ["up", "the", "bottom", "of", "to", "you"]}, {"sent": "I think the babies are all in the bottom .", "words": ["the", "all", "think", "bottom", "in", "are"]}, {"sent": "put the big one on the bottom .", "words": ["the", "put", "bottom", "on", "big"]}, {"sent": "on his bottom ?", "words": ["on", "his", "bottom"]}, {"sent": "down the bottom .", "words": ["the", "down", "bottom"]}, {"sent": "that one goes down at the bottom somewhere I think .", "words": ["the", "think", "bottom", "at", "down", "that"]}, {"sent": "yeah no that goes on the bottom .", "words": ["on", "the", "bottom", "that"]}, {"sent": "right on the bottom of your .", "words": ["the", "bottom", "your", "of", "on"]}, {"sent": "can you look down can you look at your bottom like he can ?", "words": ["can", "bottom", "your", "like", "you", "at", "he", "look", "down"]}, {"sent": "its stuck in the bottom .", "words": ["the", "in", "bottom", "stuck"]}, {"sent": "hes got his bottom sticking outof it .", "words": ["his", "it", "bottom"]}, {"sent": "it has fringes on the bottom huh ?", "words": ["on", "the", "it", "bottom"]}, {"sent": "youre gonna make ice cream on my bottom ?", "words": ["ice cream", "my", "bottom", "on", "ice", "make"]}, {"sent": "when did you bring him in the car ?", "words": ["the", "car", "bring", "you", "him", "in", "did", "when"]}, {"sent": "let us see if you can name what KennyBear uses when he goes to the bathroom .", "words": ["can", "what", "the", "if", "to", "you", "us", "he", "bathroom", "see", "when"]}, {"sent": "just when it looked as if it were all over for the young cubs Mufasa appeared and sent the hyenas flying with one swipe of his big paw .", "words": ["the", "all", "for", "if", "of", "with", "and", "his", "big", "it", "over", "when", "were"]}, {"sent": "when we were eating whatd we have on the beach ?", "words": ["the", "we", "have", "beach", "on", "when", "were"]}, {"sent": "think when Carolines gone you can have a sleep and Ill wake you up for Playdays .", "words": ["can", "up", "think", "for", "sleep", "you", "have", "and", "wake", "when", "a"]}, {"sent": "Im going to come back later on when you when youve gone to bed .", "words": ["back", "to", "bed", "you", "on", "when"]}, {"sent": "just like you did when you hurt it ?", "words": ["hurt", "like", "you", "it", "did", "when"]}, {"sent": "yesterday when William came home he saw that his brother .", "words": ["when", "he", "his", "that", "home"]}, {"sent": "remember what happens when you yell .", "words": ["what", "when", "you"]}, {"sent": "when is it Cwwws birthday ?", "words": ["it", "when", "is"]}, {"sent": "when theyre not busy right .", "words": ["when", "not"]}, {"sent": "because because because but because I cant be his friend when he cant play with me .", "words": ["be", "play", "with", "because", "he", "his", "me", "when", "but"]}, {"sent": "thats the stool I use when I wash dishes .", "words": ["the", "when", "wash"]}, {"sent": "what do you look like when you wear kayak clothes ?", "words": ["what", "do", "like", "you", "look", "when"]}, {"sent": "and yesterday when we went back to the shop they had a sale .", "words": ["the", "back", "we", "to", "and", "they", "when", "a"]}, {"sent": "and she has got to go to the vet for things like injections and when shes poorly she hasta go for tablets to make her better .", "words": ["the", "better", "for", "go", "to", "her", "like", "and", "make", "when", "she"]}, {"sent": "did we miss a spot when we were sweeping ?", "words": ["when", "we", "did", "were", "a"]}, {"sent": "did they teach you at nursery school to raise your hand when you want something .", "words": ["your", "hand", "school", "to", "you", "they", "did", "at", "when"]}, {"sent": "and when youre all done washing you can hang it up to dry .", "words": ["can", "all", "up", "to", "dry", "you", "and", "it", "when"]}, {"sent": "and when you got there you didnt have one you had two fish .", "words": ["you", "have", "there", "and", "fish", "when"]}, {"sent": "how am I sposta look for baby things when you keep sitting on the page ?", "words": ["the", "how", "for", "you", "am", "look", "on", "when"]}, {"sent": "youknow goofs off with her brothers when we tell her to stop she generally stops .", "words": ["off", "when", "we", "to", "with", "stop", "her", "she"]}, {"sent": "this is the bag we take with us when we go out .", "words": ["the", "when", "we", "is", "go", "with", "this", "us", "out", "take"]}, {"sent": "shell have her breakfast when she comes back .", "words": ["when", "back", "have", "her", "she"]}, {"sent": "when you go to the toilet .", "words": ["the", "go", "to", "you", "when"]}, {"sent": "hey Donnie what was it she said to me this morning when I put the slip on her ?", "words": ["the", "what", "put", "was", "to", "her", "this", "it", "on", "me", "when", "she"]}, {"sent": "this see this shirt Md means doctor so now hes a doctor and she has on scrubs that you wear at a hospital when youre a doctor .", "words": ["when", "so", "shirt", "and", "you", "this", "at", "on", "see", "that", "she", "a"]}, {"sent": "when it was chilly what did you do ?", "words": ["what", "do", "was", "you", "it", "did", "when"]}, {"sent": "you dont like it when I sing and dance do you ?", "words": ["do", "like", "dance", "you", "sing", "dont", "and", "it", "when"]}, {"sent": "because were halfway home when we get to Didsbury arent we ?", "words": ["when", "we", "to", "get", "were", "because", "home"]}, {"sent": "and thats so you can help Mummy in the garden during the Summer when Mummy has got to do some jobs outside .", "words": ["can", "the", "help", "do", "so", "to", "outside", "you", "and", "in", "garden", "some", "when"]}, {"sent": "and then well leave Dada here and then when Dada comes home you can show him and say look Dada .", "words": ["can", "show", "say", "and", "here", "you", "him", "look", "then", "when", "home"]}, {"sent": "and what happens when you do this ?", "words": ["what", "do", "you", "and", "this", "when"]}, {"sent": "no when youre writing you hafta follow them .", "words": ["them", "when", "you"]}, {"sent": "when did Mummy go to work ?", "words": ["work", "go", "to", "did", "when"]}, {"sent": "when I brought my boys home xxx .", "words": ["my", "when", "home"]}, {"sent": "when I tear them apart .", "words": ["tear", "when", "them"]}, {"sent": "when I let you down you can pick it up cant you ?", "words": ["can", "up", "pick", "you", "it", "down", "when"]}, {"sent": "but oh the time when youre just learning to walk .", "words": ["the", "when", "to", "walk", "but"]}, {"sent": "were your feet making a noise when you came up the stairs ?", "words": ["up", "the", "your", "when", "you", "stairs", "were", "a"]}, {"sent": "later when we come for checkout .", "words": ["for", "when", "we"]}, {"sent": "what did you do when it was snowing awhile ago ?", "words": ["what", "do", "was", "you", "it", "did", "when"]}, {"sent": "I hate it when you comb my hair when it has tangles in it .", "words": ["my", "comb", "hate", "you", "in", "it", "hair", "when"]}, {"sent": "what noise does a car make when it bashes into a pompom ?", "words": ["what", "does", "when", "into", "car", "it", "make", "a"]}, {"sent": "you cant run away when your trousers are round your ankles .", "words": ["your", "you", "run", "away", "are", "when"]}, {"sent": "or he can have a drink when hes tired .", "words": ["can", "drink", "tired", "have", "he", "when", "a"]}, {"sent": "you usually walk backwards when hes coming towards you .", "words": ["walk", "when", "you"]}, {"sent": "well ask Gwww when you can start .", "words": ["can", "when", "you"]}, {"sent": "kept them for you when you were a bit older .", "words": ["for", "them", "you", "when", "were", "a"]}, {"sent": "when you were .", "words": ["when", "were", "you"]}, {"sent": "when shes finished her biscuit .", "words": ["her", "when"]}, {"sent": "when we went in yesterday the man was serving at the counter and you said where has the lady gone didnt you ?", "words": ["the", "where", "we", "was", "and", "you", "in", "at", "when"]}, {"sent": "the boa followed his mistress when she went shopping everyone was astonished .", "words": ["the", "was", "his", "when", "she"]}, {"sent": "you had what at MrsPlattfords when your four ?", "words": ["what", "your", "you", "at", "when"]}, {"sent": "but when youre naughty .", "words": ["when", "naughty", "but"]}, {"sent": "whatre you gonna do when that turtle gets that big ?", "words": ["do", "when", "you", "big", "turtle", "that"]}, {"sent": "and then when we get off were gonna walk .", "words": ["off", "we", "and", "get", "walk", "then", "when", "were"]}, {"sent": "when I went on the soil .", "words": ["on", "the", "when"]}, {"sent": "are any of these foods that she likes when theyre real ?", "words": ["these", "when", "of", "any", "are", "that", "she"]}, {"sent": "hate it when xxx .", "words": ["it", "hate", "when"]}, {"sent": "you hafta look at Manuela when you show her your surprise face .", "words": ["your", "face", "show", "her", "you", "at", "look", "when"]}, {"sent": "when Mummy wants you to be quiet and whisper you never do .", "words": ["do", "to", "be", "you", "and", "quiet", "when"]}, {"sent": "you were in your pram when we went to the shops werent you ?", "words": ["the", "your", "when", "we", "to", "you", "in", "were"]}, {"sent": "when hes in the shed is he asleep ?", "words": ["the", "is", "asleep", "in", "he", "when"]}, {"sent": "when ?", "words": ["when"]}, {"sent": "and when you when you went swimming ?", "words": ["when", "you", "and"]}, {"sent": "when we go swimming ?", "words": ["go", "when", "we"]}, {"sent": "but sometimes when youre different you just need a different song .", "words": ["need", "you", "when", "but", "a"]}, {"sent": "remember when we forgot the brown sugar .", "words": ["the", "brown", "when", "we"]}, {"sent": "what does he like to do wh when its hot ?", "words": ["what", "does", "do", "to", "like", "hot", "he", "when"]}, {"sent": "when is your birthday ?", "words": ["your", "when", "is"]}, {"sent": "and thats where you were this morning when you woke up .", "words": ["where", "up", "when", "you", "and", "this", "were"]}, {"sent": "when you eat it ?", "words": ["eat", "it", "when", "you"]}, {"sent": "does it hurt when people hit you ?", "words": ["does", "hurt", "you", "it", "hit", "when"]}, {"sent": "it will be a biscuit when its cooked yes .", "words": ["be", "it", "will", "when", "a"]}, {"sent": "what are we gonna get when we go shopping ?", "words": ["what", "we", "go", "get", "are", "when"]}, {"sent": "Christmas tree when are we going to put ours up ?", "words": ["up", "tree", "put", "we", "to", "are", "when"]}, {"sent": "so so I wonder if they turned it off though when they did all that .", "words": ["off", "all", "if", "so", "they", "that", "it", "did", "when"]}, {"sent": "and then when you got your hair wet you didnt like it .", "words": ["wet", "your", "like", "you", "and", "it", "then", "hair", "when"]}, {"sent": "when you grow up what would you like to be when you grow up ?", "words": ["up", "what", "to", "be", "like", "you", "would", "when"]}, {"sent": "just now I just did when I grabbed when you gave it to me .", "words": ["to", "you", "it", "did", "me", "when"]}, {"sent": "Jennifer when is your birthday ?", "words": ["your", "when", "is"]}, {"sent": "and so when I first got up my back felt stiff xxx .", "words": ["up", "my", "back", "so", "first", "and", "when"]}, {"sent": "xxx when you get to school you can take it off .", "words": ["can", "off", "to", "school", "you", "get", "it", "take", "when"]}, {"sent": "and I think when Spot was trying to catch the leaves he caught that hat .", "words": ["the", "think", "to", "was", "catch", "and", "that", "he", "hat", "when"]}, {"sent": "when you were in bed you were shouting me ?", "words": ["bed", "you", "in", "me", "when", "were"]}, {"sent": "when they go what ?", "words": ["they", "what", "when", "go"]}, {"sent": "and then in December when we wanna open these little boxes .", "words": ["these", "we", "open", "and", "little", "in", "wanna", "then", "when"]}, {"sent": "when you managed to get one through .", "words": ["get", "to", "when", "you"]}, {"sent": "who were we talking to when it started to rain ?", "words": ["rain", "we", "to", "it", "who", "when", "were"]}, {"sent": "thats funny she didnt mention it when I went to get you .", "words": ["to", "you", "get", "it", "when", "she"]}, {"sent": "howve you got mud on your fingers when youve been inside ?", "words": ["your", "inside", "you", "on", "when"]}, {"sent": "and when he came to the place where the wild things are they roared their terrible roars .", "words": ["the", "where", "their", "to", "and", "they", "are", "he", "when"]}, {"sent": "you hafta ask me when you want it off .", "words": ["off", "you", "it", "me", "when"]}, {"sent": "that makes noise when you shake it .", "words": ["when", "shake", "you", "it", "that"]}, {"sent": "they turn brown when theyre dried .", "words": ["they", "when", "brown"]}, {"sent": "Ill hafta see when they arrive what their plans are wont we ?", "words": ["what", "their", "we", "they", "are", "see", "when"]}, {"sent": "its no good liking a plastic one when you willnt eat the real thing .", "words": ["the", "you", "eat", "good", "when", "a"]}, {"sent": "sorta like when you eat too much .", "words": ["like", "much", "you", "eat", "too", "when"]}, {"sent": "let Daddy have this powder please .", "words": ["have", "this"]}, {"sent": "what does this top do ?", "words": ["what", "do", "does", "this"]}, {"sent": "whats this whats this whats this ?", "words": ["this"]}, {"sent": "like this ?", "words": ["this", "like"]}, {"sent": "and this man is below the street .", "words": ["the", "is", "and", "this", "street"]}, {"sent": "now this ones definitely gonna be for me .", "words": ["this", "me", "be", "for"]}, {"sent": "this one looks more like a zebra doesnt it ?", "words": ["zebra", "like", "this", "more", "it", "a"]}, {"sent": "and you know what this is ?", "words": ["what", "is", "you", "and", "this"]}, {"sent": "come out of this box because theres lotsof .", "words": ["box", "of", "this", "out", "because"]}, {"sent": "Eleanor can read this book cant you ?", "words": ["can", "you", "this", "read", "book"]}, {"sent": "shes had like a bit of like I break off crackers and give it to her but she hasnt had anything like this .", "words": ["off", "of", "to", "her", "like", "and", "this", "it", "give", "break", "she", "but", "a"]}, {"sent": "okay you wanna put this one on ?", "words": ["put", "you", "this", "wanna", "on"]}, {"sent": "let me just stop this now .", "words": ["me", "stop", "this"]}, {"sent": "this is the states .", "words": ["the", "this", "is"]}, {"sent": "what is this ?", "words": ["what", "this", "is"]}, {"sent": "this is a picture of your car in the future .", "words": ["the", "your", "is", "of", "car", "picture", "this", "in", "a"]}, {"sent": "xxx xxx Amelias thirsty too whats this Amelia ?", "words": ["this", "too", "thirsty"]}, {"sent": "this is blue .", "words": ["this", "blue", "is"]}, {"sent": "and this ?", "words": ["this", "and"]}, {"sent": "whats this ?", "words": ["this"]}, {"sent": "ah this one looks easy .", "words": ["this"]}, {"sent": "I think this hat will stay on the side .", "words": ["the", "think", "will", "this", "on", "hat", "stay"]}, {"sent": "whats this ?", "words": ["this"]}, {"sent": "this is the way it is .", "words": ["the", "this", "it", "is"]}, {"sent": "uh this bumblebee for sure .", "words": ["this", "for"]}, {"sent": "are you going to be a better farmer than this one ?", "words": ["better", "to", "be", "you", "this", "are", "a"]}, {"sent": "I think this morning we will drive to the bank .", "words": ["the", "think", "we", "to", "drive", "this", "will"]}, {"sent": "whats this here ?", "words": ["here", "this"]}, {"sent": "I mean this didnt interest him because he doesnt really eat big chunks of meat so he probably just doesnt even recognize that .", "words": ["of", "so", "meat", "eat", "this", "him", "that", "he", "big", "because"]}, {"sent": "okay let us put this on the floor .", "words": ["the", "put", "us", "this", "on"]}, {"sent": "and this is the baby .", "words": ["the", "this", "is", "and"]}, {"sent": "did you tell Dad you went swimming this afternoon ?", "words": ["this", "did", "you"]}, {"sent": "he has got to go uh to a meeting I think this afternoon at two oclock .", "words": ["think", "go", "to", "this", "he", "at", "a"]}, {"sent": "well like this because this is meat .", "words": ["is", "like", "meat", "this", "because"]}, {"sent": "and what about this one ?", "words": ["what", "this", "about", "and"]}, {"sent": "this is a piggy .", "words": ["this", "a", "is"]}, {"sent": "this is hard .", "words": ["this", "hard", "is"]}, {"sent": "this morning .", "words": ["this"]}, {"sent": "this is longest ?", "words": ["this", "is"]}, {"sent": "this is not for listening .", "words": ["this", "for", "not", "is"]}, {"sent": "this just fell down ?", "words": ["this", "down"]}, {"sent": "no let me show you this is like Nanas fishing hat .", "words": ["is", "show", "like", "you", "this", "hat", "me"]}, {"sent": "oh this is very nice .", "words": ["this", "nice", "is"]}, {"sent": "is this cute .", "words": ["cute", "this", "is"]}, {"sent": "this one .", "words": ["this"]}, {"sent": "and this is on ?", "words": ["on", "this", "is", "and"]}, {"sent": "put this down .", "words": ["this", "put", "down"]}, {"sent": "hey Jilly look at this man .", "words": ["this", "at", "look"]}, {"sent": "does this hook under .", "words": ["does", "under", "this"]}, {"sent": "you made me this .", "words": ["me", "this", "you"]}, {"sent": "an H H is for whats this thing ?", "words": ["this", "an", "for", "is"]}, {"sent": "Im gonna shoot it this way .", "words": ["this", "it"]}, {"sent": "Cathys not very good at doing this .", "words": ["this", "not", "good", "at"]}, {"sent": "does it look like this bike on your teeshirt ?", "words": ["does", "your", "like", "this", "it", "look", "on"]}, {"sent": "this could be your desk .", "words": ["this", "could", "be", "your"]}, {"sent": "because in this magic garden .", "words": ["garden", "this", "in", "because"]}, {"sent": "I was gonna give you this .", "words": ["was", "give", "this", "you"]}, {"sent": "now this side .", "words": ["this"]}, {"sent": "what is this guy ?", "words": ["what", "this", "is"]}, {"sent": "this one .", "words": ["this"]}, {"sent": "yeah I know but who is this guy ?", "words": ["this", "is", "but", "who"]}, {"sent": "this harmless shark eats only plankton .", "words": ["this"]}, {"sent": "wheres this guy going ?", "words": ["this"]}, {"sent": "whats this ?", "words": ["this"]}, {"sent": "whats this then ?", "words": ["this", "then"]}, {"sent": "whats this ?", "words": ["this"]}, {"sent": "this is called a mouse mat Thomas .", "words": ["this", "mouse", "a", "is"]}, {"sent": "this little ducks lonely .", "words": ["this", "little"]}, {"sent": "this is a xxx .", "words": ["this", "a", "is"]}, {"sent": "wanna play toys this morning with the babies you hafta come and get dressed come on .", "words": ["the", "play", "with", "you", "and", "this", "get", "wanna", "on"]}, {"sent": "whats this .", "words": ["this"]}, {"sent": "where do you think this sun goes ?", "words": ["where", "do", "think", "sun", "you", "this"]}, {"sent": "now you tell me is this is the right way or the wrong way .", "words": ["the", "is", "you", "this", "me"]}, {"sent": "I cant believe its got in this room as well .", "words": ["this", "in", "room"]}, {"sent": "June whats this ?", "words": ["this"]}, {"sent": "look at this pumpkin .", "words": ["this", "pumpkin", "at", "look"]}, {"sent": "do you wanna play with this ?", "words": ["do", "play", "with", "you", "this", "wanna"]}, {"sent": "so this is most similar to the stuff that he plays with at home .", "words": ["the", "is", "to", "so", "with", "this", "he", "at", "that", "home"]}, {"sent": "this dog has a leash .", "words": ["this", "dog", "a"]}, {"sent": "xxx in this bowl .", "words": ["bowl", "this", "in"]}, {"sent": "oh your baby brother or sisters giving me a battering this morning Liz .", "words": ["me", "your", "this", "a"]}, {"sent": "and this is a forklift .", "words": ["this", "a", "is", "and"]}, {"sent": "this is vanilla and I think this ones strawberry .", "words": ["think", "is", "strawberry", "vanilla", "and", "this"]}, {"sent": "oh strawberries and whats this ?", "words": ["this", "and"]}, {"sent": "this has got two little blips on it and this has only one one in there .", "words": ["and", "little", "this", "in", "it", "there", "on"]}, {"sent": "and you did like this ?", "words": ["like", "you", "and", "this", "did"]}, {"sent": "okay now do this one .", "words": ["do", "this"]}, {"sent": "lets see what you hafta say about this spoon again .", "words": ["what", "say", "you", "this", "spoon", "about", "see"]}, {"sent": "this one this one this one .", "words": ["this"]}, {"sent": "watch you know William this is not a sippy cup .", "words": ["cup", "watch", "is", "you", "this", "not", "a"]}, {"sent": "do you think this might fit ?", "words": ["do", "think", "fit", "you", "this"]}, {"sent": "this is a good idea having them in the box .", "words": ["the", "box", "is", "them", "this", "in", "good", "a"]}, {"sent": "this one well this one over here .", "words": ["here", "this", "over"]}, {"sent": "I dont like the way you treat this one .", "words": ["the", "like", "you", "dont", "this"]}, {"sent": "was this in the mall ?", "words": ["the", "was", "in", "this"]}, {"sent": "this chair is cold ?", "words": ["this", "cold", "chair", "is"]}, {"sent": "okay tell you what we can tow this one .", "words": ["can", "what", "we", "you", "this"]}, {"sent": "like this or bigger ?", "words": ["this", "like"]}, {"sent": "Gabriel this is ours oh ?", "words": ["this", "is"]}, {"sent": "well you be careful you dont break anything .", "words": ["be", "you", "dont", "careful", "break"]}, {"sent": "careful because theyre all falling .", "words": ["all", "because", "careful"]}, {"sent": "you do hafta be careful of lions and tigers .", "words": ["do", "of", "be", "you", "and", "careful"]}, {"sent": "careful of the hose .", "words": ["of", "the", "hose", "careful"]}, {"sent": "you do hafta be careful on the road .", "words": ["the", "do", "be", "you", "careful", "on"]}, {"sent": "careful .", "words": ["careful"]}, {"sent": "the trains going to go if youre not careful .", "words": ["the", "if", "go", "to", "careful", "not"]}, {"sent": "you hafta be careful with these books dont you ?", "words": ["these", "be", "with", "you", "dont", "careful"]}, {"sent": "careful with those eggs .", "words": ["with", "those", "careful"]}, {"sent": "careful Honey careful .", "words": ["careful"]}, {"sent": "careful .", "words": ["careful"]}, {"sent": "be careful with that .", "words": ["be", "that", "with", "careful"]}, {"sent": "be careful .", "words": ["be", "careful"]}, {"sent": "oh careful .", "words": ["careful"]}, {"sent": "careful careful .", "words": ["careful"]}, {"sent": "be careful .", "words": ["be", "careful"]}, {"sent": "careful careful on the leg .", "words": ["on", "the", "leg", "careful"]}, {"sent": "means be careful .", "words": ["be", "careful"]}, {"sent": "be more careful .", "words": ["be", "more", "careful"]}, {"sent": "whoa careful .", "words": ["careful"]}, {"sent": "be careful .", "words": ["be", "careful"]}, {"sent": "oh be careful .", "words": ["be", "careful"]}, {"sent": "careful .", "words": ["careful"]}, {"sent": "careful honey .", "words": ["careful"]}, {"sent": "be careful .", "words": ["be", "careful"]}, {"sent": "be careful look because youll end up falling out of that chair .", "words": ["up", "of", "be", "careful", "chair", "out", "look", "that", "because"]}, {"sent": "careful .", "words": ["careful"]}, {"sent": "careful .", "words": ["careful"]}, {"sent": "be careful .", "words": ["be", "careful"]}, {"sent": "careful boo boo .", "words": ["careful"]}, {"sent": "be careful .", "words": ["be", "careful"]}, {"sent": "be careful .", "words": ["be", "careful"]}, {"sent": "oh careful .", "words": ["careful"]}, {"sent": "you hafta be careful with things .", "words": ["be", "careful", "with", "you"]}, {"sent": "oh be careful .", "words": ["be", "careful"]}, {"sent": "be careful with Purdie and that Thomas please .", "words": ["be", "with", "and", "careful", "that"]}, {"sent": "careful Adam .", "words": ["careful"]}, {"sent": "right be careful you dont fall off .", "words": ["off", "be", "fall", "you", "dont", "careful"]}, {"sent": "be careful .", "words": ["be", "careful"]}, {"sent": "it can hurt you if youre not careful .", "words": ["can", "if", "hurt", "you", "it", "careful", "not"]}, {"sent": "but what you must be careful of Thomas is when you come in the front door .", "words": ["the", "what", "is", "of", "be", "you", "in", "careful", "when", "but", "door"]}, {"sent": "oh be careful .", "words": ["be", "careful"]}, {"sent": "be careful .", "words": ["be", "careful"]}, {"sent": "be careful with that book Peter because thats Poppops .", "words": ["be", "with", "careful", "book", "that", "because"]}, {"sent": "oh youre very careful .", "words": ["careful"]}, {"sent": "youve got to be careful if theyre running along that they dont slip on any fruit .", "words": ["if", "to", "be", "any", "they", "dont", "careful", "on", "that"]}, {"sent": "be careful .", "words": ["be", "careful"]}, {"sent": "okay you be careful .", "words": ["be", "careful", "you"]}, {"sent": "careful darling .", "words": ["careful"]}, {"sent": "I told you be careful .", "words": ["be", "careful", "you"]}, {"sent": "Thomas be careful love because its .", "words": ["be", "because", "love", "careful"]}, {"sent": "be careful .", "words": ["be", "careful"]}, {"sent": "be careful .", "words": ["be", "careful"]}, {"sent": "be careful with it .", "words": ["be", "it", "with", "careful"]}, {"sent": "youre throwing oh no be careful .", "words": ["be", "careful"]}, {"sent": "now be very careful with that .", "words": ["be", "that", "with", "careful"]}, {"sent": "are you going to be careful when you turn these pages over ?", "words": ["these", "to", "be", "you", "are", "careful", "over", "when"]}, {"sent": "uhoh careful .", "words": ["careful"]}, {"sent": "well just be careful please .", "words": ["be", "careful"]}, {"sent": "oh be careful Thomas .", "words": ["be", "careful"]}, {"sent": "you hafta be very careful with it .", "words": ["be", "with", "you", "it", "careful"]}, {"sent": "you gotta be careful .", "words": ["be", "careful", "you"]}, {"sent": "careful you dont knock those flowers off .", "words": ["off", "those", "you", "dont", "knock", "careful"]}, {"sent": "careful .", "words": ["careful"]}, {"sent": "be careful you dont put your garage on top of PoohBear .", "words": ["put", "your", "garage", "be", "of", "you", "dont", "careful", "on"]}, {"sent": "be careful now .", "words": ["be", "careful"]}, {"sent": "love be careful over here .", "words": ["be", "here", "love", "careful", "over"]}, {"sent": "be careful with that .", "words": ["be", "that", "with", "careful"]}, {"sent": "be careful be careful be careful .", "words": ["be", "careful"]}, {"sent": "be careful .", "words": ["be", "careful"]}, {"sent": "oh be careful .", "words": ["be", "careful"]}, {"sent": "be careful .", "words": ["be", "careful"]}, {"sent": "be very careful .", "words": ["be", "careful"]}, {"sent": "be careful .", "words": ["be", "careful"]}, {"sent": "be careful xxx picture .", "words": ["be", "picture", "careful"]}, {"sent": "got to be careful with eggs .", "words": ["to", "be", "with", "careful"]}, {"sent": "be careful .", "words": ["be", "careful"]}, {"sent": "careful Adam .", "words": ["careful"]}, {"sent": "careful .", "words": ["careful"]}, {"sent": "careful .", "words": ["careful"]}, {"sent": "you be careful .", "words": ["be", "careful", "you"]}, {"sent": "he mustnt be very careful .", "words": ["be", "he", "careful"]}, {"sent": "careful you dont slip .", "words": ["dont", "you", "careful"]}, {"sent": "oh careful .", "words": ["careful"]}, {"sent": "careful .", "words": ["careful"]}, {"sent": "be a little careful there Boo .", "words": ["be", "little", "there", "careful", "a"]}, {"sent": "you got ta be careful okay ?", "words": ["be", "careful", "you"]}, {"sent": "be careful so you dont drop him .", "words": ["be", "so", "you", "dont", "him", "drop", "careful"]}, {"sent": "be careful please .", "words": ["be", "careful"]}, {"sent": "careful .", "words": ["careful"]}, {"sent": "just be careful wont you ?", "words": ["be", "you", "careful"]}, {"sent": "be careful you dont fall .", "words": ["be", "fall", "you", "dont", "careful"]}, {"sent": "please be careful with them Thomas .", "words": ["be", "them", "with", "careful"]}, {"sent": "gotta be careful huh ?", "words": ["be", "careful"]}, {"sent": "be careful watch .", "words": ["be", "watch", "careful"]}, {"sent": "be careful .", "words": ["be", "careful"]}, {"sent": "careful me ?", "words": ["me", "careful"]}, {"sent": "you hafta be careful on the rides dont you .", "words": ["the", "be", "you", "dont", "careful", "on"]}, {"sent": "yep you hafta put them on careful like this .", "words": ["put", "them", "like", "you", "this", "careful", "on"]}, {"sent": "whoops be careful of your fingers okay ?", "words": ["of", "be", "your", "careful"]}, {"sent": "xxx Im gonna put your apron on okay ?", "words": ["on", "put", "your"]}, {"sent": "put the umbrella on the floor .", "words": ["the", "put", "on"]}, {"sent": "put it right there .", "words": ["there", "put", "it"]}, {"sent": "put the tapes away .", "words": ["the", "put", "away"]}, {"sent": "you can put some on your face too .", "words": ["can", "put", "your", "face", "you", "too", "on", "some"]}, {"sent": "shall I put the cornflakes in a funny bowl ?", "words": ["the", "bowl", "put", "in", "a"]}, {"sent": "pot so put in the pot whatever you wanna make .", "words": ["the", "put", "so", "you", "in", "wanna", "make"]}, {"sent": "do you wanna put his tree on aswell ?", "words": ["do", "put", "tree", "you", "on", "wanna", "his"]}, {"sent": "now you go and put them wee wheels in the rest of them .", "words": ["the", "put", "go", "of", "them", "and", "you", "in"]}, {"sent": "you put it in the shed .", "words": ["the", "put", "you", "in", "it"]}, {"sent": "yeah you try and put it in that hole then .", "words": ["put", "and", "you", "in", "it", "that", "then", "try"]}, {"sent": "put the xxx before the xxx ya know .", "words": ["the", "put"]}, {"sent": "can you put them all back in the box and we can put this in the box aswell ?", "words": ["can", "all", "put", "the", "box", "back", "we", "them", "you", "and", "this", "in"]}, {"sent": "put in on the table ?", "words": ["the", "put", "table", "in", "on"]}, {"sent": "I dont have anything else to put in so well just put it back down .", "words": ["put", "back", "to", "so", "have", "dont", "in", "it", "down"]}, {"sent": "put this down Mama .", "words": ["this", "put", "down"]}, {"sent": "can we put your top on and then the bib on afterwards .", "words": ["can", "the", "put", "your", "we", "and", "bib", "on", "then"]}, {"sent": "well I suppose I probably could put these two on .", "words": ["on", "these", "could", "put"]}, {"sent": "but why do you wanna put them in the mailbox ?", "words": ["the", "do", "put", "them", "you", "in", "wanna", "but", "why"]}, {"sent": "are you going to put those away now ?", "words": ["put", "those", "to", "you", "are", "away"]}, {"sent": "put the train on the head ?", "words": ["the", "put", "train", "head", "on"]}, {"sent": "gonna put the mushroom in ?", "words": ["the", "in", "put"]}, {"sent": "put them in here .", "words": ["here", "in", "put", "them"]}, {"sent": "no we dont put this stuff in our mouth .", "words": ["put", "we", "mouth", "our", "dont", "in", "this"]}, {"sent": "I want ta put this down behind here .", "words": ["put", "behind", "here", "this", "down"]}, {"sent": "put this back in there and clean up your mess .", "words": ["clean", "up", "put", "your", "back", "and", "this", "in", "there"]}, {"sent": "oh you wanna put it on his leg ?", "words": ["put", "you", "his", "it", "leg", "wanna", "on"]}, {"sent": "did you put your letter back ?", "words": ["put", "your", "back", "you", "did"]}, {"sent": "put it in there put it in there Sally .", "words": ["in", "put", "it", "there"]}, {"sent": "put them in Carolines box ?", "words": ["in", "put", "box", "them"]}, {"sent": "shall we put the lid back on the oats ?", "words": ["the", "put", "back", "we", "on"]}, {"sent": "hand me the little dinosaur bag and let me fold it up and put it back in here so it wont get tore up .", "words": ["the", "up", "put", "back", "hand", "so", "and", "little", "here", "in", "it", "get", "me"]}, {"sent": "do you wanna put your jumper on ?", "words": ["do", "put", "your", "you", "wanna", "on"]}, {"sent": "put this one away first .", "words": ["this", "put", "away", "first"]}, {"sent": "are you gonna put the blue one there ?", "words": ["the", "put", "blue", "you", "there", "are"]}, {"sent": "put it in the puzzle .", "words": ["the", "put", "in", "it", "puzzle"]}, {"sent": "wed better put that back together .", "words": ["better", "put", "that", "back"]}, {"sent": "then we can put the basket in here cant we ?", "words": ["can", "the", "put", "we", "here", "in", "basket", "then"]}, {"sent": "oh youve put your slice of orange bread on the teaspoon as well .", "words": ["the", "put", "bread", "your", "of", "orange", "on"]}, {"sent": "can we put the spaghetti on the plate ?", "words": ["can", "the", "put", "plate", "spaghetti", "we", "on"]}, {"sent": "can I put this on you ?", "words": ["can", "put", "you", "this", "on"]}, {"sent": "now put your other hand up .", "words": ["up", "put", "your", "hand", "other"]}, {"sent": "you can put this in .", "words": ["can", "put", "you", "this", "in"]}, {"sent": "lets put the horse would you like to put the horse in the trailer ?", "words": ["the", "put", "to", "like", "you", "in", "would", "horse"]}, {"sent": "are you going to put him in ?", "words": ["put", "to", "you", "him", "are", "in"]}, {"sent": "and when his wheels slipped on the leaves and he started to go backwards the driver put the brakes on .", "words": ["the", "put", "go", "to", "and", "on", "he", "his", "when"]}, {"sent": "you gonna put that last piece in ?", "words": ["put", "last", "you", "in", "that"]}, {"sent": "put your socks on .", "words": ["on", "put", "your"]}, {"sent": "Ill put that back in the fridge because I dont think you want any more coleslaw .", "words": ["the", "put", "think", "back", "any", "you", "dont", "in", "more", "that", "because"]}, {"sent": "what shall we put under the tree ?", "words": ["the", "what", "put", "tree", "we", "under"]}, {"sent": "put some out on the plate for the for the cat .", "words": ["the", "put", "for", "plate", "cat", "out", "on", "some"]}, {"sent": "Ill put it Ill put it back over there .", "words": ["put", "back", "there", "it", "over"]}, {"sent": "he put his tail into that and its making him hurt .", "words": ["put", "into", "hurt", "and", "him", "he", "his", "that"]}, {"sent": "your what you brought this here to put it in my hair ?", "words": ["what", "put", "my", "your", "to", "you", "here", "this", "in", "it", "hair"]}, {"sent": "are you gonna put your other toys away no ?", "words": ["put", "your", "other", "you", "are", "away"]}, {"sent": "you wanna play the cards or do you want me to put them in the box ?", "words": ["the", "do", "put", "box", "to", "play", "them", "you", "in", "wanna", "me"]}, {"sent": "put it in the bin then .", "words": ["the", "put", "in", "it", "then"]}, {"sent": "oh shall we put that one away and you can make another one now if you like .", "words": ["can", "put", "if", "we", "like", "and", "you", "another", "away", "make", "that"]}, {"sent": "okay you gonna put those ones on ?", "words": ["on", "put", "those", "you"]}, {"sent": "John put it back in the freezer .", "words": ["the", "put", "back", "in", "it"]}, {"sent": "put one of the ducks down .", "words": ["of", "the", "put", "down"]}, {"sent": "put that on your ear .", "words": ["put", "your", "ear", "on", "that"]}, {"sent": "who goes first put them all down .", "words": ["all", "put", "first", "them", "who", "down"]}, {"sent": "put the pillow down .", "words": ["the", "put", "down", "pillow"]}, {"sent": "put it in the hole then .", "words": ["the", "put", "in", "it", "then"]}, {"sent": "are you going to put it away xxx and use it again ?", "words": ["put", "to", "you", "and", "are", "it", "away"]}, {"sent": "put the book back .", "words": ["the", "book", "put", "back"]}, {"sent": "we can put them right back inside their shapes .", "words": ["can", "put", "their", "back", "we", "inside", "them"]}, {"sent": "shall we put it the right way up then ?", "words": ["the", "up", "put", "we", "it", "then"]}, {"sent": "put that all together and squeeze it .", "words": ["all", "put", "and", "it", "that"]}, {"sent": "can you put it back ?", "words": ["can", "put", "back", "you", "it"]}, {"sent": "well put our hamburger in here and make our bread wait .", "words": ["put", "bread", "wait", "hamburger", "here", "our", "in", "and", "make"]}, {"sent": "would you like to put that one the table ?", "words": ["the", "put", "table", "to", "like", "you", "would", "that"]}, {"sent": "wait lets put this underneath before you put the roof on .", "words": ["the", "wait", "put", "you", "this", "roof", "on"]}, {"sent": "were gonna put your pj bottoms on before we .", "words": ["put", "your", "we", "on", "were"]}, {"sent": "we can put xxx .", "words": ["can", "put", "we"]}, {"sent": "youre going to put me in a box .", "words": ["put", "box", "to", "in", "me", "a"]}, {"sent": "you gotta put the lid on .", "words": ["the", "put", "on", "you"]}, {"sent": "here lets put them in the plate .", "words": ["the", "put", "plate", "them", "here", "in"]}, {"sent": "xxx going to put it in the cart put it in the cart .", "words": ["the", "put", "to", "in", "it"]}, {"sent": "now you wanna put the yellow one on the blue one ?", "words": ["the", "put", "you", "blue", "wanna", "on", "yellow"]}, {"sent": "you put the things where you want them .", "words": ["the", "where", "put", "them", "you"]}, {"sent": "do you wanna put it back in .", "words": ["do", "put", "back", "you", "in", "it", "wanna"]}, {"sent": "oh where shall we put Snoopy ?", "words": ["where", "put", "we"]}, {"sent": "and I put a clip .", "words": ["put", "a", "and"]}, {"sent": "shall I put your arm through your sleeve ?", "words": ["put", "your", "arm"]}, {"sent": "Im going to take all those bread crusts and put them in the food processor and press the button and it will go and theyll be turned into fine bread crumbs .", "words": ["the", "all", "bread", "put", "those", "into", "button", "go", "fine", "to", "be", "them", "and", "in", "food", "it", "will", "take"]}, {"sent": "or you could put it on as a bracelet too .", "words": ["put", "you", "it", "too", "on", "could", "a"]}, {"sent": "then put them away .", "words": ["then", "away", "them", "put"]}, {"sent": "put it down .", "words": ["down", "put", "it"]}, {"sent": "now put the last piece in and lets see what it looks like .", "words": ["the", "what", "put", "last", "like", "and", "in", "it", "see"]}, {"sent": "no no please put it back .", "words": ["put", "it", "back"]}, {"sent": "put the bucket where ?", "words": ["the", "put", "where", "bucket"]}, {"sent": "xxx put them my son put them up for me .", "words": ["up", "put", "for", "my", "them", "me"]}, {"sent": "when are you gonna put it back on ?", "words": ["put", "back", "you", "it", "are", "on", "when"]}, {"sent": "would you like me to put the food on the plates ?", "words": ["the", "put", "to", "like", "you", "food", "would", "on", "me"]}, {"sent": "I cant believe that you havent put this in there yet .", "words": ["put", "you", "this", "in", "there", "that"]}, {"sent": "are you going to put the things away in the cupboard ?", "words": ["the", "put", "to", "you", "in", "are", "away"]}, {"sent": "put the dollys away .", "words": ["the", "put", "away"]}, {"sent": "lets put some bubbles in this water .", "words": ["put", "this", "in", "water", "some", "bubbles"]}, {"sent": "well put the blanket out over here .", "words": ["the", "put", "here", "out", "blanket", "over"]}, {"sent": "come out come out .", "words": ["out"]}, {"sent": "it wouldnt be nice if we had been out now would it ?", "words": ["if", "we", "be", "nice", "it", "out", "would"]}, {"sent": "you taking them all out again ?", "words": ["all", "them", "out", "you"]}, {"sent": "you like drinking out of that little tiny cup dont you ?", "words": ["tiny", "cup", "of", "like", "you", "little", "dont", "out", "that"]}, {"sent": "well look out of the window .", "words": ["the", "of", "out", "look", "window"]}, {"sent": "are you allowed to get scissors out the drawer ?", "words": ["the", "to", "scissors", "you", "get", "are", "out", "drawer"]}, {"sent": "are you going to put these away and get the food out ?", "words": ["the", "these", "put", "to", "you", "and", "get", "food", "are", "away", "out"]}, {"sent": "sue Vanderock and BabyBlake went swimming out there ?", "words": ["there", "out", "and"]}, {"sent": "can you blow out the light ?", "words": ["can", "the", "light", "you", "out", "blow"]}, {"sent": "and together they raced out of the room .", "words": ["the", "of", "and", "they", "room", "out"]}, {"sent": "starts out with the sun coming up .", "words": ["the", "up", "sun", "with", "out"]}, {"sent": "do you want your cups out too ?", "words": ["do", "your", "you", "out", "too"]}, {"sent": "you get the cake stand out .", "words": ["the", "cake", "stand", "you", "get", "out"]}, {"sent": "now what shape did I cut out now ?", "words": ["what", "out", "did", "cut"]}, {"sent": "youve got to wash your brush out in the bit of water in between .", "words": ["the", "your", "brush", "to", "of", "in", "out", "water", "wash"]}, {"sent": "it slides out .", "words": ["it", "out"]}, {"sent": "oh his other arm has come out look .", "words": ["other", "out", "look", "his", "arm"]}, {"sent": "wanna get them out ?", "words": ["get", "out", "them", "wanna"]}, {"sent": "get out get out get out .", "words": ["get", "out"]}, {"sent": "actually were running out of cheese as well Thomas .", "words": ["of", "cheese", "were", "out"]}, {"sent": "get them out .", "words": ["get", "them", "out"]}, {"sent": "no it cant come out .", "words": ["it", "out"]}, {"sent": "watch out for that pencil .", "words": ["watch", "for", "pencil", "out", "that"]}, {"sent": "but surely if you just want a good nights sleep with a nice bed with covers youd be better to stay out of trouble and just sleep in your own bed ?", "words": ["better", "your", "if", "sleep", "be", "to", "nice", "with", "bed", "of", "you", "and", "in", "good", "out", "stay", "but", "a"]}, {"sent": "I must find out whats wrong with the motor one day .", "words": ["the", "with", "out", "find"]}, {"sent": "its probably filtering out those .", "words": ["those", "out"]}, {"sent": "ah you put some out youre so smart .", "words": ["put", "so", "you", "out", "some"]}, {"sent": "going out anyway .", "words": ["out"]}, {"sent": "this sauce came out okay didnt it ?", "words": ["this", "it", "out", "sauce"]}, {"sent": "how would you like to go out with Daddy tonight in the car .", "words": ["the", "how", "go", "to", "car", "with", "like", "you", "in", "out", "would"]}, {"sent": "take it out .", "words": ["it", "take", "out"]}, {"sent": "look look whats lying out there .", "words": ["there", "out", "look"]}, {"sent": "get out brm brm .", "words": ["get", "out"]}, {"sent": "dresser drawers she pulls them open and dumps all of the clothes out .", "words": ["the", "all", "open", "of", "them", "and", "out", "she"]}, {"sent": "and you just lift it out .", "words": ["out", "it", "you", "and"]}, {"sent": "okay you can take it out .", "words": ["can", "you", "it", "out", "take"]}, {"sent": "I hope you can get it out .", "words": ["can", "you", "get", "it", "out"]}, {"sent": "hey everybody out .", "words": ["out"]}, {"sent": "when all the flowers are out in the summer it attacks a lot of bees doesnt it ?", "words": ["the", "all", "of", "a lot", "in", "are", "it", "out", "when", "a"]}, {"sent": "the problem is you tipped all your toys out onto it didnt you before ?", "words": ["the", "all", "your", "is", "you", "it", "out"]}, {"sent": "take the screw out .", "words": ["the", "take", "out"]}, {"sent": "get out a the water Kim .", "words": ["the", "get", "out", "water", "a"]}, {"sent": "*TO1: check it out .", "words": ["it", "out"]}, {"sent": "did it just burn out ?", "words": ["it", "did", "out"]}, {"sent": "drink out of the cups .", "words": ["of", "the", "out", "drink"]}, {"sent": "Ill hafta send you out to the dentist more often Tanya .", "words": ["the", "to", "you", "more", "out"]}, {"sent": "see getting the drains out .", "words": ["the", "see", "out"]}, {"sent": "and then you get the water back out of the white box .", "words": ["the", "box", "back", "of", "you", "get", "and", "out", "water", "then", "white"]}, {"sent": "but Im not sure whether it would come out very crispy and very hard or whether it would come out and just go vrmm .", "words": ["go", "and", "it", "out", "would", "hard", "not", "but"]}, {"sent": "out of the egg came a tiny and very hungry caterpillar .", "words": ["the", "tiny", "egg", "of", "hungry", "and", "out", "a"]}, {"sent": "he wouldnt sing happy birthday and he wouldnt blow the candles out and he got very upset because he hadnt done it didnt you ?", "words": ["the", "happy", "and", "sing", "you", "it", "he", "out", "blow", "because"]}, {"sent": "I dont know maybe you can figure something out .", "words": ["can", "dont", "out", "you"]}, {"sent": "well well have Rachels toys out in a bit .", "words": ["have", "in", "out", "a"]}, {"sent": "it did fall out .", "words": ["fall", "it", "did", "out"]}, {"sent": "yes well well sort that out in a minute wont we ?", "words": ["we", "in", "out", "that", "a"]}, {"sent": "I cant figure out what your hairs gonna do .", "words": ["what", "do", "your", "out"]}, {"sent": "then um she took out a crystal ball and it showed what the Martians were gonna do .", "words": ["the", "what", "do", "and", "ball", "it", "out", "then", "were", "she", "a"]}, {"sent": "theres no garage out here .", "words": ["garage", "here", "out"]}, {"sent": "looks like a crane doesnt it with that piece sticking out ?", "words": ["with", "like", "it", "out", "that", "a"]}, {"sent": "he just ripped the feather outof her hat like that just went prump just took it out he has his own hat though .", "words": ["the", "like", "his", "that", "it", "he", "out", "hat", "her"]}, {"sent": "Mummy sort it out .", "words": ["it", "out"]}, {"sent": "they soon found out .", "words": ["they", "out"]}, {"sent": "now you can take this big crayon out .", "words": ["can", "crayon", "you", "this", "out", "take", "big"]}, {"sent": "are you going out for a ride on a sunny day are you ?", "words": ["for", "you", "are", "out", "on", "ride", "a"]}, {"sent": "and they came out the other end of the tunnel .", "words": ["the", "of", "other", "and", "they", "out"]}, {"sent": "right do you wanna get everything out ?", "words": ["do", "you", "get", "out", "wanna"]}, {"sent": "because that bits sticking out .", "words": ["out", "that", "because"]}, {"sent": "well just take them out of the cellophane .", "words": ["the", "of", "them", "out", "take"]}, {"sent": "gonna blow the candles out .", "words": ["the", "blow", "out"]}, {"sent": "lets pull out the ones we want .", "words": ["the", "we", "out", "pull"]}, {"sent": "to blow the candle out .", "words": ["the", "to", "blow", "out"]}, {"sent": "and washing her out ?", "words": ["her", "out", "and"]}, {"sent": "which one is the odd one out ?", "words": ["the", "which", "out", "is"]}, {"sent": "well just leave it on the table and well walk out .", "words": ["the", "table", "and", "it", "out", "on", "walk"]}, {"sent": "but when he came out of the shop there was a bear .", "words": ["the", "of", "was", "bear", "there", "he", "out", "when", "but", "a"]}, {"sent": "they said they were going to take you out later on .", "words": ["to", "you", "they", "out", "on", "take", "were"]}, {"sent": "if you turn the handle I think it will come out .", "words": ["the", "think", "if", "you", "it", "out", "will"]}, {"sent": "you dont needta take out anymore .", "words": ["dont", "take", "out", "you"]}, {"sent": "are you missing out ?", "words": ["are", "out", "you"]}, {"sent": "lets turn the page and find out what happens .", "words": ["the", "what", "and", "out", "find"]}, {"sent": "right hands out .", "words": ["out"]}, {"sent": "out the window .", "words": ["the", "window", "out"]}, {"sent": "you dropped it on that button and then all the money came out okay well thank you picking it up thank you very much .", "words": ["the", "all", "money", "up", "button", "much", "you", "and", "it", "out", "on", "then", "that"]}, {"sent": "do you wanna get those ones out ?", "words": ["do", "those", "you", "get", "wanna", "out"]}, {"sent": "lets take playdough out and put it away in its proper bag shall we ?", "words": ["put", "we", "and", "in", "it", "away", "out", "take"]}, {"sent": "said the midwife and she skated out of the door .", "words": ["the", "of", "and", "out", "she", "door"]}, {"sent": "I dont know get get them out .", "words": ["get", "dont", "them", "out"]}, {"sent": "there you punched out the hole .", "words": ["the", "there", "out", "you"]}, {"sent": "watch out .", "words": ["watch", "out"]}, {"sent": "I just wanna see if anybody has been out with a red pen .", "words": ["red", "if", "pen", "with", "out", "wanna", "see", "a"]}, {"sent": "youve got pieces missing out of your jigsawpuzzle Fraser .", "words": ["of", "your", "out"]}, {"sent": "I was about ready to jump out a window .", "words": ["to", "was", "jump", "out", "about", "window", "a"]}, {"sent": "its pretty chilly out there .", "words": ["there", "pretty", "out"]}, {"sent": "hold your hand out .", "words": ["your", "hold", "out", "hand"]}, {"sent": "its out at Woodside Queens .", "words": ["out", "at"]}, {"sent": "this is the mouse peeping out behind the cupboard .", "words": ["the", "is", "behind", "this", "out", "mouse"]}, {"sent": "because its time to get out now Thomas .", "words": ["get", "to", "out", "because"]}, {"sent": "oh you gotta make a little flatter there you go .", "words": ["go", "little", "you", "there", "make", "a"]}, {"sent": "to make them go fast .", "words": ["fast", "go", "to", "them", "make"]}, {"sent": "make do with that .", "words": ["do", "that", "make", "with"]}, {"sent": "Mama make sure its not hot .", "words": ["hot", "make", "not"]}, {"sent": "Id make your birthday cake out of bananas .", "words": ["your", "cake", "of", "out", "make"]}, {"sent": "when youve finished cutting the lawn you go round with that to make the lawn edge all nice and trim .", "words": ["the", "all", "go", "to", "nice", "with", "you", "and", "that", "make", "when"]}, {"sent": "what you gonna make Fraser ?", "words": ["what", "make", "you"]}, {"sent": "what had Warren done to make you cry ?", "words": ["what", "to", "cry", "you", "make"]}, {"sent": "you wanna make something ?", "words": ["make", "wanna", "you"]}, {"sent": "you make it do it .", "words": ["do", "it", "make", "you"]}, {"sent": "make this one disappear ?", "words": ["this", "make"]}, {"sent": "let me just check and make sure the basement door is shut .", "words": ["the", "basement", "is", "and", "me", "make", "door"]}, {"sent": "wanna make mommy a pretty picture ?", "words": ["picture", "pretty", "wanna", "make", "a"]}, {"sent": "you didnt hafta make her move .", "words": ["her", "make", "you"]}, {"sent": "what did you make in here ?", "words": ["what", "you", "here", "in", "did", "make"]}, {"sent": "like make the same noise she made .", "words": ["the", "like", "same", "make", "she"]}, {"sent": "think the pills will make her better .", "words": ["the", "better", "think", "her", "will", "make"]}, {"sent": "can you make squash ?", "words": ["can", "make", "you"]}, {"sent": "okay you youre gonna make it broken .", "words": ["it", "make", "broken", "you"]}, {"sent": "I can make good cartoons outof this .", "words": ["can", "this", "make", "good"]}, {"sent": "he said some people overestimate what theyre able to pay and they make adjustments later .", "words": ["what", "to", "and", "they", "he", "some", "make"]}, {"sent": "oh is that how you make hands ?", "words": ["how", "is", "you", "that", "make"]}, {"sent": "is that more brownies or did you make something else ?", "words": ["is", "you", "more", "make", "did", "that"]}, {"sent": "I thought you were gonna say you make cows .", "words": ["make", "were", "say", "you"]}, {"sent": "cause Im gonna make fried rice .", "words": ["make"]}, {"sent": "do you know what we hafta put in it to make a cinnamon roll ?", "words": ["do", "what", "put", "we", "to", "you", "in", "it", "make", "a"]}, {"sent": "no xxx well make a diamond shall we ?", "words": ["make", "we", "a"]}, {"sent": "let me see if I can make her stand up .", "words": ["can", "up", "if", "stand", "me", "her", "see", "make"]}, {"sent": "there shall we make a face now lets make a face ?", "words": ["face", "we", "there", "make", "a"]}, {"sent": "what kind are you gonna make mommy ?", "words": ["what", "are", "make", "you"]}, {"sent": "okay now you make a house .", "words": ["house", "make", "a", "you"]}, {"sent": "would you like to make some things with clay ?", "words": ["to", "with", "like", "you", "would", "some", "make"]}, {"sent": "Im gonna make a sandwich .", "words": ["make", "sandwich", "a"]}, {"sent": "I bet we could make a whole pizza with all the pizza slices .", "words": ["the", "all", "we", "with", "pizza", "could", "make", "a"]}, {"sent": "tell me precisely before I make it up again and hafta move it .", "words": ["up", "and", "it", "me", "make"]}, {"sent": "we can make a cake later .", "words": ["can", "cake", "we", "make", "a"]}, {"sent": "xxx Summer could you make an animal for the zoo ?", "words": ["the", "zoo", "for", "you", "an", "animal", "could", "make"]}, {"sent": "lets make a necklace .", "words": ["necklace", "make", "a"]}, {"sent": "well maybe we should make the fence a little bit bigger because youve got alotof animals .", "words": ["the", "we", "little", "make", "because", "a"]}, {"sent": "I can make .", "words": ["can", "make"]}, {"sent": "yeah well make a nice big house .", "words": ["nice", "make", "house", "big", "a"]}, {"sent": "Im a make me one .", "words": ["me", "make", "a"]}, {"sent": "burper what do you say after you make a big burp like that ?", "words": ["what", "do", "say", "like", "you", "big", "that", "make", "a"]}, {"sent": "make believe something what is it .", "words": ["what", "it", "make", "is"]}, {"sent": "before you make a trip to the village .", "words": ["the", "to", "you", "make", "a"]}, {"sent": "we can make a xxx outof those pillows xxx .", "words": ["can", "those", "we", "make", "a"]}, {"sent": "can you make a picture with holes ?", "words": ["can", "picture", "with", "you", "make", "a"]}, {"sent": "its the wee ones that make Christmas isnt it ?", "words": ["the", "it", "make", "that"]}, {"sent": "we dont want too many things to make .", "words": ["we", "to", "dont", "too", "make"]}, {"sent": "Im going to make him hold on first though .", "words": ["hold", "to", "first", "him", "on", "make"]}, {"sent": "its very difficult to make la .", "words": ["to", "make"]}, {"sent": "youre gonna make it like its in the jungle ?", "words": ["the", "like", "in", "it", "make"]}, {"sent": "you aint gonna make no sounds ?", "words": ["make", "you"]}, {"sent": "what did you make ?", "words": ["what", "make", "did", "you"]}, {"sent": "make eyes .", "words": ["make"]}, {"sent": "shall we make the frogs jump ?", "words": ["the", "jump", "make", "we"]}, {"sent": "we can just make this believe pretend .", "words": ["can", "we", "this", "pretend", "make"]}, {"sent": "xxx did you really make that ?", "words": ["that", "make", "did", "you"]}, {"sent": "make it go fast .", "words": ["fast", "it", "make", "go"]}, {"sent": "lorrys dont make that noise .", "words": ["dont", "that", "make"]}, {"sent": "Dan can you make an o for Dan ?", "words": ["can", "for", "you", "an", "make"]}, {"sent": "alright you make an orange alright ?", "words": ["an", "orange", "make", "you"]}, {"sent": "ha ha I can make a hundred dollars .", "words": ["can", "make", "a"]}, {"sent": "wanna make boats ?", "words": ["make", "wanna"]}, {"sent": "I was uh I was trying to make to make sure that we didnt hafta make that transition .", "words": ["we", "to", "was", "make", "that"]}, {"sent": "make some eggs .", "words": ["some", "make"]}, {"sent": "you hafta make another one here .", "words": ["another", "here", "make", "you"]}, {"sent": "that doesnt quite make sense to me .", "words": ["to", "me", "make", "that"]}, {"sent": "those are for your hair you want mommy make you pretty ?", "words": ["for", "your", "those", "you", "are", "pretty", "hair", "make"]}, {"sent": "does it make you think you hafta go to sleep ?", "words": ["does", "think", "go", "sleep", "to", "you", "it", "make"]}, {"sent": "can you make a blue circle ?", "words": ["can", "blue", "you", "make", "a"]}, {"sent": "youre going to make yourself sick if you put it that far into your mouth .", "words": ["put", "yourself", "your", "if", "into", "to", "mouth", "you", "that", "sick", "it", "make"]}, {"sent": "so we should draw him what kind of a face should we make him ?", "words": ["what", "face", "we", "of", "so", "draw", "him", "make", "a"]}, {"sent": "you know how to make believe ?", "words": ["to", "how", "make", "you"]}, {"sent": "let us see how pretty you can make it .", "words": ["can", "how", "you", "us", "it", "pretty", "see", "make"]}, {"sent": "you make it sound like an aeroplane .", "words": ["like", "you", "an", "it", "make"]}, {"sent": "make it all dark .", "words": ["all", "it", "make", "dark"]}, {"sent": "you hafta make it halfway between us .", "words": ["us", "it", "make", "you"]}, {"sent": "red and blue make violet .", "words": ["red", "blue", "make", "and"]}, {"sent": "make sure she doesnt drown again .", "words": ["make", "she"]}, {"sent": "Mummy cant make a train .", "words": ["train", "make", "a"]}, {"sent": "are you going to make big noises ?", "words": ["to", "you", "are", "make", "big"]}, {"sent": "would you make me a racing set ?", "words": ["you", "would", "me", "make", "a"]}, {"sent": "did you make a new candle ?", "words": ["you", "did", "new", "make", "a"]}, {"sent": "youre going to hit me to make me better ?", "words": ["better", "to", "me", "hit", "make"]}, {"sent": "mix it up and make it nice .", "words": ["up", "nice", "and", "it", "make"]}, {"sent": "what else can we make ?", "words": ["can", "what", "make", "we"]}, {"sent": "oh lets make you another nice new cake okay ?", "words": ["cake", "nice", "you", "another", "new", "make"]}, {"sent": "Mummys got to make it .", "words": ["to", "it", "make"]}, {"sent": "make sure you chew it alright ?", "words": ["it", "make", "you"]}, {"sent": "I dont think I can make it through there .", "words": ["can", "think", "dont", "there", "it", "make"]}, {"sent": "to make it eight oclock we need the little hand to go from the seven to the eight .", "words": ["the", "need", "we", "hand", "to", "go", "little", "it", "make"]}, {"sent": "make it towards Gail .", "words": ["it", "make"]}, {"sent": "well you can make one .", "words": ["can", "make", "you"]}, {"sent": "make this one jump out .", "words": ["this", "jump", "make", "out"]}, {"sent": "here you gotta make other stuff on there too .", "words": ["other", "here", "you", "there", "too", "on", "make"]}, {"sent": "can you make me a lion ?", "words": ["can", "lion", "you", "me", "make", "a"]}, {"sent": "you wanna make a ball ?", "words": ["you", "ball", "wanna", "make", "a"]}, {"sent": "if its mixed with milk to make porridge it tastes nice .", "words": ["if", "to", "nice", "with", "it", "milk", "make"]}, {"sent": "we dont want too much water on the floor .", "words": ["the", "we", "dont", "water", "on", "too", "much"]}, {"sent": "and Mummyll put the sheep in thankyou very much .", "words": ["the", "put", "and", "in", "sheep", "much"]}, {"sent": "oh you are too much .", "words": ["too", "are", "much", "you"]}, {"sent": "and Im going to Im going to have it examined and if theres so much as a scratch on it so much as a scratch Im going to send for MisterPlod .", "words": ["for", "if", "to", "so", "and", "have", "it", "on", "much", "a"]}, {"sent": "thank you very much for taking that off .", "words": ["off", "for", "much", "you", "that"]}, {"sent": "thats too much .", "words": ["much", "too"]}, {"sent": "thanks so much .", "words": ["so", "much"]}, {"sent": "thank you very much .", "words": ["much", "you"]}, {"sent": "how much is that ?", "words": ["that", "how", "much", "is"]}, {"sent": "Ill not hafta get very much GF .", "words": ["get", "much", "not"]}, {"sent": "its too much its too noisy .", "words": ["much", "noisy", "too"]}, {"sent": "shes more of a dress up and dance than the kitchenny stuff so I dont she doesnt see the older one do it as much so it didnt seem to I thought I just assumed youknow shed follow the older one youknow thats why this stuff is all over my house all the time .", "words": ["the", "of", "more", "dance", "it", "much", "she", "why", "up", "all", "is", "so", "to", "and", "dont", "see", "a", "do", "my", "this", "house", "over", "dress"]}, {"sent": "how much did you cost ?", "words": ["how", "much", "did", "you"]}, {"sent": "thankyou very much for doing that .", "words": ["that", "for", "much"]}, {"sent": "okay now dont interrupt too much Mare Mare .", "words": ["dont", "much", "too"]}, {"sent": "you didnt color very much .", "words": ["much", "you"]}, {"sent": "that was much better .", "words": ["better", "was", "that", "much"]}, {"sent": "thankyou very much .", "words": ["much"]}, {"sent": "wonder how much they are .", "words": ["they", "how", "are", "much"]}, {"sent": "heres my flipper I got to scrape it up now and flip it over so it doesnt cook to much .", "words": ["up", "my", "to", "so", "much", "and", "cook", "it", "over"]}, {"sent": "when Mummy was a little girl when I got a new doll perhaps for Christmas or my birthday it always smelt very much the same .", "words": ["the", "my", "for", "was", "doll", "little", "much", "it", "same", "new", "when", "a"]}, {"sent": "thats much better .", "words": ["better", "much"]}, {"sent": "oh thank you very much .", "words": ["much", "you"]}, {"sent": "theres too much hair for Daddy .", "words": ["hair", "much", "for", "too"]}, {"sent": "thankyou very much .", "words": ["much"]}, {"sent": "how much I love you .", "words": ["how", "much", "love", "you"]}, {"sent": "oh thank you so much .", "words": ["so", "much", "you"]}, {"sent": "you got too much .", "words": ["too", "much", "you"]}, {"sent": "too much swinging about .", "words": ["much", "about", "too"]}, {"sent": "thats much better .", "words": ["better", "much"]}, {"sent": "I love you so much .", "words": ["so", "much", "love", "you"]}, {"sent": "I dont like myself very much .", "words": ["dont", "much", "like", "myself"]}, {"sent": "youve not left me much uhn have you ?", "words": ["you", "have", "me", "much", "not"]}, {"sent": "oh so much for red .", "words": ["red", "so", "for", "much"]}, {"sent": "because theres too much noise .", "words": ["much", "because", "too"]}, {"sent": "Teletubbies love each other very much .", "words": ["much", "each", "love", "other"]}, {"sent": "too much stuff here thats for sure .", "words": ["here", "for", "much", "too"]}, {"sent": "thankyou very much .", "words": ["much"]}, {"sent": "its probably easier to just put it straight in the box but not as much fun is it ?", "words": ["the", "put", "box", "is", "to", "in", "it", "much", "not", "but"]}, {"sent": "to tell you the truth Barbara would have as much fun outof a couple of boxes and crayons and pens .", "words": ["the", "of", "to", "you", "have", "and", "would", "much", "a"]}, {"sent": "how much more do you want ?", "words": ["do", "how", "you", "more", "much"]}, {"sent": "or theres so much going on isnt there ?", "words": ["on", "so", "much", "there"]}, {"sent": "thank you very much my sweet .", "words": ["my", "much", "you"]}, {"sent": "thankyou very much .", "words": ["much"]}, {"sent": "well youre not missing very much .", "words": ["much", "not"]}, {"sent": "thankyou so much .", "words": ["so", "much"]}, {"sent": "yes because I know you love noodles so much .", "words": ["so", "you", "love", "noodles", "much", "because"]}, {"sent": "do they play much basketball in new Zealand ?", "words": ["do", "play", "they", "in", "new", "much"]}, {"sent": "boy youve learned so much since Ive seen you the last time .", "words": ["the", "so", "last", "you", "much"]}, {"sent": "well Im not that much of a Muggins .", "words": ["of", "much", "that", "not", "a"]}, {"sent": "well dont eat too much .", "words": ["eat", "dont", "much", "too"]}, {"sent": "theres not very much left in there .", "words": ["there", "in", "much", "not"]}, {"sent": "ah thats lovely thankyou very much .", "words": ["much"]}, {"sent": "thank you so much she loves coffee from the coffee bar .", "words": ["the", "so", "you", "coffee", "much", "she"]}, {"sent": "you dont know how much he wants .", "words": ["how", "you", "dont", "he", "much"]}, {"sent": "hm I dont think I dont know how much you really play with that but Ill just Ill put it there .", "words": ["think", "how", "put", "play", "with", "you", "dont", "that", "it", "there", "much", "but"]}, {"sent": "alright now so now how much do I owe you ?", "words": ["do", "how", "so", "you", "much"]}, {"sent": "thank you very much .", "words": ["much", "you"]}, {"sent": "thank you very much .", "words": ["much", "you"]}, {"sent": "Imean normally at home we dont hafta push the other doll on her because she likes it so much but yeah um .", "words": ["the", "push", "we", "so", "doll", "other", "much", "dont", "it", "but", "at", "on", "her", "she", "because", "home"]}, {"sent": "are you brown because you eat so much chocolate milk ?", "words": ["brown", "so", "chocolate", "you", "eat", "are", "milk", "much", "because"]}, {"sent": "this ones much makes a much better noise .", "words": ["better", "this", "much", "a"]}, {"sent": "we dont think too much of trucks .", "words": ["think", "we", "of", "dont", "too", "much"]}, {"sent": "well she didnt really play with the food that much .", "words": ["the", "play", "with", "much", "food", "that", "she"]}, {"sent": "very much .", "words": ["much"]}, {"sent": "oh I dont really eat hyenas and wildebeest too much but I suppose I could make an exception for your picnic .", "words": ["for", "your", "and", "eat", "dont", "an", "make", "too", "could", "much", "but"]}, {"sent": "oh I see he doesnt change much when you take him .", "words": ["much", "you", "him", "he", "see", "take", "when"]}, {"sent": "because she ate too much .", "words": ["much", "she", "because", "too"]}, {"sent": "youre not talking very much .", "words": ["much", "not"]}, {"sent": "how much are they ?", "words": ["they", "how", "are", "much"]}, {"sent": "what would happen if you had that much candy do you think ?", "words": ["what", "do", "think", "if", "much", "you", "would", "candy", "that"]}, {"sent": "how much you love me ?", "words": ["how", "you", "love", "me", "much"]}, {"sent": "thats not too much .", "words": ["much", "not", "too"]}, {"sent": "thank you very much .", "words": ["much", "you"]}, {"sent": "he wanted very much to see the baby it seemed a very long time and he hadta wait .", "words": ["the", "wait", "to", "long", "and", "it", "he", "see", "much", "a"]}, {"sent": "but Nomi that car is too much money .", "words": ["money", "is", "car", "much", "too", "that", "but"]}, {"sent": "coming into her own much much more isnt she ?", "words": ["into", "much", "more", "her", "she"]}, {"sent": "because theres so much white you wont be able to taste the yellow .", "words": ["the", "so", "be", "to", "taste", "you", "yellow", "white", "much", "because"]}, {"sent": "no I dont like to eat your foot thank you very much .", "words": ["your", "to", "like", "you", "eat", "dont", "foot", "much"]}, {"sent": "xxx much better okay you might just hafta keep your hand clean for awhile because you look like you have a sore on it .", "words": ["clean", "better", "for", "your", "hand", "like", "you", "have", "it", "look", "on", "much", "because", "a"]}, {"sent": "how much was all this food ?", "words": ["all", "how", "was", "this", "food", "much"]}, {"sent": "itll be much easier for you to eat .", "words": ["for", "to", "be", "you", "eat", "much"]}, {"sent": "it will be soon you dont needta bang it let me see why does it make so much noise when you bang on it ?", "words": ["does", "when", "see", "be", "so", "much", "you", "dont", "on", "it", "will", "me", "make", "why"]}, {"sent": "because therell be too much water .", "words": ["be", "water", "too", "much", "because"]}, {"sent": "pretty much is what she liked to do with them .", "words": ["what", "do", "is", "to", "with", "them", "pretty", "much", "she"]}, {"sent": "because you have so much stuff in it .", "words": ["so", "you", "have", "in", "it", "much", "because"]}, {"sent": "meaning thank you very much indeed .", "words": ["much", "you"]}, {"sent": "not much orange juice .", "words": ["juice", "orange", "much", "not"]}, {"sent": "thats pretty much it .", "words": ["it", "pretty", "much"]}, {"sent": "youre much better .", "words": ["better", "much"]}, {"sent": "there thank you so much for helping .", "words": ["for", "so", "you", "there", "much"]}, {"sent": "thank you very much .", "words": ["much", "you"]}, {"sent": "theres not much there is there ?", "words": ["there", "much", "not", "is"]}, {"sent": "the priestll come round and throw you outof mass for making too much noise .", "words": ["the", "for", "throw", "you", "and", "too", "much"]}, {"sent": "youve been sleeping too much lately havent ya ?", "words": ["much", "too"]}, {"sent": "oh thats much better .", "words": ["better", "much"]}, {"sent": "ah thankyou very much .", "words": ["much"]}, {"sent": "the pilots saying thank you very much .", "words": ["the", "much", "you"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "you couldnt be that in a hurry to go and weewee on your pottie .", "words": ["hurry", "your", "go", "be", "to", "you", "and", "in", "on", "that", "a"]}, {"sent": "better hurry up with those eggs if its already twenty past eight .", "words": ["up", "better", "hurry", "those", "if", "with"]}, {"sent": "hurry .", "words": ["hurry"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "trains dont hurry .", "words": ["dont", "hurry"]}, {"sent": "hurry hurry .", "words": ["hurry"]}, {"sent": "hurry .", "words": ["hurry"]}, {"sent": "hurry hurry see its too heavy honey .", "words": ["see", "heavy", "hurry", "too"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "catch it hurry up catch it .", "words": ["up", "it", "catch", "hurry"]}, {"sent": "hurry .", "words": ["hurry"]}, {"sent": "uh come on hurry up Derek .", "words": ["on", "hurry", "up"]}, {"sent": "hurry up and get this bear and put him in jail before somebody before he goes to hurt anybody else .", "words": ["up", "put", "hurry", "hurt", "to", "and", "get", "this", "bear", "him", "in", "he"]}, {"sent": "hurry up bee beep .", "words": ["up", "hurry", "bee"]}, {"sent": "hurry up hurry up hurry up Im gonna get xxx .", "words": ["up", "get", "hurry"]}, {"sent": "its right right at your stomach hurry up Kame .", "words": ["up", "hurry", "your", "at"]}, {"sent": "uh oh you better cover them up Megan will get mad hurry .", "words": ["up", "better", "hurry", "mad", "them", "you", "get", "cover", "will"]}, {"sent": "hurry up Bobby .", "words": ["up", "hurry"]}, {"sent": "hurry hurry .", "words": ["hurry"]}, {"sent": "in a hurry ?", "words": ["in", "hurry", "a"]}, {"sent": "xxx hurry up give it to me will ya ?", "words": ["up", "hurry", "to", "it", "will", "me", "give"]}, {"sent": "hurry .", "words": ["hurry"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "hurry up because I havent got anywhere to put the clean stuff .", "words": ["up", "the", "put", "hurry", "clean", "to", "because"]}, {"sent": "were not in a hurry .", "words": ["hurry", "in", "were", "not", "a"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "hurry hurry go get dressed put on the clothes you like best .", "words": ["the", "put", "hurry", "go", "like", "you", "get", "on"]}, {"sent": "hurry hurry load the freight .", "words": ["the", "hurry"]}, {"sent": "okay hurry up .", "words": ["up", "hurry"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "you better hurry up and play .", "words": ["up", "better", "hurry", "play", "and", "you"]}, {"sent": "would you hurry up and finish so we can get going .", "words": ["up", "can", "hurry", "we", "so", "finish", "and", "you", "get", "would"]}, {"sent": "she hasta hurry out .", "words": ["hurry", "she", "out"]}, {"sent": "well hurry up and finish .", "words": ["up", "hurry", "finish", "and"]}, {"sent": "you better hurry .", "words": ["better", "hurry", "you"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "well hurry Abe its time to eat .", "words": ["eat", "to", "hurry"]}, {"sent": "Im in kinda a hurry too .", "words": ["too", "in", "hurry", "a"]}, {"sent": "hurry up and get it before daddy gets it .", "words": ["up", "hurry", "and", "get", "it"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "whats the hurry gobbles .", "words": ["the", "hurry"]}, {"sent": "okay hurry back .", "words": ["hurry", "back"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "always in a dreadful hurry .", "words": ["in", "hurry", "a"]}, {"sent": "Jason hurry up and sit and eat so we can go to bed .", "words": ["up", "can", "hurry", "we", "go", "so", "to", "bed", "and", "eat", "sit"]}, {"sent": "hurry .", "words": ["hurry"]}, {"sent": "now hurry back hurry back hurry back .", "words": ["hurry", "back"]}, {"sent": "hurry .", "words": ["hurry"]}, {"sent": "right Ill hurry up then .", "words": ["up", "then", "hurry"]}, {"sent": "comeon hurry up where does number five ?", "words": ["up", "does", "hurry", "where"]}, {"sent": "you were telling me to hurry hurry .", "words": ["hurry", "to", "you", "me", "were"]}, {"sent": "hurry and go teetee .", "words": ["go", "hurry", "and"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "hurry up Eleanor .", "words": ["up", "hurry"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "hurry up and brush the floor .", "words": ["up", "the", "hurry", "brush", "and"]}, {"sent": "I want you to hurry up Michael .", "words": ["up", "to", "hurry", "you"]}, {"sent": "hurry up then .", "words": ["up", "then", "hurry"]}, {"sent": "come on hurry up Mickey mouse .", "words": ["on", "hurry", "up", "mouse"]}, {"sent": "Turkey Lurkey where are you going in such a hurry .", "words": ["where", "hurry", "you", "in", "are", "a"]}, {"sent": "well lets hurry .", "words": ["hurry"]}, {"sent": "hurry up theres Mickey mouse Donald duck and goofy in a cartoon .", "words": ["up", "hurry", "and", "in", "duck", "mouse", "a"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "come on drink your chocolate hurry up .", "words": ["up", "hurry", "your", "drink", "chocolate", "on"]}, {"sent": "but youre always telling me to hurry up and come in before .", "words": ["up", "hurry", "to", "and", "in", "me", "but"]}, {"sent": "Sarah hurry up .", "words": ["up", "hurry"]}, {"sent": "hurry up and finish your sandwiches .", "words": ["up", "hurry", "your", "finish", "and"]}, {"sent": "go on then hurry up .", "words": ["up", "hurry", "go", "on", "then"]}, {"sent": "oh no cried Snow White the poor thing we must hurry .", "words": ["the", "hurry", "we", "poor"]}, {"sent": "hurry hurry .", "words": ["hurry"]}, {"sent": "oh youre in a hurry to get it .", "words": ["hurry", "to", "get", "in", "it", "a"]}, {"sent": "but he was just in too much of a hurry .", "words": ["hurry", "of", "was", "in", "he", "too", "much", "but", "a"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "please hurry up .", "words": ["up", "hurry"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "well hurry up .", "words": ["up", "hurry"]}, {"sent": "hurry hurry .", "words": ["hurry"]}, {"sent": "couldnt hurry up ?", "words": ["up", "hurry"]}, {"sent": "hurry got hurry quick well put em on real quick and well come back out .", "words": ["put", "hurry", "back", "and", "out", "on"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "whats the hurry crows CockyLocky ?", "words": ["the", "hurry"]}, {"sent": "hurry up Bobby .", "words": ["up", "hurry"]}, {"sent": "hurry up with that then you can .", "words": ["up", "can", "hurry", "with", "you", "then", "that"]}, {"sent": "quick quick hurry hurry were in a rush .", "words": ["in", "hurry", "were", "a"]}, {"sent": "come on hurry up we wanna play this game .", "words": ["up", "game", "hurry", "we", "play", "this", "wanna", "on"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "oh time hurry up and fly by .", "words": ["up", "hurry", "by", "and"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "where are you going in such a hurry ?", "words": ["where", "hurry", "you", "in", "are", "a"]}, {"sent": "hurry up now .", "words": ["up", "hurry"]}, {"sent": "come here hurry up .", "words": ["here", "hurry", "up"]}, {"sent": "xxx hurry up xxx .", "words": ["up", "hurry"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "hurry up .", "words": ["up", "hurry"]}, {"sent": "hurry up mister .", "words": ["up", "hurry"]}, {"sent": "say I dont know mom .", "words": ["dont", "say"]}, {"sent": "can you say the cup is on the floor .", "words": ["can", "the", "cup", "is", "say", "you", "on"]}, {"sent": "shed say .", "words": ["say"]}, {"sent": "you say cool dude ?", "words": ["say", "you"]}, {"sent": "say Bro .", "words": ["say"]}, {"sent": "say hi .", "words": ["say"]}, {"sent": "Morgan can you say tree ?", "words": ["can", "tree", "say", "you"]}, {"sent": "you say oh my goodness .", "words": ["my", "say", "you"]}, {"sent": "I mean dont say hi .", "words": ["dont", "say"]}, {"sent": "what did you say ?", "words": ["what", "say", "did", "you"]}, {"sent": "what he say ?", "words": ["what", "he", "say"]}, {"sent": "no I just I didnt say anything .", "words": ["say"]}, {"sent": "you know what people say when they want the pigs to come and they can feed them ?", "words": ["the", "what", "can", "to", "say", "them", "you", "and", "they", "feed", "when"]}, {"sent": "and then we hadta ring Daddy and say sorry Daddy were going to very late .", "words": ["we", "to", "say", "and", "then", "were"]}, {"sent": "say turkey .", "words": ["say", "turkey"]}, {"sent": "then you have to say I spy I spy a skate .", "words": ["to", "skate", "say", "you", "have", "then", "a"]}, {"sent": "can you say .", "words": ["can", "say", "you"]}, {"sent": "you mustnt say boo .", "words": ["say", "you"]}, {"sent": "say Xavier .", "words": ["say"]}, {"sent": "well what shall I say then ?", "words": ["what", "then", "say"]}, {"sent": "Oliver wants to say hello and wish you happy birthday .", "words": ["happy", "to", "say", "wish", "you", "and"]}, {"sent": "say no .", "words": ["say"]}, {"sent": "say this dont taste like no pizza Mommy thats nasty .", "words": ["say", "taste", "like", "pizza", "dont", "this"]}, {"sent": "can you say moon ?", "words": ["can", "say", "moon", "you"]}, {"sent": "should we go say hi to the kitty cat ?", "words": ["the", "go", "we", "to", "say", "cat", "kitty"]}, {"sent": "what else did he say .", "words": ["what", "he", "did", "say"]}, {"sent": "Ive never heard him say that before .", "words": ["that", "him", "say"]}, {"sent": "go and say hello to it ?", "words": ["go", "to", "say", "and", "it"]}, {"sent": "did you say throwing ?", "words": ["say", "did", "you"]}, {"sent": "did you say I was silly ?", "words": ["was", "say", "did", "you"]}, {"sent": "xxx Id say thats good .", "words": ["say", "good"]}, {"sent": "now you wanna say something special into it ?", "words": ["into", "say", "you", "it", "wanna"]}, {"sent": "can you say yellow ?", "words": ["can", "yellow", "say", "you"]}, {"sent": "can you say yawn ?", "words": ["can", "say", "you"]}, {"sent": "so what you going to say ?", "words": ["what", "to", "so", "say", "you"]}, {"sent": "did you say that ?", "words": ["that", "say", "did", "you"]}, {"sent": "so a lot of times it takes her a week or two to get it back shell say it and then right afterwards when you say say it again its dede .", "words": ["when", "back", "of", "so", "to", "a lot", "say", "and", "get", "you", "it", "then", "her", "a"]}, {"sent": "and what do they say ?", "words": ["what", "do", "say", "and", "they"]}, {"sent": "shall we change this so it can say stole ?", "words": ["can", "we", "so", "say", "this", "it"]}, {"sent": "youve not got nought to say to us ?", "words": ["to", "say", "not", "us"]}, {"sent": "can you say Sophie ?", "words": ["can", "say", "you"]}, {"sent": "say .", "words": ["say"]}, {"sent": "do you mean to say you arent .", "words": ["to", "do", "say", "you"]}, {"sent": "can you say ball ?", "words": ["can", "ball", "say", "you"]}, {"sent": "can you say Mommy ?", "words": ["can", "say", "you"]}, {"sent": "thats a dog thats right whats a dog say ?", "words": ["say", "dog", "a"]}, {"sent": "so he can say giddyup to him .", "words": ["can", "so", "to", "say", "him", "he"]}, {"sent": "say pick me up .", "words": ["pick", "me", "up", "say"]}, {"sent": "can you say toot toot ?", "words": ["can", "say", "you"]}, {"sent": "well what did he say ?", "words": ["what", "he", "did", "say"]}, {"sent": "can you say affliction ?", "words": ["can", "say", "you"]}, {"sent": "what did he say ?", "words": ["what", "he", "did", "say"]}, {"sent": "good morning and whats the first thing you should say when you walk in the door ?", "words": ["the", "first", "say", "you", "and", "walk", "in", "good", "when", "door"]}, {"sent": "and what did you say ?", "words": ["what", "say", "you", "and", "did"]}, {"sent": "you need to say excuse me .", "words": ["need", "need to", "to", "say", "you", "me"]}, {"sent": "what does that say ?", "words": ["what", "does", "that", "say"]}, {"sent": "say bye Alex .", "words": ["say"]}, {"sent": "say broken Nomi .", "words": ["say", "broken"]}, {"sent": "what does the piggy say Patrick ?", "words": ["the", "what", "does", "say"]}, {"sent": "what did you say about the choochoo ?", "words": ["the", "what", "say", "you", "did", "about"]}, {"sent": "what do you say ?", "words": ["what", "do", "say", "you"]}, {"sent": "do you say mama ?", "words": ["do", "say", "you"]}, {"sent": "say hi to the camera .", "words": ["the", "to", "say", "camera"]}, {"sent": "and what does your toothbrush say ?", "words": ["what", "toothbrush", "does", "your", "say", "and"]}, {"sent": "say shower .", "words": ["say", "shower"]}, {"sent": "what do you say ?", "words": ["what", "do", "say", "you"]}, {"sent": "what does Grandad say to you ?", "words": ["what", "does", "to", "say", "you"]}, {"sent": "what does it say ?", "words": ["what", "does", "say", "it"]}, {"sent": "and say oh I want this and I want that right ?", "words": ["this", "that", "say", "and"]}, {"sent": "sometimes you say you dont wanna wear pants .", "words": ["say", "you", "dont", "pants", "wanna"]}, {"sent": "he used to say those things when he .", "words": ["those", "to", "say", "he", "when"]}, {"sent": "Laura say cat .", "words": ["cat", "say"]}, {"sent": "what did the mommy say ?", "words": ["the", "what", "say", "did"]}, {"sent": "can you say .", "words": ["can", "say", "you"]}, {"sent": "I joke and say its her favorite toy because she hates this thing .", "words": ["say", "and", "this", "toy", "her", "she", "because"]}, {"sent": "why do you always say no ?", "words": ["do", "say", "why", "you"]}, {"sent": "dont say that .", "words": ["dont", "that", "say"]}, {"sent": "whats a bird say ?", "words": ["bird", "say", "a"]}, {"sent": "oh you say Mama ?", "words": ["say", "you"]}, {"sent": "can you say that ?", "words": ["can", "that", "say", "you"]}, {"sent": "I dont say naughty things .", "words": ["dont", "say", "naughty"]}, {"sent": "what does CleverCat say ?", "words": ["what", "does", "say"]}, {"sent": "did you say there was ?", "words": ["was", "say", "you", "there", "did"]}, {"sent": "where did you say the fire was ?", "words": ["where", "the", "was", "say", "you", "did"]}, {"sent": "you say go .", "words": ["go", "say", "you"]}, {"sent": "what do you say ?", "words": ["what", "do", "say", "you"]}, {"sent": "whad do ya say ?", "words": ["do", "say"]}, {"sent": "say sorry .", "words": ["say"]}, {"sent": "say byebye .", "words": ["say"]}, {"sent": "what does the cow say ?", "words": ["the", "what", "does", "say", "cow"]}, {"sent": "can you say once upon a time ?", "words": ["can", "say", "a", "you"]}, {"sent": "say mommy do .", "words": ["do", "say"]}, {"sent": "whatd I say ?", "words": ["say"]}, {"sent": "is that what they say ?", "words": ["what", "is", "say", "they", "that"]}, {"sent": "you wanna say anything to her ?", "words": ["to", "say", "you", "wanna", "her"]}, {"sent": "say sorry .", "words": ["say"]}, {"sent": "the big bear doesnt say that ?", "words": ["the", "say", "bear", "big", "that"]}, {"sent": "say no need xxx .", "words": ["need", "say"]}, {"sent": "what do you say ?", "words": ["what", "do", "say", "you"]}, {"sent": "and Daddy cant say that ?", "words": ["that", "say", "and"]}, {"sent": "we needta get some more of his milk .", "words": ["milk", "we", "of", "get", "more", "his", "some"]}, {"sent": "can you color his face red ?", "words": ["can", "red", "face", "you", "his"]}, {"sent": "you cant put his you cant put his head on his feet .", "words": ["put", "head", "you", "on", "his"]}, {"sent": "so shall we put them in his ears ?", "words": ["put", "we", "so", "them", "in", "his"]}, {"sent": "he gave you his van Nina .", "words": ["his", "he", "you"]}, {"sent": "whats he got on his head ?", "words": ["on", "his", "head", "he"]}, {"sent": "watch out yelled his mother but his mother caught yelled jack but his mother caught the pancake right in the middle of the pan .", "words": ["the", "watch", "pancake", "of", "in", "out", "his", "but"]}, {"sent": "Eddie didnt know how to tie his shoes .", "words": ["his", "to", "how"]}, {"sent": "hes gonna swim to his own ocean okay and whos he gonna see there ?", "words": ["to", "swim", "and", "there", "he", "his", "see"]}, {"sent": "on his pants .", "words": ["on", "his", "pants"]}, {"sent": "the man has got back into his van hasnt he ?", "words": ["the", "into", "back", "he", "his"]}, {"sent": "the little fur father said good morning one day put on his hat and went away .", "words": ["the", "put", "little", "his", "and", "away", "good", "hat", "on"]}, {"sent": "I think the mans packing up his tool box now .", "words": ["the", "up", "think", "box", "his"]}, {"sent": "his ?", "words": ["his"]}, {"sent": "oh is he picking all his letters up from the station ?", "words": ["up", "all", "the", "is", "he", "his"]}, {"sent": "whats his mouth doing there when his nose is there ?", "words": ["is", "mouth", "there", "his", "when", "nose"]}, {"sent": "only you hafta let his feet stick up or his ears tick out or something .", "words": ["up", "stick", "you", "out", "his"]}, {"sent": "yeah on his chin .", "words": ["on", "his", "chin"]}, {"sent": "Franklin could count by twos and tie his shoes .", "words": ["his", "could", "and", "by"]}, {"sent": "I thought I was his father .", "words": ["his", "was"]}, {"sent": "whats in his mouth ?", "words": ["his", "in", "mouth"]}, {"sent": "is his name Betsy ?", "words": ["his", "is"]}, {"sent": "Sam wore his robe that was blue and his slippers were too .", "words": ["was", "and", "blue", "too", "his", "that", "were"]}, {"sent": "well its not actually a tongue I think it is his .", "words": ["think", "is", "his", "it", "tongue", "not", "a"]}, {"sent": "wheres his eye ?", "words": ["his", "eye"]}, {"sent": "Doctormwwws going to feel his stomach .", "words": ["his", "to"]}, {"sent": "oh his ear keeps falling off .", "words": ["his", "off", "ear"]}, {"sent": "oh hes not really breaking him hes just sitting on his head so he wont get wet see he puts his paws like this on top of the boys head hes not breaking him hes just going like this see ?", "words": ["the", "wet", "of", "so", "head", "like", "his", "get", "him", "this", "he", "on", "see", "not"]}, {"sent": "his custard ?", "words": ["his"]}, {"sent": "Grandpas gonna let Kit sleep in his pajama top .", "words": ["sleep", "his", "in"]}, {"sent": "what is his name ?", "words": ["his", "what", "is"]}, {"sent": "and his tongue .", "words": ["his", "tongue", "and"]}, {"sent": "dont pick it up by his tail or youll break it .", "words": ["up", "pick", "by", "dont", "it", "his", "break"]}, {"sent": "now is this monster happier again now that hes got his cardboard ?", "words": ["his", "this", "that", "is"]}, {"sent": "where did his trousers go ?", "words": ["where", "his", "did", "go"]}, {"sent": "ah hes hes gone off on his bike .", "words": ["off", "his", "on"]}, {"sent": "he eats his food all his dinner .", "words": ["his", "all", "food", "he"]}, {"sent": "CaptainFeatherSword keeps his ship tied to the dock in the harbor .", "words": ["his", "to", "in", "the"]}, {"sent": "look at his little tail .", "words": ["his", "at", "little", "look"]}, {"sent": "oh he went back to his home oh thank goodness .", "words": ["back", "to", "he", "his", "home"]}, {"sent": "tomorrow is his last day at school right Boo ?", "words": ["is", "school", "last", "at", "his"]}, {"sent": "Eddie didnt know how to tie his shoes so he asked if his mother .", "words": ["how", "if", "to", "so", "he", "his"]}, {"sent": "oh is this his cousin thats come to visit him ?", "words": ["is", "to", "this", "him", "his"]}, {"sent": "slept on his ?", "words": ["on", "his"]}, {"sent": "thats his piece of paper and his pen .", "words": ["of", "paper", "and", "pen", "his"]}, {"sent": "look at how steady his head is .", "words": ["how", "look", "is", "head", "at", "his"]}, {"sent": "he fell on his head hes crying hes crying .", "words": ["on", "his", "head", "he"]}, {"sent": "he has a hat on his hand ?", "words": ["hand", "his", "on", "he", "hat", "a"]}, {"sent": "because his name is Willy and he creeps .", "words": ["is", "and", "he", "his", "because"]}, {"sent": "come with his reindeers hadnt he ?", "words": ["his", "he", "with"]}, {"sent": "an old scarecrow was leaning on his hoe .", "words": ["old", "was", "his", "an", "on"]}, {"sent": "then he woke his parents .", "words": ["his", "then", "he"]}, {"sent": "because he has got chocolate all over his bottom .", "words": ["all", "bottom", "chocolate", "he", "his", "over", "because"]}, {"sent": "this mans got a blue bag over his shoulder .", "words": ["blue", "shoulder", "this", "his", "over", "a"]}, {"sent": "oh his bumbum ?", "words": ["his"]}, {"sent": "he eagerly washed his hands face and brushed his beak ready for Katys party .", "words": ["for", "face", "and", "party", "he", "his"]}, {"sent": "he has his hand up to his chin .", "words": ["up", "hand", "to", "chin", "he", "his"]}, {"sent": "oh thats his blanket .", "words": ["his", "blanket"]}, {"sent": "do you remember we saw Julian driving away and putting his sunglasses on ?", "words": ["do", "we", "and", "you", "on", "away", "his"]}, {"sent": "right thats right because the diver took Nemo back to his office right ?", "words": ["the", "back", "to", "his", "because"]}, {"sent": "hes playing with his his other little friend .", "words": ["his", "with", "little", "other"]}, {"sent": "on his hook .", "words": ["on", "his"]}, {"sent": "oh did his head fall off again ?", "words": ["off", "head", "fall", "did", "his"]}, {"sent": "thats his eye .", "words": ["his", "eye"]}, {"sent": "he just bopped me on the nose with his head .", "words": ["the", "head", "with", "his", "he", "on", "me", "nose"]}, {"sent": "yes well just set his Ill put his car seat back in and .", "words": ["put", "back", "car", "and", "in", "his"]}, {"sent": "take his shirt off .", "words": ["his", "shirt", "take", "off"]}, {"sent": "is that like his face with all the fur ?", "words": ["the", "all", "face", "is", "with", "like", "his", "that"]}, {"sent": "well your Daddy wants to finish his breakfast .", "words": ["his", "to", "your", "finish"]}, {"sent": "its like his interest in more in processes now of turning pages of the book or .", "words": ["the", "of", "like", "in", "more", "his", "book"]}, {"sent": "in the meadow jack put the stool next to the cow and the pail between his legs he started milking .", "words": ["the", "put", "to", "and", "in", "he", "his", "cow"]}, {"sent": "thats his mouth right there .", "words": ["his", "there", "mouth"]}, {"sent": "is the top of his hat missing ?", "words": ["the", "is", "of", "hat", "his"]}, {"sent": "and his chair goes up and down .", "words": ["up", "chair", "and", "his", "down"]}, {"sent": "Franklin packed his new pencil case with a ruler a pencil an eraser and twelve colored pencils that he had sharpened himself .", "words": ["with", "pencil", "and", "an", "he", "his", "new", "that", "a"]}, {"sent": "there was a boy who lived with his mother .", "words": ["was", "with", "there", "who", "his", "a"]}, {"sent": "are you putting his nose in now ?", "words": ["you", "in", "are", "his", "nose"]}, {"sent": "Im just putting his arm up and then it might fit a bit better .", "words": ["up", "better", "fit", "and", "it", "arm", "his", "then", "a"]}, {"sent": "you can take his hat on and off cant you ?", "words": ["can", "off", "you", "and", "on", "hat", "his", "take"]}, {"sent": "is he steering on the back with his flippers ?", "words": ["the", "back", "is", "with", "his", "he", "on"]}, {"sent": "he lost his coal tender .", "words": ["his", "he"]}, {"sent": "where is his kiss ?", "words": ["where", "his", "kiss", "is"]}, {"sent": "Melissa show Megan how he did his eyes .", "words": ["how", "show", "he", "did", "his"]}, {"sent": "now that one is that his chimney ?", "words": ["his", "that", "is"]}, {"sent": "okay lets put captain up there and then well help him find his feather sword .", "words": ["up", "put", "help", "and", "there", "him", "find", "his", "then"]}, {"sent": "what did your Daddy do for his birthday ?", "words": ["what", "do", "for", "your", "did", "his"]}, {"sent": "why hes waving his hand ?", "words": ["his", "why", "hand"]}, {"sent": "where is his crib ?", "words": ["where", "his", "crib", "is"]}, {"sent": "gone to have his bah .", "words": ["have", "to", "his"]}, {"sent": "Fraserll go eat his lunch .", "words": ["eat", "his", "go"]}, {"sent": "he bumped his head .", "words": ["his", "head", "he"]}, {"sent": "maybe he could just sit lay on his side .", "words": ["his", "he", "on", "sit", "could"]}, {"sent": "well lets get him out of his cab .", "words": ["of", "get", "him", "out", "his"]}, {"sent": "um yes he does he likes all his toys .", "words": ["all", "does", "he", "his"]}, {"sent": "is he going to go for a little ride on his boat or is he going to go make his tea ?", "words": ["for", "go", "is", "to", "boat", "little", "his", "make", "he", "on", "ride", "a"]}, {"sent": "now all his spots are blue .", "words": ["his", "all", "are", "blue"]}, {"sent": "he has got his hat on in the bath .", "words": ["the", "in", "on", "he", "hat", "his"]}, {"sent": "pecked his back .", "words": ["his", "back"]}, {"sent": "so his mummie said you .", "words": ["his", "so", "you"]}, {"sent": "is that his Mama ?", "words": ["his", "that", "is"]}, {"sent": "and well just put these two over here .", "words": ["these", "put", "and", "here", "over"]}, {"sent": "and who are these ?", "words": ["who", "are", "these", "and"]}, {"sent": "well Im not paying extra for these because you havent got enough change .", "words": ["these", "for", "you", "not", "because"]}, {"sent": "you wanna take these off ?", "words": ["off", "these", "you", "wanna", "take"]}, {"sent": "lets build another one with these blocks okay ?", "words": ["another", "these", "build", "with"]}, {"sent": "um do you think theres any of these items that he recognizes the name for ?", "words": ["the", "do", "think", "these", "for", "of", "any", "you", "he", "that"]}, {"sent": "one of these thingamajigs .", "words": ["of", "these"]}, {"sent": "take these .", "words": ["these", "take"]}, {"sent": "liking about these plates here ?", "words": ["here", "these", "about"]}, {"sent": "oh well these things happen .", "words": ["these"]}, {"sent": "so would you say that these kinds of toys at least as far as what were offering here are sor suit her personality the best ?", "words": ["the", "these", "what", "of", "so", "say", "her", "you", "here", "at", "are", "would", "that", "were"]}, {"sent": "mommy has these .", "words": ["these"]}, {"sent": "you dont want one of these ?", "words": ["of", "dont", "these", "you"]}, {"sent": "what are these Fraser ?", "words": ["what", "are", "these"]}, {"sent": "you see all these DVDs weve got here ?", "words": ["all", "these", "you", "here", "see"]}, {"sent": "xxx and shes all xxx these wee sayings .", "words": ["all", "these", "and"]}, {"sent": "lets tidy these letters up and then we can play with the shop .", "words": ["up", "these", "can", "the", "we", "play", "with", "and", "then"]}, {"sent": "I think your mom just mentioned it and I think it has something to do with one of these maybe .", "words": ["do", "think", "these", "your", "of", "to", "with", "and", "it"]}, {"sent": "thats why theres all these jungle animals look at all these little kids in this bus theyre not sitting down in their seats theyre all standing up do you know who this is ?", "words": ["up", "all", "these", "look", "do", "their", "is", "little", "you", "this", "in", "at", "who", "down", "bus", "not", "why"]}, {"sent": "there are wheels on these four .", "words": ["on", "there", "are", "these"]}, {"sent": "but the faces on these trains arent very smily are they ?", "words": ["the", "these", "they", "are", "on", "but"]}, {"sent": "Ive got to wear these trousers when we go to Sarah and Chriss tonight .", "words": ["these", "go", "we", "to", "and", "when"]}, {"sent": "what color what color are these pants .", "words": ["what", "are", "these", "pants"]}, {"sent": "you dont think its because we keep spraying all these air fresheners around that its making Purdie sneeze do you ?", "words": ["all", "think", "these", "do", "we", "you", "dont", "around", "that", "because"]}, {"sent": "these are empty because I use them as stamps .", "words": ["these", "them", "empty", "are", "because"]}, {"sent": "these this one .", "words": ["these", "this"]}, {"sent": "xxx were there any of these toys that you hadta tell her what they were this time ?", "words": ["these", "what", "of", "any", "her", "you", "there", "that", "they", "this", "were"]}, {"sent": "see all these places the ball goes to .", "words": ["the", "all", "these", "to", "ball", "see"]}, {"sent": "these icecream words are by Jerry ?", "words": ["these", "are", "by"]}, {"sent": "youre coming out in all these silly silly names .", "words": ["all", "in", "these", "out"]}, {"sent": "what are these ?", "words": ["what", "are", "these"]}, {"sent": "weve got to get these tears away .", "words": ["get", "to", "away", "these"]}, {"sent": "and what are these ?", "words": ["what", "are", "these", "and"]}, {"sent": "what are these ?", "words": ["what", "are", "these"]}, {"sent": "now these are Moroccan things .", "words": ["these", "are"]}, {"sent": "its quite nice with all these pictures here .", "words": ["all", "these", "nice", "with", "here"]}, {"sent": "sometimes these guys get a laughing thing .", "words": ["get", "these", "a"]}, {"sent": "xxx oh these shoes are for those babies not for you .", "words": ["these", "for", "those", "you", "are", "not"]}, {"sent": "well hafta have a look at these after weve had lunch .", "words": ["these", "have", "at", "look", "a"]}, {"sent": "these are the pieces we sorted out earlier on arent they ?", "words": ["the", "these", "we", "they", "are", "out", "on"]}, {"sent": "you said you have some of these books at home .", "words": ["these", "of", "you", "have", "at", "some", "home"]}, {"sent": "do these people go in the car ?", "words": ["the", "do", "these", "go", "car", "in"]}, {"sent": "these look like candy but theyre buttons .", "words": ["these", "like", "look", "candy", "but"]}, {"sent": "these are fast things .", "words": ["these", "fast", "are"]}, {"sent": "I should have flown thrown these dead flowers away shouldnt I ?", "words": ["have", "these", "away"]}, {"sent": "look at all these .", "words": ["all", "at", "these", "look"]}, {"sent": "hey look at these look at these what are these aw look .", "words": ["these", "what", "look", "are", "at"]}, {"sent": "these sounds all sound like b .", "words": ["all", "these", "like"]}, {"sent": "I could jog in these .", "words": ["in", "could", "these"]}, {"sent": "xxx get some of these things out of here .", "words": ["these", "of", "get", "here", "out", "some"]}, {"sent": "oh I cant stand these leaves .", "words": ["stand", "these"]}, {"sent": "one of these I think .", "words": ["of", "these", "think"]}, {"sent": "here put these back .", "words": ["here", "these", "put", "back"]}, {"sent": "and these are balls that the girl is holding .", "words": ["the", "these", "is", "and", "are", "that"]}, {"sent": "and so none of these other things either .", "words": ["these", "of", "so", "none", "and", "other"]}, {"sent": "look at all these people do you know these people ?", "words": ["all", "do", "these", "look", "you", "at"]}, {"sent": "you hafta practice making the shapes of these letters .", "words": ["the", "of", "these", "you"]}, {"sent": "do you know what these are called ?", "words": ["do", "what", "these", "you", "are"]}, {"sent": "these are new trousers and theyre a bit big arent they ?", "words": ["these", "and", "they", "are", "new", "big", "a"]}, {"sent": "ah what about these ?", "words": ["what", "these", "about"]}, {"sent": "you dont try and put these in the door .", "words": ["the", "these", "put", "and", "you", "dont", "in", "try", "door"]}, {"sent": "alot he has these .", "words": ["these", "he"]}, {"sent": "no I dont got these rubber these rubber Spiderman Batman .", "words": ["dont", "these"]}, {"sent": "yeah we eat these .", "words": ["eat", "these", "we"]}, {"sent": "well just put these crayons away shall we ?", "words": ["these", "put", "away", "we"]}, {"sent": "*SI1: are these string on the racket .", "words": ["on", "these", "are", "the"]}, {"sent": "dont touch it now because these are Swwws pictures .", "words": ["these", "touch", "dont", "it", "are", "because"]}, {"sent": "these are tires and wheels yep .", "words": ["these", "are", "and"]}, {"sent": "these are not very clean are they ?", "words": ["clean", "these", "they", "are", "not"]}, {"sent": "no thats where these go these .", "words": ["where", "these", "go"]}, {"sent": "can you see among these .", "words": ["can", "see", "these", "you"]}, {"sent": "how many of these are there ?", "words": ["these", "how", "of", "there", "are"]}, {"sent": "these are passengers .", "words": ["these", "are"]}, {"sent": "oh these are the cheap colors I see .", "words": ["the", "these", "are", "see"]}, {"sent": "let Mommy have two of these please .", "words": ["have", "of", "these"]}, {"sent": "what are these ?", "words": ["what", "are", "these"]}, {"sent": "you wanna look at these ?", "words": ["these", "you", "at", "wanna", "look"]}, {"sent": "well Im sure Anna wont mind you playing with them if you wanna but shall we play with these ?", "words": ["these", "if", "we", "play", "them", "with", "you", "wanna", "but"]}, {"sent": "then underneath he had these xxx line .", "words": ["these", "then", "he"]}, {"sent": "what are all these what about this whats this ?", "words": ["all", "what", "these", "this", "are", "about"]}, {"sent": "okay now do all these people wanna go find their mail ?", "words": ["all", "do", "these", "their", "go", "wanna", "find"]}, {"sent": "these are called shoots Thomas .", "words": ["these", "are"]}, {"sent": "so hes gonna come over and do these other bushes here .", "words": ["do", "these", "so", "and", "other", "here", "over"]}, {"sent": "that means I keep these too aswell .", "words": ["these", "that", "too"]}, {"sent": "these are your eyes .", "words": ["these", "are", "your"]}, {"sent": "look at these shoelaces all over the place .", "words": ["the", "all", "these", "look", "at", "over"]}, {"sent": "well look at these .", "words": ["these", "at", "look"]}, {"sent": "what are these ?", "words": ["what", "are", "these"]}, {"sent": "these must be green .", "words": ["these", "be", "green"]}, {"sent": "what do you call these ?", "words": ["what", "do", "these", "you"]}, {"sent": "are you coming to collect all these letters from the post office ?", "words": ["the", "all", "these", "to", "you", "are"]}, {"sent": "now do you want to keep these things ?", "words": ["do", "these", "to", "you", "want to"]}, {"sent": "you put these in your ears .", "words": ["these", "put", "your", "you", "in"]}, {"sent": "youll hafta slot it in between one of these others wont you ?", "words": ["these", "of", "you", "in", "it"]}, {"sent": "oh youre right so these shouldnt be green these should be red .", "words": ["red", "these", "so", "be", "green"]}, {"sent": "lets have one of these .", "words": ["have", "of", "these"]}, {"sent": "well well put these back in the basket now and Im going to take them back to the shop .", "words": ["the", "these", "put", "back", "to", "them", "and", "in", "basket", "take"]}, {"sent": "they certainly like these rides .", "words": ["they", "these", "like"]}, {"sent": "shall we turn it off ?", "words": ["off", "it", "we"]}, {"sent": "off she tumbled .", "words": ["off", "she"]}, {"sent": "do you wanna take those off now ?", "words": ["off", "do", "those", "you", "wanna", "take"]}, {"sent": "off .", "words": ["off"]}, {"sent": "oh youve licked it off havent you ?", "words": ["off", "it", "you"]}, {"sent": "no those dont come off .", "words": ["off", "dont", "those"]}, {"sent": "has it fallen off ?", "words": ["off", "it"]}, {"sent": "you tell them off .", "words": ["off", "them", "you"]}, {"sent": "Ill let you take it off .", "words": ["off", "it", "take", "you"]}, {"sent": "I wanna take your sneakers off .", "words": ["your", "take", "off", "wanna"]}, {"sent": "fell off a wall .", "words": ["off", "a"]}, {"sent": "go off the table sweetie .", "words": ["off", "the", "table", "go"]}, {"sent": "get your feet off my wall .", "words": ["get", "my", "your", "off"]}, {"sent": "its sorta hard to take off .", "words": ["off", "to", "take", "hard"]}, {"sent": "and he sailed off through the night and the day .", "words": ["off", "the", "he", "and"]}, {"sent": "get off get off .", "words": ["get", "off"]}, {"sent": "trousers socks and pants off and straight into the bath please .", "words": ["off", "the", "into", "and", "pants"]}, {"sent": "dont take your trousers off .", "words": ["your", "dont", "take", "off"]}, {"sent": "its gonna vacuum all your hair off Lenita .", "words": ["off", "all", "your", "vacuum", "hair"]}, {"sent": "get you cleaned up before you doze off .", "words": ["get", "up", "off", "you"]}, {"sent": "take them off ?", "words": ["off", "take", "them"]}, {"sent": "you never ripped your sleeve clear off your shirt ?", "words": ["off", "shirt", "your", "you"]}, {"sent": "turn it off .", "words": ["off", "it"]}, {"sent": "I turned it off .", "words": ["off", "it"]}, {"sent": "not go off ?", "words": ["off", "not", "go"]}, {"sent": "we turn the light off to go night night .", "words": ["the", "off", "we", "light", "to", "go"]}, {"sent": "dust it off see take the dust off .", "words": ["off", "the", "it", "see", "take"]}, {"sent": "off you go .", "words": ["off", "go", "you"]}, {"sent": "grandma will wipe that off in a minute .", "words": ["off", "in", "will", "wipe", "that", "a"]}, {"sent": "get off Matth .", "words": ["get", "off"]}, {"sent": "am I getting on or am I getting off ?", "words": ["on", "am", "off"]}, {"sent": "youve tipped Sue off her potty and youve tipped up Mummys tower .", "words": ["off", "up", "and", "potty", "her"]}, {"sent": "wherere you off to now ?", "words": ["off", "to", "you"]}, {"sent": "if I saw Purditer going into our fridge to get some butter and milk shed get a good telling off wouldnt she ?", "words": ["off", "if", "into", "to", "our", "get", "and", "butter", "good", "milk", "some", "she", "a"]}, {"sent": "its running off the spoon .", "words": ["off", "the", "spoon"]}, {"sent": "off thats right on .", "words": ["off", "on"]}, {"sent": "shall we pull your socks off ?", "words": ["off", "your", "pull", "we"]}, {"sent": "no I dont think they want us to throw it off their walls though .", "words": ["off", "think", "throw", "their", "to", "dont", "they", "us", "it"]}, {"sent": "okay you can turn it off now okay ?", "words": ["can", "it", "off", "you"]}, {"sent": "take your jumper off of it .", "words": ["off", "your", "of", "it", "take"]}, {"sent": "you want xxx Mommy to wash them off ?", "words": ["off", "to", "them", "you", "wash"]}, {"sent": "lets take this nappie off .", "words": ["off", "this", "take"]}, {"sent": "shall we build something up with the bricks off the er off the train ?", "words": ["up", "the", "off", "we", "train", "with", "build"]}, {"sent": "but it wasnt off .", "words": ["off", "it", "but"]}, {"sent": "which monkey fell off ?", "words": ["off", "monkey", "which"]}, {"sent": "off .", "words": ["off"]}, {"sent": "what did you take off ?", "words": ["off", "what", "you", "did", "take"]}, {"sent": "and off he ran just as a squirrel came hoping through the glade .", "words": ["off", "the", "and", "he", "squirrel", "a"]}, {"sent": "you took the handle off .", "words": ["the", "off", "you"]}, {"sent": "Perrys taking her name off and you put your name on .", "words": ["off", "put", "your", "you", "and", "on", "her"]}, {"sent": "and itll soon come off .", "words": ["off", "and"]}, {"sent": "did you pull it off in the car ?", "words": ["off", "the", "car", "pull", "you", "in", "it", "did"]}, {"sent": "he doesnt wanna get off of me .", "words": ["off", "of", "get", "he", "wanna", "me"]}, {"sent": "everybody reverse off the lift .", "words": ["off", "the"]}, {"sent": "Roman you cant put so much weight on there or theyll fall off okay ?", "words": ["off", "put", "so", "fall", "you", "there", "on", "much"]}, {"sent": "get off .", "words": ["get", "off"]}, {"sent": "when you take it off it doesnt go back so well because some of the glue comes off .", "words": ["off", "the", "when", "back", "go", "of", "so", "you", "it", "some", "take", "because", "glue"]}, {"sent": "aw she fell off xxx fell off .", "words": ["off", "she"]}, {"sent": "get off of the book .", "words": ["off", "the", "of", "get", "book"]}, {"sent": "is he going off that way ?", "words": ["off", "that", "he", "is"]}, {"sent": "where are you off to ?", "words": ["where", "off", "to", "you", "are"]}, {"sent": "scrape it off with your finger xxx .", "words": ["off", "your", "with", "finger", "it"]}, {"sent": "flew off .", "words": ["off"]}, {"sent": "are you gonna take your other boot off ?", "words": ["off", "your", "other", "you", "are", "take"]}, {"sent": "get off the xxx .", "words": ["get", "the", "off"]}, {"sent": "I just played it off .", "words": ["off", "it"]}, {"sent": "you cant take her slip off .", "words": ["off", "her", "take", "you"]}, {"sent": "take his top off aswell .", "words": ["his", "take", "off"]}, {"sent": "I think Im going to take off my shoes too .", "words": ["off", "think", "my", "to", "too", "take"]}, {"sent": "she goes off when you come .", "words": ["off", "when", "she", "you"]}, {"sent": "label off the filing cabinet .", "words": ["off", "the"]}, {"sent": "so if somebody wants to go to sleep they take the entire roof off the boat .", "words": ["the", "off", "if", "go", "sleep", "to", "so", "boat", "they", "roof", "take"]}, {"sent": "are you just peeling it off ?", "words": ["off", "it", "are", "you"]}, {"sent": "did it fall off ?", "words": ["off", "fall", "it", "did"]}, {"sent": "you wanna turn it off and were gonna go do something else ?", "words": ["off", "do", "go", "and", "you", "it", "wanna", "were"]}, {"sent": "lets just slip these things off .", "words": ["off", "these"]}, {"sent": "they might fall off the tree .", "words": ["off", "the", "tree", "fall", "they"]}, {"sent": "get off it .", "words": ["get", "it", "off"]}, {"sent": "I think you turned it off .", "words": ["off", "think", "it", "you"]}, {"sent": "because you you chucked me off chair didnt you ?", "words": ["off", "chair", "you", "me", "because"]}, {"sent": "get off .", "words": ["get", "off"]}, {"sent": "take him off .", "words": ["off", "him", "take"]}, {"sent": "wanna get off your chair ?", "words": ["off", "your", "chair", "get", "wanna"]}, {"sent": "were done here we can shut this off .", "words": ["can", "off", "we", "here", "this", "were"]}, {"sent": "the door fell right off .", "words": ["the", "off", "door"]}, {"sent": "no dont take the lid off .", "words": ["the", "dont", "take", "off"]}, {"sent": "Grandad never tells you off .", "words": ["off", "you"]}, {"sent": "finish it off a bit .", "words": ["off", "it", "finish", "a"]}, {"sent": "what are you taking off ?", "words": ["off", "what", "are", "you"]}, {"sent": "going off in a minute .", "words": ["off", "in", "a"]}, {"sent": "thats right the kittys trying to catch the bus and his hat flew off .", "words": ["the", "off", "to", "catch", "and", "hat", "his", "bus"]}, {"sent": "if we took it the plane couldnt take off .", "words": ["the", "off", "if", "we", "it", "take"]}, {"sent": "dont get xxx because when you pull it off sometimes it hurts like that .", "words": ["off", "like", "you", "get", "dont", "pull", "it", "that", "when", "because"]}, {"sent": "theyre off now .", "words": ["off"]}, {"sent": "dry him off real good .", "words": ["off", "dry", "him", "good"]}, {"sent": "do you want her to take it off now or what ?", "words": ["off", "do", "what", "to", "her", "you", "it", "take"]}, {"sent": "getting off ?", "words": ["off"]}, {"sent": "you have no business turning mine off .", "words": ["have", "mine", "off", "you"]}, {"sent": "I think this branch needs to be cut off because there are no leaves on that branch I think that means that its sick .", "words": ["off", "think", "cut", "to", "be", "this", "there", "are", "sick", "on", "that", "because"]}, {"sent": "here lets get them out .", "words": ["here", "get", "them", "out"]}, {"sent": "this one goes here .", "words": ["here", "this"]}, {"sent": "lets come over here Earl lets come over here so you can so we can see what it looks like .", "words": ["can", "what", "we", "so", "like", "you", "here", "it", "see", "over"]}, {"sent": "with a hee haw there and a hee haw here .", "words": ["with", "and", "here", "there", "a"]}, {"sent": "here Ill clear off the couch for you to sleep on .", "words": ["the", "off", "for", "sleep", "to", "couch", "you", "here", "on"]}, {"sent": "and what are they doing here ?", "words": ["what", "and", "here", "they", "are"]}, {"sent": "you make it hit here .", "words": ["you", "here", "it", "hit", "make"]}, {"sent": "get your pen up here .", "words": ["up", "your", "get", "here", "pen"]}, {"sent": "put that back in here .", "words": ["put", "back", "here", "in", "that"]}, {"sent": "right here .", "words": ["here"]}, {"sent": "here .", "words": ["here"]}, {"sent": "oh here it is .", "words": ["here", "it", "is"]}, {"sent": "look here .", "words": ["here", "look"]}, {"sent": "are you going to come and play down here ?", "words": ["to", "play", "and", "you", "here", "are", "down"]}, {"sent": "here is all the shapes .", "words": ["here", "all", "the", "is"]}, {"sent": "your waters over here .", "words": ["over", "here", "your"]}, {"sent": "here let me get these things for your mommy .", "words": ["these", "for", "your", "here", "get", "me"]}, {"sent": "here .", "words": ["here"]}, {"sent": "here look .", "words": ["here", "look"]}, {"sent": "is this your uh sculpture here ?", "words": ["here", "this", "your", "is"]}, {"sent": "Ill go on this boat here look .", "words": ["go", "here", "this", "look", "on", "boat"]}, {"sent": "here why dont you come over here Maya ?", "words": ["you", "here", "dont", "over", "why"]}, {"sent": "here get on this pot .", "words": ["here", "this", "on", "get"]}, {"sent": "that should be in here .", "words": ["here", "be", "in", "that"]}, {"sent": "here have a drink .", "words": ["here", "have", "drink", "a"]}, {"sent": "okay xxx over here .", "words": ["here", "over"]}, {"sent": "a monster here ?", "words": ["here", "a"]}, {"sent": "ah here we are then .", "words": ["here", "then", "are", "we"]}, {"sent": "theyre right here .", "words": ["here"]}, {"sent": "how can I I cant theres not enough room in here to run .", "words": ["can", "how", "to", "here", "run", "in", "room", "not"]}, {"sent": "theyre going over here .", "words": ["here", "over"]}, {"sent": "what have we got here ?", "words": ["have", "what", "here", "we"]}, {"sent": "here we go .", "words": ["here", "go", "we"]}, {"sent": "yeah you dont needta use any glue because xxx in here theres some there .", "words": ["any", "you", "here", "dont", "in", "there", "glue", "because", "some"]}, {"sent": "right over here .", "words": ["here", "over"]}, {"sent": "here .", "words": ["here"]}, {"sent": "oh its here .", "words": ["here"]}, {"sent": "here .", "words": ["here"]}, {"sent": "come here little one .", "words": ["here", "little"]}, {"sent": "how about putting her in this here ?", "words": ["how", "here", "this", "in", "about", "her"]}, {"sent": "here you are .", "words": ["here", "are", "you"]}, {"sent": "here everything is on the wall .", "words": ["here", "the", "on", "is"]}, {"sent": "Grandma and Grandpa Lastname were here yesterday looking after you .", "words": ["here", "were", "you", "and"]}, {"sent": "there are some good yoga cards here Miss Naima .", "words": ["here", "there", "are", "good", "some"]}, {"sent": "just give me a couple more minutes in here .", "words": ["here", "more", "in", "me", "give", "a"]}, {"sent": "youll hafta sit up here if you wanna see .", "words": ["up", "if", "you", "here", "wanna", "see", "sit"]}, {"sent": "get back here .", "words": ["get", "here", "back"]}, {"sent": "here they are .", "words": ["here", "they", "are"]}, {"sent": "what else can we find back here ?", "words": ["can", "what", "back", "we", "here", "find"]}, {"sent": "put it back in here ?", "words": ["put", "back", "here", "in", "it"]}, {"sent": "its got that family circle and we sit here and do some postcards dont we ?", "words": ["do", "we", "and", "here", "dont", "some", "sit", "that"]}, {"sent": "how bout over here Naima ?", "words": ["here", "how", "over"]}, {"sent": "right here we are .", "words": ["here", "are", "we"]}, {"sent": "what is this here ?", "words": ["here", "what", "this", "is"]}, {"sent": "okay here we are in Albuquerque .", "words": ["here", "in", "are", "we"]}, {"sent": "come round here look .", "words": ["here", "look"]}, {"sent": "put that right on here .", "words": ["on", "put", "here", "that"]}, {"sent": "count from here .", "words": ["here"]}, {"sent": "lookee here is lion .", "words": ["here", "lion", "is"]}, {"sent": "oh whats in here ?", "words": ["here", "in"]}, {"sent": "you stand here and Ill put these extra tins on for you .", "words": ["these", "put", "for", "stand", "and", "here", "you", "on"]}, {"sent": "here I go .", "words": ["here", "go"]}, {"sent": "come here .", "words": ["here"]}, {"sent": "you wait here your father has a marvelous surprise for you .", "words": ["wait", "for", "your", "you", "here", "a"]}, {"sent": "you sit down and xxx and Ill and Ill read you some books then it wont hurt while youre up here .", "words": ["up", "hurt", "then", "and", "you", "here", "it", "read", "some", "sit", "down"]}, {"sent": "I see something up here .", "words": ["up", "see", "here"]}, {"sent": "you gonna sleep here tonight .", "words": ["sleep", "here", "you"]}, {"sent": "here is the beehive .", "words": ["here", "the", "is"]}, {"sent": "here buddy .", "words": ["here"]}, {"sent": "here .", "words": ["here"]}, {"sent": "here it is this is our window .", "words": ["is", "our", "here", "this", "it", "window"]}, {"sent": "were there any animals here that you told him the names too ?", "words": ["the", "any", "you", "here", "there", "him", "too", "that", "were"]}, {"sent": "come on xxx in here .", "words": ["on", "in", "here"]}, {"sent": "here .", "words": ["here"]}, {"sent": "how about right here ?", "words": ["here", "how", "about"]}, {"sent": "well we needta hook it over here somewhere because xxx high for it .", "words": ["for", "we", "here", "it", "high", "over", "because"]}, {"sent": "here ?", "words": ["here"]}, {"sent": "Mamas here .", "words": ["here"]}, {"sent": "yeah the battery situation here .", "words": ["the", "here"]}, {"sent": "come here .", "words": ["here"]}, {"sent": "here you are .", "words": ["here", "are", "you"]}, {"sent": "here we are .", "words": ["here", "are", "we"]}, {"sent": "here put it on here and it will be easier to draw .", "words": ["put", "be", "to", "will", "draw", "and", "here", "it", "on"]}, {"sent": "come here .", "words": ["here"]}, {"sent": "theyve got lotsof cars here .", "words": ["here"]}, {"sent": "come back in here Lara please .", "words": ["here", "in", "back"]}, {"sent": "whats over here ?", "words": ["here", "over"]}, {"sent": "here this one .", "words": ["here", "this"]}, {"sent": "we are staying right here .", "words": ["here", "are", "we"]}, {"sent": "Im in here .", "words": ["here", "in"]}, {"sent": "the dog is over here .", "words": ["the", "is", "here", "over", "dog"]}, {"sent": "you know the easiest way to get your hands clean is to go wash them so after were done here maybe you could wash your hands .", "words": ["the", "clean", "your", "is", "go", "to", "so", "them", "you", "get", "here", "could", "wash", "were"]}, {"sent": "on the way home from school we met him playing alone and so he came here at one and stayed til four oclock right ?", "words": ["the", "we", "school", "so", "and", "here", "him", "he", "at", "on", "home"]}, {"sent": "here .", "words": ["here"]}, {"sent": "here lets put this on .", "words": ["here", "this", "put", "on"]}, {"sent": "rosie come here .", "words": ["here"]}, {"sent": "here .", "words": ["here"]}, {"sent": "ei I need some help here .", "words": ["here", "some", "help", "need"]}, {"sent": "yeah get outof here .", "words": ["get", "here"]}, {"sent": "is there a baby to give that bottle to ?", "words": ["is", "to", "there", "bottle", "give", "that", "a"]}, {"sent": "theres no room for that one there .", "words": ["there", "room", "that", "for"]}, {"sent": "there theres the flour .", "words": ["the", "there"]}, {"sent": "youre gonna fall off there Amy .", "words": ["off", "there", "fall"]}, {"sent": "there .", "words": ["there"]}, {"sent": "theres a pig in the way isnt there ?", "words": ["the", "pig", "there", "in", "a"]}, {"sent": "there it goes in there .", "words": ["there", "in", "it"]}, {"sent": "there you are .", "words": ["there", "are", "you"]}, {"sent": "there you are .", "words": ["there", "are", "you"]}, {"sent": "is there some juice in there ?", "words": ["is", "there", "in", "some", "juice"]}, {"sent": "there are three Smarties in that container .", "words": ["there", "in", "are", "that"]}, {"sent": "right there .", "words": ["there"]}, {"sent": "who is there .", "words": ["there", "is", "who"]}, {"sent": "Ill give it to you in a minute but just leave it right there for right now okay ?", "words": ["for", "to", "you", "there", "in", "it", "give", "but", "a"]}, {"sent": "what are you doing up there ?", "words": ["up", "what", "you", "there", "are"]}, {"sent": "eggs arent in there .", "words": ["there", "in"]}, {"sent": "its there .", "words": ["there"]}, {"sent": "there .", "words": ["there"]}, {"sent": "there you go .", "words": ["go", "there", "you"]}, {"sent": "can you leave that there ?", "words": ["can", "there", "that", "you"]}, {"sent": "put it there .", "words": ["there", "put", "it"]}, {"sent": "I put my hands in there now and Im going to bite you .", "words": ["put", "my", "to", "and", "you", "there", "in", "bite"]}, {"sent": "you gonna make it sit in there ?", "words": ["you", "there", "in", "it", "sit", "make"]}, {"sent": "you leave his pencil in there Adam .", "words": ["pencil", "you", "there", "in", "his"]}, {"sent": "now that machine there says put some blood on .", "words": ["put", "there", "on", "some", "that"]}, {"sent": "who put that in there ?", "words": ["put", "there", "in", "who", "that"]}, {"sent": "there .", "words": ["there"]}, {"sent": "now if you go all the way over there with that what are you gonna do with it ?", "words": ["the", "all", "what", "do", "if", "go", "with", "you", "there", "that", "are", "it", "over"]}, {"sent": "I found your book cover over there .", "words": ["your", "there", "cover", "book", "over"]}, {"sent": "whats that there ?", "words": ["there", "that"]}, {"sent": "well push the mail back in there .", "words": ["the", "push", "back", "there", "in"]}, {"sent": "there you go .", "words": ["go", "there", "you"]}, {"sent": "did you scratch your cheek there ?", "words": ["your", "cheek", "you", "there", "did"]}, {"sent": "there ya go .", "words": ["there", "go"]}, {"sent": "watch all those clothes hanging up there please because theyve just been ironed .", "words": ["up", "all", "watch", "those", "there", "because"]}, {"sent": "is he okay in there ?", "words": ["there", "in", "he", "is"]}, {"sent": "whats in there ?", "words": ["there", "in"]}, {"sent": "there does that help .", "words": ["there", "help", "does", "that"]}, {"sent": "you can leave it there if you want .", "words": ["can", "if", "you", "there", "it"]}, {"sent": "there Poohbear .", "words": ["there"]}, {"sent": "there you go .", "words": ["go", "there", "you"]}, {"sent": "was there some vegetables ?", "words": ["was", "some", "there"]}, {"sent": "there you go .", "words": ["go", "there", "you"]}, {"sent": "like if she would take like like two of these animals and say like this one goes that goes there and she would bring this one and he goes next to it and she might bring up another one and stick it like they pretend theyre talking .", "words": ["of", "like", "another", "there", "it", "pretend", "that", "take", "she", "up", "these", "stick", "if", "to", "bring", "and", "would", "say", "this", "they", "he"]}, {"sent": "there are lots of animals arent there ?", "words": ["of", "there", "are"]}, {"sent": "that goes right there .", "words": ["there", "that"]}, {"sent": "there you are .", "words": ["there", "are", "you"]}, {"sent": "there we go .", "words": ["go", "there", "we"]}, {"sent": "in the summer there it was mad with all the kids round here .", "words": ["the", "all", "was", "mad", "with", "here", "there", "in", "it"]}, {"sent": "now that dont make fit in there .", "words": ["fit", "dont", "that", "in", "there", "make"]}, {"sent": "were going to put AuntieMwwws photograph in there arent we ?", "words": ["put", "we", "to", "there", "in", "were"]}, {"sent": "red on there or would you like well use blue .", "words": ["red", "like", "you", "blue", "there", "would", "on"]}, {"sent": "see there ?", "words": ["see", "there"]}, {"sent": "there ya go .", "words": ["there", "go"]}, {"sent": "there .", "words": ["there"]}, {"sent": "there there .", "words": ["there"]}, {"sent": "is that not him there ?", "words": ["is", "there", "him", "that", "not"]}, {"sent": "there we go .", "words": ["go", "there", "we"]}, {"sent": "there .", "words": ["there"]}, {"sent": "Dan there are no more up there are there ?", "words": ["up", "there", "more", "are"]}, {"sent": "oh well maybe sometime we can go there .", "words": ["can", "there", "go", "we"]}, {"sent": "there you go .", "words": ["go", "there", "you"]}, {"sent": "there are flowers on the wall .", "words": ["on", "there", "are", "the"]}, {"sent": "no you got some more things there .", "words": ["some", "more", "there", "you"]}, {"sent": "behind there .", "words": ["there", "behind"]}, {"sent": "the baby can have a little slide down there .", "words": ["the", "can", "slide", "little", "have", "there", "down", "a"]}, {"sent": "there ?", "words": ["there"]}, {"sent": "doesnt really look like a train should but there you go .", "words": ["train", "go", "like", "you", "there", "look", "but", "a"]}, {"sent": "there we are .", "words": ["there", "are", "we"]}, {"sent": "there it goes .", "words": ["there", "it"]}, {"sent": "there are alotof spanish in there especially cubans .", "words": ["there", "in", "are"]}, {"sent": "how many are there ?", "words": ["there", "how", "are"]}, {"sent": "oh well leave it there .", "words": ["there", "it"]}, {"sent": "there .", "words": ["there"]}, {"sent": "you want one in there .", "words": ["there", "in", "you"]}, {"sent": "whos whos still over there ?", "words": ["there", "over"]}, {"sent": "there we go .", "words": ["go", "there", "we"]}, {"sent": "and then well get that in there too .", "words": ["and", "get", "there", "in", "too", "then", "that"]}, {"sent": "theres a pen Pete right there on the floor .", "words": ["the", "there", "pen", "on", "a"]}, {"sent": "well it doesnt matter but you can put it there if you like .", "words": ["can", "put", "if", "like", "you", "there", "it", "but"]}, {"sent": "Im just going to put a little drop there .", "words": ["put", "to", "little", "there", "drop", "a"]}, {"sent": "there we go .", "words": ["go", "there", "we"]}, {"sent": "there we are .", "words": ["there", "are", "we"]}, {"sent": "there it goes .", "words": ["there", "it"]}, {"sent": "there .", "words": ["there"]}, {"sent": "theres the wagon over there .", "words": ["the", "there", "over"]}, {"sent": "theres some wheat there yep .", "words": ["some", "there"]}, {"sent": "theres a little washing machine there and an oven .", "words": ["and", "little", "there", "an", "washing machine", "oven", "a"]}, {"sent": "um oh so the things she likes the best do you is there a reason you think they might be good for her personality like the kind of infant she is or not necessarily ?", "words": ["the", "do", "think", "for", "is", "of", "so", "be", "not", "like", "you", "there", "they", "good", "her", "she", "a"]}, {"sent": "whos on there ?", "words": ["on", "there"]}, {"sent": "there we are .", "words": ["there", "are", "we"]}, {"sent": "there you go .", "words": ["go", "there", "you"]}, {"sent": "to the bedroom window up there .", "words": ["the", "up", "to", "there", "window", "bedroom"]}, {"sent": "wheres that other piece over there ?", "words": ["over", "there", "that", "other"]}, {"sent": "there isnt another one there isnt another man .", "words": ["another", "there"]}, {"sent": "is there another yellow one ?", "words": ["another", "there", "yellow", "is"]}, {"sent": "let me stand that microphone up there .", "words": ["up", "stand", "there", "me", "that"]}, {"sent": "Fraser come away from there .", "words": ["there", "away"]}, {"sent": "there you go .", "words": ["go", "there", "you"]}, {"sent": "put then back over there .", "words": ["put", "back", "there", "then", "over"]}, {"sent": "you wanna take that part ?", "words": ["that", "take", "wanna", "you"]}, {"sent": "take your pants off then .", "words": ["off", "your", "pants", "then", "take"]}, {"sent": "okay Ill take off this tire .", "words": ["off", "this", "take"]}, {"sent": "no they dont take the tires off .", "words": ["off", "the", "dont", "they", "take"]}, {"sent": "just take them down .", "words": ["down", "take", "them"]}, {"sent": "take it out .", "words": ["it", "take", "out"]}, {"sent": "and then where does she take the children ?", "words": ["where", "does", "the", "and", "then", "take", "she"]}, {"sent": "you take up the red one .", "words": ["up", "the", "red", "you", "take"]}, {"sent": "what we hafta do now is take this jacket off .", "words": ["off", "what", "do", "is", "we", "this", "take", "jacket"]}, {"sent": "was that for Daddy to take into work ?", "words": ["for", "work", "into", "to", "was", "that", "take"]}, {"sent": "yeah take the suit off and dry it off .", "words": ["the", "off", "dry", "and", "it", "take"]}, {"sent": "next year Santa when you get to us you can even take a nap on the sofa if youre tired .", "words": ["can", "the", "if", "to", "you", "get", "us", "tired", "on", "sofa", "take", "when", "a"]}, {"sent": "xxx Thomas would you like to take your coat off ?", "words": ["off", "your", "to", "like", "you", "would", "take", "coat"]}, {"sent": "oh no you cant take the lid off .", "words": ["the", "take", "off", "you"]}, {"sent": "do you wanna take an apple to school ?", "words": ["do", "to", "school", "you", "an", "wanna", "apple", "take"]}, {"sent": "well dont take the crayons out because its time to take a nap .", "words": ["the", "to", "dont", "out", "take", "because", "a"]}, {"sent": "Im just having a look inside the bag that we take out with us .", "words": ["the", "we", "inside", "with", "us", "out", "look", "take", "that", "a"]}, {"sent": "just leave the jacket on for a little while more and then well be able to take it off .", "words": ["the", "off", "for", "be", "to", "little", "and", "more", "it", "on", "then", "take", "jacket", "a"]}, {"sent": "yes take your time .", "words": ["your", "take"]}, {"sent": "here lets put all the men in so you can take them for a ride okay ?", "words": ["the", "all", "put", "can", "for", "so", "them", "ride", "you", "here", "in", "take", "a"]}, {"sent": "you wanna take it off ?", "words": ["off", "you", "it", "wanna", "take"]}, {"sent": "take your take your trainers off as well .", "words": ["your", "take", "off"]}, {"sent": "should he take that thing off ?", "words": ["off", "that", "take", "he"]}, {"sent": "I got outof come up hadta take xxx .", "words": ["up", "take"]}, {"sent": "why did you take that out of there ?", "words": ["of", "you", "there", "did", "out", "take", "that", "why"]}, {"sent": "she you hafta take her to the shop because she works at the shop .", "words": ["the", "to", "you", "at", "take", "her", "she", "because"]}, {"sent": "take your fingers outof your nose then .", "words": ["your", "then", "take", "nose"]}, {"sent": "take one ?", "words": ["take"]}, {"sent": "can we take the little piece of playdough ?", "words": ["can", "the", "we", "of", "little", "take"]}, {"sent": "oh look Thomas could take the animals away couldnt he ?", "words": ["the", "away", "he", "look", "could", "take"]}, {"sent": "if you take those away nobodyd be able to sit down .", "words": ["those", "if", "be", "to", "you", "away", "sit", "down", "take"]}, {"sent": "so next time you needta blow your nose you just take your tissue out of your pocket .", "words": ["your", "tissue", "of", "so", "you", "out", "take", "blow", "nose"]}, {"sent": "take it in your hand .", "words": ["your", "hand", "in", "it", "take"]}, {"sent": "not xxx she doesnt take bottles but we babysit another baby who takes bottles .", "words": ["we", "another", "she", "who", "take", "not", "but"]}, {"sent": "but inchworm says Ill take my time .", "words": ["my", "take", "but"]}, {"sent": "want Daddy take you to bed ?", "words": ["to", "take", "bed", "you"]}, {"sent": "you wanna take it out ?", "words": ["you", "it", "out", "wanna", "take"]}, {"sent": "well Ill take that incase somebody xxx wants some .", "words": ["some", "that", "take"]}, {"sent": "and take it off .", "words": ["off", "it", "take", "and"]}, {"sent": "do you want me to take it away ?", "words": ["do", "to", "you", "it", "away", "me", "take"]}, {"sent": "thats right but take your shirt off and take your ponytails out and put them away please .", "words": ["off", "put", "your", "shirt", "them", "and", "away", "out", "take", "but"]}, {"sent": "mama said shed take you to the play ground after while .", "words": ["the", "to", "play", "you", "take"]}, {"sent": "Eleanor dont take the mop out .", "words": ["the", "dont", "out", "mop", "take"]}, {"sent": "let me see you take off your sweatshirt by yourself .", "words": ["off", "yourself", "your", "me", "you", "by", "see", "take"]}, {"sent": "come on I think I better take you back to your teacher now .", "words": ["better", "think", "your", "back", "to", "you", "on", "take"]}, {"sent": "she went to get a shot in her arm and take a blood test ?", "words": ["to", "and", "get", "in", "arm", "take", "her", "she", "a"]}, {"sent": "wont take me very long .", "words": ["me", "take", "long"]}, {"sent": "its yours you can take it home with you .", "words": ["can", "with", "you", "it", "take", "home"]}, {"sent": "take then take the .", "words": ["the", "then", "take"]}, {"sent": "here let me take it off .", "words": ["off", "here", "it", "me", "take"]}, {"sent": "do you wanna take your umbrella ?", "words": ["do", "your", "you", "wanna", "take"]}, {"sent": "would you take them off Dimitra ?", "words": ["off", "them", "you", "would", "take"]}, {"sent": "shes reading and she needs to take a break .", "words": ["break", "to", "and", "take", "she", "a"]}, {"sent": "take it how about if we put our things in here .", "words": ["how", "put", "if", "we", "our", "here", "in", "it", "about", "take"]}, {"sent": "or do you wanna take some .", "words": ["do", "you", "wanna", "some", "take"]}, {"sent": "Im gonna take you to see some more stones ?", "words": ["to", "you", "more", "see", "take", "some"]}, {"sent": "we hafta take one out .", "words": ["take", "out", "we"]}, {"sent": "should I take it ?", "words": ["it", "take"]}, {"sent": "take one out .", "words": ["take", "out"]}, {"sent": "okay lets take a break .", "words": ["break", "take", "a"]}, {"sent": "I take it youre not xxx yourself then .", "words": ["yourself", "it", "then", "take", "not"]}, {"sent": "shall we let your truck take the peas ?", "words": ["the", "your", "we", "peas", "truck", "take"]}, {"sent": "wherere you going to take him ?", "words": ["to", "him", "take", "you"]}, {"sent": "dont take his head off no .", "words": ["off", "head", "dont", "his", "take"]}, {"sent": "can you take unpinch can you take the um clothespin off ?", "words": ["can", "the", "off", "you", "take"]}, {"sent": "you can take them on the go take them on the go little lady .", "words": ["can", "the", "go", "them", "you", "little", "on", "take"]}, {"sent": "lets take the ones out first .", "words": ["the", "first", "take", "out"]}, {"sent": "you can take this whole page off .", "words": ["can", "off", "you", "this", "take"]}, {"sent": "shall we take a great big whale ?", "words": ["big", "take", "a", "we"]}, {"sent": "okay Ill take them off .", "words": ["off", "take", "them"]}, {"sent": "take take five .", "words": ["take"]}, {"sent": "you can take little bites you dont hafta shove the whole thing in your mouth at the same time .", "words": ["can", "the", "your", "mouth", "little", "you", "dont", "in", "same", "at", "take"]}, {"sent": "will you take me to the park ?", "words": ["the", "to", "you", "park", "will", "me", "take"]}, {"sent": "well now we hafta uh take a look inside the elevator and see where your class is .", "words": ["the", "where", "your", "we", "is", "inside", "and", "look", "see", "take", "a"]}, {"sent": "you take a xxx .", "words": ["take", "a", "you"]}, {"sent": "dont wanna take em out ?", "words": ["dont", "out", "take", "wanna"]}, {"sent": "you going to take a bite ?", "words": ["to", "you", "bite", "take", "a"]}, {"sent": "where does Granddad take you ?", "words": ["where", "does", "take", "you"]}, {"sent": "take that off .", "words": ["off", "that", "take"]}, {"sent": "hes going to take SantaClaus to take the childrens toys .", "words": ["the", "to", "take"]}, {"sent": "take that outof your mouth please .", "words": ["your", "that", "mouth", "take"]}, {"sent": "oh Ill take french .", "words": ["take"]}, {"sent": "you hafta take your pants off .", "words": ["off", "your", "you", "pants", "take"]}, {"sent": "its when you open them and take them off that you you know what letter youve got .", "words": ["off", "what", "open", "them", "and", "you", "that", "take", "when"]}, {"sent": "where you gonna take her ?", "words": ["where", "her", "take", "you"]}, {"sent": "try to take him out .", "words": ["try", "to", "him", "out", "take", "try to"]}, {"sent": "take that one off aswell .", "words": ["off", "that", "take"]}, {"sent": "you know I thought thats why every time we go there we take you up there to have a look at the tower .", "words": ["up", "the", "go", "we", "to", "you", "have", "there", "at", "every", "look", "take", "why", "a"]}, {"sent": "Ive just one wee measurement to take .", "words": ["to", "take"]}, {"sent": "shout one more time and Ill take it away .", "words": ["and", "more", "it", "away", "take"]}, {"sent": "you must take me to him .", "words": ["to", "you", "him", "me", "take"]}, {"sent": "we are gonna ask a girl to take they panties off and den we gonna get them and den we gonna split .", "words": ["off", "we", "to", "them", "and", "get", "they", "are", "take", "a"]}, {"sent": "take it up .", "words": ["up", "it", "take"]}, {"sent": "you can take that off if you come here for a second .", "words": ["can", "off", "for", "if", "you", "here", "take", "that", "a"]}, {"sent": "whatll happen if we take your nappie off right now ?", "words": ["off", "your", "if", "we", "take"]}, {"sent": "no just open up one and take one out .", "words": ["up", "open", "and", "out", "take"]}, {"sent": "are you looking for the I dont know if that book is in this room it might be in another room William .", "words": ["the", "for", "if", "is", "be", "you", "another", "dont", "in", "are", "this", "room", "it", "book", "that"]}, {"sent": "why doesnt she like to be copied ?", "words": ["to", "be", "like", "she", "why"]}, {"sent": "but Mummyll be home at lunchtime .", "words": ["be", "at", "but", "home"]}, {"sent": "ah hah Michael whata you wanna be when you get big ?", "words": ["be", "you", "get", "big", "wanna", "when"]}, {"sent": "what could you be ?", "words": ["what", "could", "be", "you"]}, {"sent": "nice to be in the pool on a hot day huh ?", "words": ["the", "to", "be", "nice", "pool", "in", "hot", "on", "a"]}, {"sent": "I must be in the wrong house he said .", "words": ["the", "be", "in", "house", "he"]}, {"sent": "I must be all mixed up .", "words": ["all", "be", "up"]}, {"sent": "dont be naughty towards Elmo .", "words": ["dont", "be", "naughty"]}, {"sent": "if you smash it or break it you wont be able to play with it .", "words": ["if", "be", "to", "play", "with", "you", "it", "break"]}, {"sent": "Brittany dont be so ugly let her hold one of them .", "words": ["hold", "of", "be", "so", "them", "dont", "her"]}, {"sent": "its sposta be bad luck and we dont want bad luck do we in this house ?", "words": ["do", "we", "be", "bad", "and", "dont", "in", "this", "house"]}, {"sent": "xxx Nomi be careful honey .", "words": ["be", "careful"]}, {"sent": "and who do you think they be ?", "words": ["do", "think", "be", "you", "and", "they", "who"]}, {"sent": "that must be like scaffolding that we see on building sites holding up buildings .", "words": ["up", "we", "be", "like", "on", "see", "that"]}, {"sent": "and Dad gonna be whipping your butt with it .", "words": ["your", "be", "with", "and", "it"]}, {"sent": "driver tells me it may be dangerous up there careful Toby .", "words": ["up", "be", "there", "it", "careful", "me"]}, {"sent": "I dont know what could it be ?", "words": ["what", "be", "dont", "it", "could"]}, {"sent": "cool wouldnt that be neat if they could have a flying tractor ?", "words": ["if", "be", "have", "they", "tractor", "could", "that", "a"]}, {"sent": "gonna be a very boring train .", "words": ["be", "train", "a"]}, {"sent": "oh yeah hes not his bike must be off .", "words": ["his", "be", "not", "off"]}, {"sent": "it turned out to be an apple tree .", "words": ["tree", "to", "be", "an", "it", "out", "apple"]}, {"sent": "well that can be Pingus igloo cant it ?", "words": ["can", "be", "it", "that"]}, {"sent": "well Brian that must be still hungry ?", "words": ["be", "that", "hungry"]}, {"sent": "whats Trevor gonna be ?", "words": ["be"]}, {"sent": "may be I am sitting on it .", "words": ["on", "be", "it", "am"]}, {"sent": "why dont you be SleepingBeauty ?", "words": ["dont", "be", "why", "you"]}, {"sent": "now shes gonna be five .", "words": ["be"]}, {"sent": "it might be possible to go there .", "words": ["go", "to", "be", "there", "it"]}, {"sent": "your microphone has to be put somewhere else right now .", "words": ["to", "put", "be", "your"]}, {"sent": "youll be poorly if you do that .", "words": ["do", "if", "be", "you", "that"]}, {"sent": "I think that will be the last time you wear that top .", "words": ["the", "think", "be", "last", "you", "will", "that"]}, {"sent": "this could be the market uhn couldnt it ?", "words": ["the", "be", "this", "it", "could"]}, {"sent": "do you know who might be coming today ?", "words": ["do", "who", "be", "you"]}, {"sent": "s gonna be awright .", "words": ["be"]}, {"sent": "be careful dont spill it though .", "words": ["be", "dont", "it", "spill", "careful"]}, {"sent": "be careful .", "words": ["be", "careful"]}, {"sent": "what can they be ?", "words": ["can", "what", "be", "they"]}, {"sent": "wanna be an engine driver .", "words": ["be", "an", "wanna"]}, {"sent": "no Ill think youll be here .", "words": ["here", "be", "think"]}, {"sent": "be careful now .", "words": ["be", "careful"]}, {"sent": "be careful because youve got one near your back .", "words": ["your", "back", "be", "careful", "because"]}, {"sent": "be nice to her .", "words": ["to", "be", "nice", "her"]}, {"sent": "you think shell be happier there ?", "words": ["be", "think", "there", "you"]}, {"sent": "if you were big enough you wouldnt be dropping them .", "words": ["if", "be", "them", "you", "big", "were"]}, {"sent": "everything must be absolutely perfect she says .", "words": ["be", "she"]}, {"sent": "it tends to be limited in verbal output .", "words": ["to", "in", "it", "be"]}, {"sent": "you might drop them and break them and that would be very sad .", "words": ["be", "them", "and", "you", "that", "drop", "would", "sad", "break"]}, {"sent": "Teddy Teddy useta be AuntCareys .", "words": ["be"]}, {"sent": "and then he might be better .", "words": ["better", "be", "and", "he", "then"]}, {"sent": "well it wouldnt be so bad if we could tell what it is .", "words": ["what", "if", "we", "is", "be", "so", "bad", "it", "could"]}, {"sent": "that bear better be careful .", "words": ["better", "be", "bear", "careful", "that"]}, {"sent": "I dunno what do you think you wanna be ?", "words": ["what", "do", "think", "be", "you", "wanna"]}, {"sent": "and I knew you were gonna be the same today .", "words": ["the", "be", "you", "and", "same", "were"]}, {"sent": "well he cant be in the box because the Teletubbies are in the box .", "words": ["the", "box", "be", "in", "are", "he", "because"]}, {"sent": "okay let it cool a little it may be hot .", "words": ["be", "little", "hot", "it", "a"]}, {"sent": "I might be able to do it .", "words": ["to", "be", "do", "it"]}, {"sent": "put the sheep and the cow in there he aint sposta be in here .", "words": ["the", "put", "be", "and", "here", "there", "in", "sheep", "he", "cow"]}, {"sent": "say I hope hes not gonna be like some of those drivers on the television last night .", "words": ["the", "those", "of", "be", "last", "say", "like", "on", "some", "not"]}, {"sent": "be careful where you put your feet Warren because youre just standing on them .", "words": ["where", "put", "your", "be", "them", "you", "careful", "on", "because"]}, {"sent": "itll be all ready okay ?", "words": ["all", "be"]}, {"sent": "and then itll be your turn .", "words": ["be", "then", "your", "and"]}, {"sent": "Nanette someday do you think youll be big enough to have all your toys in your bedroom ?", "words": ["do", "think", "all", "your", "be", "to", "you", "have", "in", "big", "bedroom"]}, {"sent": "youll grow up to be a big boy .", "words": ["up", "to", "be", "big", "a"]}, {"sent": "you did hafta be careful .", "words": ["be", "careful", "did", "you"]}, {"sent": "well you be eating the oatmeal with me no matter how it tastes .", "words": ["the", "how", "be", "with", "you", "it", "me"]}, {"sent": "is that gonna be another street light ?", "words": ["is", "light", "be", "another", "street", "that"]}, {"sent": "but hes got to be balanced just so hasnt he ?", "words": ["to", "be", "so", "he", "but"]}, {"sent": "this should be better .", "words": ["better", "this", "be"]}, {"sent": "youll be getting your tea in a minute .", "words": ["be", "in", "your", "a"]}, {"sent": "oh well be able to do that when youre a bit older wont we ?", "words": ["do", "when", "we", "to", "be", "that", "a"]}, {"sent": "can I be first ?", "words": ["can", "be", "first"]}, {"sent": "better be careful when we go in the garden then if theres a monster in the trees hadnt we ?", "words": ["the", "better", "if", "we", "go", "be", "in", "garden", "careful", "then", "when", "a"]}, {"sent": "I think you would be very good at iceskating .", "words": ["think", "be", "you", "at", "good", "would"]}, {"sent": "you hafta be patient and wait for Sunday morning to get here .", "words": ["wait", "for", "be", "to", "and", "you", "get", "here"]}, {"sent": "these are Mrs Woods so we hafta be very careful .", "words": ["these", "we", "so", "be", "are", "careful"]}, {"sent": "yeah there might be carrots in the vegetable hummus .", "words": ["the", "be", "there", "in", "carrots"]}, {"sent": "dont drop that on the floor again else Ill be very cross .", "words": ["the", "be", "dont", "drop", "on", "that"]}, {"sent": "that would be nice so she can drive with him .", "words": ["can", "be", "so", "nice", "with", "drive", "him", "would", "that", "she"]}, {"sent": "this must be from someone who isnt too sure of Santa .", "words": ["of", "be", "this", "too", "who"]}, {"sent": "that was going to be a chair .", "words": ["to", "was", "be", "chair", "that", "a"]}, {"sent": "like youd be familiar with already ?", "words": ["be", "with", "like"]}, {"sent": "I should be talking to you more .", "words": ["to", "be", "more", "you"]}, {"sent": "Roz will be right back shell be right back .", "words": ["will", "be", "back"]}, {"sent": "would you be able to get me a tissue ?", "words": ["tissue", "to", "be", "you", "get", "would", "me", "a"]}, {"sent": "Tom might be able to mend it .", "words": ["to", "be", "it"]}, {"sent": "she says shell be back okay .", "words": ["be", "she", "back"]}, {"sent": "be careful .", "words": ["be", "careful"]}, {"sent": "if that falls on the floor and gets dirty youll be in trouble .", "words": ["the", "if", "be", "and", "in", "dirty", "on", "that"]}, {"sent": "no I dont think she wants to be in bed all alone to tell you the truth .", "words": ["the", "all", "think", "to", "be", "bed", "you", "dont", "in", "she"]}, {"sent": "you wanna be ?", "words": ["be", "wanna", "you"]}, {"sent": "were sposta be playing with the ball not the box .", "words": ["the", "box", "be", "with", "ball", "were", "not"]}, {"sent": "they needta be sent off so that .", "words": ["off", "be", "so", "they", "that"]}, {"sent": "its been a busy day the animals go home to bed theyll soon be back to play .", "words": ["the", "back", "go", "to", "be", "play", "bed", "a", "home"]}, {"sent": "okay but hell really be made outof playdough .", "words": ["be", "but"]}, {"sent": "um hm he could be definitely some of them ate both right ?", "words": ["of", "be", "them", "he", "some", "could"]}, {"sent": "you wanna be CookieMonster and Ill be Goldie ?", "words": ["be", "and", "wanna", "you"]}, {"sent": "did you tell him to be careful ?", "words": ["to", "be", "you", "him", "did", "careful"]}, {"sent": "Dimitra is sposta be coming in the morning but weve asked her if she can hold on till the afternoon because in the morning weve got a man coming with a brand new cooker .", "words": ["the", "can", "if", "hold", "is", "be", "with", "because", "in", "on", "new", "her", "she", "but", "a"]}, {"sent": "and it has to be an edge too .", "words": ["to", "be", "and", "an", "it", "too"]}, {"sent": "because what ?", "words": ["what", "because"]}, {"sent": "because its big .", "words": ["big", "because"]}, {"sent": "they lit up quite early yesterday because it wasnt a very nice day .", "words": ["up", "nice", "they", "it", "because", "a"]}, {"sent": "yeah because you got you got to stay here and hide .", "words": ["hide", "to", "you", "here", "and", "stay", "because"]}, {"sent": "because Nanna and Granddad have a big house dont they ?", "words": ["and", "have", "dont", "they", "house", "big", "because", "a"]}, {"sent": "because its half past .", "words": ["because"]}, {"sent": "because we were looking at them .", "words": ["we", "them", "at", "were", "because"]}, {"sent": "because its kind of hot .", "words": ["of", "hot", "because"]}, {"sent": "because you wont be able to put your things out on the table if my feets in the way .", "words": ["the", "put", "my", "your", "table", "if", "to", "be", "you", "in", "out", "on", "because"]}, {"sent": "well I cant really say that because its not very polite Brain .", "words": ["that", "say", "not", "because"]}, {"sent": "oh I see you cant sleep in this car because youre too big is that right ?", "words": ["is", "sleep", "car", "you", "this", "in", "that", "too", "see", "big", "because"]}, {"sent": "you can hardly tell because it has a big hat on .", "words": ["can", "you", "on", "it", "hat", "big", "because", "a"]}, {"sent": "there are some crumbs on the floor because I havent vacuumed yet .", "words": ["the", "there", "are", "on", "some", "because"]}, {"sent": "its stuck because lids are very good when they are on arent they ?", "words": ["they", "are", "stuck", "good", "on", "when", "because"]}, {"sent": "because the train set wasnt his .", "words": ["the", "his", "because", "train"]}, {"sent": "because ones splashing himself with water .", "words": ["with", "because", "water"]}, {"sent": "just because ?", "words": ["because"]}, {"sent": "I cant because youre too heavy .", "words": ["heavy", "because", "too"]}, {"sent": "because youre getting hot .", "words": ["hot", "because"]}, {"sent": "so really this is nice because .", "words": ["is", "so", "nice", "this", "because"]}, {"sent": "because theyve got sand in them .", "words": ["in", "them", "because"]}, {"sent": "because so the people can ride on it to go onto the road .", "words": ["the", "can", "go", "so", "to", "it", "on", "ride", "because"]}, {"sent": "dada because thats what Lily always says when she like just doesnt want .", "words": ["what", "like", "when", "she", "because"]}, {"sent": "because it was the fortieth raffle prize .", "words": ["the", "was", "it", "because"]}, {"sent": "lets say hello light because I cant see .", "words": ["light", "see", "say", "because"]}, {"sent": "because Mommy hugged you ?", "words": ["because", "you"]}, {"sent": "I think thats a fib because I was there .", "words": ["think", "was", "there", "because", "a"]}, {"sent": "you can see thats milk because its got a cow on it look .", "words": ["can", "cow", "you", "on", "it", "look", "milk", "see", "because", "a"]}, {"sent": "oh but hes having trouble because its so heavy .", "words": ["so", "heavy", "because", "but"]}, {"sent": "Mommys sad because she didnt get a kiss .", "words": ["get", "sad", "kiss", "she", "because", "a"]}, {"sent": "although Daddy probably wouldnt like it that theres a turtle poem on his birthday because hes not a turtle .", "words": ["like", "his", "it", "on", "turtle", "that", "not", "because", "a"]}, {"sent": "because normally pussy cats get birds but with magpies its different .", "words": ["get", "but", "with", "because"]}, {"sent": "yeah we dont really Imean he doesnt get to chance to just play with cans and jars because theyre kind of heavy and stuff .", "words": ["we", "of", "to", "play", "with", "and", "get", "dont", "he", "heavy", "because"]}, {"sent": "so look shes happy because shes got sweeties .", "words": ["so", "because", "happy", "look"]}, {"sent": "he doesnt want you to play with it because its his and he doesnt trust you .", "words": ["to", "play", "with", "you", "and", "it", "he", "his", "because"]}, {"sent": "because he didnt spill it Nathaniel .", "words": ["it", "he", "spill", "because"]}, {"sent": "they willnt fit in with chair because these are the wrong shape .", "words": ["the", "these", "fit", "with", "chair", "they", "in", "are", "because"]}, {"sent": "thats because youre a boy .", "words": ["because", "a"]}, {"sent": "but Im not because Im playing with Kate .", "words": ["because", "with", "not", "but"]}, {"sent": "because when the tree gets blown around his house will get blown around as well .", "words": ["the", "tree", "will", "get", "house", "around", "his", "when", "because"]}, {"sent": "because if it suddenly starts to rain or anything happens if the lids on tight then no water can get in the sand .", "words": ["the", "can", "if", "rain", "to", "get", "in", "it", "water", "on", "then", "because"]}, {"sent": "we just got a letter because were really bright .", "words": ["were", "because", "a", "we"]}, {"sent": "because they is sticky .", "words": ["sticky", "they", "because", "is"]}, {"sent": "because you havent picked up .", "words": ["up", "because", "you"]}, {"sent": "spiders wait mommy needs to put this on there because she needs to hold the bag with the spiders .", "words": ["the", "wait", "put", "hold", "to", "with", "this", "there", "on", "she", "because"]}, {"sent": "thats because youre turning them .", "words": ["them", "because"]}, {"sent": "put it on there and then do it because then the glue go on the table .", "words": ["the", "do", "put", "table", "go", "and", "there", "it", "on", "glue", "then", "because"]}, {"sent": "because youve had quite a lot of unhealthy things today .", "words": ["of", "a lot", "because", "a"]}, {"sent": "because it hurts .", "words": ["it", "because"]}, {"sent": "because theyre the wrong way arent they ?", "words": ["the", "they", "because"]}, {"sent": "I tapped your hand then because you bit Mummys arm .", "words": ["your", "hand", "you", "arm", "then", "because"]}, {"sent": "think we will because youre not saying an awful lot are you ?", "words": ["think", "we", "you", "an", "are", "will", "not", "because"]}, {"sent": "you got its because you got this thing on .", "words": ["on", "this", "because", "you"]}, {"sent": "I made spaghetti because I know thats your favorite .", "words": ["your", "because", "spaghetti"]}, {"sent": "because Ive gotta cut cut something out of this .", "words": ["cut", "of", "this", "out", "because"]}, {"sent": "because people do things like that Thomas .", "words": ["do", "that", "like", "because"]}, {"sent": "because you wanted to get down .", "words": ["to", "you", "get", "down", "because"]}, {"sent": "because it gets by your elbow and you hit it by mistake .", "words": ["your", "and", "by", "you", "it", "hit", "because"]}, {"sent": "he might recognize baby because he has theres a baby doll at home we play with .", "words": ["we", "play", "with", "doll", "he", "at", "home", "because", "a"]}, {"sent": "Im going to move the water as well because we dont need the water again do we ?", "words": ["the", "do", "need", "we", "to", "dont", "water", "because"]}, {"sent": "because that Pilchards got a broken tail hasnt he ?", "words": ["broken", "he", "that", "because", "a"]}, {"sent": "shes watching her because shes little ah .", "words": ["her", "because", "little"]}, {"sent": "dont put too many in because it might break .", "words": ["put", "dont", "in", "it", "too", "break", "because"]}, {"sent": "your telling her not to move because youve heard me saying dont move havent you ?", "words": ["your", "to", "you", "dont", "me", "her", "not", "because"]}, {"sent": "because hes waving byebye .", "words": ["because"]}, {"sent": "so weve got them both because you said wed lost them uhn didnt you hm ?", "words": ["so", "them", "because", "you"]}, {"sent": "Daddy went to work late this morning because he went to watch the cricket .", "words": ["the", "watch", "work", "to", "this", "he", "because"]}, {"sent": "shes bending over because shes sweeping .", "words": ["over", "because"]}, {"sent": "because ?", "words": ["because"]}, {"sent": "well um I dont think Tamar is gonna have her party Honey because its so far after her birthday now .", "words": ["think", "is", "so", "party", "have", "dont", "her", "because"]}, {"sent": "because Im partial to PaddingtonBear birthday cake .", "words": ["to", "cake", "because"]}, {"sent": "because mommy I had the headset on and he went to the bathroom and he had the microphone .", "words": ["the", "to", "and", "he", "on", "bathroom", "because"]}, {"sent": "because .", "words": ["because"]}, {"sent": "maybe thats a very good idea because everybody else would get really scared to see a crocodile in their beds .", "words": ["their", "to", "scared", "get", "in", "good", "would", "see", "because", "a"]}, {"sent": "and um so shes obsessed with she just jumps around jumps up and down and goes wiggle wiggle wiggle wiggle because Matthew when we were down there goes wiggle he doesnt talk at all thats like his word wiggle .", "words": ["up", "all", "were", "we", "so", "with", "like", "and", "there", "at", "he", "talk", "around", "his", "down", "when", "she", "because"]}, {"sent": "because it makes a bit of a mess otherwise .", "words": ["of", "it", "because", "a"]}, {"sent": "because he wanted to go to sleep .", "words": ["go", "sleep", "to", "he", "because"]}, {"sent": "but I think its confusing Mummy because we havent got a lot of these .", "words": ["these", "think", "we", "of", "a lot", "but", "because", "a"]}, {"sent": "yeah because we have ballet class later on today .", "words": ["have", "on", "because", "we"]}, {"sent": "because its a book its a its a block .", "words": ["book", "block", "because", "a"]}, {"sent": "and which kinda surprising to me because at home she kind of ignores her babies .", "words": ["of", "to", "her", "and", "at", "me", "which", "she", "because", "home"]}, {"sent": "because thats where Daddys gonna be cooking the food .", "words": ["where", "the", "be", "food", "because"]}, {"sent": "well eat a little bit more healthily at teatime because Ive got some .", "words": ["little", "eat", "more", "at", "some", "because", "a"]}, {"sent": "because hes only tiny .", "words": ["tiny", "because"]}, {"sent": "theres no reason for you because you had from three oclock until five .", "words": ["for", "because", "you"]}, {"sent": "because then you run all over town right ?", "words": ["all", "you", "run", "then", "over", "because"]}, {"sent": "because were going to hafta go to grandmas in a bit .", "words": ["go", "to", "in", "were", "because", "a"]}, {"sent": "I think because its too cold .", "words": ["think", "cold", "because", "too"]}, {"sent": "youll feel better soon because the vets gone to get you a plaster .", "words": ["the", "better", "to", "you", "get", "because", "a"]}, {"sent": "because I knew youd have lotsof things .", "words": ["have", "because"]}, {"sent": "no its not Jwww because Jwwws on holiday .", "words": ["on", "not", "because"]}, {"sent": "because look can you see this sign here ?", "words": ["can", "you", "here", "this", "look", "see", "because"]}, {"sent": "because I dont think that theyd understand .", "words": ["dont", "think", "that", "because"]}, {"sent": "xxx because its not your stuff .", "words": ["your", "not", "because"]}, {"sent": "because he could hurt you .", "words": ["hurt", "you", "he", "could", "because"]}, {"sent": "because .", "words": ["because"]}, {"sent": "because they were trying to put them all back and its really hard trying to stuff a cupboard full of ping pong balls because you cant do it with the door open .", "words": ["the", "put", "of", "hard", "door", "with", "it", "were", "all", "full", "to", "and", "you", "because", "a", "do", "back", "open", "them", "they"]}, {"sent": "because its quite near teatime .", "words": ["because"]}, {"sent": "dont touch those things because they might fall .", "words": ["those", "fall", "touch", "dont", "they", "because"]}, {"sent": "because it will spill .", "words": ["will", "it", "spill", "because"]}, {"sent": "the plate ?", "words": ["the", "plate"]}, {"sent": "no no no put it on your plate .", "words": ["put", "your", "plate", "it", "on"]}, {"sent": "dont hit that plate Thomas .", "words": ["dont", "hit", "that", "plate"]}, {"sent": "on your plate ?", "words": ["on", "your", "plate"]}, {"sent": "should put spaghetti on a plate ?", "words": ["put", "plate", "spaghetti", "on", "a"]}, {"sent": "plate plate .", "words": ["plate"]}, {"sent": "can you put your pizza on the plate ?", "words": ["can", "the", "put", "your", "plate", "pizza", "you", "on"]}, {"sent": "shall I pop it on your plate there ?", "words": ["pop", "your", "plate", "there", "it", "on"]}, {"sent": "you put put your cup on the plate Shn .", "words": ["cup", "the", "put", "your", "plate", "you", "on"]}, {"sent": "do you wanna put it on the plate ?", "words": ["the", "do", "put", "plate", "you", "it", "wanna", "on"]}, {"sent": "there what a red plate uhn isnt it ?", "words": ["red", "what", "plate", "there", "it", "a"]}, {"sent": "you gonna put them on the plate ?", "words": ["the", "put", "plate", "them", "you", "on"]}, {"sent": "shall we get a plate ?", "words": ["get", "plate", "a", "we"]}, {"sent": "we dont eat the plate though .", "words": ["the", "plate", "we", "eat", "dont"]}, {"sent": "know what this is on the plate ?", "words": ["the", "what", "plate", "is", "this", "on"]}, {"sent": "you wanna put the spaghetti on the plate ?", "words": ["the", "put", "plate", "spaghetti", "you", "wanna", "on"]}, {"sent": "wheres the three plate ?", "words": ["the", "plate"]}, {"sent": "pop some on the Thomas plate and some on the Henry plate .", "words": ["the", "pop", "plate", "and", "on", "some"]}, {"sent": "what are you going to put on my plate ?", "words": ["what", "put", "my", "plate", "to", "you", "are", "on"]}, {"sent": "dont forget the plate .", "words": ["the", "dont", "plate"]}, {"sent": "and then you can see WinnieTherPooh on the plate underneath .", "words": ["can", "the", "plate", "you", "and", "on", "see", "then"]}, {"sent": "not on the plate .", "words": ["on", "the", "plate", "not"]}, {"sent": "you can make it in that plate .", "words": ["can", "plate", "you", "in", "it", "that", "make"]}, {"sent": "youre going to put the plate on the wall ?", "words": ["the", "put", "plate", "to", "on"]}, {"sent": "I see youre putting the plate on the couch .", "words": ["the", "plate", "couch", "on", "see"]}, {"sent": "that is a plate .", "words": ["that", "plate", "a", "is"]}, {"sent": "put it on a plate then .", "words": ["put", "plate", "it", "on", "then", "a"]}, {"sent": "theres a plate for Anne and theres a plate for mummie .", "words": ["for", "plate", "and", "a"]}, {"sent": "theres a plate for you .", "words": ["for", "plate", "you", "a"]}, {"sent": "mm delicious do you need a plate ?", "words": ["do", "need", "plate", "you", "a"]}, {"sent": "Ive eaten it outof a different plate .", "words": ["it", "plate", "a"]}, {"sent": "you got a plate ?", "words": ["plate", "a", "you"]}, {"sent": "thats a plate .", "words": ["plate", "a"]}, {"sent": "like the plate ?", "words": ["the", "plate", "like"]}, {"sent": "thats the plate we bought Purdie for her birthday .", "words": ["the", "for", "plate", "we", "her"]}, {"sent": "put it on a plate for me then .", "words": ["put", "for", "plate", "it", "on", "me", "then", "a"]}, {"sent": "Im going to have my sandwich on a plate .", "words": ["my", "plate", "to", "have", "on", "sandwich", "a"]}, {"sent": "another plate .", "words": ["another", "plate"]}, {"sent": "leave them on the plate .", "words": ["on", "the", "them", "plate"]}, {"sent": "they didnt have it you got my plate .", "words": ["my", "plate", "you", "have", "they", "it"]}, {"sent": "please do not put your tongue near the plate unless your mouth is .", "words": ["the", "do", "put", "your", "plate", "is", "mouth", "tongue", "not"]}, {"sent": "youre thinking of jelly on the plate arent you ?", "words": ["jelly", "the", "plate", "of", "you", "on"]}, {"sent": "a bowl or a plate ?", "words": ["bowl", "plate", "a"]}, {"sent": "gimme the plate .", "words": ["the", "plate"]}, {"sent": "you wanna put the bread on the plate ?", "words": ["the", "put", "bread", "plate", "you", "wanna", "on"]}, {"sent": "and I think we needta buy her a new plate Thomas .", "words": ["think", "plate", "we", "and", "new", "her", "buy", "a"]}, {"sent": "a plate .", "words": ["plate", "a"]}, {"sent": "go and get a plate .", "words": ["plate", "go", "and", "get", "a"]}, {"sent": "a plate .", "words": ["plate", "a"]}, {"sent": "theres some apple on this plate as well .", "words": ["plate", "this", "on", "apple", "some"]}, {"sent": "the doggy and the eagle are eating off the same plate oh are they gonna eat some peas ?", "words": ["the", "off", "plate", "peas", "and", "eat", "they", "are", "same", "some"]}, {"sent": "plate .", "words": ["plate"]}, {"sent": "thats a picture of the Teletubbies on the bottom of your plate .", "words": ["the", "bottom", "your", "plate", "of", "picture", "on", "a"]}, {"sent": "say plate .", "words": ["say", "plate"]}, {"sent": "plate and put all your knives and forks on there .", "words": ["all", "put", "your", "plate", "and", "there", "on"]}, {"sent": "put that guy right here and give him that little plate .", "words": ["put", "plate", "and", "here", "little", "him", "give", "that"]}, {"sent": "you have a lot of things on your plate .", "words": ["your", "plate", "of", "a lot", "you", "have", "on", "a"]}, {"sent": "Brett take that plate off the table .", "words": ["off", "the", "plate", "table", "take", "that"]}, {"sent": "oh a plate oh .", "words": ["plate", "a"]}, {"sent": "put her fork on her plate and shell tell you when .", "words": ["put", "plate", "when", "and", "you", "fork", "on", "her"]}, {"sent": "put it on your plate .", "words": ["put", "your", "plate", "it", "on"]}, {"sent": "its a plate .", "words": ["plate", "a"]}, {"sent": "yeah I think maybe youll put put some on my plate put some on my plate that seems like an awful lot .", "words": ["think", "my", "put", "plate", "like", "an", "on", "some", "that"]}, {"sent": "that is a big one so maybe it is a plate .", "words": ["plate", "is", "so", "that", "it", "big", "a"]}, {"sent": "on the plate .", "words": ["on", "the", "plate"]}, {"sent": "the plate ?", "words": ["the", "plate"]}, {"sent": "plate ?", "words": ["plate"]}, {"sent": "could you pass me the plate please .", "words": ["the", "plate", "you", "me", "could"]}, {"sent": "are you going to draw one of Purdie eating a plate of food ?", "words": ["plate", "of", "to", "draw", "you", "food", "are", "a"]}, {"sent": "oh Id like to take this plate out and you prepare me a nice meal .", "words": ["plate", "to", "nice", "like", "and", "you", "this", "out", "me", "take", "a"]}, {"sent": "that is a pretty plate .", "words": ["plate", "is", "pretty", "that", "a"]}, {"sent": "watch my plate for me .", "words": ["my", "watch", "plate", "for", "me"]}, {"sent": "theres quite a lot of coleslaw still on this plate .", "words": ["plate", "of", "a lot", "this", "on", "a"]}, {"sent": "have you picked a biscuit up from Purdies plate ?", "words": ["up", "plate", "you", "have", "a"]}, {"sent": "okay turn the plate over and put the cups on it .", "words": ["the", "put", "plate", "and", "it", "on", "over"]}, {"sent": "and this ones another plate with holly berries .", "words": ["plate", "with", "and", "another", "this"]}, {"sent": "Im putting a little more meat on my plate .", "words": ["my", "plate", "meat", "little", "more", "on", "a"]}, {"sent": "or put em on your plate when you count em .", "words": ["put", "your", "plate", "you", "on", "when"]}, {"sent": "you gotta keep that in here in that plate .", "words": ["plate", "you", "here", "in", "that"]}, {"sent": "put it on the plate .", "words": ["the", "put", "plate", "it", "on"]}, {"sent": "plate .", "words": ["plate"]}, {"sent": "you can put it on the plate .", "words": ["can", "the", "put", "plate", "you", "it", "on"]}, {"sent": "theres a waffle and spaghetti on the plate .", "words": ["the", "plate", "spaghetti", "and", "on", "a"]}, {"sent": "thats the number plate .", "words": ["the", "plate"]}, {"sent": "you wanna put the pasta on the plate ?", "words": ["the", "put", "plate", "you", "wanna", "on"]}, {"sent": "thats a red plate huh ?", "words": ["red", "plate", "a"]}, {"sent": "this is a this a plate that the grown ups eat outof .", "words": ["the", "plate", "is", "eat", "this", "that", "a"]}, {"sent": "I see a little plate and wheres the big plate ?", "words": ["the", "plate", "and", "little", "see", "big", "a"]}, {"sent": "oh heres the plate .", "words": ["the", "plate"]}, {"sent": "a plate ?", "words": ["plate", "a"]}, {"sent": "its a round plate .", "words": ["plate", "a"]}, {"sent": "it looks like AuntyMabel has made some jam tarts and theyre on the plate .", "words": ["the", "plate", "like", "and", "it", "on", "some"]}, {"sent": "I think most of the crumbs went on the plate anyway didnt they ?", "words": ["the", "think", "plate", "of", "they", "on"]}, {"sent": "xxx the plate .", "words": ["the", "plate"]}, {"sent": "is that a plate ?", "words": ["that", "plate", "a", "is"]}, {"sent": "shall I put that on the plate ?", "words": ["the", "put", "plate", "on", "that"]}, {"sent": "will you help pick them up and when you pick them up pop them straight into Purdies plate like that .", "words": ["up", "help", "pop", "plate", "into", "pick", "them", "like", "you", "and", "that", "will", "when"]}, {"sent": "yup and thats what you do with your real food huh you throw it off the plate .", "words": ["off", "what", "do", "the", "your", "throw", "plate", "with", "you", "and", "food", "it"]}, {"sent": "plate .", "words": ["plate"]}, {"sent": "its on the its on the pot plate plate .", "words": ["on", "the", "plate"]}, {"sent": "now those are empty .", "words": ["are", "those", "empty"]}, {"sent": "theyre all empty Im afraid but theyve got pictures on them .", "words": ["all", "them", "empty", "on", "but"]}, {"sent": "empty .", "words": ["empty"]}, {"sent": "is it empty ?", "words": ["empty", "it", "is"]}, {"sent": "oh its empty isnt it ?", "words": ["it", "empty"]}, {"sent": "is it empty ?", "words": ["empty", "it", "is"]}, {"sent": "well thats an empty bottle but Ive saved it .", "words": ["empty", "an", "it", "bottle", "but"]}, {"sent": "empty again now .", "words": ["empty"]}, {"sent": "almost empty .", "words": ["empty"]}, {"sent": "I think it is empty .", "words": ["empty", "think", "it", "is"]}, {"sent": "what sort of shop is this selling empty food ?", "words": ["what", "is", "of", "empty", "this", "food"]}, {"sent": "its empty .", "words": ["empty"]}, {"sent": "Ill just empty this first .", "words": ["this", "first", "empty"]}, {"sent": "it was empty Thomas .", "words": ["was", "it", "empty"]}, {"sent": "its empty .", "words": ["empty"]}, {"sent": "why dont you try an empty ice cream box ?", "words": ["ice cream", "box", "you", "empty", "dont", "an", "ice", "try", "why"]}, {"sent": "the lake was empty ?", "words": ["the", "was", "empty"]}, {"sent": "well empty them back into the box .", "words": ["the", "box", "into", "back", "them", "empty"]}, {"sent": "is it an empty cup ?", "words": ["cup", "is", "empty", "an", "it"]}, {"sent": "empty box .", "words": ["box", "empty"]}, {"sent": "which what is empty ?", "words": ["empty", "what", "which", "is"]}, {"sent": "or is it empty ?", "words": ["empty", "it", "is"]}, {"sent": "the trucks empty now and all the building bricks are on the floor .", "words": ["the", "all", "and", "empty", "are", "on"]}, {"sent": "the empty one .", "words": ["the", "empty"]}, {"sent": "yay empty .", "words": ["empty"]}, {"sent": "oh the empty washing powder box ?", "words": ["the", "box", "empty"]}, {"sent": "empty crate ?", "words": ["empty"]}, {"sent": "youve just thrown the empty tube of Smarties on the floor .", "words": ["the", "of", "on", "empty"]}, {"sent": "this ones empty .", "words": ["this", "empty"]}, {"sent": "oh dont dont empty them out .", "words": ["dont", "them", "out", "empty"]}, {"sent": "so it was was empty .", "words": ["was", "so", "it", "empty"]}, {"sent": "empty right .", "words": ["empty"]}, {"sent": "its empty now .", "words": ["empty"]}, {"sent": "is it empty or has he got something in it ?", "words": ["is", "empty", "in", "it", "he"]}, {"sent": "I wonder where this door leads she says and she enters an empty recording studio .", "words": ["where", "and", "empty", "this", "an", "she", "door"]}, {"sent": "oh empty .", "words": ["empty"]}, {"sent": "theyre all empty .", "words": ["all", "empty"]}, {"sent": "its empty .", "words": ["empty"]}, {"sent": "weve actually got two empty HappyMeal boxes so that we can practice .", "words": ["can", "we", "so", "empty", "that"]}, {"sent": "youre posting carrots in an empty cornflake box ?", "words": ["box", "empty", "in", "an", "carrots"]}, {"sent": "I think thats empty Thomas .", "words": ["think", "empty"]}, {"sent": "mommy didnt empty the dishwasher this morning so .", "words": ["the", "this", "so", "empty"]}, {"sent": "its just the empty bin theyve left there .", "words": ["the", "there", "empty"]}, {"sent": "oh Dipsys hats empty now because all the Teletubbies have gone inside the box .", "words": ["the", "all", "box", "inside", "empty", "have", "because"]}, {"sent": "empty .", "words": ["empty"]}, {"sent": "I think thats empty too isnt it ?", "words": ["too", "think", "it", "empty"]}, {"sent": "whenever you empty your washing up bowl youll always find a teaspoon in the bottom .", "words": ["up", "bowl", "the", "bottom", "your", "empty", "you", "in", "find", "a"]}, {"sent": "the honey pot was empty .", "words": ["the", "was", "empty"]}, {"sent": "its empty yes .", "words": ["empty"]}, {"sent": "see his empty plate ?", "words": ["his", "see", "plate", "empty"]}, {"sent": "and those are empty because I keep them for the stamps that theyve got .", "words": ["the", "for", "those", "them", "empty", "and", "are", "that", "because"]}, {"sent": "empty gone .", "words": ["empty"]}, {"sent": "this is an empty beaker though .", "words": ["empty", "this", "an", "is"]}, {"sent": "no my tummy is not empty .", "words": ["my", "is", "empty", "tummy", "not"]}, {"sent": "yeah I think theyre empty now because theyre gonna .", "words": ["think", "because", "empty"]}, {"sent": "and its like oh yours was empty .", "words": ["empty", "was", "like", "and"]}, {"sent": "I thought it was just an empty box .", "words": ["box", "was", "empty", "an", "it"]}, {"sent": "youre busy posting all the letters into an empty Cornflake box arent you ?", "words": ["the", "all", "box", "into", "empty", "you", "an"]}, {"sent": "its empty .", "words": ["empty"]}, {"sent": "because you need a round bit to go in an empty hole dont you ?", "words": ["need", "go", "to", "you", "empty", "dont", "in", "an", "because", "a"]}, {"sent": "no I didnt empty it .", "words": ["it", "empty"]}, {"sent": "its empty .", "words": ["empty"]}, {"sent": "an empty green cereal bowl on your head as a hat .", "words": ["bowl", "your", "head", "empty", "an", "green", "hat", "on", "cereal", "a"]}, {"sent": "the pots empty .", "words": ["the", "empty"]}, {"sent": "Ill post it through the letter box and then uh no doubt when PostmanPat comes later on hell empty the letter box wont he ?", "words": ["the", "box", "and", "empty", "it", "he", "on", "then", "when"]}, {"sent": "if you pour it out the tube is then empty .", "words": ["the", "if", "is", "pour", "you", "empty", "it", "out", "then"]}, {"sent": "its empty now .", "words": ["empty"]}, {"sent": "do you wanna empty it ?", "words": ["do", "empty", "you", "it", "wanna"]}, {"sent": "oh does the postman wanna use that bag to empty the letterbox ?", "words": ["the", "does", "to", "empty", "wanna", "that"]}, {"sent": "its empty now .", "words": ["empty"]}, {"sent": "its empty .", "words": ["empty"]}, {"sent": "is it empty ?", "words": ["empty", "it", "is"]}, {"sent": "its empty .", "words": ["empty"]}, {"sent": "oh the tissue box is empty Thomas .", "words": ["the", "box", "is", "tissue", "empty"]}, {"sent": "so the zookeepers drilled holes in an empty five gallon jug and filled it with apples grapes and raisins .", "words": ["the", "so", "with", "empty", "and", "in", "an", "it", "grapes"]}, {"sent": "well youve brought the yellow wheelie bin back so Ill probably fill it with these papers while you empty the black wheelie bin .", "words": ["the", "these", "back", "so", "with", "you", "empty", "black", "it", "yellow"]}, {"sent": "the baskets empty now .", "words": ["the", "empty"]}, {"sent": "pot of marmalade nearly empty .", "words": ["of", "empty"]}, {"sent": "they empty bins .", "words": ["they", "empty"]}, {"sent": "and he gave us some empty containers that we could bring back with us so there was never any food in any of these .", "words": ["of", "any", "could", "with", "us", "there", "food", "some", "that", "these", "so", "was", "bring", "and", "back", "we", "empty", "in", "he"]}, {"sent": "you mean an empty bottle of blackcurrant juice ?", "words": ["of", "empty", "you", "an", "bottle", "juice"]}, {"sent": "oh empty pot .", "words": ["empty"]}, {"sent": "its empty .", "words": ["empty"]}, {"sent": "its all empty .", "words": ["all", "empty"]}, {"sent": "just empty the garbage .", "words": ["the", "garbage", "empty"]}, {"sent": "where youve eaten them and the box or the bags empty .", "words": ["where", "the", "box", "them", "and", "empty"]}, {"sent": "thats empty .", "words": ["empty"]}, {"sent": "its empty .", "words": ["empty"]}, {"sent": "and Im struggling now with this because I cant remember how to empty it .", "words": ["how", "to", "with", "and", "empty", "this", "it", "because"]}, {"sent": "just an empty box ?", "words": ["an", "box", "empty"]}, {"sent": "the attic was empty .", "words": ["the", "was", "empty"]}, {"sent": "your bottle is empty ?", "words": ["empty", "bottle", "your", "is"]}, {"sent": "say empty .", "words": ["say", "empty"]}, {"sent": "you come back for this when your hands are empty .", "words": ["for", "your", "back", "you", "empty", "this", "are", "when"]}, {"sent": "well now its all empty thats hot water Shem you dont wanna drink that here you want me to get you a clean glass ?", "words": ["clean", "all", "drink", "to", "you", "empty", "dont", "hot", "here", "get", "glass", "wanna", "water", "me", "that", "a"]}, {"sent": "around the world on an empty .", "words": ["the", "empty", "an", "around", "on"]}, {"sent": "that ones empty and that ones better .", "words": ["better", "that", "and", "empty"]}, {"sent": "its just empty .", "words": ["empty"]}, {"sent": "empty the puzzle out .", "words": ["the", "puzzle", "out", "empty"]}, {"sent": "its empty .", "words": ["empty"]}, {"sent": "Ill make you a drink .", "words": ["drink", "make", "a", "you"]}, {"sent": "oh we cant drink a car .", "words": ["car", "a", "drink", "we"]}, {"sent": "you wanna drink it ?", "words": ["it", "drink", "wanna", "you"]}, {"sent": "are you giving mommy drink ?", "words": ["are", "drink", "you"]}, {"sent": "watch you do knock your drink on the floor please Thomas .", "words": ["the", "do", "watch", "your", "drink", "you", "knock", "on"]}, {"sent": "can you give him another drink ?", "words": ["can", "drink", "you", "another", "him", "give"]}, {"sent": "xxx you dont even drink you dont like coke .", "words": ["drink", "like", "you", "coke", "dont"]}, {"sent": "wanna drink it ?", "words": ["it", "drink", "wanna"]}, {"sent": "get rid of these cups so you can drink your milk .", "words": ["can", "these", "your", "drink", "of", "so", "you", "get", "milk"]}, {"sent": "take another drink .", "words": ["another", "take", "drink"]}, {"sent": "drink Tab ?", "words": ["drink"]}, {"sent": "are you gonna give it a drink ?", "words": ["drink", "you", "it", "are", "give", "a"]}, {"sent": "whos gonna drink ?", "words": ["drink"]}, {"sent": "drink water ?", "words": ["drink", "water"]}, {"sent": "give him a big drink too .", "words": ["drink", "him", "too", "give", "big", "a"]}, {"sent": "a big drink break too huh ?", "words": ["drink", "big", "too", "break", "a"]}, {"sent": "what do I hafta drink ?", "words": ["what", "do", "drink"]}, {"sent": "they dont drink that bug juice ?", "words": ["bug", "drink", "dont", "they", "juice", "that"]}, {"sent": "you wanna give the baby a drink ?", "words": ["the", "drink", "you", "wanna", "give", "a"]}, {"sent": "can the crocodile drink ?", "words": ["can", "the", "drink"]}, {"sent": "but you still drink milk at home .", "words": ["drink", "you", "at", "milk", "but", "home"]}, {"sent": "what did you drink ?", "words": ["what", "did", "drink", "you"]}, {"sent": "drink of juice .", "words": ["of", "juice", "drink"]}, {"sent": "a drink .", "words": ["drink", "a"]}, {"sent": "that isnt what you should drink .", "words": ["what", "that", "drink", "you"]}, {"sent": "every time we need a drink we just drink the rain .", "words": ["the", "need", "drink", "we", "rain", "every", "a"]}, {"sent": "you want a drink ?", "words": ["drink", "a", "you"]}, {"sent": "drink drink ?", "words": ["drink"]}, {"sent": "would you like something else to drink Thomas ?", "words": ["drink", "to", "like", "you", "would"]}, {"sent": "you can bring your own pram but they supply you with them and all and if you want becauseof the heat and theres plenty of water stops plenty of places to eat and drink .", "words": ["can", "all", "the", "your", "if", "drink", "of", "to", "with", "bring", "them", "you", "and", "they", "eat", "water", "but"]}, {"sent": "and then if you sit with a drink of milk and I have my cup of tea we can have a look at things like that cant we ?", "words": ["cup", "drink", "of", "like", "milk", "can", "with", "have", "then", "that", "look", "if", "you", "and", "a", "my", "we", "at", "sit"]}, {"sent": "what about your drink ?", "words": ["what", "drink", "your", "about"]}, {"sent": "she drink a bottle .", "words": ["bottle", "she", "drink", "a"]}, {"sent": "well do little boys drink coffee ?", "words": ["do", "coffee", "drink", "little"]}, {"sent": "the fish drink it uhn do they ?", "words": ["the", "do", "drink", "they", "it", "fish"]}, {"sent": "they take it out using their hands and then they drink it .", "words": ["their", "drink", "and", "they", "it", "out", "then", "take"]}, {"sent": "lets go in and get a drink .", "words": ["drink", "go", "and", "get", "in", "a"]}, {"sent": "I want a drink .", "words": ["drink", "a"]}, {"sent": "you want something to drink ?", "words": ["to", "drink", "you"]}, {"sent": "giving the bear a drink isnt she ?", "words": ["the", "drink", "bear", "she", "a"]}, {"sent": "a drink .", "words": ["drink", "a"]}, {"sent": "would you like chocolate milk or would you like juice to drink ?", "words": ["drink", "to", "chocolate", "like", "you", "would", "milk", "juice"]}, {"sent": "they sat in front of the fire and had a warm drink .", "words": ["the", "drink", "of", "and", "they", "in", "a"]}, {"sent": "shall we stop and have that drink now ?", "words": ["drink", "we", "stop", "and", "have", "that"]}, {"sent": "what does he drink ?", "words": ["what", "does", "he", "drink"]}, {"sent": "you want a drink ?", "words": ["drink", "a", "you"]}, {"sent": "giving Gary a drink of water .", "words": ["of", "water", "drink", "a"]}, {"sent": "you drink water out of that ?", "words": ["drink", "of", "you", "out", "water", "that"]}, {"sent": "oh is that for me to drink ?", "words": ["for", "drink", "is", "to", "me", "that"]}, {"sent": "pick up your drink .", "words": ["pick", "up", "your", "drink"]}, {"sent": "would you like a drink of tea or coffee ?", "words": ["drink", "of", "like", "you", "coffee", "would", "a"]}, {"sent": "can I have a drink of Daddys fruit juice ?", "words": ["can", "drink", "of", "have", "juice", "a"]}, {"sent": "dont you wanna drink your cranberry juice and eat your piece of .", "words": ["your", "drink", "of", "you", "and", "dont", "eat", "wanna", "juice"]}, {"sent": "now Ill just get a drink .", "words": ["get", "drink", "a"]}, {"sent": "what do you hafta drink ?", "words": ["what", "do", "drink", "you"]}, {"sent": "and then its put in bottles and then you drink the milk .", "words": ["the", "put", "drink", "you", "and", "in", "milk", "then"]}, {"sent": "juice they dont have any milk you hafta drink juice .", "words": ["drink", "any", "you", "have", "dont", "they", "milk", "juice"]}, {"sent": "dont drink all your juice because theres no juice either .", "words": ["all", "your", "drink", "dont", "juice", "because"]}, {"sent": "okay and anything anything to drink ?", "words": ["to", "drink", "and"]}, {"sent": "no you dont drink grape juice with a spoon .", "words": ["drink", "with", "you", "dont", "spoon", "juice", "a"]}, {"sent": "no youre not sposta drink that .", "words": ["that", "not", "drink"]}, {"sent": "what do you think the children would like to drink ?", "words": ["the", "what", "do", "think", "drink", "to", "like", "you", "would"]}, {"sent": "what do you drink ?", "words": ["what", "do", "drink", "you"]}, {"sent": "there drink your juice .", "words": ["there", "juice", "your", "drink"]}, {"sent": "Id take your drink away but we cant actually hear what youre going to say now .", "words": ["what", "your", "drink", "we", "to", "say", "hear", "away", "take", "but"]}, {"sent": "oh well have a good drink first .", "words": ["drink", "first", "have", "good", "a"]}, {"sent": "Mummyll just have a drink first out of the glass so there isnt quite as much to hold .", "words": ["the", "hold", "drink", "of", "so", "to", "first", "have", "there", "glass", "out", "much", "a"]}, {"sent": "and would you like a drink now ?", "words": ["drink", "like", "you", "and", "would", "a"]}, {"sent": "wow they must have some really big mosquitos in this drink .", "words": ["drink", "have", "they", "in", "this", "some", "big"]}, {"sent": "youre not to drink it yet .", "words": ["to", "it", "not", "drink"]}, {"sent": "I like we always useta drink coffee .", "words": ["coffee", "like", "drink", "we"]}, {"sent": "theres your drink .", "words": ["your", "drink"]}, {"sent": "do you want a drink ?", "words": ["do", "drink", "a", "you"]}, {"sent": "you gotta drink that juice .", "words": ["juice", "that", "drink", "you"]}, {"sent": "youre going to sit in your chair read your book and have a drink of milk .", "words": ["milk", "your", "drink", "of", "to", "chair", "and", "have", "in", "read", "book", "sit", "a"]}, {"sent": "and this drink of juice leaked again didnt it ?", "words": ["drink", "of", "and", "this", "it", "juice"]}, {"sent": "is she going to get a drink from the sink ?", "words": ["the", "drink", "is", "to", "get", "sink", "she", "a"]}, {"sent": "its a drink you mix .", "words": ["you", "drink", "a"]}, {"sent": "a drink ?", "words": ["drink", "a"]}, {"sent": "want a drink now .", "words": ["drink", "a"]}, {"sent": "like you would with the juice when you drink it .", "words": ["the", "drink", "with", "like", "you", "it", "would", "juice", "when"]}, {"sent": "xxx drink through that way ?", "words": ["that", "drink"]}, {"sent": "drink outof the cup .", "words": ["the", "cup", "drink"]}, {"sent": "and a drink of orange juice .", "words": ["drink", "of", "and", "orange", "juice", "a"]}, {"sent": "drink that good chocolate milk .", "words": ["drink", "chocolate", "good", "milk", "that"]}, {"sent": "what are you gonna drink ?", "words": ["what", "are", "drink", "you"]}, {"sent": "after his long journey Paddington felt so hungry and thirsty he didnt know which to do first eat or drink .", "words": ["do", "drink", "so", "to", "first", "hungry", "long", "and", "thirsty", "eat", "he", "his", "which"]}, {"sent": "penguins drink .", "words": ["drink"]}, {"sent": "you didnt buy a drink ?", "words": ["drink", "buy", "a", "you"]}, {"sent": "you wanna pour you a drink ?", "words": ["drink", "pour", "you", "wanna", "a"]}, {"sent": "and what has happened to your drink of juice ?", "words": ["what", "your", "drink", "of", "to", "and", "juice"]}, {"sent": "drink up .", "words": ["up", "drink"]}, {"sent": "oh drink that nicely please .", "words": ["that", "drink"]}, {"sent": "you put a drink in it .", "words": ["put", "drink", "you", "in", "it", "a"]}, {"sent": "its a bottle for a baby to drink outof .", "words": ["for", "drink", "to", "bottle", "a"]}, {"sent": "did he drink all his coffee ?", "words": ["all", "drink", "coffee", "he", "did", "his"]}, {"sent": "Ill give him a drink .", "words": ["give", "him", "drink", "a"]}, {"sent": "you could drink it up if you wanted .", "words": ["up", "if", "drink", "you", "it", "could"]}, {"sent": "you color wanna make a sun ?", "words": ["sun", "you", "wanna", "make", "a"]}, {"sent": "does he wanna look out the window like the fox ?", "words": ["the", "does", "look", "like", "he", "out", "wanna", "window"]}, {"sent": "we dont wanna open them beforehand or hell be .", "words": ["we", "open", "be", "them", "dont", "wanna"]}, {"sent": "dyou wanna look through that ?", "words": ["that", "wanna", "look"]}, {"sent": "you wanna do that ?", "words": ["do", "that", "wanna", "you"]}, {"sent": "do you wanna do you wanna play shop or something ?", "words": ["do", "play", "wanna", "you"]}, {"sent": "wanna put the lid on ?", "words": ["the", "put", "on", "wanna"]}, {"sent": "you wanna give Poy a cookie ?", "words": ["cookie", "you", "wanna", "give", "a"]}, {"sent": "hey Max heres another one you wanna get this one ?", "words": ["you", "another", "this", "get", "wanna"]}, {"sent": "sure you dont wanna play with it ?", "words": ["play", "with", "you", "dont", "it", "wanna"]}, {"sent": "do you wanna see the beetle ?", "words": ["the", "do", "you", "wanna", "see"]}, {"sent": "do you want help or do you wanna do it yourself ?", "words": ["do", "help", "yourself", "you", "it", "wanna"]}, {"sent": "I I wanna eat you up .", "words": ["eat", "up", "wanna", "you"]}, {"sent": "wanna look at some of the other ones ?", "words": ["the", "look", "of", "other", "wanna", "at", "some"]}, {"sent": "you wanna do it ?", "words": ["do", "it", "wanna", "you"]}, {"sent": "well if you wanna go see Theo we hafta go see Theo later .", "words": ["if", "go", "we", "you", "wanna", "see"]}, {"sent": "you wanna try and put this all the way together ?", "words": ["the", "all", "put", "and", "you", "this", "wanna", "try"]}, {"sent": "I wanna make it for you .", "words": ["for", "you", "it", "wanna", "make"]}, {"sent": "wanna go to the cinema and see that film ?", "words": ["the", "go", "to", "and", "wanna", "see", "that"]}, {"sent": "you wanna clean it you wanna clean the track ?", "words": ["clean", "the", "you", "it", "wanna"]}, {"sent": "do you wanna try that ?", "words": ["do", "try", "you", "wanna", "that"]}, {"sent": "you wanna make a sandwich or do you wanna make a hamburger ?", "words": ["do", "you", "hamburger", "wanna", "make", "sandwich", "a"]}, {"sent": "you wanna still talk to your Wilbur ?", "words": ["your", "to", "you", "talk", "wanna"]}, {"sent": "you wanna call me Eliza though ?", "words": ["me", "wanna", "you"]}, {"sent": "or do you wanna do some numbers ?", "words": ["do", "some", "wanna", "you"]}, {"sent": "okay what do you wanna do to your fingers ?", "words": ["what", "do", "your", "to", "you", "wanna"]}, {"sent": "I wanna get outof here .", "words": ["get", "here", "wanna"]}, {"sent": "they also wanna learn about the speech and how it develops .", "words": ["the", "how", "and", "they", "it", "about", "wanna"]}, {"sent": "we dont really wanna climb up there do we ?", "words": ["up", "do", "we", "dont", "there", "wanna", "climb"]}, {"sent": "and this is if you wanna color where you need to sit .", "words": ["where", "need", "if", "is", "need to", "to", "you", "and", "this", "wanna", "sit"]}, {"sent": "you gotta work very fast though because people wanna live in these houses and buy them so you gotta work quick uhn havent you ?", "words": ["these", "fast", "work", "so", "them", "you", "and", "in", "wanna", "buy", "because"]}, {"sent": "wanna make some fences ?", "words": ["some", "make", "wanna"]}, {"sent": "you wanna pour it into the bowl ?", "words": ["the", "bowl", "into", "pour", "you", "it", "wanna"]}, {"sent": "I think I know the only word you wanna say today .", "words": ["the", "think", "say", "you", "wanna"]}, {"sent": "wanna play with something else ?", "words": ["play", "with", "wanna"]}, {"sent": "do you not wanna play with fuzzy felts anymore ?", "words": ["do", "play", "with", "you", "wanna", "not"]}, {"sent": "you wanna put these things in the bag too ?", "words": ["the", "these", "put", "you", "in", "wanna", "too"]}, {"sent": "do you wanna get some toys can daddy go get some of your toys ?", "words": ["can", "do", "your", "go", "of", "you", "get", "wanna", "some"]}, {"sent": "you wanna do touch and tell ?", "words": ["do", "touch", "and", "you", "wanna"]}, {"sent": "when you wanna be silly .", "words": ["be", "when", "wanna", "you"]}, {"sent": "wanna listen to the recorder a little later ?", "words": ["the", "listen", "to", "little", "wanna", "a"]}, {"sent": "oh you wanna get picked up oh that was in the way .", "words": ["up", "the", "was", "you", "get", "in", "wanna", "that"]}, {"sent": "well its its like up to the people if they wanna invite me .", "words": ["up", "the", "if", "to", "like", "they", "wanna", "me"]}, {"sent": "you wanna stand up or do you wanna sit down ?", "words": ["up", "do", "stand", "you", "wanna", "sit", "down"]}, {"sent": "what ya wanna xxx ?", "words": ["what", "wanna"]}, {"sent": "you dont wanna play with toys ?", "words": ["play", "with", "you", "dont", "wanna"]}, {"sent": "here you wanna talk to Mamaw ?", "words": ["to", "you", "here", "talk", "wanna"]}, {"sent": "do you wanna take this off ?", "words": ["off", "do", "you", "this", "wanna", "take"]}, {"sent": "wanna see what I can do ?", "words": ["can", "what", "do", "wanna", "see"]}, {"sent": "you wanna color ?", "words": ["wanna", "you"]}, {"sent": "I wanna go yeah .", "words": ["wanna", "go"]}, {"sent": "do you wanna play Snap ?", "words": ["do", "play", "wanna", "you"]}, {"sent": "this oh you wanna see what I have here .", "words": ["what", "you", "have", "this", "here", "wanna", "see"]}, {"sent": "xxx you wanna do ?", "words": ["do", "wanna", "you"]}, {"sent": "what is it you wanna make ?", "words": ["what", "is", "you", "it", "wanna", "make"]}, {"sent": "wanna do this jigsawpuzzle with Mummy ?", "words": ["do", "with", "wanna", "this"]}, {"sent": "you wanna give the baby a kiss ?", "words": ["the", "you", "wanna", "give", "kiss", "a"]}, {"sent": "well what do you wanna do now just before you go back to school ?", "words": ["what", "do", "back", "go", "to", "school", "you", "wanna"]}, {"sent": "I wanna eat you .", "words": ["eat", "wanna", "you"]}, {"sent": "or do they wanna ride on the train ?", "words": ["the", "do", "train", "they", "wanna", "on", "ride"]}, {"sent": "do you wanna come see me ?", "words": ["do", "me", "you", "wanna", "see"]}, {"sent": "if you wanna stand on something stand on chair .", "words": ["if", "stand", "chair", "you", "wanna", "on"]}, {"sent": "or do you wanna have soup and then well have pizza on the side .", "words": ["the", "do", "pizza", "you", "have", "and", "on", "wanna", "soup", "then"]}, {"sent": "wanna get on ?", "words": ["get", "on", "wanna"]}, {"sent": "you wanna put little balls on the Christmas tree ?", "words": ["the", "put", "tree", "little", "you", "wanna", "on"]}, {"sent": "you wanna get over there ?", "words": ["you", "get", "there", "wanna", "over"]}, {"sent": "you wanna get the milk ?", "words": ["the", "you", "get", "wanna", "milk"]}, {"sent": "do you wanna sit down here ?", "words": ["do", "you", "here", "wanna", "sit", "down"]}, {"sent": "you wanna come outside with me ?", "words": ["outside", "with", "you", "wanna", "me"]}, {"sent": "wanna get in the window and have a look ?", "words": ["the", "and", "get", "have", "in", "wanna", "look", "window", "a"]}, {"sent": "do you wanna see what else is in here ?", "words": ["do", "what", "is", "you", "here", "in", "wanna", "see"]}, {"sent": "do you wanna show John some of your best blocks ?", "words": ["do", "your", "show", "of", "you", "wanna", "some"]}, {"sent": "wanna open the book ?", "words": ["open", "the", "book", "wanna"]}, {"sent": "go get that crayon if you wanna .", "words": ["if", "go", "crayon", "you", "get", "wanna", "that"]}, {"sent": "wanna give the chicken something to eat ?", "words": ["the", "to", "eat", "wanna", "chicken", "give"]}, {"sent": "you wanna get in there Henry ?", "words": ["you", "get", "there", "in", "wanna"]}, {"sent": "do you wanna wear this one today ?", "words": ["do", "this", "wanna", "you"]}, {"sent": "and when they wanna eat they can come out and get the food .", "words": ["can", "the", "and", "eat", "they", "get", "food", "wanna", "out", "when"]}, {"sent": "so what do you wanna do ?", "words": ["what", "do", "so", "you", "wanna"]}, {"sent": "you wanna rest ?", "words": ["wanna", "you"]}, {"sent": "wanna eat some dinner on the helicopter .", "words": ["the", "helicopter", "eat", "wanna", "on", "some"]}, {"sent": "wanna get it ?", "words": ["get", "it", "wanna"]}, {"sent": "I think they wanna come out and play .", "words": ["think", "play", "and", "they", "out", "wanna"]}, {"sent": "do they wanna go to sleep ?", "words": ["do", "go", "sleep", "to", "they", "wanna"]}, {"sent": "what do you see that you wanna play with ?", "words": ["what", "do", "play", "with", "you", "wanna", "see", "that"]}, {"sent": "you wanna use that one ?", "words": ["that", "wanna", "you"]}, {"sent": "dont you wanna see what else is in here ?", "words": ["what", "is", "you", "here", "dont", "in", "wanna", "see"]}, {"sent": "do you wanna sit down and tell me ?", "words": ["do", "you", "and", "wanna", "me", "sit", "down"]}, {"sent": "you wanna be right here ?", "words": ["here", "be", "wanna", "you"]}, {"sent": "you wanna go look out the window ?", "words": ["the", "go", "you", "out", "look", "wanna", "window"]}, {"sent": "you wanna share them ?", "words": ["them", "share", "wanna", "you"]}, {"sent": "you wanna go in the window ?", "words": ["the", "go", "you", "in", "wanna", "window"]}, {"sent": "do you wanna sit with Mummy ?", "words": ["do", "with", "you", "wanna", "sit"]}, {"sent": "oh you wanna do it ?", "words": ["do", "it", "wanna", "you"]}, {"sent": "you wanna play with your trains honey ?", "words": ["your", "play", "with", "you", "wanna"]}, {"sent": "wanna put the truck in the basket ?", "words": ["the", "put", "truck", "in", "basket", "wanna"]}, {"sent": "now what do you wanna do the shirt too ?", "words": ["the", "what", "do", "shirt", "you", "wanna", "too"]}, {"sent": "I dont wanna know what he was doing .", "words": ["what", "was", "dont", "he", "wanna"]}, {"sent": "what do you wanna do with him ?", "words": ["what", "do", "with", "you", "him", "wanna"]}, {"sent": "I wanna eat you all up .", "words": ["up", "all", "you", "eat", "wanna"]}, {"sent": "youre gonna drop it .", "words": ["it", "drop"]}, {"sent": "yeah but be careful you dont drop it .", "words": ["be", "you", "dont", "it", "drop", "careful", "but"]}, {"sent": "so maybe Ill drop it off um maybe Ill drop it off tonight .", "words": ["off", "so", "it", "drop"]}, {"sent": "if you drop your juice on the floor and the lid comes off thats the end .", "words": ["the", "off", "your", "if", "you", "and", "drop", "on", "juice"]}, {"sent": "did you drop baby ?", "words": ["drop", "did", "you"]}, {"sent": "I think its time you had a drop more .", "words": ["think", "you", "more", "drop", "a"]}, {"sent": "just drop him .", "words": ["him", "drop"]}, {"sent": "because the thing is people going on the beach often drop coins and things like that .", "words": ["the", "is", "like", "and", "beach", "drop", "on", "that", "because"]}, {"sent": "are you having a drop of this nice orange juice ?", "words": ["of", "nice", "you", "this", "are", "orange", "drop", "juice", "a"]}, {"sent": "the petals were starting to drop off .", "words": ["the", "off", "to", "drop", "were"]}, {"sent": "and did he drop the can ?", "words": ["the", "can", "and", "he", "drop", "did"]}, {"sent": "did you drop it ?", "words": ["it", "drop", "did", "you"]}, {"sent": "make sure you dont drop them .", "words": ["them", "you", "dont", "drop", "make"]}, {"sent": "xxx and make it drop into something .", "words": ["into", "and", "it", "drop", "make"]}, {"sent": "you dont needta drop them though .", "words": ["dont", "them", "drop", "you"]}, {"sent": "how did you drop it .", "words": ["how", "you", "it", "drop", "did"]}, {"sent": "drop them in .", "words": ["in", "them", "drop"]}, {"sent": "so we drop them off on the way round and pick them up on the way back .", "words": ["off", "the", "up", "back", "we", "pick", "so", "them", "and", "drop", "on"]}, {"sent": "dont drop it again .", "words": ["dont", "it", "drop"]}, {"sent": "not till weve done this because we might drop it .", "words": ["we", "this", "it", "drop", "not", "because"]}, {"sent": "make sure the box doesnt drop .", "words": ["the", "box", "make", "drop"]}, {"sent": "and dont drop it again because Im not gonna give you mine .", "words": ["and", "you", "dont", "mine", "it", "drop", "give", "not", "because"]}, {"sent": "I just drop them because it gives him a feeling of .", "words": ["of", "them", "him", "it", "drop", "because", "a"]}, {"sent": "drop .", "words": ["drop"]}, {"sent": "dont drop it okay ?", "words": ["dont", "it", "drop"]}, {"sent": "so do dont drop crayons .", "words": ["dont", "so", "do", "drop"]}, {"sent": "dont drop it because they break easy if you drop it .", "words": ["if", "you", "dont", "they", "it", "drop", "break", "because"]}, {"sent": "mind you dont drop the meat outof it .", "words": ["the", "meat", "you", "dont", "it", "drop"]}, {"sent": "did we drop a couple of things ?", "words": ["we", "of", "drop", "did", "a"]}, {"sent": "so that it doesnt slam shut and drop trap your fingers .", "words": ["your", "so", "and", "it", "drop", "that"]}, {"sent": "drop it .", "words": ["it", "drop"]}, {"sent": "hunhunh Brittany dont drop it .", "words": ["dont", "it", "drop"]}, {"sent": "lets not drop them on the floor .", "words": ["the", "them", "drop", "on", "not"]}, {"sent": "watch you dont drop them .", "words": ["watch", "them", "you", "dont", "drop"]}, {"sent": "drop .", "words": ["drop"]}, {"sent": "did you drop it over the side again ?", "words": ["the", "you", "it", "drop", "did", "over"]}, {"sent": "hes obviously waiting for the postman to drop a letter through the door .", "words": ["the", "for", "to", "drop", "door", "a"]}, {"sent": "a drop more bread .", "words": ["bread", "more", "drop", "a"]}, {"sent": "I think well just pop a drop more sugar in that Thomas .", "words": ["think", "pop", "more", "in", "drop", "that", "a"]}, {"sent": "drop something ?", "words": ["drop"]}, {"sent": "drop em .", "words": ["drop"]}, {"sent": "did you drop Daddys ring ?", "words": ["drop", "did", "you"]}, {"sent": "but sure drop the tape over .", "words": ["the", "tape", "drop", "over", "but"]}, {"sent": "probably drop Jwww off at school and then well probably go straight round there .", "words": ["off", "go", "school", "and", "there", "drop", "at", "then"]}, {"sent": "did you drop it ?", "words": ["it", "drop", "did", "you"]}, {"sent": "did you drop it on the floor ?", "words": ["the", "you", "it", "drop", "did", "on"]}, {"sent": "otherwise I will drop you .", "words": ["will", "drop", "you"]}, {"sent": "dont drop it on your toe .", "words": ["your", "toe", "dont", "it", "drop", "on"]}, {"sent": "careful watch you dont drop it darling .", "words": ["watch", "you", "dont", "it", "drop", "careful"]}, {"sent": "youve got to be holding the eggs before you can drop them .", "words": ["the", "can", "to", "be", "them", "you", "drop"]}, {"sent": "dont drop them .", "words": ["dont", "them", "drop"]}, {"sent": "dont drop your juice .", "words": ["dont", "juice", "your", "drop"]}, {"sent": "but I didnt drop anything .", "words": ["drop", "but"]}, {"sent": "dont drop it .", "words": ["dont", "it", "drop"]}, {"sent": "if you drop things into it like that .", "words": ["if", "into", "like", "you", "it", "drop", "that"]}, {"sent": "wow she gave you the good job dont drop that hold it real tight .", "words": ["the", "hold", "you", "dont", "it", "drop", "good", "that", "she"]}, {"sent": "did you drop it one on the floor ?", "words": ["the", "you", "it", "drop", "did", "on"]}, {"sent": "when Jason gets close hell drop the ball and keep running .", "words": ["the", "and", "ball", "drop", "close", "when"]}, {"sent": "dont drop that .", "words": ["dont", "that", "drop"]}, {"sent": "youve got to be very careful not to drop that torch because if you do the bulb inside will break and then it wont work .", "words": ["the", "do", "break", "work", "if", "to", "be", "inside", "you", "and", "it", "drop", "careful", "will", "then", "that", "not", "because"]}, {"sent": "you dont think I drop you off ?", "words": ["off", "think", "you", "dont", "drop"]}, {"sent": "dont drop it on the floor .", "words": ["the", "dont", "it", "drop", "on"]}, {"sent": "one drop .", "words": ["drop"]}, {"sent": "xxx wee drop of milk a wee drop of butter and throw it in the microwave .", "words": ["the", "throw", "of", "and", "in", "it", "butter", "drop", "milk", "a"]}, {"sent": "oh dont drop em pour em out .", "words": ["dont", "pour", "drop", "out"]}, {"sent": "did you drop it ?", "words": ["it", "drop", "did", "you"]}, {"sent": "oh its gonna just drop the chick over there in the forest .", "words": ["the", "there", "in", "drop", "over"]}, {"sent": "it didnt drop ?", "words": ["it", "drop"]}, {"sent": "drop the Binky ?", "words": ["the", "drop"]}, {"sent": "dont drop .", "words": ["dont", "drop"]}, {"sent": "I didnt drop any anywhere did I ?", "words": ["did", "any", "drop"]}, {"sent": "be careful not to drop it because its glass .", "words": ["to", "be", "it", "drop", "glass", "careful", "not", "because"]}, {"sent": "sure well make another one this afternoon if you drop if .", "words": ["if", "you", "another", "this", "drop", "make"]}, {"sent": "if you drop that down there you cant play with it .", "words": ["if", "play", "with", "you", "there", "it", "drop", "down", "that"]}, {"sent": "and if we drop it well break it .", "words": ["if", "we", "and", "it", "drop", "break"]}, {"sent": "youll drop them on the floor .", "words": ["on", "the", "them", "drop"]}, {"sent": "mustnt drop it .", "words": ["it", "drop"]}, {"sent": "he was just going to drop dead with a heart attack .", "words": ["to", "was", "with", "he", "drop", "a"]}, {"sent": "lotsof big lorrys drop parcels off at Stephens house and then he loads them into his car and delivers them to other people .", "words": ["off", "into", "car", "to", "them", "and", "other", "house", "drop", "he", "at", "his", "then", "big"]}, {"sent": "dont drop it .", "words": ["dont", "it", "drop"]}, {"sent": "what happens when you drop your paint can ?", "words": ["can", "what", "your", "paint", "you", "drop", "when"]}, {"sent": "dont drop it all .", "words": ["all", "dont", "it", "drop"]}, {"sent": "if you drop water on the bathroom floor .", "words": ["the", "if", "you", "drop", "water", "on", "bathroom"]}, {"sent": "because I might drop you .", "words": ["drop", "because", "you"]}, {"sent": "you drop them off there and then pick them up and the end of the morning .", "words": ["off", "up", "the", "pick", "of", "them", "you", "and", "there", "drop", "then"]}, {"sent": "did you drop one ?", "words": ["drop", "did", "you"]}, {"sent": "drop it down .", "words": ["down", "it", "drop"]}, {"sent": "dont drop them .", "words": ["dont", "them", "drop"]}, {"sent": "so they drop Barbara off on the way .", "words": ["off", "the", "so", "they", "drop", "on"]}, {"sent": "and once youve dug up all the trees you could drop them straight into the box couldnt you ?", "words": ["up", "all", "the", "box", "into", "them", "you", "and", "drop", "could"]}, {"sent": "you should never drop a cat on your big policeman brothers head .", "words": ["your", "head", "you", "cat", "drop", "on", "big", "a"]}, {"sent": "watch you dont drop them on your foot because itll hurt wont it ?", "words": ["watch", "your", "hurt", "them", "you", "dont", "foot", "it", "drop", "on", "because"]}, {"sent": "did you drop it ?", "words": ["it", "drop", "did", "you"]}, {"sent": "well pop a drop of milk into this milk jug and leave it on the table outside .", "words": ["the", "pop", "into", "table", "of", "outside", "and", "this", "on", "it", "drop", "milk", "a"]}, {"sent": "are you about to drop it on the floor again ?", "words": ["the", "to", "you", "are", "it", "drop", "about", "on"]}, {"sent": "a wee drop smidgen of bleach and that .", "words": ["of", "and", "drop", "that", "a"]}, {"sent": "whatever you do dont drop it now .", "words": ["do", "you", "dont", "it", "drop"]}, {"sent": "Eleanor dont drop it on the floor darling .", "words": ["the", "dont", "it", "drop", "on"]}, {"sent": "ei one drop in .", "words": ["in", "drop"]}, {"sent": "you can put your hands in those puppets and then they can open their mouths .", "words": ["can", "put", "your", "those", "their", "open", "you", "and", "they", "in", "then"]}, {"sent": "and now for the dancing pants doing their fabulous dance .", "words": ["the", "for", "their", "dance", "and", "pants"]}, {"sent": "you can put it on when we go to their house you can go upstairs and surprise Daddy that you know how to put the radio on .", "words": ["can", "the", "put", "how", "their", "we", "go", "to", "radio", "you", "and", "that", "it", "house", "on", "when"]}, {"sent": "I think theyre going on a trip with their suitcase .", "words": ["think", "their", "with", "on", "a"]}, {"sent": "what else do they eat now theyve finished their sandwiches .", "words": ["what", "do", "their", "eat", "they"]}, {"sent": "what are their names ?", "words": ["what", "are", "their"]}, {"sent": "have their heads coming outof the top dont they ?", "words": ["the", "their", "have", "dont", "they"]}, {"sent": "is it their aunt ?", "words": ["it", "their", "is"]}, {"sent": "babies put things in their mouth darling not you .", "words": ["put", "their", "mouth", "you", "in", "not"]}, {"sent": "unfortunately they didnt see the little girl coming down the other path walking towards their home .", "words": ["the", "their", "little", "other", "they", "see", "down", "home"]}, {"sent": "well in the morning the doctors probably hafta do their rounds .", "words": ["the", "do", "in", "their"]}, {"sent": "they eat with their mouths .", "words": ["eat", "they", "with", "their"]}, {"sent": "Im so glad that these animals eat their broccoli .", "words": ["these", "their", "so", "eat", "that"]}, {"sent": "um but theyre all breaking their kids into school .", "words": ["all", "their", "into", "school", "but"]}, {"sent": "their legs and their arms move and the head moves .", "words": ["the", "head", "their", "and"]}, {"sent": "this could be their diamond this could be their servant diamond .", "words": ["this", "could", "be", "their"]}, {"sent": "oh their friends .", "words": ["their"]}, {"sent": "um but I think the thing is obviously their interest is taken up with so My other things at the moment that .", "words": ["the", "up", "think", "their", "is", "so", "with", "other", "at", "that", "but"]}, {"sent": "sposta be their house .", "words": ["be", "house", "their"]}, {"sent": "their speech and that .", "words": ["that", "their", "and"]}, {"sent": "thats the people opposite are decorating their bathroom .", "words": ["the", "bathroom", "are", "their"]}, {"sent": "what are they gonna have with their tomato ?", "words": ["what", "their", "with", "have", "they", "are"]}, {"sent": "oh I cant see their car .", "words": ["see", "their", "car"]}, {"sent": "thats so that if they fall down they dont hurt their heads right ?", "words": ["if", "their", "hurt", "so", "fall", "they", "dont", "down", "that"]}, {"sent": "and with their spades .", "words": ["with", "their", "and"]}, {"sent": "James hadta pull their wagon and hold their royal trains and dress up as their baby .", "words": ["up", "hold", "their", "and", "pull", "dress"]}, {"sent": "I think hamsters just like to eat their seeds .", "words": ["think", "their", "to", "like", "eat"]}, {"sent": "are you going to color their tails in purple ?", "words": ["their", "to", "you", "in", "are"]}, {"sent": "what are they having at their party ?", "words": ["what", "their", "party", "they", "are", "at"]}, {"sent": "because they got squashed in the tunnel with their old car .", "words": ["the", "old", "their", "car", "with", "they", "in", "because"]}, {"sent": "so theyre looking to have someone come to work in their house .", "words": ["work", "their", "to", "so", "have", "in", "house"]}, {"sent": "Fred and Rachel have both got their cars on the drive and the men must not realize that its its ours .", "words": ["the", "their", "and", "have", "drive", "on", "that", "not"]}, {"sent": "what are their names ?", "words": ["what", "are", "their"]}, {"sent": "see thats their hiding place .", "words": ["see", "their"]}, {"sent": "oh are you gonna serve them their muffins their cupcakes ?", "words": ["are", "them", "their", "you"]}, {"sent": "do you expect their um their height and weight at this stage to still correlate with birth ?", "words": ["do", "their", "to", "with", "and", "you", "this", "at"]}, {"sent": "they cant get in their house .", "words": ["their", "get", "they", "in", "house"]}, {"sent": "two owls and a hen four larks and a wren have all built my built their nests in my beard .", "words": ["all", "my", "their", "and", "have", "in", "hen", "a"]}, {"sent": "Dipsy and Po have zips in their backs and when we unzip them theres nothing there is there ?", "words": ["their", "we", "is", "them", "and", "have", "there", "in", "when"]}, {"sent": "banged their head on some trees .", "words": ["on", "some", "head", "their"]}, {"sent": "you know whats going to happen to the people who are throwing their cups on the floor ?", "words": ["the", "their", "to", "you", "are", "who", "on"]}, {"sent": "sometimes mommy deers leave their little children and then they just go off by themselves .", "words": ["off", "their", "go", "and", "little", "they", "by", "then"]}, {"sent": "their arms dont move like that .", "words": ["dont", "that", "their", "like"]}, {"sent": "did they invite all their stuffed animals and dolls ?", "words": ["all", "their", "and", "they", "did"]}, {"sent": "what else are they having with their .", "words": ["what", "their", "with", "they", "are"]}, {"sent": "because its seven busy spiders spin their web .", "words": ["their", "because"]}, {"sent": "you know their children .", "words": ["their", "you"]}, {"sent": "oh look theyre all eating their lunch yumyum .", "words": ["all", "their", "look"]}, {"sent": "the dream went on and on and the greedy goblins drank and drank and at dawn just as their bellies were about to burst they were awakened by a distant clap of thunder .", "words": ["the", "their", "clap", "of", "to", "and", "by", "they", "about", "at", "on", "were", "a"]}, {"sent": "have they got their golden helmets on and their blue uniforms ?", "words": ["their", "and", "have", "they", "blue", "on"]}, {"sent": "so youve got to get some horse shoes and youve got to nail them onto their feet .", "words": ["their", "to", "so", "nail", "them", "and", "get", "horse", "some"]}, {"sent": "hes gonna listen to their hearts with a stethoscope .", "words": ["their", "listen", "to", "with", "a"]}, {"sent": "and their names were Flopsy Mopsy Cottontail and Peter .", "words": ["were", "their", "and"]}, {"sent": "they get to choose their prizes now ?", "words": ["get", "they", "to", "their"]}, {"sent": "I think theyll hafta go down on their own Thomas .", "words": ["think", "their", "go", "on", "down"]}, {"sent": "and they get rid of their clothes .", "words": ["their", "of", "and", "get", "they"]}, {"sent": "all these people xxx their children are in the same center .", "words": ["the", "all", "these", "their", "in", "are", "same"]}, {"sent": "whats their names ?", "words": ["their"]}, {"sent": "in their house ?", "words": ["in", "house", "their"]}, {"sent": "they both have woolly coats just like their baby lamb .", "words": ["their", "like", "have", "they", "lamb"]}, {"sent": "are they going to look after the animals while theyre in their house for the winter ?", "words": ["the", "for", "their", "to", "they", "in", "are", "house", "look"]}, {"sent": "now theyre having their dinner .", "words": ["their"]}, {"sent": "I think that they oughta make up their names .", "words": ["up", "think", "their", "they", "that", "make"]}, {"sent": "it sort of flings in their face a little bit .", "words": ["face", "their", "of", "little", "in", "it", "a"]}, {"sent": "hell s hell try to change their diapers and things like that .", "words": ["their", "to", "like", "and", "that", "try", "try to"]}, {"sent": "following their mommy duck .", "words": ["duck", "their"]}, {"sent": "all got their heads stuck down the chimney uhn havent they ?", "words": ["the", "all", "their", "they", "stuck", "down"]}, {"sent": "theyve got their shorts on havent they ?", "words": ["on", "they", "shorts", "their"]}, {"sent": "why dont we put their sweaters on ?", "words": ["put", "their", "we", "dont", "on", "why"]}, {"sent": "she closed their favorite book and they both blew out the candle .", "words": ["the", "their", "and", "they", "out", "book", "she"]}, {"sent": "xxx hyenas circled their prey licking their chops .", "words": ["their"]}, {"sent": "they went home to their house didnt they ?", "words": ["their", "to", "they", "house", "home"]}, {"sent": "and you were all shouting their name and saying wake up wake up .", "words": ["up", "all", "their", "you", "and", "wake", "were"]}, {"sent": "you probably dont remember because its a long time since weve been in their garden .", "words": ["their", "long", "you", "dont", "in", "garden", "because", "a"]}, {"sent": "this have you seen this soap its kids soap so that it doesnt slip out of their hands when they use it and it smells like berry .", "words": ["their", "when", "of", "so", "like", "soap", "you", "have", "this", "they", "it", "and", "out", "that"]}, {"sent": "and they ran and they ran through the dark wood back to their homes as quick as they could .", "words": ["the", "their", "back", "to", "and", "they", "dark", "could"]}, {"sent": "come on right theyve had their tea .", "words": ["on", "their"]}, {"sent": "just eat biscuits and they dont eat their lunch .", "words": ["their", "and", "eat", "they", "dont"]}, {"sent": "fish sleep among the green water ferns with their eyes and mouths wide open fish sleep with their eyes open isnt that funny ?", "words": ["the", "their", "sleep", "open", "with", "and", "green", "water", "that", "fish"]}, {"sent": "little yellow ducklings swim across the pond and back follow their mommy duck splish splash quack .", "words": ["the", "their", "back", "swim", "and", "little", "splash", "duck", "yellow"]}, {"sent": "because their babys are still downstairs .", "words": ["are", "their", "because"]}, {"sent": "birds use their beaks when theyre eating .", "words": ["when", "their"]}, {"sent": "breakdown trucks rescue cars and their passengers .", "words": ["their", "and"]}, {"sent": "bowing to their friends .", "words": ["to", "their"]}, {"sent": "wheres their school ?", "words": ["school", "their"]}, {"sent": "and they have some shells near their pond in their garden .", "words": ["their", "and", "have", "they", "in", "garden", "some"]}, {"sent": "you have their number ?", "words": ["have", "their", "you"]}, {"sent": "you give them their box Pete .", "words": ["box", "their", "them", "you", "give"]}, {"sent": "we still dont have their names on it .", "words": ["their", "we", "have", "dont", "it", "on"]}, {"sent": "Petey why dont you help um with their coats .", "words": ["help", "their", "with", "you", "dont", "why"]}, {"sent": "some of their tapes they told me were not .", "words": ["their", "of", "me", "they", "some", "were", "not"]}, {"sent": "thats where their home is .", "words": ["where", "their", "is", "home"]}, {"sent": "none of the other men wear belts with their pants they button suspenders onto the waist lines of their pants and snap the suspenders over their shoulders .", "words": ["the", "their", "of", "none", "with", "other", "and", "they", "pants", "over", "button"]}, {"sent": "where is their friend owl ?", "words": ["where", "owl", "their", "is"]}, {"sent": "well whats their names ?", "words": ["their"]}, {"sent": "the kitty cats were sleeping in their new bed .", "words": ["the", "their", "bed", "in", "kitty", "new", "were"]}, {"sent": "big girls wear their proper shoes without getting cross or upset .", "words": ["big", "their"]}, {"sent": "see you put their mail in their door in the slot in their door give them their mail .", "words": ["the", "put", "their", "them", "you", "in", "see", "give", "door"]}, {"sent": "they cry and they squeal they giggle and coo they bang and they splash and they clap their hands too .", "words": ["their", "clap", "cry", "and", "they", "splash", "too"]}, {"sent": "and she sometimes likes their bones haha .", "words": ["their", "she", "and"]}, {"sent": "Steven wanna make some butter ?", "words": ["some", "make", "butter", "wanna"]}, {"sent": "oh Dillon we forgot the butter .", "words": ["the", "butter", "we"]}, {"sent": "or would you just like some butter on this ?", "words": ["like", "you", "this", "butter", "would", "on", "some"]}, {"sent": "we take off our butter xxx change .", "words": ["off", "we", "our", "butter", "take"]}, {"sent": "oh peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "because things like milk and orange juice and butter .", "words": ["like", "and", "orange", "butter", "milk", "juice", "because"]}, {"sent": "what happened to the the stick David you put the real other butter in the back brought one of yours right ?", "words": ["the", "what", "stick", "put", "back", "of", "to", "you", "other", "in", "butter"]}, {"sent": "yeah were making bread and butter for snack .", "words": ["bread", "for", "and", "butter", "were"]}, {"sent": "see the butter up there .", "words": ["the", "up", "there", "butter", "see"]}, {"sent": "what do you put butter on ?", "words": ["what", "do", "put", "you", "butter", "on"]}, {"sent": "butter ?", "words": ["butter"]}, {"sent": "Im getting the butter out of the fridge .", "words": ["the", "of", "butter", "out"]}, {"sent": "almond butter .", "words": ["butter"]}, {"sent": "thats peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "butter ?", "words": ["butter"]}, {"sent": "you dont kiss peanut butter ?", "words": ["you", "dont", "peanut butter", "butter", "kiss"]}, {"sent": "shall I make you a butter .", "words": ["make", "butter", "a", "you"]}, {"sent": "the butter is yellow .", "words": ["the", "yellow", "butter", "is"]}, {"sent": "get some peanut butter out .", "words": ["get", "butter", "out", "peanut butter", "some"]}, {"sent": "I thought that was the butter .", "words": ["the", "was", "that", "butter"]}, {"sent": "Cathy if she knew there was one p off butter shed get the bus to Ballymena .", "words": ["off", "the", "if", "was", "to", "get", "there", "butter", "bus", "she"]}, {"sent": "okay here who wants a taste of the butter ?", "words": ["the", "of", "taste", "here", "butter", "who", "a"]}, {"sent": "where do you put the peanut butter ?", "words": ["where", "do", "put", "the", "you", "butter", "peanut butter"]}, {"sent": "I see the butter I thought Ruby uses butter to make the cake .", "words": ["the", "cake", "to", "butter", "see", "make"]}, {"sent": "theres some cracker with almond buttered apple butter on it .", "words": ["cracker", "with", "it", "butter", "on", "apple", "some"]}, {"sent": "dyou want fresh apple with almond butter ?", "words": ["apple", "with", "butter"]}, {"sent": "so are we going to put some butter on the toast .", "words": ["the", "put", "toast", "we", "to", "so", "are", "butter", "on", "some"]}, {"sent": "who wants the peanut butter ?", "words": ["the", "peanut butter", "butter", "who"]}, {"sent": "on the top theyve put some more butter cream .", "words": ["the", "put", "more", "butter", "on", "some"]}, {"sent": "thats lotsof butter isnt it ?", "words": ["it", "butter"]}, {"sent": "you like peanut butter ?", "words": ["peanut butter", "butter", "like", "you"]}, {"sent": "I want peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "butter .", "words": ["butter"]}, {"sent": "do you want peanut butter and crackers ?", "words": ["do", "and", "you", "peanut butter", "butter"]}, {"sent": "put some butter on it .", "words": ["put", "it", "butter", "on", "some"]}, {"sent": "mm peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "or is it the butter ?", "words": ["the", "it", "butter", "is"]}, {"sent": "Im so hungry said jack this is after he made the butter lets make the pancake but his mother said I need some kindling wood to build up the fire .", "words": ["the", "up", "pancake", "need", "is", "so", "to", "hungry", "this", "he", "butter", "build", "his", "some", "make", "but"]}, {"sent": "peanut butter makes me thirsty .", "words": ["me", "peanut butter", "butter", "thirsty"]}, {"sent": "thats a butter tray .", "words": ["tray", "butter", "a"]}, {"sent": "does this butter taste as good as the butter you made Gabriella with AuntBetty ?", "words": ["the", "does", "with", "taste", "you", "this", "butter", "good"]}, {"sent": "peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "knife to put the butter on .", "words": ["the", "put", "to", "knife", "butter", "on"]}, {"sent": "butter .", "words": ["butter"]}, {"sent": "when do you need peanut butter ?", "words": ["do", "need", "you", "butter", "peanut butter", "when"]}, {"sent": "well have some butter .", "words": ["have", "some", "butter"]}, {"sent": "um peanut butter and jelly .", "words": ["jelly", "peanut butter", "butter", "and"]}, {"sent": "are you trying to tell me that you want peanut butter on there too ?", "words": ["to", "you", "there", "are", "butter", "peanut butter", "too", "on", "me", "that"]}, {"sent": "Im just going to butter this .", "words": ["to", "butter", "this"]}, {"sent": "oh you want more almond butter on bagel ?", "words": ["on", "more", "butter", "you"]}, {"sent": "as well as the peanut butter .", "words": ["the", "butter", "peanut butter"]}, {"sent": "right and hes there with his mother and theyre making the butter right ?", "words": ["the", "with", "and", "there", "butter", "his"]}, {"sent": "Mummys going to put some butter on her toast .", "words": ["put", "toast", "to", "butter", "on", "some", "her"]}, {"sent": "xxx xxx peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "have you got butter ?", "words": ["have", "butter", "you"]}, {"sent": "what would you eat if you didnt have peanut butter ?", "words": ["what", "if", "you", "eat", "have", "butter", "peanut butter", "would"]}, {"sent": "would you please give me one quart of milk and one pound of butter ?", "words": ["of", "you", "and", "butter", "would", "milk", "me", "give"]}, {"sent": "so I didnt want you to get it in the butter .", "words": ["the", "to", "so", "you", "get", "in", "it", "butter"]}, {"sent": "xxx butter xxx .", "words": ["butter"]}, {"sent": "Ill get you some more butter .", "words": ["you", "get", "more", "butter", "some"]}, {"sent": "well it looks like butter .", "words": ["it", "butter", "like"]}, {"sent": "peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "heres peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "this little piggy ate bread and butter .", "words": ["bread", "and", "little", "this", "butter"]}, {"sent": "you put butter on toast .", "words": ["put", "toast", "you", "butter", "on"]}, {"sent": "margarine is a kind of butter .", "words": ["of", "butter", "a", "is"]}, {"sent": "butter .", "words": ["butter"]}, {"sent": "probably the spoons and banging things on the peanut butter jar .", "words": ["the", "and", "butter", "peanut butter", "on", "jar"]}, {"sent": "well I would prepare the onion and the garlic on their own with a wee bit of butter .", "words": ["the", "their", "of", "with", "and", "butter", "would", "on", "a"]}, {"sent": "because were making bread and butter for snack .", "words": ["bread", "for", "and", "butter", "were", "because"]}, {"sent": "Chester and Wilson wanted toast with jam and peanut butter for breakfast the next morning .", "words": ["the", "for", "toast", "with", "and", "butter", "peanut butter"]}, {"sent": "butter .", "words": ["butter"]}, {"sent": "thats butter can I have some butter .", "words": ["can", "some", "have", "butter"]}, {"sent": "butter butter butter butter .", "words": ["butter"]}, {"sent": "can I put the butter away ?", "words": ["can", "the", "put", "away", "butter"]}, {"sent": "how bout Kamut with almond butter n apple butter on it .", "words": ["how", "with", "it", "butter", "on", "apple"]}, {"sent": "heavy peanut butter .", "words": ["heavy", "butter", "peanut butter"]}, {"sent": "its like butter .", "words": ["butter", "like"]}, {"sent": "hey Wes you dropped your peanut butter .", "words": ["peanut butter", "your", "butter", "you"]}, {"sent": "Jeffrey is gonna read the butter side down .", "words": ["the", "is", "read", "butter", "down"]}, {"sent": "youre always telling me to put more butter on your food arent you ?", "words": ["put", "your", "to", "you", "more", "food", "butter", "on", "me"]}, {"sent": "you wanna build something with butter ?", "words": ["with", "you", "build", "butter", "wanna"]}, {"sent": "only one butter .", "words": ["butter"]}, {"sent": "its a peanut butter cookie .", "words": ["cookie", "peanut butter", "butter", "a"]}, {"sent": "Dad left the butter out all night .", "words": ["the", "all", "butter", "out"]}, {"sent": "do you want some butter ?", "words": ["do", "butter", "some", "you"]}, {"sent": "which ones do you like the peanut butter or the chocolate ?", "words": ["the", "do", "chocolate", "like", "you", "butter", "peanut butter", "which"]}, {"sent": "dyou want some almond butter with your fresh pear ?", "words": ["your", "some", "with", "butter"]}, {"sent": "they were all bravely marching with banners a flutter down a hole for their country and right side up butter .", "words": ["up", "all", "for", "their", "with", "and", "they", "butter", "down", "were", "a"]}, {"sent": "is that peanut butter ?", "words": ["peanut butter", "that", "butter", "is"]}, {"sent": "Im putting some butter on toast .", "words": ["on", "some", "butter", "toast"]}, {"sent": "did you find the peanut butter again ?", "words": ["the", "you", "butter", "did", "find", "peanut butter"]}, {"sent": "butter .", "words": ["butter"]}, {"sent": "I do not know what that is but I think it might be peanut butter .", "words": ["do", "what", "think", "is", "be", "it", "butter", "peanut butter", "that", "not", "but"]}, {"sent": "I see you want another one with butter ?", "words": ["with", "you", "another", "butter", "see"]}, {"sent": "can you throw it ?", "words": ["can", "it", "throw", "you"]}, {"sent": "wait mummy wants to throw the ball to you .", "words": ["the", "wait", "throw", "to", "you", "ball"]}, {"sent": "dont throw .", "words": ["dont", "throw"]}, {"sent": "throw the pizza .", "words": ["the", "throw", "pizza"]}, {"sent": "yes I think we oughta throw it away .", "words": ["think", "throw", "we", "away", "it"]}, {"sent": "thats a very good throw .", "words": ["throw", "good", "a"]}, {"sent": "that is a good throw .", "words": ["throw", "is", "good", "that", "a"]}, {"sent": "okay throw it again .", "words": ["it", "throw"]}, {"sent": "not throw it down .", "words": ["down", "throw", "it", "not"]}, {"sent": "what you throw ?", "words": ["what", "throw", "you"]}, {"sent": "lets go throw that away .", "words": ["away", "that", "throw", "go"]}, {"sent": "you throw yourself on the bed like that .", "words": ["the", "yourself", "throw", "bed", "like", "you", "on", "that"]}, {"sent": "throw it down at Robbie .", "words": ["down", "it", "throw", "at"]}, {"sent": "throw it to her .", "words": ["to", "it", "her", "throw"]}, {"sent": "oh look how well you throw it .", "words": ["how", "throw", "you", "it", "look"]}, {"sent": "ah good throw .", "words": ["throw", "good"]}, {"sent": "you cant just throw the .", "words": ["the", "throw", "you"]}, {"sent": "well then dont throw them okay ?", "words": ["dont", "then", "them", "throw"]}, {"sent": "throw it ?", "words": ["it", "throw"]}, {"sent": "oh thats a good throw ready .", "words": ["throw", "good", "a"]}, {"sent": "oh dont throw them at mommy please .", "words": ["dont", "them", "throw", "at"]}, {"sent": "dont throw it .", "words": ["dont", "it", "throw"]}, {"sent": "ohno we dont throw them .", "words": ["dont", "them", "throw", "we"]}, {"sent": "oh dont throw it .", "words": ["dont", "it", "throw"]}, {"sent": "if you throw it again Warren that bag of toys goes in the hall .", "words": ["the", "throw", "if", "of", "you", "in", "it", "that"]}, {"sent": "dont throw .", "words": ["dont", "throw"]}, {"sent": "can you throw it ?", "words": ["can", "it", "throw", "you"]}, {"sent": "that was a good throw though .", "words": ["throw", "was", "good", "that", "a"]}, {"sent": "you cant just throw him off .", "words": ["off", "him", "throw", "you"]}, {"sent": "throw it here .", "words": ["here", "it", "throw"]}, {"sent": "throw it to mommy .", "words": ["to", "it", "throw"]}, {"sent": "throw it to mummy .", "words": ["to", "it", "throw"]}, {"sent": "throw it to me .", "words": ["to", "me", "it", "throw"]}, {"sent": "you gonna roll it or or or throw it ?", "words": ["it", "throw", "you"]}, {"sent": "oh dont throw it away give it to me .", "words": ["throw", "to", "dont", "away", "it", "me", "give"]}, {"sent": "throw the ball .", "words": ["the", "ball", "throw"]}, {"sent": "throw it .", "words": ["it", "throw"]}, {"sent": "throw it to her throw it to her throw it to her throw it to her .", "words": ["to", "it", "her", "throw"]}, {"sent": "you throw it ?", "words": ["it", "throw", "you"]}, {"sent": "throw it in here .", "words": ["here", "in", "it", "throw"]}, {"sent": "and oh she loves to throw them thats the latest thing .", "words": ["the", "throw", "to", "them", "and", "she"]}, {"sent": "dont throw it at me .", "words": ["throw", "dont", "it", "at", "me"]}, {"sent": "the monkey would throw mangos down to the crocodile .", "words": ["the", "throw", "to", "monkey", "would", "down"]}, {"sent": "xxx throw it .", "words": ["it", "throw"]}, {"sent": "no no dont throw it .", "words": ["dont", "it", "throw"]}, {"sent": "throw it real hard .", "words": ["it", "throw", "hard"]}, {"sent": "throw it to me .", "words": ["to", "me", "it", "throw"]}, {"sent": "I was going to throw it away I know a little boy who might very well .", "words": ["throw", "to", "was", "little", "it", "away", "who", "a"]}, {"sent": "throw it .", "words": ["it", "throw"]}, {"sent": "throw the ball some more .", "words": ["the", "throw", "ball", "more", "some"]}, {"sent": "what if I throw you .", "words": ["what", "throw", "if", "you"]}, {"sent": "Max knew Ruby would find a reason to throw out his popsicle .", "words": ["throw", "to", "would", "out", "find", "his", "popsicle", "a"]}, {"sent": "throw it back to Mummy .", "words": ["to", "it", "throw", "back"]}, {"sent": "Ill throw it the handest and it will .", "words": ["the", "throw", "and", "it", "will"]}, {"sent": "oh dont throw them .", "words": ["dont", "them", "throw"]}, {"sent": "we can throw the ball in the box .", "words": ["can", "the", "box", "throw", "we", "ball", "in"]}, {"sent": "here now you throw it .", "words": ["here", "it", "throw", "you"]}, {"sent": "good throw .", "words": ["throw", "good"]}, {"sent": "Eleanor dont throw it .", "words": ["dont", "it", "throw"]}, {"sent": "dont throw it at her .", "words": ["throw", "dont", "it", "at", "her"]}, {"sent": "okay throw it to mummy .", "words": ["to", "it", "throw"]}, {"sent": "dont throw them down .", "words": ["dont", "down", "them", "throw"]}, {"sent": "oh dont throw that .", "words": ["dont", "that", "throw"]}, {"sent": "dont throw those around Fraser .", "words": ["dont", "those", "throw", "around"]}, {"sent": "no Ive Im gonna throw it away .", "words": ["away", "it", "throw"]}, {"sent": "poor Jimbo you just throw em .", "words": ["throw", "you", "poor"]}, {"sent": "throw the ball to me .", "words": ["the", "throw", "to", "ball", "me"]}, {"sent": "how bout you you throw it ?", "words": ["how", "it", "throw", "you"]}, {"sent": "can you throw it to me ?", "words": ["can", "throw", "to", "you", "it", "me"]}, {"sent": "if you throw Bunny Bunnyll hurt himself Eleanor and then hell cry look .", "words": ["throw", "if", "hurt", "cry", "and", "you", "look", "then"]}, {"sent": "dont throw .", "words": ["dont", "throw"]}, {"sent": "should mommy throw that away ?", "words": ["away", "that", "throw"]}, {"sent": "throw the ball to Mommy .", "words": ["the", "to", "ball", "throw"]}, {"sent": "hey did you throw the indian ?", "words": ["the", "throw", "did", "you"]}, {"sent": "well you grab it go ahead get more I really feel like we should throw this stuff away its yucky .", "words": ["throw", "go", "we", "yucky", "like", "you", "get", "this", "more", "it", "away"]}, {"sent": "no do you want me to throw that away for you ?", "words": ["do", "for", "throw", "to", "you", "away", "me", "that"]}, {"sent": "you dont just throw do you ?", "words": ["dont", "do", "throw", "you"]}, {"sent": "I throw you way up high .", "words": ["up", "throw", "high", "you"]}, {"sent": "throw it in the toilet ?", "words": ["the", "in", "it", "throw"]}, {"sent": "okay okay dont throw it cause you could hit yourself in the head .", "words": ["the", "yourself", "throw", "head", "you", "dont", "in", "it", "could", "hit"]}, {"sent": "throw it throw it .", "words": ["it", "throw"]}, {"sent": "here throw it to daddy .", "words": ["here", "to", "it", "throw"]}, {"sent": "me throw it to you ?", "words": ["throw", "to", "you", "it", "me"]}, {"sent": "uhuh no throw .", "words": ["throw"]}, {"sent": "but its not fair to throw them on the floor and damage them .", "words": ["the", "throw", "to", "them", "and", "on", "not", "but"]}, {"sent": "dont throw them on the floor .", "words": ["the", "throw", "them", "dont", "on"]}, {"sent": "just throw your clothes downstairs ?", "words": ["your", "throw"]}, {"sent": "you can throw a ball .", "words": ["can", "throw", "you", "ball", "a"]}, {"sent": "you cant have anything if youre gonna throw it do you hear me ?", "words": ["do", "throw", "if", "you", "have", "hear", "it", "me"]}, {"sent": "throw the ball to Daddy .", "words": ["the", "to", "ball", "throw"]}, {"sent": "here sweetheart throw the ball here .", "words": ["here", "the", "ball", "throw"]}, {"sent": "throw stones at telephone poles .", "words": ["throw", "telephone", "at"]}, {"sent": "throw it to mummy .", "words": ["to", "it", "throw"]}, {"sent": "youre not your Mummyll throw me out .", "words": ["your", "throw", "out", "me", "not"]}, {"sent": "throw Mommy the ball .", "words": ["the", "ball", "throw"]}, {"sent": "do you want me to throw it to you ?", "words": ["do", "throw", "to", "you", "it", "me"]}, {"sent": "go throw it up .", "words": ["up", "it", "throw", "go"]}, {"sent": "oh if youre going to throw it down thats not a nice boy is it ?", "words": ["throw", "if", "is", "to", "nice", "it", "down", "not", "a"]}, {"sent": "okay throw the ball at the hippo .", "words": ["the", "ball", "throw", "at"]}, {"sent": "you try looking through that piece of cheese Ive just cut off the block for you .", "words": ["off", "the", "try", "block", "cut", "for", "of", "you", "cheese", "that"]}, {"sent": "look cheese .", "words": ["cheese", "look"]}, {"sent": "this is regular delicious macaroni and cheese .", "words": ["this", "cheese", "and", "is"]}, {"sent": "I think you bought some cheese did you ?", "words": ["think", "you", "did", "some", "cheese"]}, {"sent": "ah is it cheese ?", "words": ["it", "cheese", "is"]}, {"sent": "cheese and then the other piece of bread .", "words": ["the", "bread", "of", "other", "and", "then", "cheese"]}, {"sent": "did he want cottage cheese ?", "words": ["cheese", "he", "did"]}, {"sent": "it isnt cheese .", "words": ["it", "cheese"]}, {"sent": "may I have some more cheese ?", "words": ["have", "some", "more", "cheese"]}, {"sent": "cheese and oh I want this I want that yeah .", "words": ["this", "that", "cheese", "and"]}, {"sent": "some cheese .", "words": ["some", "cheese"]}, {"sent": "have your piece of cheese .", "words": ["have", "of", "cheese", "your"]}, {"sent": "Im just going to put this fridge eh cheese in the fridge .", "words": ["the", "put", "to", "this", "in", "cheese"]}, {"sent": "cheese is yellow .", "words": ["yellow", "cheese", "is"]}, {"sent": "you put cheese on the bread ?", "words": ["the", "put", "bread", "you", "on", "cheese"]}, {"sent": "do you want a piece of cheese ?", "words": ["do", "of", "you", "cheese", "a"]}, {"sent": "oh you like the cheese dont you ?", "words": ["the", "like", "you", "dont", "cheese"]}, {"sent": "no cheese .", "words": ["cheese"]}, {"sent": "and cheese ?", "words": ["cheese", "and"]}, {"sent": "oh the cheese smells strong .", "words": ["the", "cheese"]}, {"sent": "that cheese .", "words": ["cheese", "that"]}, {"sent": "sliced cheese .", "words": ["cheese"]}, {"sent": "cheese .", "words": ["cheese"]}, {"sent": "you ate up all the cheese miss miss .", "words": ["up", "all", "the", "you", "cheese"]}, {"sent": "cheese ?", "words": ["cheese"]}, {"sent": "cheese yummy .", "words": ["cheese"]}, {"sent": "yeah cheese .", "words": ["cheese"]}, {"sent": "we dont have any cheese honey .", "words": ["we", "any", "have", "dont", "cheese"]}, {"sent": "put the cheese on .", "words": ["the", "put", "cheese", "on"]}, {"sent": "lets have some cheese .", "words": ["have", "some", "cheese"]}, {"sent": "it doesnt eat cheese at all .", "words": ["all", "eat", "it", "at", "cheese"]}, {"sent": "see hes walking on the cheese .", "words": ["on", "see", "cheese", "the"]}, {"sent": "like uh the green beans and uh cheese corn stuff like that .", "words": ["the", "like", "green beans", "and", "that", "beans", "green", "corn", "cheese"]}, {"sent": "want me to feed baby cheese ?", "words": ["to", "me", "cheese", "feed"]}, {"sent": "strip cheese or big cheese ?", "words": ["big", "cheese"]}, {"sent": "and the cheese I usually like .", "words": ["the", "cheese", "like", "and"]}, {"sent": "do you not want some cheese ?", "words": ["do", "you", "some", "cheese", "not"]}, {"sent": "cheese .", "words": ["cheese"]}, {"sent": "theres some toast with cheese on it .", "words": ["toast", "with", "it", "on", "some", "cheese"]}, {"sent": "cheese ?", "words": ["cheese"]}, {"sent": "is it real or is it toy cheese ?", "words": ["toy", "it", "cheese", "is"]}, {"sent": "and the cheese .", "words": ["the", "cheese", "and"]}, {"sent": "she says Poppadum cheese and Poppadum I cant believe it I dont know where she got it from .", "words": ["where", "and", "dont", "it", "cheese", "she"]}, {"sent": "you neednt have the cheese but youve eaten nearly all the cheese .", "words": ["the", "all", "you", "have", "cheese", "but"]}, {"sent": "look how much cheese they got .", "words": ["how", "much", "they", "look", "cheese"]}, {"sent": "and cheese sandwich ?", "words": ["cheese", "sandwich", "and"]}, {"sent": "she says cheese and papadums .", "words": ["cheese", "she", "and"]}, {"sent": "tell me to say cheese .", "words": ["to", "me", "cheese", "say"]}, {"sent": "you can put that with your cheese to make a sandwich .", "words": ["can", "put", "your", "to", "with", "you", "make", "cheese", "that", "sandwich", "a"]}, {"sent": "cheese .", "words": ["cheese"]}, {"sent": "right there is cheese .", "words": ["there", "cheese", "is"]}, {"sent": "the cheese is all done ?", "words": ["the", "all", "cheese", "is"]}, {"sent": "what about cheese is that soft enough for her to eat ?", "words": ["what", "soft", "for", "is", "to", "her", "eat", "about", "cheese", "that"]}, {"sent": "would you like a cheese sandwich ?", "words": ["like", "you", "would", "cheese", "sandwich", "a"]}, {"sent": "cheese yeah dont eat it for real just pretend .", "words": ["for", "eat", "dont", "it", "pretend", "cheese"]}, {"sent": "who doesnt love cheese ?", "words": ["cheese", "love", "who"]}, {"sent": "what the cheese ?", "words": ["the", "what", "cheese"]}, {"sent": "cheese pizza .", "words": ["cheese", "pizza"]}, {"sent": "youve got your cheese dancing across your highchair table have you ?", "words": ["your", "table", "you", "have", "cheese"]}, {"sent": "we take the cheese out dont we ?", "words": ["the", "we", "dont", "out", "take", "cheese"]}, {"sent": "ham and cheese .", "words": ["cheese", "and"]}, {"sent": "yeah and cheese .", "words": ["cheese", "and"]}, {"sent": "yeah but you can wipe your cream cheese .", "words": ["can", "your", "you", "wipe", "cheese", "but"]}, {"sent": "you stay here and Ill get you a triangle cheese .", "words": ["and", "here", "get", "you", "stay", "cheese", "a"]}, {"sent": "well cheese spread I mean dairylea is you know soft cheese spread but I mean squirty cheese in a tube .", "words": ["soft", "is", "you", "in", "cheese", "but", "a"]}, {"sent": "no its not cheese .", "words": ["cheese", "not"]}, {"sent": "what thats cheese you like cheese .", "words": ["what", "cheese", "like", "you"]}, {"sent": "you see a piece of cheese ?", "words": ["of", "you", "see", "cheese", "a"]}, {"sent": "its too big of a piece of cheese for that hamburger .", "words": ["for", "of", "hamburger", "big", "that", "too", "cheese", "a"]}, {"sent": "cheese .", "words": ["cheese"]}, {"sent": "okay I think youll like it better with cheese on it .", "words": ["better", "think", "with", "like", "it", "on", "cheese"]}, {"sent": "eat the cheese with a spoon ?", "words": ["the", "with", "eat", "spoon", "cheese", "a"]}, {"sent": "you can have piece of cheese on toast while the pastas cooking .", "words": ["can", "the", "toast", "of", "you", "have", "on", "cheese"]}, {"sent": "thats some cheese .", "words": ["some", "cheese"]}, {"sent": "cheese ?", "words": ["cheese"]}, {"sent": "now that you have a mouth full of cheese .", "words": ["full", "of", "mouth", "you", "have", "cheese", "that", "a"]}, {"sent": "wheres the cheese ?", "words": ["the", "cheese"]}, {"sent": "how about cheese ?", "words": ["how", "cheese", "about"]}, {"sent": "youve got some bread and heres some cheese .", "words": ["some", "bread", "cheese", "and"]}, {"sent": "xxx cheese .", "words": ["cheese"]}, {"sent": "goat cheese with cranberries .", "words": ["cheese", "with"]}, {"sent": "you do like cheese .", "words": ["do", "cheese", "like", "you"]}, {"sent": "do you know how to make cheese sauce Naima ?", "words": ["do", "how", "sauce", "to", "you", "cheese", "make"]}, {"sent": "when I gave it I said cheese and she she bit this .", "words": ["when", "and", "this", "it", "cheese", "she"]}, {"sent": "Ill get you some cheese .", "words": ["get", "some", "cheese", "you"]}, {"sent": "its cheese .", "words": ["cheese"]}, {"sent": "got a packet of Dairylea strip cheese .", "words": ["of", "cheese", "a"]}, {"sent": "cheese cheese cheese what ?", "words": ["what", "cheese"]}, {"sent": "were gonna have a cheese sandwich .", "words": ["sandwich", "have", "cheese", "were", "a"]}, {"sent": "yeah give me that tomato and cheese .", "words": ["and", "that", "me", "give", "cheese"]}, {"sent": "put the cheese on the bread .", "words": ["the", "put", "bread", "on", "cheese"]}, {"sent": "have you eaten the cheese from the top of the drinking cup ?", "words": ["the", "cup", "of", "you", "have", "cheese"]}, {"sent": "cheese yummy .", "words": ["cheese"]}, {"sent": "and guess who we could invite to your cheese house ?", "words": ["your", "we", "to", "and", "house", "who", "could", "cheese"]}, {"sent": "you cant wash cheese Thomas .", "words": ["cheese", "wash", "you"]}, {"sent": "thin strips of cheese .", "words": ["of", "cheese"]}, {"sent": "nice spreading cheese .", "words": ["nice", "cheese"]}, {"sent": "have some more cheese .", "words": ["have", "some", "more", "cheese"]}, {"sent": "well we must buy some more cheese from the supermarket .", "words": ["the", "we", "more", "some", "cheese", "buy"]}, {"sent": "so I guess like a sandwich Ill give them bread and cheese and meat .", "words": ["bread", "so", "them", "like", "meat", "and", "give", "cheese", "sandwich", "a"]}, {"sent": "one two three tomato sauce .", "words": ["sauce"]}, {"sent": "well I mean the tomato sauce wont work for my dad .", "words": ["the", "my", "for", "work", "sauce"]}, {"sent": "this sauce is not so exciting .", "words": ["is", "sauce", "so", "this", "not"]}, {"sent": "youre gonna make coffee and spaghetti and are you gonna make spaghetti sauce ?", "words": ["spaghetti", "sauce", "and", "you", "coffee", "are", "make"]}, {"sent": "and thats a very basic wee sauce .", "words": ["sauce", "a", "and"]}, {"sent": "a scoop of icecream with a sauce over it .", "words": ["sauce", "of", "with", "it", "over", "a"]}, {"sent": "its cheese sauce .", "words": ["cheese", "sauce"]}, {"sent": "well theres sauce all over it but the part Im giving you is cheesy part anyway .", "words": ["the", "all", "sauce", "is", "you", "it", "over", "but"]}, {"sent": "you quite like those without a sauce or anything on them .", "words": ["those", "sauce", "them", "like", "you", "on", "a"]}, {"sent": "or you do want the sauce part ?", "words": ["the", "do", "sauce", "you"]}, {"sent": "what do you grate up and put on top of the tomato sauce on pizza ?", "words": ["up", "what", "do", "put", "the", "sauce", "of", "pizza", "and", "you", "on"]}, {"sent": "would you like some red sauce ?", "words": ["red", "sauce", "like", "you", "would", "some"]}, {"sent": "it kinda looks like spaghetti sauce though doesnt it ?", "words": ["it", "like", "sauce", "spaghetti"]}, {"sent": "spaghetti and tomato sauce and toast .", "words": ["toast", "sauce", "and", "spaghetti"]}, {"sent": "the spaghetti have sauce on it ?", "words": ["the", "sauce", "spaghetti", "have", "it", "on"]}, {"sent": "xxx noodles and red sauce .", "words": ["noodles", "red", "sauce", "and"]}, {"sent": "xxx hot sauce on it .", "words": ["on", "hot", "it", "sauce"]}, {"sent": "one two three tomato sauce ?", "words": ["sauce"]}, {"sent": "its probably not good tomato sauce .", "words": ["not", "good", "sauce"]}, {"sent": "its in your spaghetti sauce .", "words": ["in", "your", "sauce", "spaghetti"]}, {"sent": "I think youre enjoying the tomato sauce as well arent you ?", "words": ["the", "think", "you", "sauce"]}, {"sent": "two all beef patties special sauce lettuce cheese .", "words": ["all", "cheese", "sauce"]}, {"sent": "youre covered with spaghetti sauce .", "words": ["with", "sauce", "spaghetti"]}, {"sent": "one has strawberry sauce one has chocolate .", "words": ["chocolate", "sauce", "strawberry"]}, {"sent": "mm Bert sauce sounds good .", "words": ["good", "sauce"]}, {"sent": "youre drinking the sauce ?", "words": ["the", "sauce"]}, {"sent": "any tomato sauce in here ?", "words": ["here", "in", "any", "sauce"]}, {"sent": "drinking sauce .", "words": ["sauce"]}, {"sent": "Im just trying to quietly scrape my plate because I want this pasta sauce .", "words": ["my", "plate", "sauce", "to", "this", "because"]}, {"sent": "sauce for the chicken .", "words": ["the", "chicken", "for", "sauce"]}, {"sent": "its tomato sauce .", "words": ["sauce"]}, {"sent": "and I think this is spaghetti and sauce but .", "words": ["think", "spaghetti", "is", "sauce", "and", "this", "but"]}, {"sent": "have it with red sauce .", "words": ["red", "sauce", "with", "have", "it"]}, {"sent": "um theres one that has the meat sauce but he wasnt playing so much with that one .", "words": ["the", "sauce", "so", "with", "meat", "much", "he", "that", "but"]}, {"sent": "spaghetti sauce in your eyebrows .", "words": ["in", "your", "sauce", "spaghetti"]}, {"sent": "I know its in a sauce so its different .", "words": ["so", "in", "sauce", "a"]}, {"sent": "then you made apple sauce mhm .", "words": ["apple", "then", "sauce", "you"]}, {"sent": "Mommy the sauce xxx .", "words": ["the", "sauce"]}, {"sent": "what texture is your tomato sauce ?", "words": ["what", "your", "sauce", "is"]}, {"sent": "how about a tiny drop of strawberry sauce on top ?", "words": ["tiny", "how", "strawberry", "of", "sauce", "drop", "about", "on", "a"]}, {"sent": "spaghetti with sauce .", "words": ["with", "sauce", "spaghetti"]}, {"sent": "sauce .", "words": ["sauce"]}, {"sent": "not without tomato sauce so .", "words": ["so", "not", "sauce"]}, {"sent": "that what we cook duck sauce .", "words": ["what", "we", "sauce", "cook", "duck", "that"]}, {"sent": "two all beef patties special sauce lettuce cheese pickles onions on a sesame seed bun .", "words": ["all", "sauce", "on", "cheese", "a"]}, {"sent": "we have a little chicken tikka sauce and I wonder if we could make a vinaigrette out of this although this sauce does not excite me so much but we could probably do that right ?", "words": ["of", "me", "could", "not", "does", "have", "chicken", "much", "that", "if", "sauce", "so", "little", "and", "make", "a", "do", "we", "this", "out", "but"]}, {"sent": "whose sauce ?", "words": ["sauce"]}, {"sent": "marinara sauce .", "words": ["sauce"]}, {"sent": "xxx do you want tomato sauce on your pasta or not ?", "words": ["do", "your", "sauce", "you", "on", "not"]}, {"sent": "not really sauce is it ?", "words": ["it", "not", "is", "sauce"]}, {"sent": "its got sauce ?", "words": ["sauce"]}, {"sent": "you like cranberry sauce .", "words": ["like", "sauce", "you"]}, {"sent": "with cranberry sauce .", "words": ["with", "sauce"]}, {"sent": "we cook duck sauce today .", "words": ["duck", "cook", "sauce", "we"]}, {"sent": "tomato sauce ?", "words": ["sauce"]}, {"sent": "June can you say apple sauce ?", "words": ["can", "sauce", "say", "you", "apple"]}, {"sent": "well well put some sauce with the pasta .", "words": ["the", "put", "sauce", "with", "some"]}, {"sent": "its brown sauce you dont like .", "words": ["sauce", "brown", "like", "you", "dont"]}, {"sent": "um he gets apple sauce at home .", "words": ["sauce", "he", "at", "apple", "home"]}, {"sent": "its got coleslaw sauce on it .", "words": ["on", "it", "sauce"]}, {"sent": "you got spaghetti and pasta sauce .", "words": ["spaghetti", "sauce", "and", "you"]}, {"sent": "red sauce .", "words": ["red", "sauce"]}, {"sent": "the sauce is pretty hot .", "words": ["the", "is", "sauce", "hot", "pretty"]}, {"sent": "my mother makes hot sauce .", "words": ["hot", "my", "sauce"]}, {"sent": "is that a good combination yogurt and apple blueberry sauce ?", "words": ["is", "sauce", "yogurt", "and", "good", "apple", "that", "a"]}, {"sent": "wouldnt you eat the apple sauce ?", "words": ["the", "sauce", "you", "eat", "apple"]}, {"sent": "lets go look in a couple of cookbooks and see we could do I feel like we have peanut sauce a lot .", "words": ["do", "go", "of", "we", "sauce", "a lot", "like", "and", "have", "in", "look", "see", "could", "a"]}, {"sent": "you like the pasta with the red sauce on it ?", "words": ["the", "red", "sauce", "with", "like", "you", "it", "on"]}, {"sent": "that looks like a microphone hey like this thats when theyre like could you please tell my how much this spaghetti sauce costs in isle five ?", "words": ["my", "how", "spaghetti", "sauce", "like", "much", "you", "this", "that", "in", "could", "when", "a"]}, {"sent": "its a macaroni with sauce on it .", "words": ["sauce", "with", "it", "on", "a"]}, {"sent": "can you find me any tomato sauce ?", "words": ["can", "sauce", "any", "you", "find", "me"]}, {"sent": "Ive got some sauce .", "words": ["some", "sauce"]}, {"sent": "thats a marinara sauce this is with meat .", "words": ["is", "sauce", "with", "meat", "this", "a"]}, {"sent": "I eat hot sauce on everything .", "words": ["eat", "hot", "on", "sauce"]}, {"sent": "is it brown sauce or is it tomato sauce ?", "words": ["brown", "it", "sauce", "is"]}, {"sent": "the cheese sauce now .", "words": ["the", "cheese", "sauce"]}, {"sent": "tomato sauce .", "words": ["sauce"]}, {"sent": "yes I mixed it into a large amount of pasta thats not a lot of tomato sauce thats a small amount .", "words": ["into", "sauce", "of", "a lot", "it", "not", "a"]}, {"sent": "sauce ?", "words": ["sauce"]}, {"sent": "apple sauce whats that ?", "words": ["apple", "that", "sauce"]}, {"sent": "well make your own sauce and boil your .", "words": ["your", "make", "and", "sauce"]}, {"sent": "Ill xxx get this mince sauce out .", "words": ["get", "this", "out", "sauce"]}, {"sent": "tomato sauce sauce ?", "words": ["sauce"]}, {"sent": "have they got salt and vinegar on or red sauce ?", "words": ["red", "sauce", "and", "have", "they", "salt", "on"]}, {"sent": "thats perhaps to pour the sauce out .", "words": ["the", "sauce", "to", "pour", "out"]}, {"sent": "and tomato sauce .", "words": ["sauce", "and"]}, {"sent": "what kind of sauce ?", "words": ["of", "what", "sauce"]}, {"sent": "more apple sauce .", "words": ["apple", "more", "sauce"]}, {"sent": "used to feed you spaghetti sauce every afternoon at four oclock ?", "words": ["spaghetti", "sauce", "to", "you", "at", "every", "feed"]}, {"sent": "the sauce .", "words": ["the", "sauce"]}, {"sent": "brown sauce .", "words": ["sauce", "brown"]}, {"sent": "its sauce .", "words": ["sauce"]}, {"sent": "what dyou have where dyou put your sauce ?", "words": ["where", "what", "put", "your", "sauce", "have"]}, {"sent": "milk and brown sauce .", "words": ["milk", "brown", "sauce", "and"]}, {"sent": "um if youre gonna tomato sauce .", "words": ["if", "sauce"]}, {"sent": "apple sauce doesnt have the skin does it ?", "words": ["the", "does", "sauce", "have", "it", "apple"]}, {"sent": "and then xxx the sauce ?", "words": ["the", "then", "sauce", "and"]}, {"sent": "special sauce .", "words": ["sauce"]}, {"sent": "Daddy has gone to work hasnt he ?", "words": ["he", "to", "work"]}, {"sent": "how does it work ?", "words": ["does", "how", "it", "work"]}, {"sent": "*AD1: we xxx work today .", "words": ["work", "we"]}, {"sent": "oh yeah well your daddy went to work today but your mommy didnt go to work so shes gonna be at home when we get there .", "words": ["your", "work", "go", "we", "to", "so", "be", "get", "there", "at", "when", "but", "home"]}, {"sent": "on the work bench .", "words": ["on", "the", "work", "bench"]}, {"sent": "yeah we glued it on but I guess it didnt work very well .", "words": ["work", "we", "it", "on", "but"]}, {"sent": "well he had better get back to work hadnt he ?", "words": ["better", "work", "back", "to", "get", "he"]}, {"sent": "you gotta work cmon you gotta work .", "words": ["work", "you"]}, {"sent": "who comes to look after you when Mummys at work on Wednesdays ?", "words": ["work", "to", "you", "at", "look", "on", "who", "when"]}, {"sent": "you hafta work and pick up heavy things .", "words": ["up", "work", "pick", "and", "you", "heavy"]}, {"sent": "this way if you have to do all the work .", "words": ["the", "do", "all", "work", "if", "to", "you", "have", "this"]}, {"sent": "make it work .", "words": ["work", "it", "make"]}, {"sent": "that very nice work .", "words": ["work", "nice", "that"]}, {"sent": "daddys going to work over there .", "words": ["over", "to", "work", "there"]}, {"sent": "Im going to go back to work .", "words": ["to", "work", "back", "go"]}, {"sent": "all this work I hafta do this morning .", "words": ["all", "do", "work", "this"]}, {"sent": "you know what what his plan is what time he has to work until or what you know .", "words": ["what", "work", "is", "to", "you", "he", "his"]}, {"sent": "why didnt it work ?", "words": ["it", "work", "why"]}, {"sent": "because its hard work .", "words": ["hard", "work", "because"]}, {"sent": "is he going back to work ?", "words": ["work", "back", "is", "to", "he"]}, {"sent": "uh uh uh uh hes got to work .", "words": ["to", "work"]}, {"sent": "does this work this lift ?", "words": ["does", "work", "this"]}, {"sent": "because Mummy will be at work wont she ?", "words": ["work", "be", "at", "will", "she", "because"]}, {"sent": "I guess that wasnt made to work .", "words": ["to", "that", "work"]}, {"sent": "because she hadta go to work .", "words": ["work", "go", "to", "she", "because"]}, {"sent": "Im going off to work to get some pennies .", "words": ["off", "work", "to", "get", "some"]}, {"sent": "good work .", "words": ["work", "good"]}, {"sent": "its hard work isnt it ?", "words": ["it", "work", "hard"]}, {"sent": "Sue has gone to work .", "words": ["to", "work"]}, {"sent": "do you help daddy with his work do you ?", "words": ["do", "help", "work", "with", "you", "his"]}, {"sent": "these binoculars work .", "words": ["these", "work"]}, {"sent": "no it doesnt work .", "words": ["it", "work"]}, {"sent": "no it wont work .", "words": ["it", "work"]}, {"sent": "Sue and I will be work mother at your school again right ?", "words": ["work", "your", "be", "school", "and", "at", "will"]}, {"sent": "are you gonna do some work ?", "words": ["do", "work", "you", "are", "some"]}, {"sent": "whistle while you work work work .", "words": ["work", "you"]}, {"sent": "and he wont be home from work tonight until half past eight nine oclock .", "words": ["work", "be", "and", "he", "home"]}, {"sent": "okay make it work .", "words": ["work", "it", "make"]}, {"sent": "yeah but then somebody will work the elevator and what will happen to your tiger ?", "words": ["the", "what", "work", "your", "to", "and", "tiger", "will", "then", "but"]}, {"sent": "another work .", "words": ["another", "work"]}, {"sent": "you work out how to take it off then .", "words": ["off", "how", "work", "to", "you", "it", "out", "then", "take"]}, {"sent": "your legs dont work ?", "words": ["dont", "your", "work"]}, {"sent": "is that going to work ?", "words": ["to", "that", "work", "is"]}, {"sent": "where does he work ?", "words": ["where", "does", "he", "work"]}, {"sent": "that didnt work did it ?", "words": ["work", "it", "that", "did"]}, {"sent": "and I wanna see how they work .", "words": ["how", "work", "and", "they", "wanna", "see"]}, {"sent": "this will work take this piece of grass go like this .", "words": ["work", "go", "of", "like", "this", "will", "take", "grass"]}, {"sent": "those trains dont really work on this train track very well do they ?", "words": ["do", "those", "work", "train", "dont", "this", "they", "on"]}, {"sent": "these dont work either .", "words": ["dont", "these", "work"]}, {"sent": "are you able to make it work ?", "words": ["work", "to", "you", "it", "are", "make"]}, {"sent": "I hope to be done with my clinical work fast enough so that I can do some Nobotics .", "words": ["can", "do", "my", "fast", "work", "to", "be", "so", "with", "some", "that"]}, {"sent": "and now theyll work .", "words": ["work", "and"]}, {"sent": "and theres like work book .", "words": ["book", "work", "like", "and"]}, {"sent": "wind can help us work windmills can pump water from wells deep in the ground .", "words": ["can", "the", "help", "work", "us", "in", "water", "wind"]}, {"sent": "mommys gonna go in the car to go to work .", "words": ["the", "work", "go", "car", "to", "in"]}, {"sent": "where does Daddy work ?", "words": ["where", "does", "work"]}, {"sent": "you want ta work on it ?", "words": ["on", "it", "work", "you"]}, {"sent": "what what work you did with her .", "words": ["what", "work", "with", "you", "did", "her"]}, {"sent": "how about that one can that one work ?", "words": ["can", "how", "work", "about", "that"]}, {"sent": "well sometimes things dont work out just right .", "words": ["dont", "work", "out"]}, {"sent": "thats extra work for Mommy and Daddy when they wanna clean up .", "words": ["clean", "up", "for", "work", "and", "they", "wanna", "when"]}, {"sent": "Daddys still at work isnt he ?", "words": ["he", "work", "at"]}, {"sent": "we need does that work on there ?", "words": ["does", "need", "work", "we", "there", "on", "that"]}, {"sent": "tell me about your work .", "words": ["work", "me", "your", "about"]}, {"sent": "hes um he does contract work for PhilipsWhirlpool .", "words": ["does", "for", "he", "work"]}, {"sent": "but that doesnt work .", "words": ["work", "that", "but"]}, {"sent": "that didnt work at all .", "words": ["all", "work", "that", "at"]}, {"sent": "Dada hasta go to work today .", "words": ["to", "work", "go"]}, {"sent": "what do where do you work ?", "words": ["where", "what", "do", "work", "you"]}, {"sent": "um Sue Sues home from work isnt she ?", "words": ["work", "she", "home"]}, {"sent": "children going to work yeah .", "words": ["to", "work"]}, {"sent": "okay so thisll work .", "words": ["so", "work"]}, {"sent": "right now shes at work .", "words": ["work", "at"]}, {"sent": "thats what makes it work .", "words": ["what", "it", "work"]}, {"sent": "I thought something went up the vacuum but I couldnt work out what it was .", "words": ["up", "the", "what", "work", "was", "vacuum", "it", "out", "but"]}, {"sent": "so I didnt work Monday then .", "words": ["so", "then", "work"]}, {"sent": "oh how is she gonna get to work ?", "words": ["how", "work", "is", "to", "get", "she"]}, {"sent": "I dont think thatll work .", "words": ["dont", "think", "work"]}, {"sent": "and that doesnt work either .", "words": ["work", "that", "and"]}, {"sent": "I dont think thats going to work .", "words": ["to", "dont", "think", "work"]}, {"sent": "I have something thatll work .", "words": ["have", "work"]}, {"sent": "what kind of work are they going to do ?", "words": ["what", "do", "work", "of", "to", "they", "are"]}, {"sent": "good work .", "words": ["work", "good"]}, {"sent": "its a pity the lift doesnt work isnt it ?", "words": ["the", "it", "work", "a"]}, {"sent": "thats Daddys for work .", "words": ["for", "work"]}, {"sent": "I cant think how it would work really Thomas .", "words": ["think", "how", "work", "it", "would"]}, {"sent": "xxx work xxx .", "words": ["work"]}, {"sent": "mommys going to work ?", "words": ["to", "work"]}, {"sent": "what day does Mummy go to choir uh to work ?", "words": ["what", "does", "work", "go", "to"]}, {"sent": "did Daddy hafta work this week ?", "words": ["this", "work", "did"]}, {"sent": "we hafta do some of your work in your book today .", "words": ["do", "your", "work", "we", "of", "in", "book", "some"]}, {"sent": "you did lots of work today .", "words": ["of", "work", "did", "you"]}, {"sent": "from Mummys work .", "words": ["work"]}, {"sent": "oops that uh doesnt work very well but .", "words": ["work", "that", "but"]}, {"sent": "well see if you can make that light work .", "words": ["can", "work", "if", "light", "you", "that", "see", "make"]}, {"sent": "well hafta work out what well be doing soon .", "words": ["what", "be", "work", "out"]}, {"sent": "doesnt work like that at all xxx .", "words": ["all", "work", "like", "at", "that"]}, {"sent": "so it doesnt work any more no .", "words": ["work", "so", "any", "more", "it"]}, {"sent": "its hard work .", "words": ["work", "hard"]}, {"sent": "those are carrots in a bag .", "words": ["those", "in", "are", "carrots", "a"]}, {"sent": "just because its not potato and carrots .", "words": ["and", "carrots", "not", "because", "potato"]}, {"sent": "no Laura cant have the carrots .", "words": ["have", "the", "carrots"]}, {"sent": "I can see two carrots as well .", "words": ["can", "see", "carrots"]}, {"sent": "what else do you have with potatoes and carrots ?", "words": ["what", "do", "with", "you", "have", "and", "carrots"]}, {"sent": "Kim and carrots build the bird feeder with seeds .", "words": ["the", "bird", "with", "and", "build", "carrots"]}, {"sent": "he worked in the garden and helped Mr McGregor with his carrots .", "words": ["the", "with", "and", "garden", "in", "he", "carrots", "his"]}, {"sent": "well young carrots you can but that carrot had been in the fridge for a good couple of weeks .", "words": ["can", "the", "for", "of", "you", "in", "carrots", "good", "that", "but", "a"]}, {"sent": "mhm mhm carrots .", "words": ["carrots"]}, {"sent": "how many carrots have we got ?", "words": ["have", "how", "carrots", "we"]}, {"sent": "Rabbit has a garden he grows carrots I wish he could grow honey .", "words": ["wish", "garden", "he", "carrots", "could", "a"]}, {"sent": "and some carrots .", "words": ["some", "carrots", "and"]}, {"sent": "and then at the end of it we end of it we ended up with four carrots when in actual fact weve got one two three four five .", "words": ["the", "up", "we", "of", "with", "and", "in", "it", "carrots", "at", "then", "when"]}, {"sent": "but they also eat carrots dont they ?", "words": ["eat", "they", "dont", "carrots", "but"]}, {"sent": "you want me to give you carrots so you can feed him ?", "words": ["can", "to", "so", "you", "him", "carrots", "feed", "me", "give"]}, {"sent": "do we have any carrots ?", "words": ["do", "we", "any", "have", "carrots"]}, {"sent": "did she tell her Mummy that Nacnac ate the carrots ?", "words": ["the", "her", "did", "carrots", "that", "she"]}, {"sent": "who wants carrots ?", "words": ["carrots", "who"]}, {"sent": "carrots ?", "words": ["carrots"]}, {"sent": "oh I dont usually have carrots in my hair Thomas .", "words": ["my", "have", "dont", "in", "carrots", "hair"]}, {"sent": "and all the nice carrots .", "words": ["the", "all", "nice", "and", "carrots"]}, {"sent": "who ate my carrots ?", "words": ["my", "carrots", "who"]}, {"sent": "you like carrots .", "words": ["like", "carrots", "you"]}, {"sent": "are you tipping it up are you tipping the trailer up with all the carrots in ?", "words": ["up", "the", "all", "with", "you", "in", "it", "are", "carrots"]}, {"sent": "Ill wear carrots and peas a little more please .", "words": ["peas", "little", "and", "more", "carrots", "a"]}, {"sent": "who ate the carrots ?", "words": ["the", "carrots", "who"]}, {"sent": "thats Peter rabbit eating carrots and whats on top of the spade .", "words": ["the", "of", "and", "carrots", "on"]}, {"sent": "would you likes me to grind up some sesame to have on top of this spinach and carrots ?", "words": ["up", "of", "to", "you", "have", "this", "and", "carrots", "would", "on", "me", "some"]}, {"sent": "they eat carrots .", "words": ["eat", "they", "carrots"]}, {"sent": "oh its carrots now is it ?", "words": ["it", "carrots", "is"]}, {"sent": "he likes carrots .", "words": ["he", "carrots"]}, {"sent": "and carrots .", "words": ["carrots", "and"]}, {"sent": "two carrots ah theyre running everywhere uhn arent they ?", "words": ["they", "carrots"]}, {"sent": "well I dont know if there are any more carrots left .", "words": ["if", "any", "dont", "there", "are", "more", "carrots"]}, {"sent": "remember eat carrots ?", "words": ["eat", "carrots"]}, {"sent": "she might not be able to take two carrots at once .", "words": ["to", "be", "at", "carrots", "she", "take", "not"]}, {"sent": "the carrots have got to go into the freezer .", "words": ["the", "into", "go", "to", "have", "carrots"]}, {"sent": "carrots .", "words": ["carrots"]}, {"sent": "were they have they been eating the carrots ?", "words": ["the", "have", "they", "carrots", "were"]}, {"sent": "you like carrots too ?", "words": ["too", "like", "carrots", "you"]}, {"sent": "carrots .", "words": ["carrots"]}, {"sent": "wheres carrots ?", "words": ["carrots"]}, {"sent": "whenever I give you peas and carrots you dont eat them .", "words": ["peas", "them", "and", "you", "dont", "eat", "carrots", "give"]}, {"sent": "what is it are you eating carrots ?", "words": ["what", "is", "you", "it", "are", "carrots"]}, {"sent": "maybe we could pickle some carrots make them sour .", "words": ["we", "them", "pickle", "carrots", "some", "could", "make"]}, {"sent": "the rabbit eats carrots .", "words": ["the", "carrots"]}, {"sent": "and carrots ?", "words": ["carrots", "and"]}, {"sent": "those are carrots .", "words": ["are", "those", "carrots"]}, {"sent": "some more carrots .", "words": ["some", "more", "carrots"]}, {"sent": "thats alotof carrots isnt it ?", "words": ["it", "carrots"]}, {"sent": "and she eats carrots .", "words": ["she", "carrots", "and"]}, {"sent": "yeah and what youre coloring right there is carrots .", "words": ["what", "is", "and", "there", "carrots"]}, {"sent": "so carrots and um squash and sweet potato .", "words": ["so", "carrots", "potato", "and"]}, {"sent": "hehe carrots carrots .", "words": ["carrots"]}, {"sent": "hes peeling carrots .", "words": ["carrots"]}, {"sent": "how can they be carrots I said ?", "words": ["can", "how", "be", "they", "carrots"]}, {"sent": "and carrots .", "words": ["carrots", "and"]}, {"sent": "do you do you eat carrots ?", "words": ["eat", "do", "carrots", "you"]}, {"sent": "yumyum carrots .", "words": ["carrots"]}, {"sent": "why does he like to eat juicy carrots ?", "words": ["does", "to", "like", "eat", "he", "carrots", "why"]}, {"sent": "some things are easy to choke on like carrots .", "words": ["to", "like", "are", "carrots", "on", "some"]}, {"sent": "those are carrots .", "words": ["are", "those", "carrots"]}, {"sent": "those are carrots .", "words": ["are", "those", "carrots"]}, {"sent": "does Nia say that she ate the carrots ?", "words": ["the", "does", "say", "carrots", "that", "she"]}, {"sent": "where is my carrots ?", "words": ["where", "my", "carrots", "is"]}, {"sent": "is he dreaming about carrots ?", "words": ["he", "carrots", "about", "is"]}, {"sent": "big carrots and make little pieces out of them with the grater ?", "words": ["the", "of", "them", "with", "little", "and", "big", "carrots", "out", "make"]}, {"sent": "another carrots .", "words": ["another", "carrots"]}, {"sent": "what color are carrots ?", "words": ["what", "are", "carrots"]}, {"sent": "Nia didnt eat the carrots .", "words": ["eat", "the", "carrots"]}, {"sent": "yeah maybe theyre gonna make something to eat with the carrots ?", "words": ["the", "to", "with", "eat", "carrots", "make"]}, {"sent": "we dont want carrots either do we ?", "words": ["dont", "do", "carrots", "we"]}, {"sent": "you wanna watch the one about carrots .", "words": ["the", "watch", "you", "carrots", "about", "wanna"]}, {"sent": "you eat your carrots and your string beans .", "words": ["your", "and", "eat", "you", "beans", "carrots"]}, {"sent": "do you want carrots Darren ?", "words": ["do", "carrots", "you"]}, {"sent": "he does have like um um apples and carrots and pears and eggs and youknow .", "words": ["does", "like", "and", "have", "he", "carrots"]}, {"sent": "carrots orange .", "words": ["orange", "carrots"]}, {"sent": "of course youre eating carrots so you willnt say anything will you Becky ?", "words": ["of", "so", "say", "you", "carrots", "will"]}, {"sent": "lets have the mac and cheese and you can have carrots from a jar too if you want .", "words": ["the", "can", "if", "jar", "you", "have", "and", "carrots", "too", "cheese", "a"]}, {"sent": "Im not sure if we have any carrots .", "words": ["if", "we", "any", "have", "carrots", "not"]}, {"sent": "maybe they like orange color carrots .", "words": ["they", "orange", "like", "carrots"]}, {"sent": "six orange carrots .", "words": ["orange", "carrots"]}, {"sent": "carrots .", "words": ["carrots"]}, {"sent": "do you just want carrots for your dinner ?", "words": ["do", "for", "your", "you", "carrots"]}, {"sent": "and carrots .", "words": ["carrots", "and"]}, {"sent": "oh boy you got carrots in your ear .", "words": ["your", "you", "in", "carrots", "ear"]}, {"sent": "I think you could make potatoes and sweet and carrots .", "words": ["think", "and", "you", "carrots", "could", "make"]}, {"sent": "carrots .", "words": ["carrots"]}, {"sent": "what color are carrots ?", "words": ["what", "are", "carrots"]}, {"sent": "the carrots .", "words": ["the", "carrots"]}, {"sent": "orange carrots .", "words": ["orange", "carrots"]}, {"sent": "and carrots .", "words": ["carrots", "and"]}, {"sent": "here have some carrots parrot .", "words": ["here", "some", "have", "carrots"]}, {"sent": "carrots .", "words": ["carrots"]}, {"sent": "horses like to eat apples and carrots .", "words": ["to", "like", "and", "eat", "carrots"]}, {"sent": "hes eating the carrots from this book before the wee mice get them .", "words": ["the", "them", "get", "this", "carrots", "book"]}, {"sent": "a few dashed away and returned with fresh mushrooms noodles pea pods and carrots .", "words": ["with", "and", "away", "carrots", "noodles", "a"]}, {"sent": "rabbits eat carrots .", "words": ["eat", "carrots"]}, {"sent": "we need carrots .", "words": ["need", "carrots", "we"]}, {"sent": "that night at dinner pj sat and ate his mashed carrots .", "words": ["and", "carrots", "at", "his", "that"]}, {"sent": "lets try another one lets start another one .", "words": ["another", "try"]}, {"sent": "why dont you try putting in .", "words": ["you", "dont", "in", "try", "why"]}, {"sent": "lets just try this .", "words": ["this", "try"]}, {"sent": "try again .", "words": ["try"]}, {"sent": "let me try to finish reading this .", "words": ["to", "finish", "this", "me", "try", "try to"]}, {"sent": "so we try and use the rubbish dont we ?", "words": ["the", "we", "so", "and", "dont", "try"]}, {"sent": "*TO2: want to try another ?", "words": ["another", "to", "try", "want to"]}, {"sent": "lets try to move a little bit .", "words": ["to", "little", "try", "try to", "a"]}, {"sent": "try .", "words": ["try"]}, {"sent": "n try try and try try and try .", "words": ["try", "and"]}, {"sent": "wanna try another one ?", "words": ["another", "try", "wanna"]}, {"sent": "here let us try the other end .", "words": ["the", "other", "here", "us", "try"]}, {"sent": "here try that .", "words": ["here", "that", "try"]}, {"sent": "well now we try again with this tape recorder .", "words": ["we", "with", "tape", "this", "try"]}, {"sent": "now try .", "words": ["try"]}, {"sent": "try try and put it on like a pair or trousers or a pair of shorts and it might go on easier eh ?", "words": ["put", "shorts", "go", "of", "like", "and", "it", "on", "try", "a"]}, {"sent": "here now try you try .", "words": ["here", "try", "you"]}, {"sent": "dwan try writing it yourself ?", "words": ["yourself", "it", "try"]}, {"sent": "and try to turn hell shove a screwdriver into everything .", "words": ["into", "to", "and", "try", "try to", "a"]}, {"sent": "oh lets try to keep the crayons on the table though .", "words": ["the", "table", "to", "on", "try", "try to"]}, {"sent": "right lets try again .", "words": ["try"]}, {"sent": "you try .", "words": ["try", "you"]}, {"sent": "try this one .", "words": ["this", "try"]}, {"sent": "okay try and get that Tellytubbies book out for me .", "words": ["try", "for", "and", "get", "out", "book", "me", "that"]}, {"sent": "did he say lets try again ?", "words": ["try", "he", "did", "say"]}, {"sent": "try it .", "words": ["it", "try"]}, {"sent": "try it with the fork .", "words": ["the", "with", "it", "fork", "try"]}, {"sent": "try it now .", "words": ["it", "try"]}, {"sent": "yes youre getting very excited I think Ill try to think of something else look at this let me get that wheres the other piece of the slide ?", "words": ["the", "think", "slide", "of", "to", "other", "get", "this", "that", "at", "look", "me", "try", "try to"]}, {"sent": "okay let me try it .", "words": ["me", "it", "try"]}, {"sent": "try .", "words": ["try"]}, {"sent": "you try one more time .", "words": ["more", "try", "you"]}, {"sent": "well I think you probably do you should try .", "words": ["do", "think", "try", "you"]}, {"sent": "Gaspard and Lisa our teacher said please ty try to stay out of trouble .", "words": ["of", "to", "our", "and", "out", "stay", "try", "try to"]}, {"sent": "lets have a lets have a try .", "words": ["have", "try", "a"]}, {"sent": "try it .", "words": ["it", "try"]}, {"sent": "wanna try green wanna try green try green .", "words": ["try", "green", "wanna"]}, {"sent": "what are you going to try and make outof it ?", "words": ["what", "to", "you", "and", "are", "make", "it", "try"]}, {"sent": "wait till you try icecream .", "words": ["wait", "try", "you"]}, {"sent": "why dont you try and actually fit two pieces together because it they move about dont they ?", "words": ["fit", "because", "and", "you", "dont", "they", "it", "about", "try", "why"]}, {"sent": "lets try that again .", "words": ["that", "try"]}, {"sent": "try to think back .", "words": ["think", "back", "to", "try", "try to"]}, {"sent": "sometimes hell try to put the toy in my mouth after he takes it .", "words": ["the", "put", "my", "to", "mouth", "toy", "in", "he", "it", "try", "try to"]}, {"sent": "try this one .", "words": ["this", "try"]}, {"sent": "so are you going to try and get round some Wednesday ?", "words": ["to", "so", "you", "and", "get", "are", "some", "try"]}, {"sent": "you try it .", "words": ["it", "try", "you"]}, {"sent": "try again .", "words": ["try"]}, {"sent": "try it on the playdough .", "words": ["on", "the", "it", "try"]}, {"sent": "the elephants try to get you ?", "words": ["the", "to", "you", "get", "try", "try to"]}, {"sent": "lets try another one .", "words": ["another", "try"]}, {"sent": "how do you try these ?", "words": ["do", "how", "these", "you", "try"]}, {"sent": "now you try putting that on .", "words": ["on", "that", "try", "you"]}, {"sent": "I think you might hafta try and .", "words": ["think", "try", "and", "you"]}, {"sent": "wanna try that one ?", "words": ["that", "try", "wanna"]}, {"sent": "alright Ill try one more time .", "words": ["more", "try"]}, {"sent": "try him again .", "words": ["him", "try"]}, {"sent": "well lets try one of these little square ones .", "words": ["of", "these", "try", "little"]}, {"sent": "let me take some time to to cut out some shapes for you because its a little bit difficult for you to cut out shapes unless you wanna try .", "words": ["for", "try", "cut", "a", "to", "you", "little", "out", "wanna", "me", "take", "because", "some"]}, {"sent": "try it on your hand .", "words": ["your", "hand", "it", "on", "try"]}, {"sent": "wanna try something else ?", "words": ["try", "wanna"]}, {"sent": "we can give that a try .", "words": ["can", "try", "we", "give", "that", "a"]}, {"sent": "you wanna try ?", "words": ["try", "wanna", "you"]}, {"sent": "okay lets try a different story okay ?", "words": ["story", "try", "a"]}, {"sent": "lets try it that way round then .", "words": ["that", "it", "try", "then"]}, {"sent": "why dont you try that like when you go hiking with Daddy ?", "words": ["try", "when", "go", "with", "like", "you", "dont", "that", "why"]}, {"sent": "try that bit along the side .", "words": ["the", "that", "try"]}, {"sent": "try try and lift that one up without breaking it .", "words": ["up", "try", "and", "it", "that"]}, {"sent": "does she try to pretty up herself or other people ?", "words": ["up", "does", "to", "other", "pretty", "try to", "try", "she"]}, {"sent": "lets try and make these things .", "words": ["these", "make", "try", "and"]}, {"sent": "lets go try and have some .", "words": ["go", "and", "have", "some", "try"]}, {"sent": "yeah some of them she didnt try but the Pullaphone Pullahphone .", "words": ["the", "of", "them", "some", "try", "she", "but"]}, {"sent": "nice try baby .", "words": ["nice", "try"]}, {"sent": "you wanna try that ?", "words": ["that", "try", "wanna", "you"]}, {"sent": "lets try up here a little higher .", "words": ["up", "little", "here", "try", "a"]}, {"sent": "I guess you can try though .", "words": ["can", "try", "you"]}, {"sent": "let me try that one .", "words": ["me", "that", "try"]}, {"sent": "well shall we try it ?", "words": ["it", "try", "we"]}, {"sent": "now try again .", "words": ["try"]}, {"sent": "do you wanna try that ?", "words": ["do", "try", "you", "wanna", "that"]}, {"sent": "you wanna try this one ?", "words": ["this", "try", "wanna", "you"]}, {"sent": "here try it again .", "words": ["here", "it", "try"]}, {"sent": "alright try this .", "words": ["this", "try"]}, {"sent": "you wanna try this one ?", "words": ["this", "try", "wanna", "you"]}, {"sent": "try and put it over the shape .", "words": ["the", "put", "try", "and", "it", "over"]}, {"sent": "and try it again .", "words": ["it", "try", "and"]}, {"sent": "try it one more time .", "words": ["more", "it", "try"]}, {"sent": "you want me to try and put a battery in him or no ?", "words": ["put", "to", "and", "you", "in", "him", "me", "try", "a"]}, {"sent": "can you try it ?", "words": ["can", "it", "try", "you"]}, {"sent": "you wanna try this drink ?", "words": ["drink", "you", "this", "wanna", "try"]}, {"sent": "you wanna try ?", "words": ["try", "wanna", "you"]}, {"sent": "shell try everything she eats Thomas .", "words": ["try", "she"]}, {"sent": "yeah lets try .", "words": ["try"]}, {"sent": "so I bought so I ordered six dresses from this company just to try them all on .", "words": ["all", "to", "so", "them", "this", "on", "try"]}, {"sent": "okay I think you wanna try this kinda cracker ?", "words": ["think", "cracker", "you", "this", "wanna", "try"]}, {"sent": "try it .", "words": ["it", "try"]}, {"sent": "try the back .", "words": ["the", "try", "back"]}, {"sent": "well you can try .", "words": ["can", "try", "you"]}, {"sent": "let me try .", "words": ["me", "try"]}, {"sent": "you wanna try mommys backpack ?", "words": ["try", "wanna", "you"]}, {"sent": "now try .", "words": ["try"]}, {"sent": "that mine its my book .", "words": ["book", "mine", "my", "that"]}, {"sent": "can I put mine on now ?", "words": ["can", "mine", "put", "on"]}, {"sent": "youre going to punch mine up ?", "words": ["up", "to", "mine"]}, {"sent": "oh mine ?", "words": ["mine"]}, {"sent": "that ones mine .", "words": ["mine", "that"]}, {"sent": "no mine mine mine mine .", "words": ["mine"]}, {"sent": "I thought that was mine .", "words": ["was", "that", "mine"]}, {"sent": "because VanNich had a sputter exactly like mine and he yelled .", "words": ["like", "and", "mine", "he", "because", "a"]}, {"sent": "is this mine ?", "words": ["this", "mine", "is"]}, {"sent": "oh no she doesnt want any of mine .", "words": ["of", "mine", "any", "she"]}, {"sent": "mines full of grey Imean I mine is .", "words": ["of", "mine", "full", "is"]}, {"sent": "whats in mine ?", "words": ["mine", "in"]}, {"sent": "yeah theyre mine .", "words": ["mine"]}, {"sent": "I still iron mine .", "words": ["mine"]}, {"sent": "Ive done mine .", "words": ["mine"]}, {"sent": "its mine .", "words": ["mine"]}, {"sent": "can I have yours and you have mine ?", "words": ["can", "you", "have", "mine", "and"]}, {"sent": "and mine is in Florida .", "words": ["mine", "in", "is", "and"]}, {"sent": "mine ?", "words": ["mine"]}, {"sent": "but I dont hafta take the skin off mine do I ?", "words": ["the", "off", "do", "dont", "mine", "take", "but"]}, {"sent": "I wanna do mine the same .", "words": ["the", "do", "mine", "same", "wanna"]}, {"sent": "look mine .", "words": ["mine", "look"]}, {"sent": "not mine .", "words": ["mine", "not"]}, {"sent": "is it mine ?", "words": ["mine", "it", "is"]}, {"sent": "glad that tongues not mine .", "words": ["mine", "that", "not"]}, {"sent": "meaning youre pointing to Mommys when you say mine .", "words": ["to", "say", "you", "mine", "when"]}, {"sent": "you make mine that big .", "words": ["you", "mine", "big", "make", "that"]}, {"sent": "yours is right there see mine goes here .", "words": ["is", "here", "there", "mine", "see"]}, {"sent": "mine fell out my mouth .", "words": ["mine", "my", "mouth", "out"]}, {"sent": "you dont need mine .", "words": ["dont", "need", "mine", "you"]}, {"sent": "oh thankyou what kind is mine ?", "words": ["what", "mine", "is"]}, {"sent": "mine kinda looks like a .", "words": ["mine", "like", "a"]}, {"sent": "youre pointing to mine .", "words": ["to", "mine"]}, {"sent": "Ill put yours next to mine because then I can reach it .", "words": ["can", "put", "to", "mine", "it", "then", "because"]}, {"sent": "mine .", "words": ["mine"]}, {"sent": "Ive got to polish mine here .", "words": ["here", "to", "mine"]}, {"sent": "I think yours is bigger than mine .", "words": ["mine", "think", "is"]}, {"sent": "if you have the um insurance cards in your wallet for the kids or mine I dunno .", "words": ["the", "for", "your", "if", "you", "have", "mine", "in"]}, {"sent": "it cried soon and crashed into mine .", "words": ["mine", "it", "into", "and"]}, {"sent": "oh this is mine .", "words": ["this", "mine", "is"]}, {"sent": "wheres mine ?", "words": ["mine"]}, {"sent": "did you say mind or mine ?", "words": ["mine", "say", "did", "you"]}, {"sent": "mine .", "words": ["mine"]}, {"sent": "its mine mommy .", "words": ["mine"]}, {"sent": "thats mine .", "words": ["mine"]}, {"sent": "your birthdays nearly the same day as mine .", "words": ["the", "same", "your", "mine"]}, {"sent": "I feed mine on corn .", "words": ["on", "mine", "corn", "feed"]}, {"sent": "its no good saying sorry Mummy its mine now .", "words": ["mine", "good"]}, {"sent": "is he going down the mine ?", "words": ["the", "is", "mine", "he", "down"]}, {"sent": "no you cant eat mine either .", "words": ["eat", "mine", "you"]}, {"sent": "oh is that mine ?", "words": ["mine", "that", "is"]}, {"sent": "is this mine ?", "words": ["this", "mine", "is"]}, {"sent": "look at the inside of mine .", "words": ["the", "look", "of", "inside", "mine", "at"]}, {"sent": "xxx all mine .", "words": ["all", "mine"]}, {"sent": "Im making mine xxx ah .", "words": ["mine"]}, {"sent": "I gave him mine .", "words": ["mine", "him"]}, {"sent": "these are mine too ?", "words": ["these", "are", "mine", "too"]}, {"sent": "is that mine ?", "words": ["mine", "that", "is"]}, {"sent": "which ones mine ?", "words": ["mine", "which"]}, {"sent": "not mine .", "words": ["mine", "not"]}, {"sent": "mine paper .", "words": ["mine", "paper"]}, {"sent": "mine ?", "words": ["mine"]}, {"sent": "oh I found mine .", "words": ["mine"]}, {"sent": "I just made a nose on mine too .", "words": ["mine", "too", "on", "a", "nose"]}, {"sent": "well if weve got trucks I dont even know what Ive got now in mine .", "words": ["what", "if", "dont", "in", "mine"]}, {"sent": "I I might put mine there .", "words": ["mine", "put", "there"]}, {"sent": "yeah why dont you play with mine ?", "words": ["play", "with", "you", "dont", "mine", "why"]}, {"sent": "on mine .", "words": ["on", "mine"]}, {"sent": "down into the mine shaft .", "words": ["the", "mine", "down", "into"]}, {"sent": "and I go to mine .", "words": ["go", "to", "mine", "and"]}, {"sent": "mine .", "words": ["mine"]}, {"sent": "does it suit me this new hat of mine ?", "words": ["does", "of", "this", "mine", "it", "hat", "me", "new"]}, {"sent": "mine .", "words": ["mine"]}, {"sent": "here this ones mine .", "words": ["here", "this", "mine"]}, {"sent": "I did mine in green .", "words": ["mine", "in", "did", "green"]}, {"sent": "oh thats mine thats mine thats mine .", "words": ["mine"]}, {"sent": "oh is that mine ?", "words": ["mine", "that", "is"]}, {"sent": "Im gonna take mine apart .", "words": ["mine", "take"]}, {"sent": "no it is mine .", "words": ["mine", "it", "is"]}, {"sent": "not all mine .", "words": ["all", "not", "mine"]}, {"sent": "mommy cut mine in four pieces .", "words": ["mine", "in", "cut"]}, {"sent": "lets turn mine over ?", "words": ["mine", "over"]}, {"sent": "well this is mine .", "words": ["this", "mine", "is"]}, {"sent": "no that is mine .", "words": ["mine", "that", "is"]}, {"sent": "its not your fathers its mine .", "words": ["mine", "your", "not"]}, {"sent": "see mine look .", "words": ["see", "mine", "look"]}, {"sent": "mine .", "words": ["mine"]}, {"sent": "listen to mine .", "words": ["to", "mine", "listen"]}, {"sent": "well you aint caught a rabbit you aint no friend a mine .", "words": ["mine", "a", "you"]}, {"sent": "if you end up with two xxx ones mine .", "words": ["up", "if", "with", "you", "mine"]}, {"sent": "is that one of them mine ?", "words": ["is", "of", "them", "mine", "that"]}, {"sent": "you take mine .", "words": ["mine", "take", "you"]}, {"sent": "but Im gonna put mine back on Im just fixing it .", "words": ["put", "back", "mine", "it", "on", "but"]}, {"sent": "I know what mine is .", "words": ["what", "mine", "is"]}, {"sent": "I can roll mine up by .", "words": ["can", "mine", "up", "by"]}, {"sent": "Dandogs mine now .", "words": ["mine"]}, {"sent": "looka mine .", "words": ["mine"]}, {"sent": "its mine .", "words": ["mine"]}, {"sent": "I know I I just wanna hold it thats not his thats not mine .", "words": ["hold", "mine", "it", "wanna", "his", "not"]}, {"sent": "want me to help you ?", "words": ["to", "me", "help", "you"]}, {"sent": "you want me to build it .", "words": ["to", "you", "it", "build", "me"]}, {"sent": "thanks for teaching me that Em .", "words": ["me", "for", "that"]}, {"sent": "can you get me a block that color ?", "words": ["can", "block", "you", "get", "me", "that", "a"]}, {"sent": "why dont you tell me what everything is ?", "words": ["what", "is", "you", "dont", "me", "why"]}, {"sent": "let me just see .", "words": ["see", "me"]}, {"sent": "why dont you tell me about that ride .", "words": ["ride", "you", "dont", "about", "me", "that", "why"]}, {"sent": "do it for me please ?", "words": ["do", "me", "it", "for"]}, {"sent": "so I would say pick me up at two thirty .", "words": ["up", "pick", "so", "say", "at", "would", "me"]}, {"sent": "psst uhoh Nicki help me .", "words": ["me", "help"]}, {"sent": "Id get me a real nice hot xxx .", "words": ["nice", "get", "hot", "me", "a"]}, {"sent": "pour pour some orange juice for me ?", "words": ["for", "me", "pour", "orange", "some", "juice"]}, {"sent": "how come youre calling me Taylor today ?", "words": ["me", "how"]}, {"sent": "youre biting me yes .", "words": ["me"]}, {"sent": "will you pat me to sleep ?", "words": ["sleep", "to", "you", "will", "me"]}, {"sent": "and I could say excuse me MrWaiter please could I have a bowl of cat food for my pussy cat and youd give it to me and I could put it outside for her ?", "words": ["bowl", "my", "for", "give", "put", "of", "to", "outside", "say", "and", "have", "cat", "food", "it", "me", "could", "her", "a"]}, {"sent": "I thought maybe you were doing it for me .", "words": ["for", "you", "it", "me", "were"]}, {"sent": "me .", "words": ["me"]}, {"sent": "are you copying me coughing ?", "words": ["me", "are", "you"]}, {"sent": "me take these blocks home for supper .", "words": ["these", "for", "me", "take", "home"]}, {"sent": "tell me about it .", "words": ["me", "it", "about"]}, {"sent": "its no good getting cross with me .", "words": ["me", "with", "good"]}, {"sent": "you tell me .", "words": ["me", "you"]}, {"sent": "how am I sposta do it when youre hanging on me ?", "words": ["do", "how", "it", "am", "on", "me", "when"]}, {"sent": "cmere want me to read you want Mommy to read it for you ?", "words": ["for", "to", "you", "it", "read", "me"]}, {"sent": "want me to open it ?", "words": ["to", "me", "it", "open"]}, {"sent": "let me mop your face up .", "words": ["up", "your", "face", "mop", "me"]}, {"sent": "let me have the didey .", "words": ["have", "me", "the"]}, {"sent": "okay and then when you feel better you let me know .", "words": ["better", "you", "and", "me", "then", "when"]}, {"sent": "me ?", "words": ["me"]}, {"sent": "you wanna make me happy ?", "words": ["happy", "you", "wanna", "me", "make"]}, {"sent": "Father Sheehan gave me them for you .", "words": ["me", "for", "them", "you"]}, {"sent": "me .", "words": ["me"]}, {"sent": "wanna tell me done ?", "words": ["me", "wanna"]}, {"sent": "excuse me man with the guitar .", "words": ["the", "me", "with"]}, {"sent": "are you copying me ?", "words": ["me", "are", "you"]}, {"sent": "can you help me make it taller ?", "words": ["can", "help", "you", "it", "me", "make"]}, {"sent": "let me see xxx .", "words": ["see", "me"]}, {"sent": "for me and .", "words": ["me", "for", "and"]}, {"sent": "can you clean this in the sink for me ?", "words": ["can", "clean", "the", "for", "you", "this", "in", "sink", "me"]}, {"sent": "you just hafta give me my keys thats all .", "words": ["all", "my", "you", "me", "give", "keys"]}, {"sent": "excuse me excuse me .", "words": ["me"]}, {"sent": "here let me open that up for ya .", "words": ["up", "for", "open", "here", "me", "that"]}, {"sent": "let me do it .", "words": ["me", "do", "it"]}, {"sent": "now wheres the piano that youve been telling me about ?", "words": ["the", "me", "that", "about"]}, {"sent": "have you come to see me ?", "words": ["see", "to", "you", "have", "me"]}, {"sent": "give me something good to eat .", "words": ["to", "eat", "good", "me", "give"]}, {"sent": "Im a go home and tell your grandmother that you didnt wanna come home with me .", "words": ["your", "go", "with", "and", "you", "wanna", "me", "that", "a", "home"]}, {"sent": "youre asking me what the choochoos doing .", "words": ["the", "me", "what"]}, {"sent": "let me look .", "words": ["me", "look"]}, {"sent": "throw the ball to me .", "words": ["the", "throw", "to", "ball", "me"]}, {"sent": "will you remind me to get syrup ?", "words": ["to", "you", "get", "will", "me"]}, {"sent": "tell me what you did last night .", "words": ["what", "last", "you", "did", "me"]}, {"sent": "Claire says to me .", "words": ["to", "me"]}, {"sent": "do you want me to do it ?", "words": ["do", "to", "you", "it", "me"]}, {"sent": "let me see your teeth .", "words": ["see", "me", "your"]}, {"sent": "I know well you sure woke me up .", "words": ["up", "me", "you"]}, {"sent": "theres no point negotiating with me to find one of them .", "words": ["of", "to", "with", "them", "find", "me"]}, {"sent": "right you try against me then .", "words": ["me", "then", "try", "you"]}, {"sent": "bring me the rabbits .", "words": ["the", "me", "bring"]}, {"sent": "not me .", "words": ["me", "not"]}, {"sent": "how about you read to me ?", "words": ["how", "to", "you", "read", "about", "me"]}, {"sent": "Adam let me see .", "words": ["see", "me"]}, {"sent": "give me the baby and Mommy fix it .", "words": ["the", "fix", "and", "it", "me", "give"]}, {"sent": "lemme see show me show me .", "words": ["show", "see", "me"]}, {"sent": "you want me to put the man in again ?", "words": ["the", "put", "to", "you", "in", "me"]}, {"sent": "she asked me to .", "words": ["to", "me", "she"]}, {"sent": "come show me where the refrigerator is .", "words": ["where", "the", "refrigerator", "is", "show", "me"]}, {"sent": "let me show you look .", "words": ["show", "me", "look", "you"]}, {"sent": "thats kind of that reminds me of a pool ladder .", "words": ["of", "pool", "ladder", "me", "that", "a"]}, {"sent": "let me see .", "words": ["see", "me"]}, {"sent": "Gary come and let me comb your hair .", "words": ["your", "comb", "and", "me", "hair"]}, {"sent": "can you pick up your sweater and fold it for me ?", "words": ["can", "up", "for", "your", "sweater", "pick", "you", "and", "it", "me"]}, {"sent": "why dont you wanna tell me ?", "words": ["you", "dont", "wanna", "me", "why"]}, {"sent": "want me to come and open it ?", "words": ["open", "to", "and", "it", "me"]}, {"sent": "let me go see who this is .", "words": ["go", "is", "me", "this", "who", "see"]}, {"sent": "you tell me the story .", "words": ["the", "me", "story", "you"]}, {"sent": "are you going to tell me the story ?", "words": ["the", "to", "you", "are", "me", "story"]}, {"sent": "theyre killing me ae .", "words": ["me"]}, {"sent": "you told me I was in the way .", "words": ["the", "was", "you", "in", "me"]}, {"sent": "hey wait let me get em .", "words": ["get", "me", "wait"]}, {"sent": "dis is for me but my mommy just got me this .", "words": ["my", "for", "is", "this", "me", "but"]}, {"sent": "well let me have a look .", "words": ["have", "me", "look", "a"]}, {"sent": "want me do that one ?", "words": ["me", "do", "that"]}, {"sent": "what did you tell me out in the kitchen whatd you do with Erika ?", "words": ["the", "what", "do", "with", "you", "in", "did", "out", "kitchen", "me"]}, {"sent": "nobody wants to play with the ball with me .", "words": ["the", "to", "play", "with", "ball", "me"]}, {"sent": "throw me the ball .", "words": ["the", "me", "ball", "throw"]}, {"sent": "now will you help me ?", "words": ["will", "me", "help", "you"]}, {"sent": "watch me jump over you .", "words": ["watch", "you", "jump", "me", "over"]}, {"sent": "wanna finish being tested with me ?", "words": ["me", "with", "finish", "wanna"]}, {"sent": "you want me to throw it higher ?", "words": ["throw", "to", "you", "it", "me"]}, {"sent": "can you tell me a story about what happens on this on this picture ?", "words": ["can", "what", "picture", "you", "this", "about", "on", "me", "story", "a"]}, {"sent": "would you help me close the door ?", "words": ["the", "help", "you", "would", "me", "close", "door"]}, {"sent": "she squeezes me too tight .", "words": ["me", "she", "too"]}, {"sent": "what do you want me to do with it ?", "words": ["what", "do", "to", "with", "you", "it", "me"]}, {"sent": "can you find me an n ?", "words": ["can", "you", "an", "find", "me"]}, {"sent": "let me get those pictures .", "words": ["get", "me", "those"]}, {"sent": "tell me about it .", "words": ["me", "it", "about"]}, {"sent": "goodness me .", "words": ["me"]}, {"sent": "let me do it one time .", "words": ["me", "do", "it"]}, {"sent": "let us talk about the picture .", "words": ["the", "picture", "us", "about", "talk"]}, {"sent": "talk ?", "words": ["talk"]}, {"sent": "otherwise Mama talk to you .", "words": ["to", "talk", "you"]}, {"sent": "um we talk a lot about ducks .", "words": ["we", "a lot", "about", "talk", "a"]}, {"sent": "youll hafta talk to Ursula about that .", "words": ["to", "about", "that", "talk"]}, {"sent": "talk nicely yeah .", "words": ["talk"]}, {"sent": "and well talk to you when we get back .", "words": ["back", "we", "to", "you", "and", "get", "talk", "when"]}, {"sent": "they made small talk .", "words": ["they", "talk"]}, {"sent": "are you too tired to talk ?", "words": ["to", "tired", "you", "are", "talk", "too"]}, {"sent": "dont dont talk like that .", "words": ["dont", "that", "like", "talk"]}, {"sent": "well your in here just to talk a little bit right ?", "words": ["your", "to", "little", "here", "in", "talk", "a"]}, {"sent": "so you can sit and talk to me cant you ?", "words": ["can", "so", "to", "and", "you", "talk", "me", "sit"]}, {"sent": "did your echo talk to you ?", "words": ["your", "to", "you", "did", "talk"]}, {"sent": "use real talk .", "words": ["talk"]}, {"sent": "oh youre gonna talk on the telephone ?", "words": ["on", "the", "telephone", "talk"]}, {"sent": "talk properly please .", "words": ["talk"]}, {"sent": "no no no you dont talk .", "words": ["dont", "talk", "you"]}, {"sent": "now talk .", "words": ["talk"]}, {"sent": "what shall we talk about ?", "words": ["what", "about", "talk", "we"]}, {"sent": "Brett can I talk in or on ?", "words": ["can", "in", "on", "talk"]}, {"sent": "in their cabs and talk to each other through the window ?", "words": ["the", "their", "each", "to", "and", "other", "in", "talk", "window"]}, {"sent": "daddys got to talk to the man .", "words": ["the", "to", "talk"]}, {"sent": "yes but dont you like to talk with maam though ?", "words": ["to", "with", "like", "you", "dont", "talk", "but"]}, {"sent": "i a lot of its probably new vocabulary when we talk about it other than colors .", "words": ["i", "we", "of", "a lot", "other", "it", "talk", "about", "new", "when", "a"]}, {"sent": "how does that duck talk ?", "words": ["does", "how", "talk", "duck", "that"]}, {"sent": "pokey talk ?", "words": ["talk"]}, {"sent": "you wanna talk about the building site ?", "words": ["the", "you", "about", "talk", "wanna"]}, {"sent": "you talk to Amy while I get er .", "words": ["get", "to", "talk", "you"]}, {"sent": "talk .", "words": ["talk"]}, {"sent": "shes listening to you talk in the living room and shes in the kitchen .", "words": ["the", "to", "living room", "you", "and", "in", "room", "talk", "kitchen"]}, {"sent": "talk to Kenny ?", "words": ["to", "talk"]}, {"sent": "she can talk now .", "words": ["can", "she", "talk"]}, {"sent": "would you like to talk to Uncle Bob first ?", "words": ["to", "first", "like", "you", "talk", "would"]}, {"sent": "can you talk regular ?", "words": ["can", "talk", "you"]}, {"sent": "it doesnt talk ?", "words": ["it", "talk"]}, {"sent": "oh your monkey doesnt talk ?", "words": ["monkey", "your", "talk"]}, {"sent": "oh what are you gonna talk about ?", "words": ["what", "you", "are", "about", "talk"]}, {"sent": "I cant understand you when you talk in that silly voice .", "words": ["when", "you", "in", "talk", "that"]}, {"sent": "talk to talk talk to Barry Patrick .", "words": ["to", "talk"]}, {"sent": "no when you talk the both of them go on .", "words": ["the", "go", "of", "them", "you", "talk", "on", "when"]}, {"sent": "its very hard for you to talk into the microphone when youre drinking Nomi .", "words": ["the", "for", "into", "to", "you", "talk", "hard", "when"]}, {"sent": "do you wanna talk to Daddy for a minute while I go make Jwww a sandwich ?", "words": ["do", "for", "go", "to", "you", "talk", "wanna", "make", "sandwich", "a"]}, {"sent": "you want the baby to talk to you ?", "words": ["the", "to", "talk", "you"]}, {"sent": "come on you stay here and talk to me .", "words": ["to", "me", "and", "here", "you", "talk", "on", "stay"]}, {"sent": "Laras forgotten how to talk .", "words": ["to", "how", "talk"]}, {"sent": "hey wai wai wait I gotta talk to Summer .", "words": ["to", "wait", "talk"]}, {"sent": "do you wanna talk to him ?", "words": ["do", "to", "you", "him", "talk", "wanna"]}, {"sent": "but that wont make sense mommy because if she listens to herself talk shes gonna sit there and shell say silly things to hear herself act silly .", "words": ["if", "to", "say", "and", "hear", "there", "that", "but", "talk", "sit", "make", "she", "because"]}, {"sent": "I can take this from you while I talk to your mom .", "words": ["can", "your", "to", "you", "this", "talk", "take"]}, {"sent": "you will hafta talk .", "words": ["will", "talk", "you"]}, {"sent": "youre what are you gonna talk to me about ?", "words": ["what", "to", "you", "are", "talk", "about", "me"]}, {"sent": "should we bring them into the other room so you can talk to Manuela ?", "words": ["the", "can", "into", "we", "so", "to", "them", "bring", "other", "you", "room", "talk"]}, {"sent": "come in then we cant hear you when you talk .", "words": ["we", "you", "hear", "in", "talk", "then", "when"]}, {"sent": "well you can talk to me .", "words": ["can", "to", "you", "talk", "me"]}, {"sent": "well I shall talk to HennyPenny .", "words": ["to", "talk"]}, {"sent": "its nought its just listening to you talk .", "words": ["to", "talk", "you"]}, {"sent": "are you gonna talk properly please now ?", "words": ["are", "talk", "you"]}, {"sent": "are we going to talk about this book ?", "words": ["we", "to", "this", "are", "about", "talk", "book"]}, {"sent": "like inside sometimes it makes some noises so when you talk you cant hear what youre saying you can only hear the noises in the machine .", "words": ["can", "what", "the", "so", "inside", "like", "you", "hear", "in", "it", "talk", "some", "when"]}, {"sent": "if you wanna talk to Joey come on over here .", "words": ["if", "to", "you", "here", "talk", "wanna", "on", "over"]}, {"sent": "and talk clear .", "words": ["talk", "and"]}, {"sent": "can the baby talk on the phone ?", "words": ["can", "the", "on", "talk"]}, {"sent": "Mummys making Barney talk .", "words": ["talk"]}, {"sent": "it all depends on it you talk or not because if you talk then you can hear your voice .", "words": ["can", "all", "your", "if", "you", "hear", "it", "talk", "on", "then", "not", "because"]}, {"sent": "can you talk to it ?", "words": ["can", "to", "you", "it", "talk"]}, {"sent": "are you talk talking to her ?", "words": ["to", "you", "are", "talk", "her"]}, {"sent": "and I got out and I looked at that tire and neither one of us could even talk .", "words": ["of", "and", "us", "out", "at", "talk", "could", "that"]}, {"sent": "why cant we talk about it ?", "words": ["we", "it", "about", "talk", "why"]}, {"sent": "you wanna talk about another book .", "words": ["you", "another", "about", "talk", "wanna", "book"]}, {"sent": "are you gonna talk to me ?", "words": ["to", "you", "are", "talk", "me"]}, {"sent": "youre not going to talk to anyone on your telephone ?", "words": ["your", "to", "talk", "on", "not", "telephone"]}, {"sent": "are you gonna talk properly ?", "words": ["are", "talk", "you"]}, {"sent": "if you dont talk to me I have let Sophia come out .", "words": ["if", "to", "you", "have", "dont", "talk", "out", "me"]}, {"sent": "can you talk to Mr Rabbit ?", "words": ["can", "to", "talk", "you"]}, {"sent": "I dont wanna talk to you when youre being rude .", "words": ["to", "you", "dont", "talk", "wanna", "when"]}, {"sent": "Mummy wants to talk to you .", "words": ["to", "talk", "you"]}, {"sent": "when you talk through that ?", "words": ["that", "when", "talk", "you"]}, {"sent": "some days people would come here and talk to him over Christmas .", "words": ["to", "and", "here", "him", "talk", "would", "some", "over"]}, {"sent": "um I talk I try to talk to him as I give him new items and try to explain what they are .", "words": ["what", "to", "and", "they", "him", "are", "talk", "give", "new", "try", "try to"]}, {"sent": "talk nice and loudly .", "words": ["nice", "talk", "and"]}, {"sent": "you will not talk about breakfast when youve eaten breakfast .", "words": ["you", "about", "talk", "will", "when", "not"]}, {"sent": "I want someone to talk to .", "words": ["to", "talk"]}, {"sent": "I cant hear you when you talk with your mouth full .", "words": ["your", "full", "mouth", "with", "you", "hear", "talk", "when"]}, {"sent": "you cant play yet you hafta talk to me .", "words": ["to", "play", "you", "talk", "me"]}, {"sent": "so that must mean that the people in the lighthouse can talk through a radio to people in the in the air mustnt it ?", "words": ["the", "can", "radio", "so", "to", "in", "it", "talk", "that", "a"]}, {"sent": "um can you talk a little bit about that .", "words": ["can", "little", "you", "talk", "about", "that", "a"]}, {"sent": "oh you wanna talk to Amelia okay .", "words": ["to", "talk", "wanna", "you"]}, {"sent": "talk to me .", "words": ["to", "me", "talk"]}, {"sent": "and talk a bit more when we come back .", "words": ["back", "we", "and", "more", "talk", "when", "a"]}, {"sent": "then Im keeping quiet and you talk the rest of the time .", "words": ["the", "of", "you", "and", "talk", "quiet", "then"]}, {"sent": "and were recording Jwww but he doesnt wanna talk this morning .", "words": ["and", "this", "he", "wanna", "talk", "were", "but"]}, {"sent": "dont talk back .", "words": ["dont", "back", "talk"]}, {"sent": "oh Lala Billy doesnt wanna talk .", "words": ["talk", "wanna"]}, {"sent": "I need you to talk to me .", "words": ["need", "to", "you", "talk", "me"]}, {"sent": "talk or sing a song or something .", "words": ["sing", "talk", "a"]}, {"sent": "Nana wants to talk to you .", "words": ["to", "talk", "you"]}, {"sent": "if you dont if you dont talk properly .", "words": ["dont", "if", "talk", "you"]}, {"sent": "take it off and and talk youre not nobodys you youre not gonna hear anything if you just sit there listening .", "words": ["off", "if", "and", "you", "hear", "there", "it", "talk", "sit", "take", "not"]}, {"sent": "why dont you talk for her too now Naomi ?", "words": ["for", "you", "dont", "talk", "too", "her", "why"]}, {"sent": "could not talk her outof it .", "words": ["it", "talk", "could", "her", "not"]}, {"sent": "and it hadta be baked for an hour and a half .", "words": ["for", "be", "and", "an", "it", "a"]}, {"sent": "it doesnt look like an a Ive seen before .", "words": ["like", "an", "it", "look", "a"]}, {"sent": "heres an eight .", "words": ["an"]}, {"sent": "you have an elephant ?", "words": ["have", "an", "elephant", "you"]}, {"sent": "should get an elastic band for them shouldnt we really ?", "words": ["for", "we", "them", "get", "an"]}, {"sent": "it was probably an accident .", "words": ["was", "an", "it"]}, {"sent": "lets just get an eggcup of water and put the daisys and the buttercup that weve picked .", "words": ["the", "put", "of", "and", "get", "an", "water", "that"]}, {"sent": "I think youre going to hafta have an injection .", "words": ["have", "to", "think", "an"]}, {"sent": "the heater is an alright .", "words": ["the", "an", "is"]}, {"sent": "honey thats not a pear thats an avocado .", "words": ["an", "not", "a"]}, {"sent": "an elderly gentleman with a walking stick .", "words": ["an", "with", "stick", "a"]}, {"sent": "you talk about custard an awful lot Thomas .", "words": ["an", "about", "talk", "you"]}, {"sent": "a wise old owl lived in an oak .", "words": ["old", "owl", "in", "an", "a"]}, {"sent": "yeah that is an airplane .", "words": ["airplane", "an", "that", "is"]}, {"sent": "Mommy needs an envelope .", "words": ["an"]}, {"sent": "you have an apple right there .", "words": ["you", "have", "there", "an", "apple"]}, {"sent": "you having an off time ?", "words": ["off", "an", "you"]}, {"sent": "and thats an agg .", "words": ["an", "and"]}, {"sent": "yes now we have an orange house .", "words": ["we", "have", "an", "orange", "house"]}, {"sent": "get the red one and color it in and it will be an apple .", "words": ["the", "red", "be", "and", "get", "in", "it", "an", "will", "apple"]}, {"sent": "Im gonna draw an elephant .", "words": ["an", "draw", "elephant"]}, {"sent": "guess we can make that into an ax .", "words": ["can", "into", "we", "an", "make", "that"]}, {"sent": "so tomorrow is kind of an interesting day because its Mattys last day of school .", "words": ["is", "of", "so", "last", "school", "an", "because"]}, {"sent": "Im gonna be uh an elephant .", "words": ["be", "an", "elephant"]}, {"sent": "is he an elephant ?", "words": ["an", "elephant", "he", "is"]}, {"sent": "an than I answer your question .", "words": ["an", "your"]}, {"sent": "well Stuarts mummie bought him an activity centre .", "words": ["him", "an"]}, {"sent": "an umbrella .", "words": ["an"]}, {"sent": "an our um uh lobster .", "words": ["an", "our"]}, {"sent": "its an easel .", "words": ["an"]}, {"sent": "did you have an icecream for your breakfast ?", "words": ["for", "your", "you", "have", "an", "did"]}, {"sent": "an electric toothbrush .", "words": ["toothbrush", "an"]}, {"sent": "I dont believe theres an eighteen .", "words": ["dont", "an"]}, {"sent": "in fact weve got an exercise bicycle in our bedroom .", "words": ["bicycle", "our", "in", "an", "bedroom"]}, {"sent": "so the last letter is an a right ?", "words": ["the", "is", "so", "last", "an", "a"]}, {"sent": "oh there is an aeroplane .", "words": ["there", "an", "is"]}, {"sent": "and the jigsaw has an identical picture .", "words": ["the", "an", "picture", "and"]}, {"sent": "a chick in an egg .", "words": ["in", "egg", "an", "a"]}, {"sent": "you need an infant sleeping pill or the equivalent of that .", "words": ["the", "need", "of", "you", "an", "that"]}, {"sent": "it needs to be ready in an emergency doesnt it ?", "words": ["to", "be", "in", "it", "an"]}, {"sent": "thats an apple but it doesnt work .", "words": ["work", "an", "it", "apple", "but"]}, {"sent": "hands pick an apple .", "words": ["pick", "apple", "an"]}, {"sent": "that doesnt look like an a .", "words": ["like", "an", "look", "that", "a"]}, {"sent": "now its just an eep .", "words": ["an"]}, {"sent": "*IN2: yeah its an orange but that doesnt work either that just falls off the lid .", "words": ["off", "the", "work", "an", "orange", "that", "but"]}, {"sent": "Zeek and Zach hopped into the yard and pretended to lay an egg .", "words": ["the", "into", "egg", "to", "and", "an"]}, {"sent": "oh like an uh an orange like that you eat xxx my gosh .", "words": ["my", "like", "you", "eat", "an", "orange", "that"]}, {"sent": "I have an idea .", "words": ["have", "an"]}, {"sent": "an apple to bite and a moustache of white .", "words": ["of", "to", "and", "an", "bite", "apple", "white", "a"]}, {"sent": "Im gonna make an ice cream soda and youll be finished in ice cream soda .", "words": ["ice cream", "soda", "be", "and", "an", "in", "ice", "make"]}, {"sent": "thats an antenna .", "words": ["an"]}, {"sent": "you have quite an arm dont you .", "words": ["you", "have", "dont", "an", "arm"]}, {"sent": "an elephant .", "words": ["an", "elephant"]}, {"sent": "an wheres it going ?", "words": ["an", "it"]}, {"sent": "and that might be an ambulance .", "words": ["be", "an", "that", "and"]}, {"sent": "thats an apple .", "words": ["apple", "an"]}, {"sent": "look at that Reese an ice cream cone .", "words": ["ice cream", "look", "an", "at", "ice", "that"]}, {"sent": "its got an eye .", "words": ["an", "eye"]}, {"sent": "Mia do you have an ace ?", "words": ["have", "do", "an", "you"]}, {"sent": "and Ill give you a bottle in half an hour .", "words": ["you", "and", "in", "an", "bottle", "give", "a"]}, {"sent": "and as they left the shop we heard an almighty .", "words": ["the", "we", "and", "they", "an"]}, {"sent": "an eel .", "words": ["an"]}, {"sent": "youre an ice cream man and youre too busy are you ?", "words": ["ice cream", "and", "you", "an", "are", "too", "ice"]}, {"sent": "theres an ugly bag .", "words": ["an"]}, {"sent": "and hes holding an umbrella because its raining .", "words": ["an", "because", "and"]}, {"sent": "well an octopus has eight and a squid has more than eight .", "words": ["an", "more", "a", "and"]}, {"sent": "dont think cows run an awful lot .", "words": ["dont", "think", "run", "an"]}, {"sent": "and there isnt an aeroplane is there ?", "words": ["there", "an", "is", "and"]}, {"sent": "Im sure hes such an adaptable wee boy .", "words": ["an"]}, {"sent": "its an apple .", "words": ["apple", "an"]}, {"sent": "dont you want an apple ?", "words": ["apple", "dont", "an", "you"]}, {"sent": "oh then an easy access came .", "words": ["then", "an"]}, {"sent": "theres just petrol an elevator and a lift and a car wash isnt there ?", "words": ["car", "and", "there", "an", "wash", "a"]}, {"sent": "that is not an apple .", "words": ["is", "an", "apple", "that", "not"]}, {"sent": "no this isnt an owowow .", "words": ["this", "an"]}, {"sent": "mommy sheep is called an ewe and daddy is a ram .", "words": ["is", "and", "an", "sheep", "a"]}, {"sent": "do you want sweeties or do you want an egg ?", "words": ["do", "an", "egg", "you"]}, {"sent": "are you making an omelette ?", "words": ["an", "are", "you"]}, {"sent": "have an ice pole eh ?", "words": ["have", "an", "ice"]}, {"sent": "an old tramp .", "words": ["old", "an"]}, {"sent": "theres been an accident .", "words": ["an"]}, {"sent": "had you better get an ambulance of a fireengine out ?", "words": ["better", "of", "you", "get", "an", "out", "a"]}, {"sent": "this is an airplane .", "words": ["airplane", "this", "an", "is"]}, {"sent": "its an anchor .", "words": ["an"]}, {"sent": "thats a toy not an icecream you SillyBilly .", "words": ["you", "toy", "an", "not", "a"]}, {"sent": "Stewart Millers and hes in wanting an ice lolly and crisps and a drink .", "words": ["drink", "and", "in", "an", "ice", "a"]}, {"sent": "has Fraser been on an aeroplane before ?", "words": ["on", "an"]}, {"sent": "thats not an eating nut .", "words": ["an", "not"]}, {"sent": "did you see phoebe told me there was an anemone there was there ?", "words": ["was", "me", "you", "there", "an", "did", "see"]}, {"sent": "oh thats an xray .", "words": ["an"]}, {"sent": "an egg .", "words": ["an", "egg"]}, {"sent": "you dont need an explanation .", "words": ["dont", "an", "need", "you"]}, {"sent": "thats an alligator .", "words": ["an", "alligator"]}, {"sent": "thats an .", "words": ["an"]}, {"sent": "can Eleanor draw an aeroplane ?", "words": ["can", "an", "draw"]}, {"sent": "xxx yeah well well have to call it an elephant .", "words": ["to", "elephant", "have", "an", "it"]}, {"sent": "picked an apple from the tree .", "words": ["the", "apple", "an", "tree"]}, {"sent": "I might be about an hour .", "words": ["be", "an", "about"]}, {"sent": "I didnt think it was that bad Ellens like ooh youre in trouble pediatrician says half an hour a day Im like xxx .", "words": ["think", "was", "like", "bad", "in", "it", "an", "that", "a"]}, {"sent": "why cant you make an engine shed ?", "words": ["an", "make", "why", "you"]}, {"sent": "yeah some I think sometimes theyre like some of the little boys like if I feel like alot of the girls dont go for it as much xxx kinda look at it and then they wanna deal with the cooking or something .", "words": ["the", "think", "go", "of", "like", "with", "it", "some", "then", "much", "for", "look", "if", "little", "and", "dont", "they", "wanna", "at"]}, {"sent": "what do you do then when youre naughty ?", "words": ["what", "do", "you", "then", "when", "naughty"]}, {"sent": "and then ?", "words": ["then", "and"]}, {"sent": "all right then .", "words": ["all", "then"]}, {"sent": "and then one day I went into Mothercare .", "words": ["then", "into", "and"]}, {"sent": "you bring your crane over here then and Ill help .", "words": ["help", "your", "bring", "you", "here", "and", "then", "over"]}, {"sent": "okay alright then til just come with them then .", "words": ["then", "with", "them"]}, {"sent": "and then you know what Mamas going to do ?", "words": ["what", "do", "to", "you", "and", "then"]}, {"sent": "o and then y .", "words": ["then", "and"]}, {"sent": "what do you wanna do then ?", "words": ["what", "do", "you", "wanna", "then"]}, {"sent": "then what does the little girl do ?", "words": ["the", "what", "does", "do", "little", "then"]}, {"sent": "wipe your nose and then see if your pizzas cooked .", "words": ["your", "if", "and", "see", "then", "wipe", "nose"]}, {"sent": "thats the newer ones then .", "words": ["the", "then"]}, {"sent": "and then it hurts doesnt it ?", "words": ["then", "it", "and"]}, {"sent": "thats another thing then isnt it ?", "words": ["another", "then", "it"]}, {"sent": "right now then what goes in there ?", "words": ["what", "then", "in", "there"]}, {"sent": "pick up then .", "words": ["pick", "then", "up"]}, {"sent": "then comes aunties birthday .", "words": ["then"]}, {"sent": "okay then .", "words": ["then"]}, {"sent": "and then at you know like when you went to the aquarium ?", "words": ["the", "to", "like", "you", "and", "at", "then", "when"]}, {"sent": "then you can make your own thing .", "words": ["can", "your", "you", "then", "make"]}, {"sent": "and then press it down on another bit .", "words": ["and", "another", "it", "on", "then", "down"]}, {"sent": "who was at Toddlers then ?", "words": ["was", "then", "at", "who"]}, {"sent": "you going to have your lunch then .", "words": ["your", "to", "you", "have", "then"]}, {"sent": "but then I got those in a wallet too .", "words": ["those", "in", "too", "then", "but", "a"]}, {"sent": "that looks like an old fashioned train at the front and then it looks like the rattly train at the back and the middle looks like an ordinary train .", "words": ["old", "the", "back", "train", "like", "and", "an", "it", "at", "then", "that"]}, {"sent": "lets say that maybe that you guys were sleeping and then the meat eater sneaked up on them .", "words": ["the", "up", "say", "them", "meat", "and", "you", "on", "then", "that", "were"]}, {"sent": "no theyre not because you are not concentrating and you did not concentrate then .", "words": ["and", "you", "are", "did", "then", "not", "because"]}, {"sent": "and then read it ?", "words": ["then", "it", "read", "and"]}, {"sent": "and then the fireman .", "words": ["the", "then", "and"]}, {"sent": "and then its put into bottles .", "words": ["then", "into", "put", "and"]}, {"sent": "wanna play with something else then ?", "words": ["then", "play", "with", "wanna"]}, {"sent": "wait till Ive finished then xxx .", "words": ["wait", "then"]}, {"sent": "then the cow can eat it aswell cant she .", "words": ["the", "can", "eat", "it", "then", "she", "cow"]}, {"sent": "give you your hot milk and then get Amy another bottle .", "words": ["your", "then", "you", "and", "get", "hot", "another", "bottle", "milk", "give"]}, {"sent": "what do you want then ?", "words": ["what", "do", "then", "you"]}, {"sent": "because the gorilla died and xxx pretend we came back after riding then we made him alive and xxx okay ?", "words": ["the", "back", "we", "then", "and", "him", "pretend", "because"]}, {"sent": "are we going to look at this book first then ?", "words": ["we", "to", "first", "this", "are", "at", "look", "book", "then"]}, {"sent": "come on then .", "words": ["on", "then"]}, {"sent": "right I was eating cake and cider then I saw a little .", "words": ["cake", "was", "and", "little", "then", "a"]}, {"sent": "well press it down properly then its fixed okay .", "words": ["down", "then", "it"]}, {"sent": "come on then .", "words": ["on", "then"]}, {"sent": "well shall we wash your hands and then you can get down from your high chair ?", "words": ["can", "your", "we", "chair", "and", "you", "get", "high chair", "high", "then", "down", "wash"]}, {"sent": "and its my birthday in January can I come out then ?", "words": ["can", "my", "and", "in", "out", "then"]}, {"sent": "stand up then .", "words": ["up", "stand", "then"]}, {"sent": "and then on your birthday we went to see Jeannine as well .", "words": ["your", "we", "to", "and", "on", "see", "then"]}, {"sent": "when you were doing that then what ?", "words": ["what", "when", "you", "then", "that", "were"]}, {"sent": "and then have some breakfast .", "words": ["have", "some", "then", "and"]}, {"sent": "I looked at them all and then I could tell they all had no fear and now all was well they all went away they all waved good bye so I sat by the lake and looked at the sky .", "words": ["the", "all", "sky", "was", "so", "them", "and", "by", "they", "away", "good", "at", "then", "could"]}, {"sent": "and then you needta .", "words": ["then", "you", "and"]}, {"sent": "because then if the keys are stolen .", "words": ["the", "if", "are", "then", "keys", "because"]}, {"sent": "Tweeniesll be on now then .", "words": ["on", "be", "then"]}, {"sent": "then go get the Big Bird book okay ?", "words": ["the", "go", "get", "book", "then"]}, {"sent": "right then .", "words": ["then"]}, {"sent": "go on then .", "words": ["on", "then", "go"]}, {"sent": "come on then .", "words": ["on", "then"]}, {"sent": "and then wed hafta take you to hospital .", "words": ["to", "you", "and", "then", "take"]}, {"sent": "I had jelly and xxx then Polly came in in the middle xxx .", "words": ["jelly", "the", "and", "in", "then"]}, {"sent": "whatre we gonna do then ?", "words": ["do", "then", "we"]}, {"sent": "and then you gotta go then we gotta go back the other way after you get the rope .", "words": ["the", "back", "go", "we", "you", "and", "other", "get", "then"]}, {"sent": "you put that one in the bin then .", "words": ["the", "put", "you", "in", "then", "that"]}, {"sent": "Ill hafta read it then and see .", "words": ["and", "it", "read", "see", "then"]}, {"sent": "go on then .", "words": ["on", "then", "go"]}, {"sent": "can Mama cut a square for you and then you can color the square in ?", "words": ["can", "the", "for", "cut", "you", "and", "in", "then", "a"]}, {"sent": "go on then .", "words": ["on", "then", "go"]}, {"sent": "youd be upstairs having a wee wee by then ?", "words": ["be", "then", "by", "a"]}, {"sent": "then you went to sleep uhhuh .", "words": ["sleep", "to", "then", "you"]}, {"sent": "then you can tell some more ?", "words": ["can", "you", "more", "some", "then"]}, {"sent": "right then .", "words": ["then"]}, {"sent": "and then he pulls out .", "words": ["then", "he", "out", "and"]}, {"sent": "and then well find well circle all the roses .", "words": ["the", "all", "and", "find", "then"]}, {"sent": "let me have them then .", "words": ["have", "me", "then", "them"]}, {"sent": "put it down then .", "words": ["down", "put", "it", "then"]}, {"sent": "and then he put the hand set down .", "words": ["the", "put", "hand", "and", "he", "then", "down"]}, {"sent": "because your nails are nice and soft then arent they ?", "words": ["soft", "your", "nice", "and", "they", "are", "then", "because"]}, {"sent": "but then that woman had no brothers and sisters .", "words": ["then", "that", "but", "and"]}, {"sent": "you rolled off the bed then Fraser .", "words": ["off", "the", "bed", "you", "then"]}, {"sent": "and then well have .", "words": ["have", "then", "and"]}, {"sent": "that one he needs some petrol in there and then he comes round .", "words": ["and", "there", "in", "he", "some", "then", "that"]}, {"sent": "and so then Andrew learned to be a good babysitter .", "words": ["to", "so", "be", "and", "good", "then", "a"]}, {"sent": "Eeyore bottle then .", "words": ["bottle", "then"]}, {"sent": "you put it on the dolly then .", "words": ["the", "put", "you", "it", "on", "then"]}, {"sent": "shall I do it then ?", "words": ["do", "then", "it"]}, {"sent": "alright then .", "words": ["then"]}, {"sent": "alright then .", "words": ["then"]}, {"sent": "come on then .", "words": ["on", "then"]}, {"sent": "and then youd say .", "words": ["then", "say", "and"]}, {"sent": "do you wanna put this puzzle away and then well take it off ?", "words": ["off", "do", "put", "you", "and", "this", "away", "puzzle", "it", "wanna", "then", "take"]}, {"sent": "and then is the i going to ride the train ?", "words": ["the", "i", "is", "train", "to", "and", "then", "ride"]}, {"sent": "whats that then ?", "words": ["then", "that"]}, {"sent": "come on then .", "words": ["on", "then"]}, {"sent": "is dat alotof peanut xxx and then its my turn .", "words": ["my", "then", "and", "is"]}, {"sent": "why dont you try another piece until then .", "words": ["you", "another", "dont", "then", "try", "why"]}, {"sent": "shall we just see how you go with this and then if you need anything else .", "words": ["how", "need", "if", "go", "we", "with", "you", "and", "this", "see", "then"]}, {"sent": "lets put it there then .", "words": ["there", "put", "it", "then"]}, {"sent": "so have you anything else to tell me before I go then ?", "words": ["go", "so", "to", "you", "have", "me", "then"]}, {"sent": "then what Thomas ?", "words": ["what", "then"]}, {"sent": "whatve you made there then ?", "words": ["there", "then", "you"]}, {"sent": "so we could do with finding the green one before we do the same to that .", "words": ["the", "do", "we", "so", "to", "with", "same", "green", "could", "that"]}, {"sent": "not the same place but the same .", "words": ["the", "same", "not", "but"]}, {"sent": "shes from the same place with the Eiffel tower right ?", "words": ["the", "same", "with"]}, {"sent": "want her to ride on the same horse ?", "words": ["the", "to", "ride", "horse", "same", "on", "her"]}, {"sent": "yeah alotof the words the same huh .", "words": ["the", "same"]}, {"sent": "is it on the same street that the Apollo theater is on ?", "words": ["the", "is", "it", "same", "on", "street", "that"]}, {"sent": "about the same people .", "words": ["the", "same", "about"]}, {"sent": "no when our Adam started school it was the same .", "words": ["the", "was", "school", "our", "it", "same", "when"]}, {"sent": "all your animals sound very much the same Thomas .", "words": ["the", "all", "your", "same", "much"]}, {"sent": "was it pretty much the same stuff .", "words": ["the", "was", "it", "pretty", "same", "much"]}, {"sent": "this is the same blanket .", "words": ["the", "is", "this", "same", "blanket"]}, {"sent": "and the see through racing driver that we wind up with the same key .", "words": ["the", "up", "we", "with", "and", "same", "see", "that", "wind"]}, {"sent": "like that its the same thing .", "words": ["the", "same", "that", "like"]}, {"sent": "thats actually the same .", "words": ["the", "same"]}, {"sent": "thats the same .", "words": ["the", "same"]}, {"sent": "its not the same dog as Pepper .", "words": ["the", "dog", "same", "not"]}, {"sent": "thats the same foot Jamar you have your right foot xxx .", "words": ["the", "your", "you", "have", "foot", "same"]}, {"sent": "shes starting school the same year as him then .", "words": ["the", "school", "him", "same", "then"]}, {"sent": "not exactly the same but .", "words": ["the", "same", "not", "but"]}, {"sent": "the same store where we saw SantaClaus .", "words": ["the", "where", "we", "store", "same"]}, {"sent": "these shoes are the same thing .", "words": ["the", "these", "are", "same"]}, {"sent": "turn it off now were going to have a bath in a minute just had a shower and its on the same date as before .", "words": ["off", "the", "shower", "to", "and", "have", "in", "it", "same", "on", "were", "a"]}, {"sent": "they both landed on the same days .", "words": ["on", "they", "same", "the"]}, {"sent": "they both have the same name Naima ?", "words": ["have", "they", "same", "the"]}, {"sent": "same stuff she cares about at home that she does here .", "words": ["does", "here", "same", "about", "at", "that", "she", "home"]}, {"sent": "and its the same giraffe .", "words": ["the", "giraffe", "same", "and"]}, {"sent": "aw dyou want him to be in the same class as you ?", "words": ["the", "to", "be", "you", "him", "in", "same"]}, {"sent": "its the same .", "words": ["the", "same"]}, {"sent": "while Mummy washes her blouse I think well hafta put your trousers in the washingmachine at the same time wont we ?", "words": ["the", "think", "put", "your", "we", "in", "same", "at", "her"]}, {"sent": "theyre not the same are they ?", "words": ["the", "they", "are", "same", "not"]}, {"sent": "dollys dress is nearly the same as your dress .", "words": ["the", "your", "is", "same", "dress"]}, {"sent": "the same name as you .", "words": ["the", "same", "you"]}, {"sent": "can you find another shoe thats the same ?", "words": ["can", "the", "shoe", "another", "you", "same", "find"]}, {"sent": "its the same thing isnt it Michael ?", "words": ["the", "it", "same"]}, {"sent": "theyre the same what ?", "words": ["the", "what", "same"]}, {"sent": "oh it was the same one .", "words": ["the", "was", "it", "same"]}, {"sent": "need another one the same .", "words": ["another", "the", "need", "same"]}, {"sent": "do the same thing Chi Yan .", "words": ["the", "do", "same"]}, {"sent": "same as you would .", "words": ["same", "would", "you"]}, {"sent": "and then youre going to see how many stones and shells it takes in this pan to be the same weight as the marbles at the other side .", "words": ["the", "how", "to", "be", "and", "other", "this", "in", "it", "same", "at", "see", "then"]}, {"sent": "we had the exact same toys .", "words": ["the", "same", "we"]}, {"sent": "the same kind right .", "words": ["the", "same"]}, {"sent": "xxx the same time ?", "words": ["the", "same"]}, {"sent": "get the sets back together in the same room .", "words": ["the", "back", "get", "in", "room", "same"]}, {"sent": "its the same color now Fraser .", "words": ["the", "same"]}, {"sent": "you have the same book .", "words": ["the", "you", "have", "same", "book"]}, {"sent": "I bet they do exactly the same this year .", "words": ["the", "do", "they", "this", "same"]}, {"sent": "yeah a dragon says the same thing .", "words": ["the", "same", "a"]}, {"sent": "were on the same one again .", "words": ["on", "the", "were", "same"]}, {"sent": "more of the same bullshit .", "words": ["of", "the", "more", "same"]}, {"sent": "or the same furniture but theyre just moving it from their old house to their new house right ?", "words": ["the", "old", "their", "to", "it", "same", "house", "new", "but"]}, {"sent": "you could play the same game .", "words": ["the", "game", "play", "you", "same", "could"]}, {"sent": "the vase the same way wouldnt it ?", "words": ["the", "it", "same"]}, {"sent": "same time again ?", "words": ["same"]}, {"sent": "thats all the same thickness isnt it ?", "words": ["the", "all", "it", "same"]}, {"sent": "its the same size .", "words": ["the", "same"]}, {"sent": "I dont think any of them are the same .", "words": ["the", "think", "of", "any", "them", "dont", "are", "same"]}, {"sent": "not the same is it ?", "words": ["the", "is", "it", "same", "not"]}, {"sent": "I think its the same one isnt it ?", "words": ["the", "think", "it", "same"]}, {"sent": "same as that .", "words": ["that", "same"]}, {"sent": "they dont put the whole thing in they dont put the same animals in the same in the same in the same cage .", "words": ["the", "put", "dont", "they", "in", "same"]}, {"sent": "shes got the same color hair as Daddy .", "words": ["the", "hair", "same"]}, {"sent": "well theyre all kept in the same box .", "words": ["the", "all", "box", "in", "same"]}, {"sent": "you had exactly the same ones didnt you ?", "words": ["the", "same", "you"]}, {"sent": "so are there um do you think shes familiar with any of the ones here do you have the same set or .", "words": ["the", "do", "think", "of", "so", "with", "any", "you", "here", "there", "have", "are", "same"]}, {"sent": "thats the same dog I see mm .", "words": ["the", "see", "same", "dog"]}, {"sent": "same color as Jesss eyes .", "words": ["same"]}, {"sent": "can you see theyre the same color ?", "words": ["can", "the", "you", "same", "see"]}, {"sent": "same thing .", "words": ["same"]}, {"sent": "he had got trousers and a top and theyre all the same colors arent they ?", "words": ["the", "all", "and", "they", "he", "same", "a"]}, {"sent": "he has the same one ?", "words": ["the", "same", "he"]}, {"sent": "and its the same feeling .", "words": ["the", "same", "and"]}, {"sent": "its the same old song .", "words": ["the", "same", "old"]}, {"sent": "xxx is dat the same one ?", "words": ["the", "same", "is"]}, {"sent": "the same day we were using it we got it .", "words": ["the", "we", "it", "same", "were"]}, {"sent": "at the same time .", "words": ["the", "same", "at"]}, {"sent": "but the legs hafta be the same .", "words": ["the", "be", "same", "but"]}, {"sent": "and theres a baby with the same expression on his face .", "words": ["the", "face", "with", "and", "his", "same", "on", "a"]}, {"sent": "I was wondering the same thing but I figured as I go through this entire table I will find it .", "words": ["the", "table", "go", "was", "this", "it", "same", "find", "will", "but"]}, {"sent": "I thought it was the same ball .", "words": ["the", "was", "ball", "it", "same"]}, {"sent": "same car same car same car same car .", "words": ["car", "same"]}, {"sent": "do they look the same ?", "words": ["the", "do", "they", "same", "look"]}, {"sent": "hey I bet there are more things that are the same .", "words": ["the", "there", "more", "are", "same", "that"]}, {"sent": "I guess just the same way we cry .", "words": ["the", "cry", "same", "we"]}, {"sent": "your watch plays the same tune as the ice cream man ?", "words": ["the", "ice cream", "watch", "your", "same", "ice"]}, {"sent": "and if the pictures are the same .", "words": ["the", "if", "and", "are", "same"]}, {"sent": "Im sure we saw two either the same day or very soon after one another .", "words": ["the", "same", "another", "we"]}, {"sent": "because you keep singing the same ones over and over again .", "words": ["the", "you", "and", "same", "over", "because"]}, {"sent": "in the bathroom again same thing .", "words": ["the", "bathroom", "in", "same"]}, {"sent": "same with the bread .", "words": ["the", "bread", "same", "with"]}, {"sent": "you got to make it so they all bend the same way .", "words": ["the", "all", "to", "so", "you", "they", "it", "same", "make"]}, {"sent": "if were going the same direction as the as the numbers on the clock .", "words": ["the", "if", "clock", "same", "on", "were"]}, {"sent": "I think the frames the same color int it ?", "words": ["the", "think", "it", "same"]}, {"sent": "thats the same one that he blew over there Fraser .", "words": ["the", "there", "he", "same", "over", "that"]}, {"sent": "the same one ?", "words": ["the", "same"]}, {"sent": "yes the same time ?", "words": ["the", "same"]}, {"sent": "two of them are the same .", "words": ["the", "of", "them", "are", "same"]}, {"sent": "oh they look the same dont they ?", "words": ["the", "they", "dont", "same", "look"]}, {"sent": "thats a very very big church .", "words": ["church", "big", "a"]}, {"sent": "a church tower .", "words": ["church", "a"]}, {"sent": "okay and the balloon wanted to go into church too .", "words": ["the", "into", "go", "to", "church", "and", "balloon", "too"]}, {"sent": "there isnt a church on that page is there ?", "words": ["is", "church", "there", "on", "that", "a"]}, {"sent": "coming home from church ?", "words": ["church", "home"]}, {"sent": "what does the church clock strike ?", "words": ["the", "what", "does", "clock", "church"]}, {"sent": "was that the church ?", "words": ["the", "was", "that", "church"]}, {"sent": "you know Suzy who goes to church ?", "words": ["to", "who", "church", "you"]}, {"sent": "no Ill be the big church and youll be in the sunday school and Christopher will be in the nursery with pat .", "words": ["the", "be", "church", "school", "with", "and", "in", "will", "big"]}, {"sent": "a church ?", "words": ["church", "a"]}, {"sent": "its a church song that isnt it ?", "words": ["church", "it", "that", "a"]}, {"sent": "wasnt that awful the way theyd only just found out and they sent them off to church on Sunday morning ?", "words": ["the", "off", "to", "church", "them", "and", "they", "out", "on", "that"]}, {"sent": "no Im not talking about the church one .", "words": ["the", "church", "not", "about"]}, {"sent": "theyll be at church .", "words": ["be", "church", "at"]}, {"sent": "we saw that big church didnt we ?", "words": ["church", "big", "that", "we"]}, {"sent": "um the little boy and the balloon are going past a church steeple .", "words": ["the", "church", "and", "little", "are", "balloon", "a"]}, {"sent": "Nana and Grandad will be coming to listen to your nursery sing a long and then theyll be going to church to the Christingel service .", "words": ["the", "your", "listen", "be", "to", "church", "long", "and", "sing", "will", "then", "a"]}, {"sent": "is that what you say in church ?", "words": ["what", "is", "church", "say", "you", "in", "that"]}, {"sent": "you dont go to church today .", "words": ["go", "to", "church", "you", "dont"]}, {"sent": "its still four oclock according to the church clock .", "words": ["the", "to", "church", "clock"]}, {"sent": "but they all had pictures of the church clock on them .", "words": ["the", "all", "clock", "of", "church", "them", "they", "on", "but"]}, {"sent": "she went to church not long ago .", "words": ["to", "church", "long", "she", "not"]}, {"sent": "can you see the windows in the church Thomas ?", "words": ["can", "the", "church", "you", "in", "see"]}, {"sent": "theres the church .", "words": ["the", "church"]}, {"sent": "do you wanna put the bell tower here next to the church ?", "words": ["the", "do", "put", "to", "church", "you", "here", "wanna"]}, {"sent": "like your church .", "words": ["church", "your", "like"]}, {"sent": "you think it looks like a church ?", "words": ["think", "church", "like", "you", "it", "a"]}, {"sent": "he called the church ah I thought he was home .", "words": ["the", "church", "was", "he", "home"]}, {"sent": "swimming on the pond behind the church ?", "words": ["on", "the", "behind", "church"]}, {"sent": "thats the church .", "words": ["the", "church"]}, {"sent": "and then the next morning the little boy went with his grandmother to church .", "words": ["the", "to", "church", "with", "little", "and", "his", "then"]}, {"sent": "here is the church .", "words": ["here", "the", "church", "is"]}, {"sent": "theyre doing a lot of repairs to the church roof though arent they ?", "words": ["the", "of", "to", "church", "a lot", "they", "roof", "a"]}, {"sent": "xxx in the church .", "words": ["the", "church", "in"]}, {"sent": "it must be a big digger to pick a whole church up and put it in the skip .", "words": ["up", "the", "put", "pick", "to", "be", "church", "and", "in", "it", "big", "a"]}, {"sent": "a tower on top of a church .", "words": ["on", "of", "church", "a"]}, {"sent": "oh what kind of church is that ?", "words": ["what", "is", "of", "church", "that"]}, {"sent": "oh he got that at church on Sunday .", "words": ["church", "he", "at", "on", "that"]}, {"sent": "weve got the church magazines .", "words": ["the", "church"]}, {"sent": "God is inside of the people who think about him when they go to church .", "words": ["the", "think", "is", "of", "go", "inside", "to", "church", "they", "him", "about", "who", "when"]}, {"sent": "when we play church theres always a crowd brothers the preacher he talks very loud .", "words": ["the", "we", "church", "play", "he", "when", "loud", "a"]}, {"sent": "and the church clock seems to have stopped at four oclock .", "words": ["the", "clock", "church", "to", "and", "have", "at"]}, {"sent": "why is that a church hat ?", "words": ["is", "church", "hat", "that", "why", "a"]}, {"sent": "or shall we make a church ?", "words": ["church", "make", "a", "we"]}, {"sent": "you go to church to say goodbye when somebody dies .", "words": ["go", "to", "church", "say", "you", "when"]}, {"sent": "is it a church with a steeple ?", "words": ["is", "church", "with", "it", "a"]}, {"sent": "he went to church then .", "words": ["to", "then", "he", "church"]}, {"sent": "and then the next morning he went to church with his grandmother .", "words": ["the", "to", "church", "with", "and", "he", "his", "then"]}, {"sent": "um the next morning the little boy was seen we saw the little boy walking with his grandmother to church .", "words": ["the", "we", "was", "to", "church", "with", "little", "his"]}, {"sent": "that what a church steeple is ?", "words": ["what", "is", "church", "that", "a"]}, {"sent": "um the little boy is walking past a church .", "words": ["the", "is", "church", "little", "a"]}, {"sent": "a church dont you sometimes ?", "words": ["dont", "church", "you", "a"]}, {"sent": "the next morning the little boy went with his grandmother to church .", "words": ["the", "to", "church", "with", "little", "his"]}, {"sent": "climbing the church .", "words": ["the", "church"]}, {"sent": "but now I know like the older children that go to that summer activity schemes in the church youknow ?", "words": ["the", "go", "to", "church", "like", "in", "that", "but"]}, {"sent": "and Matthew threw that much of a fit she hadta go into church and Matthew hadta sit in the car with Jack .", "words": ["the", "into", "go", "of", "church", "fit", "car", "with", "much", "and", "in", "sit", "that", "she", "a"]}, {"sent": "you wanna start with the church part ?", "words": ["the", "church", "with", "you", "wanna"]}, {"sent": "no thats the church where we useta go to church a long time ago .", "words": ["the", "where", "go", "we", "church", "to", "long", "a"]}, {"sent": "hope itll stop before they got to go down the church .", "words": ["the", "go", "to", "church", "stop", "they", "down"]}, {"sent": "at the church service .", "words": ["the", "church", "at"]}, {"sent": "she might see them at church but .", "words": ["church", "them", "at", "see", "she", "but"]}, {"sent": "whats our church called Thomas ?", "words": ["church", "our"]}, {"sent": "and can you see that big big church behind it ?", "words": ["can", "church", "behind", "you", "and", "big", "it", "see", "that"]}, {"sent": "that wasnt at the church .", "words": ["the", "church", "that", "at"]}, {"sent": "what else do we do at church ?", "words": ["what", "do", "we", "church", "at"]}, {"sent": "and then on the village it tells you where to put the grocers the bakers shops the church .", "words": ["the", "where", "put", "to", "church", "and", "you", "it", "on", "then"]}, {"sent": "church .", "words": ["church"]}, {"sent": "walking near the church .", "words": ["the", "church"]}, {"sent": "what happens in church when people give money ?", "words": ["money", "what", "church", "in", "give", "when"]}, {"sent": "wheres the church ?", "words": ["the", "church"]}, {"sent": "and this is for Rory in church .", "words": ["for", "is", "church", "and", "this", "in"]}, {"sent": "leave church .", "words": ["church"]}, {"sent": "they went to church ?", "words": ["to", "they", "church"]}, {"sent": "especially not in church .", "words": ["church", "in", "not"]}, {"sent": "take me to the church take me to the church on time Im getting married in the morning .", "words": ["the", "to", "church", "in", "on", "me", "take"]}, {"sent": "it was a pen actually from church wasnt it ?", "words": ["was", "church", "it", "pen", "a"]}, {"sent": "could we just have one in there and one in the church please .", "words": ["the", "we", "church", "and", "have", "there", "in", "could"]}, {"sent": "church bell sound .", "words": ["church"]}, {"sent": "thats a church .", "words": ["church", "a"]}, {"sent": "the little boy was walking with his grandmother to a church .", "words": ["the", "was", "to", "church", "with", "little", "his", "a"]}, {"sent": "could it go at the front of the church ?", "words": ["the", "go", "of", "church", "it", "at", "could"]}, {"sent": "a church ?", "words": ["church", "a"]}, {"sent": "oh thats church music ?", "words": ["church"]}, {"sent": "ya go to church on sundays .", "words": ["on", "to", "church", "go"]}, {"sent": "near the church ?", "words": ["the", "church"]}, {"sent": "well its in a church honey .", "words": ["church", "in", "a"]}, {"sent": "thats why I think it might be a little church .", "words": ["think", "be", "church", "little", "it", "why", "a"]}, {"sent": "hickory dickory dock the church mouse ran up the clock the clock struck ten the mouse said amen hickory dickory dock .", "words": ["the", "up", "clock", "church", "mouse"]}, {"sent": "is that like at church ?", "words": ["is", "church", "like", "at", "that"]}, {"sent": "no ReverendTimms in that church .", "words": ["church", "in", "that"]}, {"sent": "was there a red one in the church ?", "words": ["red", "the", "was", "church", "there", "in", "a"]}, {"sent": "um the little boy and the balloon are going past a church steeple .", "words": ["the", "church", "and", "little", "are", "balloon", "a"]}, {"sent": "after the boy went to church with his grandmother ?", "words": ["the", "to", "church", "with", "his"]}, {"sent": "its a church is it ?", "words": ["church", "it", "is", "a"]}, {"sent": "so the church is still there .", "words": ["the", "is", "so", "church", "there"]}, {"sent": "is she gonna go to church or to school ?", "words": ["go", "is", "to", "church", "school", "she"]}, {"sent": "Nana didnt take you to church .", "words": ["to", "take", "church", "you"]}, {"sent": "are you going to church ?", "words": ["to", "are", "church", "you"]}, {"sent": "I could make one for every finger if you want .", "words": ["for", "if", "you", "finger", "every", "could", "make"]}, {"sent": "maybe he could have his own chair .", "words": ["chair", "have", "he", "his", "could"]}, {"sent": "you could cut the grass ?", "words": ["the", "cut", "you", "could", "grass"]}, {"sent": "we could put it down here .", "words": ["put", "we", "here", "it", "could", "down"]}, {"sent": "Nora you could come in .", "words": ["in", "could", "you"]}, {"sent": "I knew you could do it by yourself .", "words": ["do", "yourself", "by", "you", "it", "could"]}, {"sent": "yes that could be a driver .", "words": ["be", "could", "that", "a"]}, {"sent": "maybe you could hook it to the tow truck .", "words": ["the", "to", "truck", "you", "it", "could"]}, {"sent": "I could bring cars the next time .", "words": ["the", "could", "bring"]}, {"sent": "she did say I could play with them .", "words": ["play", "say", "with", "them", "did", "could", "she"]}, {"sent": "you could be Sneezy .", "words": ["be", "could", "you"]}, {"sent": "who could go on there ?", "words": ["go", "there", "who", "on", "could"]}, {"sent": "I ran as east as I could run I told the man .", "words": ["the", "run", "could"]}, {"sent": "I suppose it could double up as a cup if you really must .", "words": ["up", "cup", "if", "you", "it", "could", "a"]}, {"sent": "we could pop some pips in a pot .", "words": ["pop", "we", "in", "some", "could", "a"]}, {"sent": "now could you just fix my xxx ?", "words": ["fix", "could", "my", "you"]}, {"sent": "could it be .", "words": ["be", "could", "it"]}, {"sent": "could I have a piece a paper please ?", "words": ["have", "could", "paper", "a"]}, {"sent": "threw the ball as fast as she could and hard as she could .", "words": ["the", "fast", "and", "ball", "hard", "could", "she"]}, {"sent": "well could I play with these picture pairs here ?", "words": ["these", "play", "with", "picture", "here", "could"]}, {"sent": "we could clean that would be something huh ?", "words": ["clean", "we", "be", "would", "could", "that"]}, {"sent": "I wonder if I could make a pail out of .", "words": ["if", "of", "out", "could", "make", "a"]}, {"sent": "well thats okay too I said you could go in Catherines room .", "words": ["go", "you", "in", "room", "too", "could"]}, {"sent": "I could understand maybe if they put one on her at nighttime .", "words": ["put", "if", "they", "at", "on", "could", "her"]}, {"sent": "otherwise she said they would have loved to visit been wanting to take the kids to pops and this would have been a good excuse as any you could have sold four tickets .", "words": ["the", "to", "any", "and", "have", "they", "this", "you", "good", "would", "could", "take", "she", "a"]}, {"sent": "and I hafta wake her up most of the time or she could shes a sleeper and if I dont wake her up she could just probably slept till eleven or something you know .", "words": ["up", "the", "if", "of", "and", "you", "dont", "wake", "could", "her", "she", "a"]}, {"sent": "how could I go down and put that child in the car ?", "words": ["the", "how", "put", "go", "car", "and", "in", "could", "down", "that"]}, {"sent": "because see like when she first came over from Scotland I dont think she could say a word .", "words": ["think", "first", "say", "like", "dont", "over", "see", "could", "when", "she", "because", "a"]}, {"sent": "you could say it and I couldnt .", "words": ["say", "and", "you", "it", "could"]}, {"sent": "thats where I could find the jam .", "words": ["where", "the", "could", "find"]}, {"sent": "you could rock him on your rocking ah look at this here .", "words": ["your", "you", "here", "this", "him", "at", "look", "on", "could", "rock"]}, {"sent": "and we suddenly realized the men were there and we quickly undid the straps of your chair and rushed you to the window so you could see them .", "words": ["the", "your", "we", "of", "to", "so", "them", "chair", "and", "you", "there", "see", "window", "were", "could"]}, {"sent": "you could make a house out of wood .", "words": ["of", "you", "house", "out", "could", "make", "a"]}, {"sent": "know what we could do ?", "words": ["what", "could", "do", "we"]}, {"sent": "and xxx could be a reason it could be the slightest of reasons but .", "words": ["the", "of", "be", "and", "it", "could", "but", "a"]}, {"sent": "I think you could manage to make a bit better .", "words": ["better", "think", "to", "you", "could", "make", "a"]}, {"sent": "couldnt find it could they ?", "words": ["they", "could", "it", "find"]}, {"sent": "we could really do with planting it somewhere .", "words": ["do", "we", "with", "it", "could"]}, {"sent": "Trina could you get your face outta there please ?", "words": ["your", "face", "you", "get", "there", "could"]}, {"sent": "right could you start putting some more things in the bin bag please Thomas .", "words": ["the", "you", "more", "in", "some", "could"]}, {"sent": "but I could read it right now if you want just so you know one last offer .", "words": ["if", "so", "last", "you", "it", "read", "could", "but"]}, {"sent": "wonder if I could write in crayon .", "words": ["if", "crayon", "write", "in", "could"]}, {"sent": "oh we could have put that in the letter couldnt we ?", "words": ["the", "put", "we", "have", "in", "could", "that"]}, {"sent": "you know who else could go in this train dont you ?", "words": ["go", "train", "you", "this", "in", "dont", "who", "could"]}, {"sent": "we could buy a Happy meal for one pound ninety nine with our winnings couldnt we ?", "words": ["for", "we", "with", "our", "could", "buy", "a"]}, {"sent": "your mommy said maybe we could have a little piece if we drank all our milk .", "words": ["all", "your", "if", "we", "little", "have", "our", "milk", "could", "a"]}, {"sent": "could you could you ask mummy for a piece of kitchen roll for daddy please ?", "words": ["for", "of", "you", "kitchen", "could", "a"]}, {"sent": "it could keep them it could keep them in there with the big horrible xxx .", "words": ["the", "them", "with", "there", "in", "it", "could", "big"]}, {"sent": "could you come and show me where these go ?", "words": ["where", "these", "go", "show", "and", "you", "me", "could"]}, {"sent": "we could write one about Thomas Henry and Po .", "words": ["we", "write", "and", "about", "could"]}, {"sent": "could you please tell him describe to him what hes sposta do for brushing his teeth ?", "words": ["what", "do", "for", "to", "you", "him", "his", "could"]}, {"sent": "we could have a look see if we can get you it .", "words": ["can", "if", "we", "you", "have", "get", "it", "look", "see", "could", "a"]}, {"sent": "we could get the cookie .", "words": ["the", "cookie", "we", "get", "could"]}, {"sent": "you could you could give me one of your yur .", "words": ["your", "of", "you", "me", "give", "could"]}, {"sent": "you could have porridge .", "words": ["have", "could", "you"]}, {"sent": "because I walked past this and thought oh we could read that seeing as were playing with diggers .", "words": ["were", "we", "with", "and", "this", "read", "could", "that", "because"]}, {"sent": "how many do you suppose we could squeeze in here ?", "words": ["do", "how", "we", "you", "here", "in", "could"]}, {"sent": "so we could you you could relax .", "words": ["so", "could", "you", "we"]}, {"sent": "I wish I could hear about them .", "words": ["them", "hear", "wish", "about", "could"]}, {"sent": "do you think you could heat it up a bit more for me ?", "words": ["up", "do", "think", "for", "you", "more", "it", "me", "could", "a"]}, {"sent": "oh lets see I wonder what Daisy could eat .", "words": ["eat", "see", "what", "could"]}, {"sent": "hello hey do you have any milk that I could buy ?", "words": ["do", "any", "you", "have", "milk", "could", "that", "buy"]}, {"sent": "if you put if you load everything onto the tray and carry the tray you could make just one trip to the kitchen couldnt you ?", "words": ["the", "put", "if", "to", "you", "and", "carry", "make", "kitchen", "could", "tray"]}, {"sent": "could she sleep in there ?", "words": ["sleep", "there", "in", "could", "she"]}, {"sent": "it c could was just thrown sothat perhaps he didnt really mean it ?", "words": ["was", "could", "it", "he"]}, {"sent": "do you think she could blow bubbles ?", "words": ["blow", "do", "think", "you", "could", "she", "bubbles"]}, {"sent": "so I dont think you could have hurt it .", "words": ["think", "hurt", "so", "you", "have", "dont", "it", "could"]}, {"sent": "we could get a page .", "words": ["get", "could", "a", "we"]}, {"sent": "you could put your hand through .", "words": ["put", "your", "hand", "you", "could"]}, {"sent": "I suppose really we should have said he could have stayed but I dont know whether I could have coped with two little boys .", "words": ["we", "with", "little", "have", "dont", "he", "could", "but"]}, {"sent": "could you not ?", "words": ["could", "not", "you"]}, {"sent": "you could color her Wellingtons yellow like yours .", "words": ["like", "you", "yellow", "could", "her"]}, {"sent": "and we could even put some chopsticks in it someday .", "words": ["put", "we", "and", "in", "it", "some", "could"]}, {"sent": "he could read .", "words": ["could", "he", "read"]}, {"sent": "well I dont know what you could do .", "words": ["what", "do", "you", "dont", "could"]}, {"sent": "if there was a cow I could have got some milk .", "words": ["if", "was", "have", "there", "milk", "some", "could", "cow", "a"]}, {"sent": "I wonder who that could be .", "words": ["be", "could", "that", "who"]}, {"sent": "maybe you could tie it on .", "words": ["on", "could", "it", "you"]}, {"sent": "kee ker I kee crowed the rooster jack woke up and looked out the window to see the sunrise he stretched his arms and thought how hungry I am I could eat a large pancake for breakfast .", "words": ["the", "up", "how", "for", "pancake", "to", "hungry", "and", "eat", "rooster", "he", "am", "out", "his", "see", "window", "a", "could"]}, {"sent": "could play that game if you wanted .", "words": ["game", "if", "play", "you", "could", "that"]}, {"sent": "and then you could cook a meal .", "words": ["then", "you", "and", "cook", "could", "a"]}, {"sent": "maybe you could drink some of the milk .", "words": ["the", "drink", "of", "you", "milk", "some", "could"]}, {"sent": "so could we do it privately without anyone knowing ?", "words": ["do", "we", "so", "it", "could"]}, {"sent": "what could we make ?", "words": ["what", "could", "make", "we"]}, {"sent": "we could walk to xxx .", "words": ["to", "could", "walk", "we"]}, {"sent": "because he said we could shut it off in half an hour and I dont know but maybe .", "words": ["off", "we", "and", "dont", "in", "it", "he", "an", "but", "could", "because"]}, {"sent": "could you tell daddy what you saw on the slides ?", "words": ["the", "what", "you", "on", "could"]}, {"sent": "oh you sat down so we could read together ?", "words": ["we", "so", "you", "read", "could", "down"]}, {"sent": "you could do it on the table if you want .", "words": ["the", "do", "table", "if", "you", "it", "on", "could"]}, {"sent": "we could hear the men laughing at something couldnt we ?", "words": ["the", "we", "hear", "at", "could"]}, {"sent": "we could but it would be kind of small .", "words": ["we", "of", "be", "it", "would", "could", "but"]}, {"sent": "you could put this in a tray ?", "words": ["put", "you", "this", "in", "could", "tray", "a"]}, {"sent": "and we could hear .", "words": ["could", "hear", "we", "and"]}, {"sent": "it could be orangejuice .", "words": ["be", "could", "it"]}, {"sent": "right please could I have the .", "words": ["have", "the", "could"]}, {"sent": "yes it could probably push that lorry that truck couldnt it ?", "words": ["push", "truck", "it", "could", "that"]}, {"sent": "have fun you could have fun playing a game .", "words": ["game", "you", "have", "could", "a"]}, {"sent": "um I was just wondering if you could help .", "words": ["help", "if", "was", "you", "could"]}, {"sent": "I made you a waffle by the way .", "words": ["the", "by", "a", "you"]}, {"sent": "by the train .", "words": ["the", "train", "by"]}, {"sent": "and its not going to be here by tea time .", "words": ["to", "be", "and", "here", "by", "not"]}, {"sent": "Roz do you think it is easier to match them up by colors or by shapes or what or by design ?", "words": ["up", "do", "think", "what", "is", "to", "them", "you", "by", "it"]}, {"sent": "how about down by the bay ?", "words": ["the", "how", "by", "about", "down"]}, {"sent": "well I cant finish it by myself .", "words": ["it", "finish", "myself", "by"]}, {"sent": "by accident .", "words": ["by"]}, {"sent": "I dont know how much time has gone by .", "words": ["dont", "how", "much", "by"]}, {"sent": "pretend I get dead by myself .", "words": ["get", "pretend", "myself", "by"]}, {"sent": "and on his way home the little boy passed by a bakery shop .", "words": ["the", "and", "his", "little", "by", "on", "a", "home"]}, {"sent": "hafta do it all by myself ?", "words": ["all", "do", "by", "it", "myself"]}, {"sent": "so they took it away and were married next day by the turkey who lives on a hill .", "words": ["the", "so", "and", "by", "they", "away", "it", "turkey", "on", "who", "were", "a"]}, {"sent": "when I travel I go mostly by car .", "words": ["car", "when", "by", "go"]}, {"sent": "sit down by .", "words": ["sit", "down", "by"]}, {"sent": "I meant to say to you by the way .", "words": ["the", "to", "say", "by", "you"]}, {"sent": "who of all people should chance to fly by .", "words": ["all", "of", "to", "by", "who"]}, {"sent": "everybody promenade two by two .", "words": ["by"]}, {"sent": "do you know what I mean by tartan ?", "words": ["do", "what", "by", "you"]}, {"sent": "or can you do it all by yourself ?", "words": ["can", "do", "all", "yourself", "you", "by", "it"]}, {"sent": "hey I think Albert wants to do it by himself .", "words": ["do", "think", "to", "by", "it"]}, {"sent": "I can tell by all the tears and .", "words": ["can", "all", "the", "and", "by"]}, {"sent": "theyre hiding by the seat .", "words": ["the", "by"]}, {"sent": "you see it by the door .", "words": ["the", "by", "you", "it", "see", "door"]}, {"sent": "okay she must be finished by now .", "words": ["be", "she", "by"]}, {"sent": "just stand by the blue gate .", "words": ["the", "stand", "blue", "by"]}, {"sent": "Im gonna see this picture over by the tv .", "words": ["the", "picture", "by", "this", "tv", "see", "over"]}, {"sent": "its down by Caroline isnt it ?", "words": ["it", "down", "by"]}, {"sent": "by Lindseys foot .", "words": ["foot", "by"]}, {"sent": "dyou want your shovel back by the way ?", "words": ["the", "your", "back", "by", "shovel"]}, {"sent": "like she likes to organize things by color and when she messes something up she usually puts it away .", "words": ["up", "to", "like", "by", "and", "it", "away", "when", "she"]}, {"sent": "can you ride your bicycle by yourself or does Maggie push you ?", "words": ["can", "does", "yourself", "your", "push", "bicycle", "you", "by", "ride"]}, {"sent": "have him go by and put mail in the mail box ?", "words": ["the", "put", "box", "go", "by", "have", "and", "him", "in"]}, {"sent": "its written by the same people .", "words": ["the", "same", "by"]}, {"sent": "can you get in this chair by yourself ?", "words": ["can", "yourself", "chair", "you", "get", "this", "in", "by"]}, {"sent": "by the pond .", "words": ["the", "by"]}, {"sent": "by the time the story was over Spot was fast asleep .", "words": ["the", "fast", "was", "by", "asleep", "story", "over"]}, {"sent": "and see cars n trucks go by .", "words": ["go", "see", "by", "and"]}, {"sent": "theres some building work going on near by isnt there ?", "words": ["work", "by", "there", "on", "some"]}, {"sent": "where are the legos by the way ?", "words": ["where", "the", "are", "by"]}, {"sent": "that seems to me that slumber parties are pretty outdated by the time you get to the university dont you think ?", "words": ["the", "think", "to", "by", "you", "get", "dont", "are", "pretty", "me", "that"]}, {"sent": "she baked me my bread she brewed me my ale she sat by the fire and told a fine tale .", "words": ["the", "bread", "my", "fine", "by", "and", "me", "she", "a"]}, {"sent": "I think by half and hour just gradually up but .", "words": ["up", "think", "and", "by", "but"]}, {"sent": "so is it worse to break something by accident or to break something when youre doing something naughty ?", "words": ["when", "is", "to", "so", "by", "it", "break", "naughty"]}, {"sent": "by the cat puppet .", "words": ["the", "cat", "by"]}, {"sent": "and that lamp that I can read by that light .", "words": ["can", "light", "and", "by", "read", "lamp", "that"]}, {"sent": "what about the one about the rabbit running by ?", "words": ["the", "what", "about", "by"]}, {"sent": "well you beat him by mile Zoe .", "words": ["him", "by", "you"]}, {"sent": "what have you been frightened by ?", "words": ["have", "what", "by", "you"]}, {"sent": "over there by the horsie .", "words": ["the", "there", "over", "by"]}, {"sent": "he seems to be fascinated by the sound .", "words": ["the", "to", "be", "by", "he"]}, {"sent": "we walked by it before ?", "words": ["it", "by", "we"]}, {"sent": "put the little man by the green one ?", "words": ["the", "put", "by", "little", "green"]}, {"sent": "are you learning and learning by playing with this laundry basket ?", "words": ["with", "by", "and", "you", "this", "are", "basket"]}, {"sent": "oh well if youre not going to play with me Aran Ill er read a book by myself .", "words": ["if", "to", "play", "with", "by", "read", "myself", "book", "me", "not", "a"]}, {"sent": "you should wear the microphone and let the bear be by himself .", "words": ["the", "be", "bear", "and", "you", "by"]}, {"sent": "take your trousers off and you can go by yourself .", "words": ["off", "can", "yourself", "your", "go", "you", "and", "by", "take"]}, {"sent": "slithered and slithered on by or something .", "words": ["on", "by", "and"]}, {"sent": "um on his way out from church the boy passed by a bakery shop .", "words": ["the", "church", "by", "his", "out", "on", "a"]}, {"sent": "by the cakes .", "words": ["the", "by"]}, {"sent": "watch the xxx Michele you cant sit by yourself .", "words": ["the", "watch", "yourself", "by", "you", "sit"]}, {"sent": "there so you put that one on by in there .", "words": ["put", "so", "you", "by", "there", "in", "on", "that"]}, {"sent": "come on by ?", "words": ["on", "by"]}, {"sent": "theyre all over the floor down there by the radiator .", "words": ["the", "all", "by", "there", "down", "over"]}, {"sent": "you fell over by the garbage ?", "words": ["the", "by", "you", "over", "garbage"]}, {"sent": "and then he swims by and I hafta go ooh got you .", "words": ["go", "by", "and", "you", "he", "then"]}, {"sent": "you can go by foot and you can go by car but MarvinKMooney will you please go now .", "words": ["can", "go", "car", "by", "you", "and", "foot", "will", "but"]}, {"sent": "youre gonna go by yourself ?", "words": ["yourself", "by", "go"]}, {"sent": "down down baby down by the rollercoaster .", "words": ["the", "down", "by"]}, {"sent": "Cindy looked at Cromwell sitting by himself in the wagon and suddenly she felt mad .", "words": ["the", "mad", "by", "and", "in", "at", "she"]}, {"sent": "you stay by the toilet now .", "words": ["the", "stay", "by", "you"]}, {"sent": "by me okay .", "words": ["me", "by"]}, {"sent": "two bins side by side .", "words": ["by"]}, {"sent": "by the legs of the chair .", "words": ["the", "of", "chair", "by"]}, {"sent": "is he taking him by the hand ?", "words": ["the", "is", "hand", "by", "him", "he"]}, {"sent": "can you show me the daddy being kissed by the mommy ?", "words": ["can", "the", "show", "you", "by", "me"]}, {"sent": "that must be empty by now .", "words": ["be", "that", "by", "empty"]}, {"sent": "why dont you go through your lil tunnel because I cant get by there with you xxx little feet .", "words": ["your", "go", "with", "because", "you", "get", "dont", "by", "there", "little", "why"]}, {"sent": "yeah up by her head .", "words": ["up", "head", "her", "by"]}, {"sent": "do you think he needs to perhaps sit by the toilet .", "words": ["the", "do", "think", "to", "you", "by", "he", "sit"]}, {"sent": "Ill swing by my ankles shell cling to your knees as you hang by your nose from high .", "words": ["my", "your", "swing", "to", "by", "you", "high", "nose"]}, {"sent": "dont go by this this is Davids .", "words": ["is", "go", "by", "dont", "this"]}, {"sent": "not by the hair .", "words": ["the", "hair", "not", "by"]}, {"sent": "step by step a step by step .", "words": ["a", "by"]}, {"sent": "and by the end of it he told us all about the Zoo so he did .", "words": ["the", "all", "of", "so", "by", "and", "us", "it", "he", "did", "about"]}, {"sent": "if youre not in here by the time you get to three you wont be having a bath with me and Amy .", "words": ["the", "if", "to", "be", "with", "by", "here", "you", "in", "get", "and", "me", "not", "a"]}, {"sent": "see if you can do that one by yourself .", "words": ["can", "do", "yourself", "if", "you", "by", "see", "that"]}, {"sent": "by the table ya sit in your chair by the table .", "words": ["the", "your", "table", "chair", "by", "in", "sit"]}, {"sent": "you go by Bash and not Sebastian ?", "words": ["go", "by", "and", "you", "not"]}, {"sent": "I usually I if every thing going well by about eleven oclock in the morning I am through with the school part of it .", "words": ["the", "if", "of", "school", "with", "by", "in", "it", "every", "about", "am"]}, {"sent": "there is a piece down there on the floor Joey the ah by the blue cabinet .", "words": ["the", "is", "by", "blue", "there", "on", "down", "a"]}, {"sent": "shall we put shall we put your cake on a plate and then you can give Morag and Inga some cant you because they must have finished their pizza by now .", "words": ["put", "plate", "can", "their", "pizza", "by", "have", "on", "some", "then", "cake", "and", "you", "because", "a", "your", "we", "they", "give"]}, {"sent": "theyre up there by my purse .", "words": ["up", "my", "purse", "by", "there"]}, {"sent": "all by myself .", "words": ["all", "myself", "by"]}, {"sent": "and let me trim this fingernail before you get scratched by it .", "words": ["you", "get", "this", "and", "by", "it", "me"]}, {"sent": "when he feels like his brothers entertained by him hes really happy about that so he does try to entertain Scott alot .", "words": ["does", "try", "happy", "so", "to", "like", "by", "him", "that", "he", "about", "his", "when", "try to"]}, {"sent": "well I think AuntieMabel has got Pippin on a lead by the looks of it .", "words": ["the", "think", "of", "by", "it", "on", "a"]}, {"sent": "its side by side .", "words": ["by"]}, {"sent": "for if I do my mother will say did ya ever see a goose kissing a moose down by the bay .", "words": ["goose", "do", "my", "for", "the", "if", "did ya", "say", "moose", "by", "did", "will", "see", "down", "a"]}, {"sent": "were gonna have a bunny by our tree .", "words": ["tree", "bunny", "by", "have", "our", "were", "a"]}, {"sent": "I must admit he looks flat enough to have been run over by a steam roller doesnt he ?", "words": ["to", "by", "have", "run", "he", "over", "a"]}, {"sent": "when that one is all finished well put another one on .", "words": ["all", "put", "when", "is", "another", "on", "that"]}, {"sent": "wheres another purple one ?", "words": ["another"]}, {"sent": "is that another Mama book ?", "words": ["another", "book", "that", "is"]}, {"sent": "lets find another picture .", "words": ["another", "picture", "find"]}, {"sent": "theres another piggy there .", "words": ["another", "there"]}, {"sent": "you want another cookie ?", "words": ["another", "cookie", "you"]}, {"sent": "can you find another monkey ?", "words": ["can", "monkey", "you", "another", "find"]}, {"sent": "Ill get another .", "words": ["get", "another"]}, {"sent": "so weve been really weve done tons of stuff we went hiking a bunch of times and we hm Im trying to think about another thing you did that you told me about that was very interesting do you wanna tell the story about that girl at your school and the cheese ?", "words": ["the", "think", "of", "school", "about", "me", "another", "that", "so", "to", "was", "and", "you", "cheese", "a", "do", "your", "we", "did", "wanna", "at", "story"]}, {"sent": "make me another one .", "words": ["another", "me", "make"]}, {"sent": "thats another finger .", "words": ["another", "finger"]}, {"sent": "another circle .", "words": ["another"]}, {"sent": "do you wanna look at another page ?", "words": ["do", "you", "another", "at", "wanna", "look"]}, {"sent": "have you got another giraffe downstairs like that ?", "words": ["giraffe", "like", "you", "have", "another", "that"]}, {"sent": "lets look at another book .", "words": ["another", "book", "at", "look"]}, {"sent": "thats another flower .", "words": ["another", "flower"]}, {"sent": "what ya looking for another .", "words": ["another", "what", "for"]}, {"sent": "if youve lost it well see about getting another one .", "words": ["if", "another", "it", "about", "see"]}, {"sent": "and another one over there .", "words": ["another", "there", "over", "and"]}, {"sent": "I need another bolt though another yellow bolt .", "words": ["another", "yellow", "need"]}, {"sent": "oh you saw another one ?", "words": ["another", "you"]}, {"sent": "oh we got another box alright .", "words": ["another", "box", "we"]}, {"sent": "theres another icecream cone .", "words": ["another"]}, {"sent": "another a .", "words": ["another", "a"]}, {"sent": "shall we put another people in there ?", "words": ["put", "we", "another", "there", "in"]}, {"sent": "heres another book .", "words": ["another", "book"]}, {"sent": "I dont think weve got another one have we ?", "words": ["think", "we", "another", "dont", "have"]}, {"sent": "wanna do another one ?", "words": ["another", "do", "wanna"]}, {"sent": "because we make another lolly outof with the stick didnt we ?", "words": ["the", "stick", "we", "with", "another", "make", "because"]}, {"sent": "I think Ill make another tower .", "words": ["another", "think", "make"]}, {"sent": "well hafta get another pen out wont I ?", "words": ["get", "out", "another", "pen"]}, {"sent": "lets turn them over and see if we can find another one .", "words": ["can", "if", "we", "them", "and", "another", "find", "see", "over"]}, {"sent": "thats another bale of hay isnt it ?", "words": ["another", "of", "it"]}, {"sent": "another one I know .", "words": ["another"]}, {"sent": "try another color .", "words": ["another", "try"]}, {"sent": "shall we do it on another piece of paper ?", "words": ["do", "we", "of", "paper", "another", "it", "on"]}, {"sent": "then another truck will come along with the tarmac .", "words": ["the", "truck", "with", "another", "will", "then"]}, {"sent": "can you find another stick ?", "words": ["can", "stick", "you", "another", "find"]}, {"sent": "it is another busy day at the station .", "words": ["the", "is", "another", "it", "at"]}, {"sent": "another book .", "words": ["another", "book"]}, {"sent": "theres one spoon in the truck and theres another spoon there by the train .", "words": ["the", "train", "truck", "and", "another", "there", "in", "by", "spoon"]}, {"sent": "and they want me back for another one .", "words": ["for", "back", "and", "another", "they", "me"]}, {"sent": "theres another piece of cheese wheres another piece of cheese ?", "words": ["another", "of", "cheese"]}, {"sent": "would you like to play another game ?", "words": ["game", "to", "play", "like", "you", "another", "would"]}, {"sent": "could you go and get Avril another toy or so ?", "words": ["go", "so", "and", "get", "you", "another", "toy", "could"]}, {"sent": "heres another man riding on the back of the elephant .", "words": ["the", "back", "of", "elephant", "another", "on"]}, {"sent": "I didnt realize there was another one .", "words": ["was", "there", "another"]}, {"sent": "can I have another slice of bread on my sandwich ?", "words": ["can", "bread", "my", "of", "have", "another", "on", "sandwich"]}, {"sent": "so the next time you are in we are going to do another one of these lab visits and then another play session again and one of the things we will be looking at is how things have changed .", "words": ["the", "of", "be", "another", "have", "are", "then", "these", "how", "is", "so", "to", "play", "you", "and", "will", "do", "we", "in", "at"]}, {"sent": "if its if its a big fire you might need another fire engine .", "words": ["need", "if", "you", "another", "big", "a"]}, {"sent": "and we can change it on another picture .", "words": ["can", "we", "picture", "and", "another", "it", "on"]}, {"sent": "another one those are red ones .", "words": ["another", "are", "those", "red"]}, {"sent": "thats another blue one .", "words": ["another", "blue"]}, {"sent": "its another bath .", "words": ["another"]}, {"sent": "another one ?", "words": ["another"]}, {"sent": "another man ?", "words": ["another"]}, {"sent": "cool another table .", "words": ["another", "table"]}, {"sent": "this is another a picture of a pussycat but this is a cartoon .", "words": ["is", "of", "picture", "another", "this", "but", "a"]}, {"sent": "now Im putting another girl thats going to xxx and now she goes down to go in the bus .", "words": ["the", "go", "to", "and", "another", "in", "down", "bus", "she"]}, {"sent": "um can we find another orange ?", "words": ["can", "we", "another", "orange", "find"]}, {"sent": "I dont want another one .", "words": ["another", "dont"]}, {"sent": "heres another ring .", "words": ["another"]}, {"sent": "thats another little boy isnt it ?", "words": ["another", "it", "little"]}, {"sent": "try another door .", "words": ["another", "try", "door"]}, {"sent": "theres another table leg .", "words": ["another", "table", "leg"]}, {"sent": "you want another green pepper ?", "words": ["another", "green", "you"]}, {"sent": "another worm yeah .", "words": ["another"]}, {"sent": "wanna make another story with these guys ?", "words": ["these", "with", "another", "wanna", "story", "make"]}, {"sent": "would you like another bar ?", "words": ["another", "like", "you", "would"]}, {"sent": "I think theres another one coming on actually in a minute .", "words": ["think", "another", "in", "on", "a"]}, {"sent": "well just get another clean diaper .", "words": ["get", "diaper", "another", "clean"]}, {"sent": "oh theres another yellow mommy .", "words": ["another", "yellow"]}, {"sent": "another one .", "words": ["another"]}, {"sent": "he needs another leg doesnt he ?", "words": ["another", "he", "leg"]}, {"sent": "and another mailman .", "words": ["another", "and"]}, {"sent": "then bring a book ba another book back for me or .", "words": ["for", "back", "bring", "another", "book", "me", "then", "a"]}, {"sent": "and then Ill pick up another one and do it with him .", "words": ["up", "do", "pick", "with", "and", "another", "him", "it", "then"]}, {"sent": "wanna pick another one ?", "words": ["pick", "another", "wanna"]}, {"sent": "so it grows another arm then .", "words": ["so", "another", "it", "arm", "then"]}, {"sent": "another one by the kennel look .", "words": ["another", "the", "look", "by"]}, {"sent": "another truck .", "words": ["another", "truck"]}, {"sent": "right well Ill say another one .", "words": ["another", "say"]}, {"sent": "and then youve to give them another one about an hour later or so .", "words": ["give", "to", "so", "them", "and", "another", "an", "about", "then"]}, {"sent": "you can have one another time .", "words": ["can", "have", "another", "you"]}, {"sent": "is that another baby ?", "words": ["another", "that", "is"]}, {"sent": "Mummy better nip and get another tissue .", "words": ["better", "tissue", "and", "get", "another"]}, {"sent": "another man coming ?", "words": ["another"]}, {"sent": "have another look .", "words": ["have", "another", "look"]}, {"sent": "and I put his dodie in and keep him for another wee while .", "words": ["put", "for", "and", "another", "in", "him", "his"]}, {"sent": "theres another truck .", "words": ["another", "truck"]}, {"sent": "another tin of strawberrys .", "words": ["another", "of"]}, {"sent": "shall I get another one ?", "words": ["get", "another"]}, {"sent": "another one .", "words": ["another"]}, {"sent": "do you think she rec thought uh the doll was another baby when interacting with it ?", "words": ["the", "do", "think", "was", "with", "doll", "you", "another", "it", "when", "she"]}, {"sent": "ah you making another thats a good idea .", "words": ["another", "good", "a", "you"]}, {"sent": "moving to another .", "words": ["another", "to"]}, {"sent": "another day .", "words": ["another"]}, {"sent": "xxx only been to Macdonalds about three or four times but each time they xxx another toy .", "words": ["each", "to", "another", "they", "toy", "about", "but"]}, {"sent": "another line ?", "words": ["another"]}, {"sent": "Lilys gonna be in in school by the time I dont have an appointment on Friday morning .", "words": ["the", "be", "school", "by", "have", "dont", "in", "an", "on"]}, {"sent": "you dont like to go to school ?", "words": ["go", "to", "school", "like", "you", "dont"]}, {"sent": "what do you do when you go to school Adam ?", "words": ["what", "do", "go", "to", "school", "you", "when"]}, {"sent": "you made a room a school room .", "words": ["school", "room", "a", "you"]}, {"sent": "the boys in school what did they think about all the your shirt ?", "words": ["the", "what", "think", "all", "your", "school", "shirt", "they", "in", "did", "about"]}, {"sent": "but you hafta tell me all about school .", "words": ["all", "school", "you", "about", "me", "but"]}, {"sent": "I am school so I cant come to school ?", "words": ["to", "school", "so", "am"]}, {"sent": "has she got some special friends at the school at the creche .", "words": ["the", "school", "at", "some", "she"]}, {"sent": "he has probably already left for school but when you finish eating .", "words": ["for", "school", "finish", "you", "he", "when", "but"]}, {"sent": "if you get there too late you might miss something thats going to happen at school .", "words": ["if", "to", "school", "you", "get", "there", "at", "too"]}, {"sent": "have you got any animals in your school ?", "words": ["your", "school", "any", "you", "have", "in"]}, {"sent": "oh Helens going to school ?", "words": ["to", "school"]}, {"sent": "oh okay theyre gonna go to different school .", "words": ["to", "school", "go"]}, {"sent": "school you mean ?", "words": ["school", "you"]}, {"sent": "went to school .", "words": ["to", "school"]}, {"sent": "do you play golf at school at all ?", "words": ["do", "all", "school", "play", "you", "at"]}, {"sent": "Lina who are your friends at school ?", "words": ["your", "school", "are", "at", "who"]}, {"sent": "you know what at school .", "words": ["what", "school", "at", "you"]}, {"sent": "that goes in the school .", "words": ["the", "school", "in", "that"]}, {"sent": "how was school today ?", "words": ["was", "how", "school"]}, {"sent": "he wont have gone to school today .", "words": ["have", "to", "school", "he"]}, {"sent": "say school .", "words": ["school", "say"]}, {"sent": "from school .", "words": ["school"]}, {"sent": "you gonna take him to school ?", "words": ["to", "school", "you", "him", "take"]}, {"sent": "are you gonna share when you go to school ?", "words": ["share", "go", "to", "school", "you", "are", "when"]}, {"sent": "heres the school I dont know where the doggie is .", "words": ["the", "where", "is", "school", "dont"]}, {"sent": "what did you do at school today ?", "words": ["what", "do", "school", "you", "did", "at"]}, {"sent": "is it time to go home from school now ?", "words": ["go", "is", "to", "school", "it", "home"]}, {"sent": "you made a snail did you make a snail at school Shem ?", "words": ["school", "you", "did", "at", "make", "a"]}, {"sent": "xxx going to school .", "words": ["to", "school"]}, {"sent": "and there had been the problem with the pigeon on the way to school .", "words": ["the", "to", "school", "with", "and", "there", "on"]}, {"sent": "hes gone to big school hasnt he ?", "words": ["he", "to", "school", "big"]}, {"sent": "well I did have my last day of school but its another semester just like you you had your last day of school and now youre starting a new school year right ?", "words": ["my", "your", "of", "school", "last", "like", "you", "have", "another", "and", "did", "new", "but", "a"]}, {"sent": "does she wanna take a giraffe to school ?", "words": ["does", "giraffe", "to", "school", "wanna", "take", "she", "a"]}, {"sent": "everyone at school wanted it .", "words": ["school", "it", "at"]}, {"sent": "you cant go to school like that everyone will look at you .", "words": ["go", "to", "school", "like", "you", "at", "look", "will", "that"]}, {"sent": "is he at school with everyone ?", "words": ["is", "school", "with", "he", "at"]}, {"sent": "yeah have you um do you have alphabits at school ?", "words": ["do", "school", "you", "have", "at"]}, {"sent": "do you write in school Sarah ?", "words": ["do", "school", "write", "you", "in"]}, {"sent": "picking picking the children up from school .", "words": ["the", "school", "up"]}, {"sent": "maybe in my school hafta ?", "words": ["school", "in", "my"]}, {"sent": "right I went to school in California .", "words": ["to", "school", "in"]}, {"sent": "lets go to school .", "words": ["to", "school", "go"]}, {"sent": "school .", "words": ["school"]}, {"sent": "I didnt get you a bowl from school .", "words": ["bowl", "school", "you", "get", "a"]}, {"sent": "youll have school tomorrow William .", "words": ["have", "school"]}, {"sent": "tell me about school .", "words": ["me", "school", "about"]}, {"sent": "did someone read this book to you at school ?", "words": ["to", "school", "you", "this", "read", "did", "at", "book"]}, {"sent": "xxx big girl because you go to big school as well dont you ?", "words": ["go", "to", "school", "you", "dont", "big", "because"]}, {"sent": "that Adam that went to school ?", "words": ["to", "school", "that"]}, {"sent": "because youve not started school yet .", "words": ["school", "not", "because"]}, {"sent": "do you wanna take your lunch to school today ?", "words": ["do", "your", "to", "school", "you", "wanna", "take"]}, {"sent": "you were left with Lynn and Charlotte because Ben was at nursery school .", "words": ["was", "school", "with", "and", "you", "at", "were", "because"]}, {"sent": "and youll get the diarrhea in the chu at school .", "words": ["the", "school", "and", "get", "in", "at"]}, {"sent": "when are you gonna start school do you know ?", "words": ["do", "school", "you", "are", "when"]}, {"sent": "you you do that in school huh ?", "words": ["do", "school", "you", "in", "that"]}, {"sent": "do they cook foods at at your at the school that you ?", "words": ["the", "do", "your", "school", "you", "they", "cook", "at", "that"]}, {"sent": "hes gonna be dynamite when he goes to school .", "words": ["be", "to", "school", "he", "when"]}, {"sent": "theres Mummy saying get ready for school now Quigly .", "words": ["get", "school", "for"]}, {"sent": "why dont you tell me what you did at school today ?", "words": ["what", "school", "you", "dont", "did", "at", "me", "why"]}, {"sent": "so anyway whats your favorite thing to do at school ?", "words": ["do", "your", "to", "so", "school", "at"]}, {"sent": "so the school is a success xxx .", "words": ["the", "is", "so", "school", "a"]}, {"sent": "um Thomass friend from school .", "words": ["school"]}, {"sent": "Jwwws at school and Daddys gone to work now .", "words": ["work", "to", "school", "and", "at"]}, {"sent": "no because you guys dont have school today right ?", "words": ["school", "you", "have", "dont", "because"]}, {"sent": "xxx eat that cake or did she take it to to school or what ?", "words": ["what", "cake", "to", "school", "eat", "it", "did", "take", "that", "she"]}, {"sent": "is he here at school ?", "words": ["is", "school", "here", "he", "at"]}, {"sent": "is that your school bag ?", "words": ["your", "school", "that", "is"]}, {"sent": "do books go inside the school bus ?", "words": ["the", "do", "go", "school", "inside", "bus"]}, {"sent": "Lily was really really sorry hm Lilys mother wrote a note and Lilys father baked some tasty snacks for Lily to take to school the next day .", "words": ["the", "for", "was", "to", "school", "and", "some", "take", "a"]}, {"sent": "hooray everyone cheers as Bruno and Miss Honey come out of the school .", "words": ["the", "of", "school", "and", "out"]}, {"sent": "and um I couldnt take the school bus .", "words": ["the", "school", "and", "take", "bus"]}, {"sent": "do you like school ?", "words": ["do", "school", "like", "you"]}, {"sent": "no what Im gonna do is first I drop you xxx I take you to school .", "words": ["what", "do", "is", "to", "school", "first", "you", "drop", "take"]}, {"sent": "xxx at school .", "words": ["school", "at"]}, {"sent": "now tell me about in school when youre youre doing your work .", "words": ["your", "work", "school", "in", "about", "me", "when"]}, {"sent": "so your brother must go to a big school then .", "words": ["your", "go", "to", "so", "school", "then", "big", "a"]}, {"sent": "mm and do they go to school ?", "words": ["do", "go", "to", "school", "and", "they"]}, {"sent": "when you finish school are you going to go to her ?", "words": ["go", "school", "to", "her", "finish", "you", "are", "when"]}, {"sent": "what was the last painting you did in school ?", "words": ["the", "what", "was", "last", "school", "you", "in", "did"]}, {"sent": "you went to baby school and then you came home .", "words": ["to", "school", "and", "you", "then", "home"]}, {"sent": "Rwwws not at school .", "words": ["school", "not", "at"]}, {"sent": "Im not really sure what time you do things at school to be honest .", "words": ["what", "do", "school", "to", "be", "you", "at", "not"]}, {"sent": "do they have pictures of dogs at school ?", "words": ["do", "of", "school", "have", "they", "at"]}, {"sent": "do you go to school ?", "words": ["do", "go", "school", "to", "you"]}, {"sent": "Rabbit and Beaver will learn new things at school and so will you .", "words": ["school", "so", "and", "you", "at", "will", "new"]}, {"sent": "its like when youre at school as well .", "words": ["school", "when", "like", "at"]}, {"sent": "did you walk all the way to school today ?", "words": ["the", "all", "to", "school", "you", "did", "walk"]}, {"sent": "so I hadta stay with her a couple of weeks for GB and Sunday school .", "words": ["for", "of", "so", "school", "with", "and", "stay", "her", "a"]}, {"sent": "cause youre gonna be in school when Mommy gets her hair cut .", "words": ["cut", "be", "school", "her", "in", "hair", "when"]}, {"sent": "he goes to school too .", "words": ["to", "school", "he", "too"]}, {"sent": "what do you go to school for ?", "words": ["what", "do", "for", "go", "to", "school", "you"]}, {"sent": "here comes a school bus can they go to school Roman ?", "words": ["can", "go", "school", "to", "here", "they", "bus", "a"]}, {"sent": "well if you have a birthday cake at school your friends will sing HappyBirthday .", "words": ["your", "if", "cake", "school", "you", "have", "sing", "at", "will", "a"]}, {"sent": "we dont come to school tamarra what is tamarra ?", "words": ["what", "we", "is", "to", "school", "dont"]}, {"sent": "who can tell me what happens in school ?", "words": ["can", "what", "school", "in", "who", "me"]}, {"sent": "your overalls for school .", "words": ["school", "for", "your"]}, {"sent": "do you wanna go to school ?", "words": ["do", "go", "to", "school", "you", "wanna"]}, {"sent": "but its interesting it has a lot of its about um a um black law school professor .", "words": ["of", "school", "a lot", "black", "it", "about", "but", "a"]}, {"sent": "so tell me what you did on your first day at school ?", "words": ["what", "your", "so", "school", "first", "you", "did", "at", "on", "me"]}, {"sent": "do you wanna set up some of the train ?", "words": ["up", "do", "the", "train", "of", "you", "wanna", "some"]}, {"sent": "oh watch out train .", "words": ["watch", "out", "train"]}, {"sent": "youve found a train have you ?", "words": ["have", "train", "you", "a"]}, {"sent": "its not a train is it ?", "words": ["train", "is", "it", "not", "a"]}, {"sent": "Drybones train .", "words": ["train"]}, {"sent": "thats a train thats a train thats a train thats a train .", "words": ["train", "a"]}, {"sent": "is that the train then ?", "words": ["the", "train", "is", "then", "that"]}, {"sent": "what do you think they put in that train ?", "words": ["what", "do", "think", "put", "train", "you", "they", "in", "that"]}, {"sent": "to drive the train .", "words": ["drive", "to", "train", "the"]}, {"sent": "so the spoon is riding round inside the box with the juice like the carriage of a train .", "words": ["the", "box", "is", "of", "so", "inside", "train", "with", "like", "spoon", "juice", "a"]}, {"sent": "are you pushing the train ?", "words": ["the", "train", "are", "you"]}, {"sent": "can he drive the train now ?", "words": ["can", "the", "train", "drive", "he"]}, {"sent": "oh hes tired of riding the train ?", "words": ["of", "the", "train", "tired"]}, {"sent": "Alex fix your train .", "words": ["fix", "your", "train"]}, {"sent": "train .", "words": ["train"]}, {"sent": "youre lifting the lid of the train and youre giving the train driver some crisps arent you ?", "words": ["the", "train", "of", "and", "you", "some"]}, {"sent": "well hes on the back of the train isnt he ?", "words": ["the", "back", "train", "of", "he", "on"]}, {"sent": "Im going to wash this train as well .", "words": ["to", "wash", "this", "train"]}, {"sent": "train .", "words": ["train"]}, {"sent": "which small train ?", "words": ["which", "train"]}, {"sent": "theres the three beggars in the train .", "words": ["the", "in", "train"]}, {"sent": "you could put the key on the train and see if itll go through the tunnel .", "words": ["the", "put", "if", "train", "go", "you", "and", "on", "see", "could"]}, {"sent": "what does my train .", "words": ["what", "my", "does", "train"]}, {"sent": "does that go to the train ?", "words": ["the", "does", "go", "train", "to", "that"]}, {"sent": "a train ?", "words": ["train", "a"]}, {"sent": "Im gonna play with your train if youre playing with that .", "words": ["your", "if", "train", "play", "with", "that"]}, {"sent": "the train .", "words": ["the", "train"]}, {"sent": "the train track runs all the way around the mountains .", "words": ["the", "all", "around", "train"]}, {"sent": "those things are parts of the train I see .", "words": ["the", "those", "train", "of", "are", "see"]}, {"sent": "dont you get me train dont you get me train no no no no no no pull pull .", "words": ["train", "you", "get", "dont", "pull", "me"]}, {"sent": "theres nobody left to drive the train though .", "words": ["drive", "to", "train", "the"]}, {"sent": "make train ?", "words": ["make", "train"]}, {"sent": "drive the train into the garage .", "words": ["the", "into", "train", "garage", "drive"]}, {"sent": "choochoo train ?", "words": ["train"]}, {"sent": "well Burnage station it is isnt it its one platform when you want trains to the airport and and another platform when you want a train to Manchester .", "words": ["the", "is", "train", "to", "you", "and", "another", "it", "when", "a"]}, {"sent": "is it a train set ?", "words": ["train", "it", "a", "is"]}, {"sent": "I dont think the train wants a biscuit Dominic .", "words": ["the", "think", "train", "dont", "a"]}, {"sent": "because youve got the same train set at home .", "words": ["the", "train", "same", "at", "because", "home"]}, {"sent": "have you found another train ?", "words": ["have", "train", "another", "you"]}, {"sent": "that train is far away now .", "words": ["away", "that", "is", "train"]}, {"sent": "why did he miss his train ?", "words": ["train", "he", "did", "his", "why"]}, {"sent": "hold the train ?", "words": ["the", "hold", "train"]}, {"sent": "whos driving the train now ?", "words": ["the", "train"]}, {"sent": "so theres no bit of the track where a train can come off .", "words": ["where", "the", "can", "off", "train", "of", "so", "a"]}, {"sent": "this is called a puffing xxx steam train cake .", "words": ["cake", "train", "is", "this", "a"]}, {"sent": "it goes in the train puzzle that is true .", "words": ["the", "train", "is", "in", "it", "puzzle", "that"]}, {"sent": "I think he was actually knocked down by a train by the looks of him because his heads .", "words": ["the", "think", "train", "of", "was", "by", "him", "he", "his", "down", "because", "a"]}, {"sent": "would you will I bring along a train set the next time ?", "words": ["the", "train", "bring", "you", "would", "will", "a"]}, {"sent": "think those children are going to school on the train ?", "words": ["the", "think", "those", "train", "to", "school", "are", "on"]}, {"sent": "what train ?", "words": ["what", "train"]}, {"sent": "wheres the train going to ?", "words": ["the", "to", "train"]}, {"sent": "Im building your train up .", "words": ["up", "your", "train"]}, {"sent": "thats for the train .", "words": ["the", "for", "train"]}, {"sent": "are you driving the train Fraser ?", "words": ["the", "train", "are", "you"]}, {"sent": "want a train the back another bus .", "words": ["the", "back", "train", "another", "bus", "a"]}, {"sent": "tell Jeannine you went on a train .", "words": ["on", "train", "a", "you"]}, {"sent": "the train on top of the tower xxx xxx xxx whoa .", "words": ["the", "of", "on", "train"]}, {"sent": "whats the train drive on ?", "words": ["the", "on", "drive", "train"]}, {"sent": "train .", "words": ["train"]}, {"sent": "do you sit by mommy on the train ?", "words": ["the", "do", "train", "by", "you", "on", "sit"]}, {"sent": "gosh that train sounds loud .", "words": ["that", "loud", "train"]}, {"sent": "train .", "words": ["train"]}, {"sent": "look that baby has a lil train .", "words": ["train", "that", "a", "look"]}, {"sent": "caboose too big we need the little train we need the tender .", "words": ["the", "need", "we", "train", "little", "too", "big"]}, {"sent": "going on a train .", "words": ["on", "train", "a"]}, {"sent": "but she had a really enjoyable journey on the train didnt she ?", "words": ["the", "train", "on", "she", "but", "a"]}, {"sent": "my cars bigger than your train .", "words": ["my", "your", "train"]}, {"sent": "and then you got a train here .", "words": ["train", "you", "and", "here", "then", "a"]}, {"sent": "I started directing the train so can you see the controller ?", "words": ["the", "can", "train", "so", "you", "see"]}, {"sent": "that was the that train .", "words": ["the", "was", "that", "train"]}, {"sent": "its a bit of a collapsing train this .", "words": ["of", "this", "train", "a"]}, {"sent": "well I hope he xxx train .", "words": ["he", "train"]}, {"sent": "oh oh no a train .", "words": ["train", "a"]}, {"sent": "train carriages Thomas please ?", "words": ["train"]}, {"sent": "shall we have her waiting here for the train ?", "words": ["the", "for", "we", "train", "have", "here", "her"]}, {"sent": "and the trucks going down to the train .", "words": ["the", "train", "to", "and", "down"]}, {"sent": "give the tiger some bones or hell eat the train driver .", "words": ["the", "train", "eat", "tiger", "some", "give"]}, {"sent": "oh the girls on the train ?", "words": ["the", "on", "train"]}, {"sent": "wheres the choochoo train going ?", "words": ["the", "train"]}, {"sent": "or is it just going to be a train ?", "words": ["is", "train", "to", "be", "it", "a"]}, {"sent": "theres a little man driving the train .", "words": ["the", "train", "little", "a"]}, {"sent": "whos going on the train ?", "words": ["on", "the", "train"]}, {"sent": "like to be a York train and go to York .", "words": ["train", "go", "to", "be", "like", "and", "a"]}, {"sent": "oh it looks like its about a train .", "words": ["train", "like", "it", "about", "a"]}, {"sent": "the mail train .", "words": ["the", "train"]}, {"sent": "whats happened to the train station ?", "words": ["the", "to", "train"]}, {"sent": "this persons going to wait for the train .", "words": ["the", "wait", "for", "train", "to", "this"]}, {"sent": "a quiet train .", "words": ["train", "quiet", "a"]}, {"sent": "the train went overhead .", "words": ["the", "train"]}, {"sent": "I think its a train isnt it ?", "words": ["it", "think", "train", "a"]}, {"sent": "okay right are we going to carry on with this train track ?", "words": ["we", "train", "to", "with", "carry", "this", "are", "on"]}, {"sent": "I think the train might be in the kitchen .", "words": ["the", "think", "train", "be", "in", "kitchen"]}, {"sent": "oh perhaps we could put the tiger on the circus train couldnt we ?", "words": ["the", "put", "we", "train", "tiger", "on", "could"]}, {"sent": "was it a train ?", "words": ["train", "was", "it", "a"]}, {"sent": "theyve been on the train uhn havent they ?", "words": ["on", "the", "they", "train"]}, {"sent": "and the train was going too fast ?", "words": ["the", "fast", "train", "was", "and", "too"]}, {"sent": "and thats a train .", "words": ["train", "a", "and"]}, {"sent": "ooh Im waiting for the train .", "words": ["the", "for", "train"]}, {"sent": "chasing the lady train yes .", "words": ["the", "train"]}, {"sent": "who is driving the train ?", "words": ["the", "train", "is", "who"]}, {"sent": "oh the hippopotamuses are walking up the stairs .", "words": ["the", "stairs", "are", "up"]}, {"sent": "do you play on nanas stairs at her house ?", "words": ["do", "play", "you", "house", "at", "on", "stairs", "her"]}, {"sent": "you can up the stairs .", "words": ["can", "the", "up", "you", "stairs"]}, {"sent": "maybe we should make some stairs that go up to the treehouse what do you think ?", "words": ["up", "the", "what", "do", "think", "we", "go", "to", "you", "make", "some", "that", "stairs"]}, {"sent": "okay can we put you gonna start up the stairs ?", "words": ["can", "up", "put", "the", "we", "you", "stairs"]}, {"sent": "so up the stairs she went .", "words": ["up", "the", "so", "stairs", "she"]}, {"sent": "I think its on the stairs .", "words": ["on", "the", "think", "stairs"]}, {"sent": "you have a cough she said and she dashed down the stairs and up again with .", "words": ["the", "up", "with", "and", "have", "you", "stairs", "down", "she", "a"]}, {"sent": "its somebody on the stairs watching you .", "words": ["on", "the", "stairs", "you"]}, {"sent": "you wanna go up the stairs and under the bridge and down again ?", "words": ["the", "up", "go", "you", "and", "under", "wanna", "stairs", "down"]}, {"sent": "its Purdie on the stairs .", "words": ["on", "the", "stairs"]}, {"sent": "are you doing stairs to climb up ?", "words": ["up", "to", "you", "are", "stairs", "climb"]}, {"sent": "please come down the stairs .", "words": ["the", "stairs", "down"]}, {"sent": "theyre running down the stairs .", "words": ["the", "stairs", "down"]}, {"sent": "and Mummy was standing in the kitchen talking to Grandma on the telephone and suddenly heard you coming down the stairs .", "words": ["the", "was", "to", "and", "you", "in", "kitchen", "on", "stairs", "down", "telephone"]}, {"sent": "shes talking about when you came up the stairs was it last time or two times ago ?", "words": ["up", "the", "was", "last", "you", "it", "about", "stairs", "when"]}, {"sent": "hes going up the stairs .", "words": ["up", "the", "stairs"]}, {"sent": "what are you doing up those stairs ?", "words": ["up", "what", "those", "you", "are", "stairs"]}, {"sent": "you fell down the stairs the other day didnt you because youd got two legs in one leg hole .", "words": ["the", "other", "you", "in", "leg", "stairs", "down", "because"]}, {"sent": "just sitting on the stairs looking down .", "words": ["on", "the", "down", "stairs"]}, {"sent": "oh look whos coming down the stairs .", "words": ["the", "stairs", "down", "look"]}, {"sent": "up the stairs oh aye up the stairs its good up there .", "words": ["up", "the", "there", "good", "stairs"]}, {"sent": "I can climb stairs .", "words": ["can", "stairs", "climb"]}, {"sent": "we go down the stairs ?", "words": ["the", "go", "we", "stairs", "down"]}, {"sent": "climb up the stairs .", "words": ["up", "the", "climb", "stairs"]}, {"sent": "stairs ?", "words": ["stairs"]}, {"sent": "chairs too big for the stairs .", "words": ["the", "for", "too", "stairs", "big"]}, {"sent": "stairs .", "words": ["stairs"]}, {"sent": "is he going up the stairs ?", "words": ["up", "the", "is", "he", "stairs"]}, {"sent": "at the flight of stairs .", "words": ["the", "of", "stairs", "at"]}, {"sent": "why dont you eat the chocolate biscuit thats on the stairs if you want something to eat ?", "words": ["the", "if", "to", "chocolate", "you", "eat", "dont", "on", "stairs", "why"]}, {"sent": "she wont go shell stand at the top of the stairs Mommy never tries to go down them .", "words": ["the", "go", "of", "stand", "to", "them", "at", "stairs", "down", "she"]}, {"sent": "why are you bumping down the stairs ?", "words": ["the", "you", "are", "stairs", "down", "why"]}, {"sent": "oh you wanna make stairs okay .", "words": ["stairs", "make", "wanna", "you"]}, {"sent": "youve put all this fruit all this plastic fruit under the stairs havent you ?", "words": ["the", "all", "put", "you", "under", "this", "stairs"]}, {"sent": "do you think the baby would get hurt if she fell down the stairs ?", "words": ["the", "do", "think", "if", "hurt", "you", "get", "would", "stairs", "down", "she"]}, {"sent": "youre posting them through the stairs ?", "words": ["the", "stairs", "them"]}, {"sent": "see this is this is the post office stairs .", "words": ["the", "is", "this", "see", "stairs"]}, {"sent": "whats this under the stairs ?", "words": ["the", "this", "under", "stairs"]}, {"sent": "you remember the stairs at their house ?", "words": ["the", "their", "you", "house", "at", "stairs"]}, {"sent": "pottie down that stairs .", "words": ["stairs", "down", "that"]}, {"sent": "more stairs .", "words": ["stairs", "more"]}, {"sent": "and then they walk up the stairs again .", "words": ["up", "the", "and", "they", "stairs", "then", "walk"]}, {"sent": "whos coming down the stairs .", "words": ["the", "stairs", "down"]}, {"sent": "move your toys so Mummy can get down the stairs .", "words": ["can", "the", "your", "so", "get", "stairs", "down"]}, {"sent": "well I think she might generalize the idea of the bricks on top of one another to going up the stairs .", "words": ["the", "up", "think", "of", "to", "another", "on", "stairs", "she"]}, {"sent": "did they have stairs ?", "words": ["have", "they", "did", "stairs"]}, {"sent": "and then he rushed up the stairs to get him to the .", "words": ["up", "the", "to", "and", "get", "him", "he", "stairs", "then"]}, {"sent": "youre gonna go down stairs and get Meike .", "words": ["go", "and", "get", "stairs", "down"]}, {"sent": "see theres no stairs here .", "words": ["here", "see", "stairs"]}, {"sent": "they cant go up the stairs backwards .", "words": ["up", "the", "go", "they", "stairs"]}, {"sent": "the puppys walking up the stairs right .", "words": ["the", "stairs", "up"]}, {"sent": "yeah the stairs .", "words": ["the", "stairs"]}, {"sent": "shes half way up the stairs .", "words": ["up", "the", "stairs"]}, {"sent": "be careful going down the stairs .", "words": ["the", "be", "careful", "stairs", "down"]}, {"sent": "go up the stairs and knock on mummys door .", "words": ["up", "the", "go", "and", "knock", "on", "stairs", "door"]}, {"sent": "hm they run down the stairs .", "words": ["the", "they", "run", "stairs", "down"]}, {"sent": "get your best smile out from under the stairs .", "words": ["the", "your", "get", "under", "out", "stairs", "smile"]}, {"sent": "feet down the stairs .", "words": ["the", "stairs", "down"]}, {"sent": "aw theres no stairs ?", "words": ["stairs"]}, {"sent": "oh do not turn around when were going down the stairs .", "words": ["the", "do", "around", "stairs", "down", "when", "not", "were"]}, {"sent": "but my stairs are very are really very steep .", "words": ["stairs", "my", "are", "but"]}, {"sent": "wanna go down the stairs ?", "words": ["the", "go", "wanna", "stairs", "down"]}, {"sent": "no dont kick it down the stairs please .", "words": ["the", "dont", "kick", "it", "stairs", "down"]}, {"sent": "on the stairs .", "words": ["on", "the", "stairs"]}, {"sent": "climb up the stairs .", "words": ["up", "the", "climb", "stairs"]}, {"sent": "does she count up the stairs ?", "words": ["up", "does", "the", "stairs", "she"]}, {"sent": "eight stairs .", "words": ["stairs"]}, {"sent": "whos gonna use those stairs do you think ?", "words": ["do", "think", "those", "you", "stairs"]}, {"sent": "whats Grandma got on her stairs ?", "words": ["on", "stairs", "her"]}, {"sent": "is he gonna be careful when he climbs up the stairs ?", "words": ["up", "the", "is", "be", "he", "careful", "stairs", "when"]}, {"sent": "we do one two three four five up the stairs .", "words": ["up", "do", "the", "we", "stairs"]}, {"sent": "stairs who do you have a picnic with ?", "words": ["do", "with", "you", "have", "who", "stairs", "a"]}, {"sent": "jump the stairs .", "words": ["the", "jump", "stairs"]}, {"sent": "pretending stairs ?", "words": ["stairs"]}, {"sent": "I know well the stairs go round in a spiral big wooden stairs .", "words": ["the", "go", "in", "stairs", "big", "a"]}, {"sent": "why are they going up the stairs ?", "words": ["up", "the", "they", "are", "stairs", "why"]}, {"sent": "Missus Bear and Sam held hands on the stairs .", "words": ["on", "the", "stairs", "and"]}, {"sent": "can I show you one other thing to do with those stairs ?", "words": ["can", "do", "those", "show", "to", "with", "other", "you", "stairs"]}, {"sent": "is the stairs broken ?", "words": ["the", "stairs", "broken", "is"]}, {"sent": "okay stickers and well put these games down stairs .", "words": ["these", "put", "and", "stairs", "down"]}, {"sent": "as I said lifted up the stairs .", "words": ["up", "the", "stairs"]}, {"sent": "will you put those peas on the stairs as well ?", "words": ["the", "put", "those", "peas", "you", "on", "will", "stairs"]}, {"sent": "the boy fell down stairs the .", "words": ["the", "stairs", "down"]}, {"sent": "lets put that on the stairs as well .", "words": ["the", "put", "on", "stairs", "that"]}, {"sent": "going up the stairs yes she might do .", "words": ["up", "the", "do", "stairs", "she"]}, {"sent": "xxx stairs .", "words": ["stairs"]}, {"sent": "make him go up the stairs .", "words": ["up", "the", "go", "him", "stairs", "make"]}, {"sent": "oh youre Daddys down stairs .", "words": ["stairs", "down"]}, {"sent": "its in the hallway under the stairs isnt it ?", "words": ["the", "under", "in", "it", "stairs"]}, {"sent": "you said the man wasnt down stairs .", "words": ["the", "stairs", "down", "you"]}, {"sent": "and he comes down the stairs again .", "words": ["the", "and", "he", "stairs", "down"]}, {"sent": "the gate whatever it was the top of our stairs must have been a different size xxx .", "words": ["the", "of", "was", "our", "have", "it", "stairs", "a"]}, {"sent": "oh theyre good stairs arent they ?", "words": ["stairs", "they", "good"]}, {"sent": "ups the stairs and in the bathtub .", "words": ["the", "and", "in", "bathtub", "stairs"]}, {"sent": "mind the stairs .", "words": ["the", "stairs"]}, {"sent": "xxx her paws pattering on the stairs .", "words": ["on", "the", "her", "stairs"]}, {"sent": "dont do it by the stairs .", "words": ["the", "do", "by", "dont", "it", "stairs"]}, {"sent": "wait are you calling that I call that yellow .", "words": ["wait", "you", "are", "yellow", "that"]}, {"sent": "wait .", "words": ["wait"]}, {"sent": "youve got to wait for Santas elfs to come havent you ?", "words": ["to", "wait", "for", "you"]}, {"sent": "okay wait one second .", "words": ["wait"]}, {"sent": "wait can we can we invite can we invite baby bear as well ?", "words": ["can", "wait", "bear", "we"]}, {"sent": "well shes going to have a long wait Thomas .", "words": ["wait", "to", "long", "have", "a"]}, {"sent": "wait a minute .", "words": ["wait", "a"]}, {"sent": "right you wait here .", "words": ["here", "wait", "you"]}, {"sent": "wait wait wait .", "words": ["wait"]}, {"sent": "oh no wait a minute .", "words": ["wait", "a"]}, {"sent": "maybe she can wait .", "words": ["can", "wait", "she"]}, {"sent": "oh no Ill uh Ill um wait until we see .", "words": ["see", "wait", "we"]}, {"sent": "wait a minute .", "words": ["wait", "a"]}, {"sent": "wait .", "words": ["wait"]}, {"sent": "wait .", "words": ["wait"]}, {"sent": "okay wait for a green light ?", "words": ["wait", "for", "light", "green", "a"]}, {"sent": "hey wait wait wait wait wait wait .", "words": ["wait"]}, {"sent": "well wait for you .", "words": ["wait", "for", "you"]}, {"sent": "come on cakes whispered Sam I cant wait I cant wait I cant wait .", "words": ["on", "wait"]}, {"sent": "you wait here with Andy then .", "words": ["wait", "with", "you", "here", "then"]}, {"sent": "fantastic xxx oh wait a second we got a we got to xxx .", "words": ["to", "wait", "we", "a"]}, {"sent": "wait wait wait wait wait .", "words": ["wait"]}, {"sent": "wait which way you gonna do him ?", "words": ["do", "wait", "you", "him", "which"]}, {"sent": "wait wait wait .", "words": ["wait"]}, {"sent": "sum wait a minute sum come on .", "words": ["on", "wait", "a"]}, {"sent": "wait a minute .", "words": ["wait", "a"]}, {"sent": "look wait until you see .", "words": ["see", "wait", "you", "look"]}, {"sent": "sure well wait for you .", "words": ["wait", "for", "you"]}, {"sent": "wait a minute wait a minute wait a minute .", "words": ["wait", "a"]}, {"sent": "okay Ill wait xxx .", "words": ["wait"]}, {"sent": "oh wait is that a turtle thats a turtle right there .", "words": ["wait", "is", "there", "turtle", "that", "a"]}, {"sent": "wait until I I finish .", "words": ["wait", "finish"]}, {"sent": "now wait a minute second .", "words": ["wait", "a"]}, {"sent": "wait .", "words": ["wait"]}, {"sent": "wait till we get down there .", "words": ["wait", "we", "get", "there", "down"]}, {"sent": "wait wait wait wait wait wait watch .", "words": ["wait", "watch"]}, {"sent": "wait a minute mike .", "words": ["wait", "a"]}, {"sent": "youre gonna hafta wait until she comes back and then youll hafta tell her huh ?", "words": ["wait", "back", "and", "then", "her", "she"]}, {"sent": "wait Adam .", "words": ["wait"]}, {"sent": "wait a minute .", "words": ["wait", "a"]}, {"sent": "wait wait wait wait wait wait wait wait wait wait wait wait wait wait wait lemme do it .", "words": ["do", "wait", "it"]}, {"sent": "youll hafta wait until next week .", "words": ["wait"]}, {"sent": "wait wait .", "words": ["wait"]}, {"sent": "wait where where are you gonna go hunting ?", "words": ["where", "wait", "go", "you", "are"]}, {"sent": "wait wait watch Meikes face .", "words": ["wait", "watch", "face"]}, {"sent": "wait a minute I think its this one lemme see .", "words": ["wait", "think", "this", "see", "a"]}, {"sent": "wait lemme lemme see move back .", "words": ["see", "wait", "back"]}, {"sent": "wait a second .", "words": ["wait", "a"]}, {"sent": "so well wait to clean you up thats fine .", "words": ["clean", "up", "wait", "fine", "to", "so", "you"]}, {"sent": "*TO2: yes Elmo cant wait to xxx .", "words": ["to", "wait"]}, {"sent": "wait a minute .", "words": ["wait", "a"]}, {"sent": "wait wait wait wait wait for me .", "words": ["me", "wait", "for"]}, {"sent": "wait a second .", "words": ["wait", "a"]}, {"sent": "wait a second .", "words": ["wait", "a"]}, {"sent": "wait a minute let Granny do you know where the um dustpan and brush is ?", "words": ["where", "do", "wait", "the", "is", "brush", "you", "and", "a"]}, {"sent": "wait .", "words": ["wait"]}, {"sent": "wait a minute .", "words": ["wait", "a"]}, {"sent": "here here wait wait here .", "words": ["here", "wait"]}, {"sent": "oh wait a minute .", "words": ["wait", "a"]}, {"sent": "you hafta wait a long time .", "words": ["long", "wait", "a", "you"]}, {"sent": "wait let us try this hand .", "words": ["wait", "hand", "us", "this", "try"]}, {"sent": "Bea wait wait wait wait wait .", "words": ["wait"]}, {"sent": "we hafta wait .", "words": ["wait", "we"]}, {"sent": "wait .", "words": ["wait"]}, {"sent": "wait a minute .", "words": ["wait", "a"]}, {"sent": "wait a minute .", "words": ["wait", "a"]}, {"sent": "hafta wait because its too hot .", "words": ["hot", "wait", "because", "too"]}, {"sent": "can you wait till we get back to Pittsburgh ?", "words": ["can", "wait", "back", "we", "to", "you", "get"]}, {"sent": "you hafta wait for Mummy dont you ?", "words": ["dont", "wait", "for", "you"]}, {"sent": "wait .", "words": ["wait"]}, {"sent": "wait first you gotta go to the .", "words": ["the", "wait", "go", "to", "first", "you"]}, {"sent": "you hafta wait until Ive finished feeding Amy dont you sweetheart ?", "words": ["dont", "wait", "you"]}, {"sent": "wait wait wait wait .", "words": ["wait"]}, {"sent": "oh wait a minute wait xxx .", "words": ["wait", "a"]}, {"sent": "wait wait wait let Mey see .", "words": ["see", "wait"]}, {"sent": "wait a minute .", "words": ["wait", "a"]}, {"sent": "wait .", "words": ["wait"]}, {"sent": "wait a minute .", "words": ["wait", "a"]}, {"sent": "well wait a minute theres her legs .", "words": ["wait", "her", "a"]}, {"sent": "wait .", "words": ["wait"]}, {"sent": "wait a second .", "words": ["wait", "a"]}, {"sent": "so it works out well if you can wait that long right ?", "words": ["can", "wait", "if", "so", "long", "you", "it", "out", "that"]}, {"sent": "wait wait .", "words": ["wait"]}, {"sent": "wait wait wait no no no hes not to play baseball with .", "words": ["wait", "to", "play", "with", "not"]}, {"sent": "wait can I just say something ?", "words": ["can", "wait", "say"]}, {"sent": "just wait a minute .", "words": ["wait", "a"]}, {"sent": "okay wait just a minute .", "words": ["wait", "a"]}, {"sent": "wait hold on you dont like honey but its your favorite thing ?", "words": ["wait", "your", "hold", "like", "you", "dont", "on", "but"]}, {"sent": "wait a second .", "words": ["wait", "a"]}, {"sent": "got to wait .", "words": ["to", "wait"]}, {"sent": "wait a minute .", "words": ["wait", "a"]}, {"sent": "will you wait for me ?", "words": ["wait", "for", "you", "will", "me"]}, {"sent": "wait until Roz comes back and you can ask her .", "words": ["can", "wait", "back", "you", "and", "her"]}, {"sent": "maybe we could let you press this little thing so the numbers that would go away wait first before we press it wait before we press it .", "words": ["the", "wait", "we", "go", "so", "first", "little", "you", "this", "away", "it", "would", "could", "that"]}, {"sent": "wait .", "words": ["wait"]}, {"sent": "wait .", "words": ["wait"]}, {"sent": "wait a minute .", "words": ["wait", "a"]}, {"sent": "wait .", "words": ["wait"]}, {"sent": "wait wait for me said a little voice ?", "words": ["wait", "for", "little", "me", "a"]}, {"sent": "wait .", "words": ["wait"]}, {"sent": "I wont Ill just eat a couple .", "words": ["eat", "a"]}, {"sent": "eat eggs how many eggs would you like ?", "words": ["how", "like", "you", "eat", "would"]}, {"sent": "now I think that youre gonna need to eat something before your swim class so .", "words": ["think", "need", "your", "need to", "to", "so", "swim", "eat", "that"]}, {"sent": "eat your turkeyburger .", "words": ["eat", "your"]}, {"sent": "you dont eat the bird all away .", "words": ["the", "all", "bird", "you", "eat", "dont", "away"]}, {"sent": "Im gonna eat the waffle .", "words": ["eat", "the"]}, {"sent": "let her in xxx to eat ?", "words": ["eat", "to", "in", "her"]}, {"sent": "no you have to stay here and eat your lunch .", "words": ["your", "to", "you", "have", "here", "and", "eat", "stay"]}, {"sent": "doesnt really well she was interested in the spaghetti for some real she doesnt eat any finger foods at all she only has purees so .", "words": ["the", "all", "for", "spaghetti", "was", "so", "any", "eat", "finger", "in", "at", "some", "she"]}, {"sent": "I wanna eat .", "words": ["eat", "wanna"]}, {"sent": "if he dunna want it Ill eat it .", "words": ["eat", "it", "he", "if"]}, {"sent": "oh I didnt eat it over here .", "words": ["eat", "it", "over", "here"]}, {"sent": "what dyou what dyou wanna eat in the cafe ?", "words": ["the", "what", "eat", "in", "wanna"]}, {"sent": "so we just we just call it an apple and shell eat it .", "words": ["we", "so", "and", "eat", "an", "it", "apple"]}, {"sent": "he did not eat the cake from Olga .", "words": ["the", "cake", "eat", "he", "did", "not"]}, {"sent": "please dont eat it .", "words": ["eat", "dont", "it"]}, {"sent": "you should stay at the table and eat Keisha .", "words": ["the", "table", "you", "and", "eat", "at", "stay"]}, {"sent": "big bears eat .", "words": ["eat", "big"]}, {"sent": "did we eat all our raisins ?", "words": ["all", "we", "our", "eat", "did"]}, {"sent": "is he going to eat something ?", "words": ["eat", "to", "he", "is"]}, {"sent": "dont eat the crayons .", "words": ["eat", "dont", "the"]}, {"sent": "what textures do you like to eat ?", "words": ["what", "do", "to", "like", "you", "eat"]}, {"sent": "something to eat .", "words": ["eat", "to"]}, {"sent": "and cows eat grass .", "words": ["eat", "grass", "and"]}, {"sent": "would he eat you ?", "words": ["eat", "he", "you", "would"]}, {"sent": "you wanna eat lunch ?", "words": ["eat", "wanna", "you"]}, {"sent": "if that were a real waffle I wouldnt wanna eat it .", "words": ["if", "eat", "it", "wanna", "that", "were", "a"]}, {"sent": "but at teatime youve got to eat a proper meal .", "words": ["to", "eat", "at", "but", "a"]}, {"sent": "right are you going to sit up there and eat up ?", "words": ["up", "to", "you", "and", "there", "eat", "are", "sit"]}, {"sent": "hell hell eat your hand later when hes xxx .", "words": ["eat", "your", "when", "hand"]}, {"sent": "who has got nothing to eat ?", "words": ["eat", "to", "who"]}, {"sent": "Ill eat yours .", "words": ["eat"]}, {"sent": "she al she also doesnt eat any sweets .", "words": ["eat", "any", "she"]}, {"sent": "are they more delicious when you eat them .", "words": ["them", "you", "eat", "they", "more", "are", "when"]}, {"sent": "and these people eat see ?", "words": ["eat", "these", "see", "and"]}, {"sent": "yeah eat it up .", "words": ["eat", "it", "up"]}, {"sent": "eat it or leave it .", "words": ["eat", "it"]}, {"sent": "share half with me and well both eat it all up .", "words": ["up", "all", "share", "with", "and", "eat", "it", "me"]}, {"sent": "I I dont wanna eat any of my dessert xxx .", "words": ["my", "of", "any", "eat", "dont", "wanna"]}, {"sent": "I dont wanna eat it .", "words": ["eat", "dont", "it", "wanna"]}, {"sent": "no youre gonna eat that too .", "words": ["eat", "that", "too"]}, {"sent": "and did you eat all of your food ?", "words": ["all", "your", "of", "you", "eat", "and", "food", "did"]}, {"sent": "we dont eat dinner until five oclock xxx .", "words": ["eat", "dont", "we"]}, {"sent": "help him eat them .", "words": ["eat", "help", "them", "him"]}, {"sent": "yeah you eat all the food up .", "words": ["the", "all", "up", "you", "eat", "food"]}, {"sent": "okay you can use that fork to eat your cake .", "words": ["can", "your", "cake", "to", "you", "eat", "fork", "that"]}, {"sent": "cant eat them .", "words": ["eat", "them"]}, {"sent": "are you gonna eat are you gonna eat it for supper .", "words": ["for", "you", "eat", "it", "are"]}, {"sent": "where did we eat it that day ?", "words": ["where", "we", "eat", "it", "did", "that"]}, {"sent": "now eat your breakfast everybody .", "words": ["eat", "your"]}, {"sent": "thats okay well eat them .", "words": ["eat", "them"]}, {"sent": "ha wanna eat now or go eat in the car ?", "words": ["the", "go", "car", "eat", "in", "wanna"]}, {"sent": "because you didnt eat much at dinner .", "words": ["you", "eat", "at", "much", "because"]}, {"sent": "eat .", "words": ["eat"]}, {"sent": "I dont want it you eat it .", "words": ["eat", "dont", "it", "you"]}, {"sent": "you gonna eat your sandwich then ?", "words": ["your", "you", "eat", "then", "sandwich"]}, {"sent": "you gonna eat your soup thats in the dish ?", "words": ["the", "your", "you", "eat", "dish", "in", "soup"]}, {"sent": "oh no dont eat the icecream cone .", "words": ["eat", "dont", "the"]}, {"sent": "here eat a cookie .", "words": ["here", "cookie", "eat", "a"]}, {"sent": "I hope theres some tasty monkeys to eat .", "words": ["to", "some", "eat"]}, {"sent": "wouldjou like something else to eat ?", "words": ["eat", "to", "like"]}, {"sent": "whatd you eat ?", "words": ["eat", "you"]}, {"sent": "all the people eat ?", "words": ["the", "all", "eat"]}, {"sent": "are you going to eat something else .", "words": ["eat", "to", "are", "you"]}, {"sent": "eat the peas .", "words": ["eat", "the", "peas"]}, {"sent": "eat it I got to eat him ?", "words": ["eat", "to", "him", "it"]}, {"sent": "look at all these things youre not allowed to eat yet huh .", "words": ["all", "these", "look", "to", "eat", "at", "not"]}, {"sent": "you eat bones ?", "words": ["eat", "you"]}, {"sent": "oh whats baby having to eat then ?", "words": ["eat", "to", "then"]}, {"sent": "oh you eat your Cornflakes .", "words": ["eat", "your", "you"]}, {"sent": "eat that properly .", "words": ["eat", "that"]}, {"sent": "I guess we should all eat these blueberries tomorrow .", "words": ["eat", "all", "these", "we"]}, {"sent": "then Lola saw some peas I dont eat peas said Lola .", "words": ["peas", "eat", "dont", "some", "then"]}, {"sent": "what will we eat with it for dinner ?", "words": ["what", "for", "we", "with", "eat", "it", "will"]}, {"sent": "what would you like to eat ?", "words": ["what", "to", "like", "you", "eat", "would"]}, {"sent": "just eat as well .", "words": ["eat"]}, {"sent": "youve got to learn that when youve finished a meal that you push to one side what you dont wanna eat .", "words": ["what", "push", "when", "to", "you", "eat", "dont", "wanna", "that", "a"]}, {"sent": "you dont get very messy when all you eat is finger food .", "words": ["all", "is", "you", "get", "dont", "eat", "finger", "food", "when"]}, {"sent": "youve got to eat something if youre hungry .", "words": ["eat", "to", "hungry", "if"]}, {"sent": "would you like to eat some apple ?", "words": ["to", "like", "you", "eat", "would", "apple", "some"]}, {"sent": "we dont eat golf balls any more than we eat anything else .", "words": ["we", "any", "eat", "dont", "more"]}, {"sent": "dogs dont normally eat oranges .", "words": ["eat", "dont"]}, {"sent": "about to eat a pancake .", "words": ["pancake", "to", "eat", "about", "a"]}, {"sent": "ah you know I think she would eat she would eat shell eat anything .", "words": ["think", "you", "eat", "would", "she"]}, {"sent": "what else do you like to eat ?", "words": ["what", "do", "to", "like", "you", "eat"]}, {"sent": "you know how to eat it .", "words": ["how", "to", "you", "eat", "it"]}, {"sent": "do we eat peanutbutter right from the jar ?", "words": ["the", "do", "we", "eat", "jar"]}, {"sent": "that look like our pizza that we eat sometimes ?", "words": ["we", "like", "pizza", "our", "eat", "look", "that"]}, {"sent": "eat ?", "words": ["eat"]}, {"sent": "whynt you xxx and eat your cereal like a good girl .", "words": ["your", "like", "and", "eat", "you", "good", "cereal", "a"]}, {"sent": "do you want upstairs and eat ?", "words": ["eat", "do", "and", "you"]}, {"sent": "Thomas eat .", "words": ["eat"]}, {"sent": "youre gonna eat the spoon youre silly .", "words": ["eat", "the", "spoon"]}, {"sent": "what kind of meat do we like to eat ?", "words": ["what", "do", "we", "of", "to", "like", "meat", "eat"]}, {"sent": "is that nice to eat ?", "words": ["is", "to", "nice", "eat", "that"]}, {"sent": "does he eat it eat it out of the bottom of the cone ?", "words": ["the", "does", "bottom", "of", "eat", "it", "he", "out"]}, {"sent": "they eat trees .", "words": ["eat", "they"]}, {"sent": "well eat it but quickly before you drop it .", "words": ["you", "eat", "it", "drop", "but"]}, {"sent": "thats very nice to eat .", "words": ["eat", "to", "nice"]}, {"sent": "okay Ill eat it .", "words": ["eat", "it"]}, {"sent": "big boy .", "words": ["big"]}, {"sent": "is it too big ?", "words": ["too", "it", "big", "is"]}, {"sent": "big whale xxx .", "words": ["big"]}, {"sent": "Ive got big ears ?", "words": ["big"]}, {"sent": "it had a big red star shaped thing on the front and you kept saying it looked like the sun and you were very excited .", "words": ["red", "the", "star", "sun", "like", "and", "you", "it", "on", "big", "were", "a"]}, {"sent": "why not put a big one on because theyre all going to fall down ?", "words": ["all", "put", "to", "because", "fall", "on", "down", "big", "not", "why", "a"]}, {"sent": "there was a big splash .", "words": ["was", "there", "splash", "big", "a"]}, {"sent": "that was a big apple wasnt it ?", "words": ["was", "that", "it", "apple", "big", "a"]}, {"sent": "they probably lived in a big white house xxx .", "words": ["they", "in", "house", "white", "big", "a"]}, {"sent": "way too big .", "words": ["big", "too"]}, {"sent": "is there a big yellow thing in there Fraser ?", "words": ["is", "there", "in", "yellow", "big", "a"]}, {"sent": "its a pretty big sandwich to eat I think .", "words": ["think", "to", "eat", "pretty", "big", "sandwich", "a"]}, {"sent": "hes so big he cant even fit in the house .", "words": ["the", "so", "fit", "in", "he", "house", "big"]}, {"sent": "when you and Isabel were running around the park there was a big tractor wasnt there ?", "words": ["the", "when", "was", "and", "you", "there", "big", "park", "tractor", "around", "were", "a"]}, {"sent": "xxx thats right a big big bridge and theres tracks on it too .", "words": ["and", "it", "too", "on", "big", "a"]}, {"sent": "got a big bruise on my foot Thomas .", "words": ["my", "foot", "on", "big", "a"]}, {"sent": "what about that great big doll out in the hall ?", "words": ["the", "what", "doll", "that", "in", "about", "out", "big"]}, {"sent": "a big cheeky boy .", "words": ["big", "a"]}, {"sent": "that is a a big truck .", "words": ["is", "truck", "that", "big", "a"]}, {"sent": "wait I think we have a big big holly tree here too .", "words": ["wait", "think", "tree", "we", "have", "here", "too", "big", "a"]}, {"sent": "thats a big one isnt it .", "words": ["it", "big", "a"]}, {"sent": "you got the big ball ?", "words": ["the", "ball", "big", "you"]}, {"sent": "too big boot .", "words": ["big", "too"]}, {"sent": "NinNin big splash .", "words": ["splash", "big"]}, {"sent": "you can sit on the big chair when youre playing with things on the table but .", "words": ["can", "the", "when", "table", "with", "chair", "you", "on", "sit", "big", "but"]}, {"sent": "a big circle .", "words": ["big", "a"]}, {"sent": "the mummies are going in this big field arent they ?", "words": ["the", "this", "in", "are", "they", "big"]}, {"sent": "it was so big right Matty ?", "words": ["was", "big", "it", "so"]}, {"sent": "two boys two big boys .", "words": ["big"]}, {"sent": "and the next to the wedding was the big fat cow .", "words": ["the", "to", "was", "and", "big", "cow"]}, {"sent": "with the big spike on his back ?", "words": ["the", "back", "with", "on", "his", "big"]}, {"sent": "well youre going to the big school but on sunday Im going to put Christopher in the nursery .", "words": ["the", "put", "to", "school", "in", "on", "big", "but"]}, {"sent": "are you a big brother or little brother ?", "words": ["little", "you", "are", "big", "a"]}, {"sent": "and big .", "words": ["big", "and"]}, {"sent": "has he got a big red nose as well ?", "words": ["red", "he", "big", "a", "nose"]}, {"sent": "are you only small or are you big ?", "words": ["are", "big", "you"]}, {"sent": "thats MrHappy because hes got a big smiley face .", "words": ["big", "face", "because", "a"]}, {"sent": "I had a big pen and Ive lost it .", "words": ["and", "it", "pen", "big", "a"]}, {"sent": "if you hold that that big ball of blue tack .", "words": ["hold", "if", "of", "you", "ball", "blue", "that", "big"]}, {"sent": "did I take big bite ?", "words": ["bite", "big", "take", "did"]}, {"sent": "has Dada got a little nose or a big nose ?", "words": ["nose", "big", "little", "a"]}, {"sent": "big long one .", "words": ["big", "long"]}, {"sent": "a big hurd .", "words": ["big", "a"]}, {"sent": "big blue buttons .", "words": ["big", "blue"]}, {"sent": "big boy .", "words": ["big"]}, {"sent": "youre okay big girl .", "words": ["big"]}, {"sent": "we hafta go in the big plane to go and see her dont we ?", "words": ["the", "go", "we", "to", "her", "and", "dont", "in", "see", "big"]}, {"sent": "how big was it ?", "words": ["was", "how", "it", "big"]}, {"sent": "just one big track .", "words": ["big"]}, {"sent": "shall we turn the torch off and put the big light back on ?", "words": ["the", "off", "put", "back", "we", "light", "and", "on", "big"]}, {"sent": "lookit that big pumpkin .", "words": ["pumpkin", "big", "that"]}, {"sent": "thats a big clam shell .", "words": ["big", "a"]}, {"sent": "heres a big dinosaur coming .", "words": ["big", "a"]}, {"sent": "but she has big ears .", "words": ["big", "she", "but"]}, {"sent": "a big red wood box .", "words": ["red", "box", "big", "a"]}, {"sent": "its not that big .", "words": ["big", "that", "not"]}, {"sent": "this works with the big kids .", "words": ["the", "this", "big", "with"]}, {"sent": "want that big green one ?", "words": ["big", "that", "green"]}, {"sent": "yeah the big horse maybe the big horse can sleep in the bathtub .", "words": ["the", "can", "sleep", "in", "bathtub", "horse", "big"]}, {"sent": "xxx on her face just like yours is a big pretty face .", "words": ["face", "is", "like", "big", "pretty", "on", "her", "a"]}, {"sent": "now you watch this big man .", "words": ["this", "watch", "big", "you"]}, {"sent": "shes a bit big isnt she ?", "words": ["big", "she", "a"]}, {"sent": "you are a big boy .", "words": ["are", "big", "a", "you"]}, {"sent": "that is big enough that he can chew on it and it doesnt really and Im not too worried about him being able to chew through it .", "words": ["can", "is", "to", "and", "that", "it", "he", "him", "about", "too", "on", "big", "not"]}, {"sent": "some day Tuesday theyre going to fix the great big witch right ?", "words": ["the", "to", "fix", "some", "big"]}, {"sent": "hes a big caveman .", "words": ["big", "a"]}, {"sent": "um and then like s even stuffed animals shes not so big with .", "words": ["so", "with", "like", "and", "then", "big", "not"]}, {"sent": "too big .", "words": ["big", "too"]}, {"sent": "a big truck coming ?", "words": ["truck", "big", "a"]}, {"sent": "that was a bit of a big sneeze wasnt it ?", "words": ["of", "was", "that", "it", "big", "a"]}, {"sent": "big storks .", "words": ["big"]}, {"sent": "thats a big hill isnt it ?", "words": ["it", "big", "a"]}, {"sent": "a big tunnel .", "words": ["big", "a"]}, {"sent": "is that a big car ?", "words": ["is", "car", "that", "big", "a"]}, {"sent": "big big big huge circle .", "words": ["big"]}, {"sent": "oh big kiss for the dog .", "words": ["the", "for", "big", "kiss", "dog"]}, {"sent": "is that a big horse or a little horse ?", "words": ["is", "little", "that", "horse", "big", "a"]}, {"sent": "lets make the man sit in this big block .", "words": ["the", "block", "this", "in", "big", "sit", "make"]}, {"sent": "a big jigsaw puzzle .", "words": ["puzzle", "big", "a"]}, {"sent": "it looks like a circle and a crayon and a big g .", "words": ["crayon", "like", "and", "it", "big", "a"]}, {"sent": "that is a big farm .", "words": ["big", "that", "a", "is"]}, {"sent": "is it a big bicycle or a little bicycle ?", "words": ["is", "bicycle", "little", "it", "big", "a"]}, {"sent": "five pound note and a big ones a .", "words": ["big", "a", "and"]}, {"sent": "you can give him a big hug cant you ?", "words": ["can", "you", "him", "hug", "give", "big", "a"]}, {"sent": "the big brown bear .", "words": ["the", "bear", "big", "brown"]}, {"sent": "it was a snake it was seriously how big would you say that snake was ?", "words": ["how", "was", "say", "you", "that", "it", "would", "big", "a"]}, {"sent": "big white van .", "words": ["white", "big"]}, {"sent": "because dont forget were making a big big track .", "words": ["because", "dont", "big", "were", "a"]}, {"sent": "I think shes a little big for that car isnt she ?", "words": ["think", "for", "car", "little", "that", "big", "she", "a"]}, {"sent": "are you playing with your big red bus ?", "words": ["red", "your", "with", "you", "are", "big", "bus"]}, {"sent": "thats my big girl .", "words": ["my", "big"]}, {"sent": "youre gonna make a big circle ?", "words": ["big", "make", "a"]}, {"sent": "youll be in big trouble .", "words": ["be", "in", "big"]}, {"sent": "oh theres a big bear there .", "words": ["bear", "there", "big", "a"]}, {"sent": "but when she was a lot younger than she is now Mummy lived in a very very big house with very big ceilings .", "words": ["is", "a lot", "was", "with", "in", "big", "house", "when", "she", "but", "a"]}, {"sent": "youre a big girl .", "words": ["big", "a"]}, {"sent": "elephant has a big trunk .", "words": ["big", "a", "elephant"]}, {"sent": "today must be a big day for ya rush around .", "words": ["for", "be", "around", "big", "a"]}, {"sent": "very big cat isnt it ?", "words": ["cat", "it", "big"]}, {"sent": "the big tradewinds came back .", "words": ["the", "big", "back"]}, {"sent": "somebodys got her ear in there .", "words": ["there", "in", "her", "ear"]}, {"sent": "this ear goes razzledazzle .", "words": ["this", "ear"]}, {"sent": "hes whispering in your ear .", "words": ["in", "your", "ear"]}, {"sent": "xxx just give your ear a quick clean .", "words": ["clean", "your", "ear", "give", "a"]}, {"sent": "where your ear at ?", "words": ["where", "at", "your", "ear"]}, {"sent": "I was listening to music at a concert and I walked to close to the speakers and then my ear really staring hurting after that and it hurt all the way home .", "words": ["the", "all", "my", "hurt", "to", "was", "then", "and", "that", "it", "at", "home", "close", "ear", "a"]}, {"sent": "pick up his ear on the floor .", "words": ["up", "the", "pick", "on", "ear", "his"]}, {"sent": "ear .", "words": ["ear"]}, {"sent": "does it go on an ear ?", "words": ["does", "go", "an", "it", "ear", "on"]}, {"sent": "xxx youre putting it in your ear oh oh we have to blow your nose pumpkin .", "words": ["your", "we", "to", "have", "in", "it", "ear", "pumpkin", "blow", "nose"]}, {"sent": "heres your ear .", "words": ["your", "ear"]}, {"sent": "shes pulling his ear .", "words": ["his", "ear"]}, {"sent": "is it this boo this ear or this ear ?", "words": ["this", "it", "ear", "is"]}, {"sent": "wheres your ear Ellisy ?", "words": ["your", "ear"]}, {"sent": "and his other ear ?", "words": ["his", "other", "ear", "and"]}, {"sent": "its your ear .", "words": ["your", "ear"]}, {"sent": "thats her ear .", "words": ["her", "ear"]}, {"sent": "ear .", "words": ["ear"]}, {"sent": "youve got dirt in your ear ?", "words": ["in", "your", "ear"]}, {"sent": "now wheres the dogs ear ?", "words": ["the", "ear"]}, {"sent": "you shouldnt scream in my ear like that .", "words": ["my", "like", "you", "in", "ear", "that"]}, {"sent": "can you say ear ?", "words": ["can", "ear", "say", "you"]}, {"sent": "dont go and belch in my ear Neil .", "words": ["my", "go", "and", "dont", "in", "ear"]}, {"sent": "heres the other ear .", "words": ["the", "ear", "other"]}, {"sent": "I see your ear .", "words": ["see", "your", "ear"]}, {"sent": "thats ear phones like daddys stereo phones and you put ear phones on and you listen .", "words": ["put", "listen", "like", "you", "and", "ear", "on"]}, {"sent": "Im only asking because your ear looks so red .", "words": ["red", "your", "so", "ear", "because"]}, {"sent": "two ear booboos and a lantern in the belly .", "words": ["the", "and", "in", "ear", "a"]}, {"sent": "Im just taking it out of my ear .", "words": ["my", "of", "it", "out", "ear"]}, {"sent": "that no fair whispering in mommys ear .", "words": ["in", "that", "ear"]}, {"sent": "and then you may have another ear .", "words": ["you", "have", "another", "and", "ear", "then"]}, {"sent": "yeah you can put the hat over the left or the right ear .", "words": ["can", "the", "put", "you", "ear", "hat", "over"]}, {"sent": "hes gonna get your ear .", "words": ["get", "your", "ear"]}, {"sent": "not in your ear .", "words": ["in", "your", "not", "ear"]}, {"sent": "has he got an ear ?", "words": ["an", "he", "ear"]}, {"sent": "why are you putting it in your ear Thomas ?", "words": ["your", "you", "in", "it", "are", "ear", "why"]}, {"sent": "this is for the ear .", "words": ["the", "for", "is", "this", "ear"]}, {"sent": "wheres froggies ear ?", "words": ["ear"]}, {"sent": "are you sure thats your left ear ?", "words": ["ear", "are", "your", "you"]}, {"sent": "ear after ear after ear after ear after ear a tear tear .", "words": ["tear", "a", "ear"]}, {"sent": "your nose or your ear thats your ear sweetie .", "words": ["ear", "your", "nose"]}, {"sent": "hes rubbing his ear .", "words": ["his", "ear"]}, {"sent": "ear .", "words": ["ear"]}, {"sent": "pull the other ear out .", "words": ["the", "other", "pull", "out", "ear"]}, {"sent": "want me leave your ear alone ?", "words": ["me", "your", "ear"]}, {"sent": "no were not gonna color your ear .", "words": ["your", "were", "not", "ear"]}, {"sent": "are you gonna whisper in his ear ?", "words": ["you", "in", "are", "ear", "his"]}, {"sent": "Im gonna get your ear .", "words": ["get", "your", "ear"]}, {"sent": "not in your ear .", "words": ["in", "your", "not", "ear"]}, {"sent": "if I give you this big ear of corn youll play with it .", "words": ["if", "of", "play", "with", "you", "this", "it", "corn", "ear", "give", "big"]}, {"sent": "two ear .", "words": ["ear"]}, {"sent": "oh to look inside your ear ?", "words": ["look", "your", "to", "inside", "ear"]}, {"sent": "she whispered in your ear ?", "words": ["in", "your", "she", "ear"]}, {"sent": "pull your ear ?", "words": ["ear", "your", "pull"]}, {"sent": "thats BirthdayBears ear .", "words": ["ear"]}, {"sent": "thats an ear not an eye .", "words": ["an", "not", "eye", "ear"]}, {"sent": "thats sposta be his ear .", "words": ["his", "be", "ear"]}, {"sent": "the ear .", "words": ["the", "ear"]}, {"sent": "it looks like Morgans ear only bigger .", "words": ["it", "like", "ear"]}, {"sent": "and another ear .", "words": ["another", "ear", "and"]}, {"sent": "oh her ear .", "words": ["her", "ear"]}, {"sent": "will you please look in my ear ?", "words": ["my", "you", "in", "look", "will", "ear"]}, {"sent": "does this ear hurt Thomas ?", "words": ["hurt", "does", "this", "ear"]}, {"sent": "why is your ear itching ?", "words": ["ear", "your", "why", "is"]}, {"sent": "just pop your ear on Mummys chest .", "words": ["on", "pop", "your", "ear"]}, {"sent": "does that go on your ear ?", "words": ["does", "your", "go", "ear", "on", "that"]}, {"sent": "which ear hurts ?", "words": ["which", "ear"]}, {"sent": "you put your ear here .", "words": ["put", "your", "you", "here", "ear"]}, {"sent": "I can just see the top of its ear .", "words": ["can", "the", "of", "ear", "see"]}, {"sent": "and Mommys left ear .", "words": ["ear", "and"]}, {"sent": "one ear .", "words": ["ear"]}, {"sent": "I was just stroking your ear .", "words": ["was", "your", "ear"]}, {"sent": "Im gonna tell you a secret in your ear .", "words": ["your", "you", "in", "ear", "a"]}, {"sent": "heres Mommys ear .", "words": ["ear"]}, {"sent": "itll make your ear better .", "words": ["your", "better", "make", "ear"]}, {"sent": "yeah ear .", "words": ["ear"]}, {"sent": "but I can see one ear .", "words": ["can", "see", "but", "ear"]}, {"sent": "thats a a reindeers ear .", "words": ["ear", "a"]}, {"sent": "have you still got an itchy ear ?", "words": ["have", "an", "ear", "you"]}, {"sent": "and you landed on your ear ?", "words": ["your", "you", "and", "ear", "on"]}, {"sent": "triangle ear .", "words": ["ear"]}, {"sent": "where the cats other ear ?", "words": ["where", "the", "ear", "other"]}, {"sent": "okay you children are gonna hafta stand up because youll uh xxx against your ear alright ah .", "words": ["up", "your", "stand", "you", "are", "ear", "because"]}, {"sent": "I Ill den you touched me in the ear and blue and blue and white .", "words": ["the", "you", "and", "blue", "in", "ear", "me", "white"]}, {"sent": "on your ear .", "words": ["on", "your", "ear"]}, {"sent": "ear ?", "words": ["ear"]}, {"sent": "theres theres one ear there .", "words": ["there", "ear"]}, {"sent": "you give somebody a ear ache .", "words": ["give", "ear", "a", "you"]}, {"sent": "theres its ear .", "words": ["ear"]}, {"sent": "his ear .", "words": ["his", "ear"]}, {"sent": "does your ear bother you huh ?", "words": ["does", "your", "you", "ear"]}, {"sent": "a seagull came along and picked him up by the ear .", "words": ["up", "the", "by", "and", "him", "ear", "a"]}, {"sent": "ear after ear ?", "words": ["ear"]}, {"sent": "how about your ear ?", "words": ["how", "your", "about", "ear"]}, {"sent": "are you going to put his other ear in here ?", "words": ["put", "to", "you", "other", "here", "in", "are", "ear", "his"]}, {"sent": "in dollys ear ?", "words": ["in", "ear"]}, {"sent": "what a pretty little ear .", "words": ["what", "little", "pretty", "ear", "a"]}, {"sent": "hes also he has a real ear .", "words": ["ear", "he", "a"]}, {"sent": "youll blow somebodys ear off .", "words": ["off", "blow", "ear"]}, {"sent": "you wouldnt like somebody to take off your ear would you ?", "words": ["off", "your", "to", "like", "would", "you", "ear", "take"]}, {"sent": "hes not happy and he wants his twin .", "words": ["happy", "and", "he", "his", "not"]}, {"sent": "one happy hippo .", "words": ["happy"]}, {"sent": "how are you today looking so lovely and happy .", "words": ["how", "happy", "so", "and", "you", "are"]}, {"sent": "who are we singing happy birthday to today ?", "words": ["happy", "we", "to", "are", "who"]}, {"sent": "shes not happy .", "words": ["not", "happy"]}, {"sent": "was your pumpkin a happy or a sad pumpkin ?", "words": ["your", "happy", "was", "sad", "pumpkin", "a"]}, {"sent": "see he is happy .", "words": ["see", "he", "happy", "is"]}, {"sent": "if youre happy and you know it .", "words": ["if", "happy", "you", "and", "it"]}, {"sent": "you know why shes happy ?", "words": ["happy", "why", "you"]}, {"sent": "I would be happy to help .", "words": ["help", "happy", "to", "be", "would"]}, {"sent": "youre happy with that .", "words": ["that", "with", "happy"]}, {"sent": "you werent happy that night were you ?", "words": ["were", "that", "happy", "you"]}, {"sent": "we wish you a merry christmas and a happy new year .", "words": ["happy", "we", "and", "wish", "you", "new", "a"]}, {"sent": "has that got any happy people on it ?", "words": ["happy", "any", "it", "on", "that"]}, {"sent": "happy birthday teddy they all cheered .", "words": ["all", "they", "happy"]}, {"sent": "if youre happy and you know it touch your eyebrow .", "words": ["your", "if", "happy", "touch", "you", "and", "it"]}, {"sent": "are you happy ?", "words": ["are", "happy", "you"]}, {"sent": "she looks so happy doesnt she ?", "words": ["so", "she", "happy"]}, {"sent": "I was pretty scared especially when I saw the police coming but my parents were with them and they were happy to see me .", "words": ["the", "my", "were", "happy", "was", "to", "me", "with", "them", "scared", "and", "they", "pretty", "see", "when", "but"]}, {"sent": "if youre happy and you know it then your life will surely show it .", "words": ["your", "if", "happy", "show", "you", "and", "it", "will", "then"]}, {"sent": "is he happy is he happy ?", "words": ["he", "happy", "is"]}, {"sent": "now theyre happy now arent they ?", "words": ["they", "happy"]}, {"sent": "I think during the day well be able to see some pictures on the television because one or two places such as Australia will be saying happy new year to each other a long time before we will .", "words": ["the", "think", "happy", "each", "a", "be", "to", "will", "we", "long", "other", "on", "see", "new", "because", "some"]}, {"sent": "he says dont worry be happy .", "words": ["dont", "be", "he", "happy"]}, {"sent": "ah look at that no thats happy look at the smile he made .", "words": ["the", "look", "happy", "he", "at", "that", "smile"]}, {"sent": "och Cathy happy xxx .", "words": ["happy"]}, {"sent": "if youre happy and you know it jump up and down .", "words": ["up", "if", "happy", "you", "and", "jump", "it", "down"]}, {"sent": "hes getting a big hug so hes very happy .", "words": ["happy", "so", "hug", "big", "a"]}, {"sent": "and they all came out from under the mushroom and they were all happy do you know what this is ?", "words": ["the", "all", "do", "what", "happy", "is", "and", "you", "they", "under", "this", "out", "were"]}, {"sent": "happy happy happy ?", "words": ["happy"]}, {"sent": "whos happy ?", "words": ["happy"]}, {"sent": "I cant sing happy birthday to myself who ever heard of a person singing happy birthday to himself ?", "words": ["happy", "of", "to", "sing", "myself", "who", "a"]}, {"sent": "is he happy ?", "words": ["he", "happy", "is"]}, {"sent": "dont wanna say happy birthday to big Jake ?", "words": ["happy", "to", "say", "dont", "wanna", "big"]}, {"sent": "are you gonna sing happy birthday ?", "words": ["sing", "are", "happy", "you"]}, {"sent": "is this a happy birthday present ?", "words": ["happy", "present", "is", "this", "a"]}, {"sent": "you could be happy ?", "words": ["be", "could", "happy", "you"]}, {"sent": "no dont touch that hon tell me about the happy birthday cake .", "words": ["the", "happy", "cake", "touch", "dont", "about", "me", "that"]}, {"sent": "the koalas happy to be home just like youre happy to be home from Marthas Vineyard ?", "words": ["the", "happy", "to", "be", "like", "home"]}, {"sent": "because weve been saying happy birthday all weekend .", "words": ["all", "happy", "because"]}, {"sent": "Theos so happy .", "words": ["so", "happy"]}, {"sent": "Grandma and Grandpa are going to be so happy to see you when they see at the end of the month .", "words": ["the", "happy", "of", "to", "so", "be", "and", "you", "they", "are", "at", "see", "when"]}, {"sent": "Mummy wasnt happy was she ?", "words": ["was", "she", "happy"]}, {"sent": "happy .", "words": ["happy"]}, {"sent": "and a happy Halloween face .", "words": ["happy", "face", "a", "and"]}, {"sent": "if youre happy and you know it .", "words": ["if", "happy", "you", "and", "it"]}, {"sent": "happy birthday .", "words": ["happy"]}, {"sent": "happy birthday to you .", "words": ["to", "happy", "you"]}, {"sent": "what do you do when youre happy ?", "words": ["what", "do", "happy", "you", "when"]}, {"sent": "oh youre so happy now .", "words": ["so", "happy"]}, {"sent": "hes happy now is he ?", "words": ["he", "happy", "is"]}, {"sent": "hes a happy young boy .", "words": ["happy", "a"]}, {"sent": "well first we had problems with our started and then they fixed it but then they didnt fix it entirely right and we hadta send it back and you were so happy right ?", "words": ["back", "we", "happy", "fix", "so", "first", "with", "our", "and", "they", "you", "it", "then", "were", "but"]}, {"sent": "shes far too happy .", "words": ["happy", "too"]}, {"sent": "happy .", "words": ["happy"]}, {"sent": "is she happy ?", "words": ["she", "happy", "is"]}, {"sent": "if you wake wake up tomorrow morning and you and and your legs are gone and then you have a tail are you going to be happy ?", "words": ["up", "your", "if", "happy", "to", "be", "and", "you", "have", "are", "wake", "then", "a"]}, {"sent": "and happy Christmas to Thomas from BuzzLightyear .", "words": ["to", "happy", "and"]}, {"sent": "Honey Bunny Funny Bunny was very happy .", "words": ["was", "happy"]}, {"sent": "whos happy ?", "words": ["happy"]}, {"sent": "hes happy yay hes happy and EmilyElizabeth is happy yay .", "words": ["is", "happy", "and"]}, {"sent": "happy morning .", "words": ["happy"]}, {"sent": "why is she happy ?", "words": ["happy", "she", "why", "is"]}, {"sent": "if youre happy and you know it .", "words": ["if", "happy", "you", "and", "it"]}, {"sent": "Jwwws happy ?", "words": ["happy"]}, {"sent": "because it doesnt seem terribly happy .", "words": ["happy", "it", "because"]}, {"sent": "if youre happy and you know it hop on your knee .", "words": ["knee", "your", "if", "happy", "you", "and", "it", "on"]}, {"sent": "well she might not be happy letting you have keys .", "words": ["happy", "be", "you", "have", "she", "keys", "not"]}, {"sent": "Im happy to go now .", "words": ["to", "happy", "go"]}, {"sent": "that ones happy .", "words": ["that", "happy"]}, {"sent": "and say happy birthday .", "words": ["say", "happy", "and"]}, {"sent": "and Ill be really happy .", "words": ["be", "happy", "and"]}, {"sent": "ohdear is he happy ?", "words": ["he", "happy", "is"]}, {"sent": "hes quite happy sailing in that little boat .", "words": ["happy", "little", "in", "that", "boat"]}, {"sent": "sad face happy face .", "words": ["sad", "face", "happy"]}, {"sent": "are you quite happy playing with your trucks while Mummy finishes her toast ?", "words": ["your", "happy", "toast", "with", "you", "are", "her"]}, {"sent": "Spicey was all happy .", "words": ["all", "was", "happy"]}, {"sent": "does she look happy ?", "words": ["does", "she", "happy", "look"]}, {"sent": "dinosaurs happy .", "words": ["happy"]}, {"sent": "happy birthday to you .", "words": ["to", "happy", "you"]}, {"sent": "there is my happy boy .", "words": ["there", "my", "happy", "is"]}, {"sent": "and the dogs happy .", "words": ["the", "happy", "and"]}, {"sent": "whos happy ?", "words": ["happy"]}, {"sent": "if youre happy you laugh .", "words": ["if", "happy", "you"]}, {"sent": "can you go in and say happy halloween to Manuela ?", "words": ["can", "happy", "go", "to", "say", "and", "you", "in"]}, {"sent": "why are you not happy ?", "words": ["happy", "you", "are", "not", "why"]}, {"sent": "happy birthday dear Jonathan .", "words": ["happy"]}, {"sent": "you didnt seem very happy at playgroup this morning .", "words": ["this", "at", "happy", "you"]}, {"sent": "she sure is happy .", "words": ["she", "happy", "is"]}, {"sent": "what does that hafta do with making you happy at valentines .", "words": ["what", "do", "does", "happy", "with", "you", "at", "that"]}, {"sent": "are you gonna sing happy birthday to me ?", "words": ["happy", "to", "you", "sing", "are", "me"]}, {"sent": "you really make me happy Abe .", "words": ["me", "make", "happy", "you"]}, {"sent": "if youre happy and you know it then your life will surely show it .", "words": ["your", "if", "happy", "show", "you", "and", "it", "will", "then"]}, {"sent": "happy birthday Thomas .", "words": ["happy"]}, {"sent": "they sang happy birthday ?", "words": ["they", "happy"]}, {"sent": "Edwards happy but Ducks not .", "words": ["but", "not", "happy"]}, {"sent": "hes very happy now .", "words": ["happy"]}, {"sent": "happy Hendrika .", "words": ["happy"]}, {"sent": "yes Im so happy Im so excited .", "words": ["so", "happy"]}, {"sent": "alright you ready for happy babies ?", "words": ["for", "happy", "you"]}, {"sent": "go and play go on .", "words": ["on", "play", "and", "go"]}, {"sent": "awoh here would you like me to play too ?", "words": ["to", "play", "like", "you", "here", "would", "me", "too"]}, {"sent": "um pizza hell eat pizza but he didnt really play with the pizza .", "words": ["the", "play", "with", "pizza", "eat", "he", "but"]}, {"sent": "lets play cards ?", "words": ["play"]}, {"sent": "do you play games ?", "words": ["do", "play", "you"]}, {"sent": "I like to play with my ball .", "words": ["my", "to", "play", "with", "like", "ball"]}, {"sent": "play with some of Lorraines toys now .", "words": ["of", "some", "play", "with"]}, {"sent": "lets play time to erase .", "words": ["to", "play"]}, {"sent": "go and play with that .", "words": ["go", "play", "with", "and", "that"]}, {"sent": "uh do you guys play with stuffed animals at home ?", "words": ["do", "play", "with", "you", "at", "home"]}, {"sent": "probably play with toys more than look at a book .", "words": ["look", "play", "with", "more", "at", "book", "a"]}, {"sent": "hurry where we can go play .", "words": ["where", "can", "hurry", "go", "we", "play"]}, {"sent": "to play .", "words": ["to", "play"]}, {"sent": "you wanna play ?", "words": ["play", "wanna", "you"]}, {"sent": "play play play play play .", "words": ["play"]}, {"sent": "who came to play ?", "words": ["to", "play", "who"]}, {"sent": "you can go play .", "words": ["can", "go", "play", "you"]}, {"sent": "we can play play with my big lion .", "words": ["can", "my", "we", "lion", "play", "with", "big"]}, {"sent": "I think he tends to like play with one thing for youknow ten minutes or something before .", "words": ["think", "for", "to", "play", "with", "like", "he"]}, {"sent": "let us play .", "words": ["us", "play"]}, {"sent": "and work and play ?", "words": ["play", "work", "and"]}, {"sent": "now I know who gave it to you to play with .", "words": ["to", "play", "with", "you", "it", "who"]}, {"sent": "you hafta play where you are ?", "words": ["where", "play", "are", "you"]}, {"sent": "you hafta play with as many different colors as you have players .", "words": ["have", "play", "with", "you"]}, {"sent": "Ive got to play with Gordon ?", "words": ["to", "play", "with"]}, {"sent": "oh shes going to play with that penguin is she ?", "words": ["is", "to", "play", "with", "penguin", "that", "she"]}, {"sent": "were going to play some nice games today .", "words": ["to", "nice", "play", "some", "were"]}, {"sent": "and then when we finish up with this then we can play it if its okay with your mom awright ?", "words": ["up", "can", "your", "if", "we", "play", "with", "finish", "and", "this", "it", "then", "when"]}, {"sent": "well you can play with the envelope .", "words": ["can", "the", "play", "with", "you"]}, {"sent": "here lets play with this stuff .", "words": ["here", "this", "play", "with"]}, {"sent": "I wanna go and play some .", "words": ["go", "play", "and", "wanna", "some"]}, {"sent": "did you play at all ?", "words": ["all", "play", "you", "did", "at"]}, {"sent": "now were going to play with horse and tiger .", "words": ["to", "play", "with", "and", "tiger", "horse", "were"]}, {"sent": "say something and Mommyll play it back .", "words": ["back", "play", "say", "and", "it"]}, {"sent": "wee play .", "words": ["play"]}, {"sent": "this is to play with .", "words": ["is", "to", "play", "with", "this"]}, {"sent": "are we not going to play ?", "words": ["we", "to", "play", "are", "not"]}, {"sent": "well play with it in a minute .", "words": ["play", "with", "in", "it", "a"]}, {"sent": "dont play with that .", "words": ["dont", "that", "play", "with"]}, {"sent": "you wanna play peepbo ?", "words": ["play", "wanna", "you"]}, {"sent": "play with their friends and every thing .", "words": ["their", "play", "with", "and", "every"]}, {"sent": "sure you can play .", "words": ["can", "play", "you"]}, {"sent": "you play with your computer .", "words": ["your", "play", "with", "you"]}, {"sent": "what did we play ?", "words": ["what", "play", "did", "we"]}, {"sent": "okay come on Bob its time to play with Lofty oh Lofty wants to play ?", "words": ["on", "to", "play", "with"]}, {"sent": "play with the helicopter ?", "words": ["the", "helicopter", "play", "with"]}, {"sent": "what do you wanna play with ?", "words": ["what", "do", "play", "with", "you", "wanna"]}, {"sent": "xxx help the goldfish play today .", "words": ["the", "help", "play"]}, {"sent": "but you can touch a play stove because its just pretend .", "words": ["can", "play", "because", "touch", "you", "stove", "pretend", "but", "a"]}, {"sent": "he wants to play with you .", "words": ["to", "play", "with", "you", "he"]}, {"sent": "now lets play felt .", "words": ["play"]}, {"sent": "um so we are going to see your school play tomorrow .", "words": ["your", "we", "to", "so", "school", "play", "are", "see"]}, {"sent": "thats where you play with Rwww .", "words": ["where", "play", "with", "you"]}, {"sent": "maybe Ill make a heart for valentines day out of play dough .", "words": ["for", "of", "play", "out", "make", "play dough", "a"]}, {"sent": "how do you play soccer ?", "words": ["do", "how", "play", "you"]}, {"sent": "does she play with you ?", "words": ["does", "play", "with", "you", "she"]}, {"sent": "what should we play with ?", "words": ["what", "play", "with", "we"]}, {"sent": "play Pete ?", "words": ["play"]}, {"sent": "do you wanna play with these blocks ?", "words": ["do", "these", "play", "with", "you", "wanna"]}, {"sent": "bring them all the way out now so we can play .", "words": ["the", "all", "can", "we", "so", "play", "them", "bring", "out"]}, {"sent": "can you remember how to play Becky ?", "words": ["can", "how", "to", "play", "you"]}, {"sent": "here I have it turned it around now so that you can play with the mailbox .", "words": ["can", "the", "so", "play", "with", "you", "here", "have", "it", "around", "that"]}, {"sent": "what do you wanna play ?", "words": ["what", "do", "play", "you", "wanna"]}, {"sent": "youre going to play with mommy next .", "words": ["to", "play", "with"]}, {"sent": "are you gonna play soccer next year ?", "words": ["are", "play", "you"]}, {"sent": "love to do it on him and play with him and yes .", "words": ["do", "to", "play", "with", "and", "him", "it", "love", "on"]}, {"sent": "let her play with it then please .", "words": ["play", "with", "it", "then", "her"]}, {"sent": "play with .", "words": ["play", "with"]}, {"sent": "will you show me how to play with it Thomas ?", "words": ["how", "show", "to", "play", "with", "you", "it", "will", "me"]}, {"sent": "how did you guys play with the baby ?", "words": ["the", "how", "play", "with", "you", "did"]}, {"sent": "if given the chance she might play with the books a little more .", "words": ["the", "if", "play", "with", "little", "more", "she", "a"]}, {"sent": "did you guys play out in the sun a lot this summer ?", "words": ["the", "a lot", "sun", "play", "you", "this", "in", "did", "out", "a"]}, {"sent": "Jilly did you wanna play um with this more ?", "words": ["play", "with", "you", "this", "more", "did", "wanna"]}, {"sent": "well well play with that in a while .", "words": ["play", "with", "in", "that", "a"]}, {"sent": "I know you like Mommys purse but I dont want you to play with it .", "words": ["purse", "to", "play", "with", "like", "you", "dont", "it", "but"]}, {"sent": "can we can we play a different game next time ?", "words": ["can", "game", "we", "play", "a"]}, {"sent": "bring the guitar over here and well play .", "words": ["the", "play", "bring", "and", "here", "over"]}, {"sent": "you wanna go and play ?", "words": ["go", "play", "and", "you", "wanna"]}, {"sent": "what did you play in the playground .", "words": ["the", "what", "play", "you", "in", "did"]}, {"sent": "and did you play in the balls ?", "words": ["the", "play", "you", "and", "in", "did"]}, {"sent": "right well play .", "words": ["play"]}, {"sent": "and if Daddy comes in you cant play with them .", "words": ["if", "play", "with", "them", "you", "and", "in"]}, {"sent": "yeah in terms of her play food .", "words": ["of", "play", "in", "food", "her"]}, {"sent": "do you wanna play with your food ?", "words": ["do", "your", "play", "with", "you", "food", "wanna"]}, {"sent": "I tried to get him to play with the animals .", "words": ["the", "to", "play", "with", "get", "him"]}, {"sent": "wanna play some more with the crayons ?", "words": ["the", "play", "with", "more", "wanna", "some"]}, {"sent": "shes going to come and have her lunch tomorrow and play isnt she ?", "words": ["to", "play", "and", "have", "her", "she"]}, {"sent": "Nanette would you like to would you like to play with the car or read a story ?", "words": ["the", "to", "car", "play", "with", "like", "you", "read", "would", "story", "a"]}, {"sent": "know what else we can play with ?", "words": ["can", "what", "we", "play", "with"]}, {"sent": "what do you guys play ?", "words": ["what", "do", "play", "you"]}, {"sent": "who are you in the play then .", "words": ["the", "play", "you", "in", "are", "who", "then"]}, {"sent": "well why dont we play a different game today ?", "words": ["game", "we", "play", "dont", "why", "a"]}, {"sent": "are you just gonna play with your toys today ?", "words": ["your", "play", "with", "you", "are"]}, {"sent": "maybe hed play with those mazes .", "words": ["play", "with", "those"]}, {"sent": "you wanna play with that ?", "words": ["play", "with", "you", "wanna", "that"]}, {"sent": "today is July fourth two thousand and ten and this is the eighteen month play session .", "words": ["the", "is", "play", "and", "this"]}, {"sent": "like part of the little people play set theres like a lamp post and theres a little platform and you push it down .", "words": ["the", "push", "of", "play", "like", "little", "and", "you", "it", "lamp", "down", "a"]}, {"sent": "what do ya wanna play with ?", "words": ["what", "do", "play", "with", "wanna"]}, {"sent": "go see if you can sit down and play for a minute when Mommys done with this then you can have it okay ?", "words": ["can", "for", "if", "go", "play", "with", "then", "you", "and", "this", "have", "it", "see", "sit", "down", "when", "a"]}, {"sent": "no I just felt sick .", "words": ["sick"]}, {"sent": "no point having it if its going to make you sick .", "words": ["if", "to", "you", "it", "sick", "make"]}, {"sent": "tell Carl that you was a very sick little boy right ?", "words": ["was", "you", "little", "sick", "that", "a"]}, {"sent": "wanna stop you from being sick really .", "words": ["sick", "stop", "wanna", "you"]}, {"sent": "Thomas was very cheerful its a beautiful day but James was sick .", "words": ["was", "sick", "but", "a"]}, {"sent": "the witch is sick in his house ?", "words": ["the", "is", "in", "sick", "house", "his"]}, {"sent": "is that mean hes sick ?", "words": ["sick", "that", "is"]}, {"sent": "come on now Lynette you been getting away with sick now you hafta eat something .", "words": ["with", "you", "eat", "away", "sick", "on"]}, {"sent": "yeah you dont nursie as much when youre sick right .", "words": ["when", "you", "dont", "sick", "much"]}, {"sent": "six sick bricks .", "words": ["sick"]}, {"sent": "Imean Im sick of .", "words": ["of", "sick"]}, {"sent": "you getting sick off wrapping presents like me ?", "words": ["off", "like", "you", "sick", "me"]}, {"sent": "howd he get sick ?", "words": ["get", "sick", "he"]}, {"sent": "hes sick ?", "words": ["sick"]}, {"sent": "what Mummy and Daddy were saying is you dont eat all the chocolate egg at once because you might be poorly sick .", "words": ["the", "what", "all", "egg", "is", "be", "chocolate", "you", "and", "dont", "eat", "sick", "at", "were", "because"]}, {"sent": "what happened when you were sick do you remember ?", "words": ["what", "do", "you", "sick", "when", "were"]}, {"sent": "do you still feel sick Nicole ?", "words": ["do", "sick", "you"]}, {"sent": "what Zoie Lenita was out sick too ?", "words": ["what", "was", "sick", "out", "too"]}, {"sent": "and does the baby get sick ?", "words": ["the", "does", "and", "get", "sick"]}, {"sent": "and some people get air sick .", "words": ["get", "some", "sick", "and"]}, {"sent": "that makes Zic very sick .", "words": ["sick", "that"]}, {"sent": "youll be sick if I do this .", "words": ["do", "if", "be", "this", "sick"]}, {"sent": "that means they visit all the sick patients .", "words": ["the", "all", "they", "sick", "that"]}, {"sent": "who am I going to really see also whos sick ?", "words": ["to", "sick", "am", "who", "see"]}, {"sent": "I remember that almost more than being sick .", "words": ["more", "sick", "that"]}, {"sent": "youre sick of Gordon are you ?", "words": ["of", "sick", "are", "you"]}, {"sent": "shes not being sick is she ?", "words": ["she", "sick", "not", "is"]}, {"sent": "oh well yes and then theyre sick .", "words": ["then", "sick", "and"]}, {"sent": "yeah Angie is sick .", "words": ["sick", "is"]}, {"sent": "you know how sick people act before they before they .", "words": ["they", "how", "sick", "you"]}, {"sent": "oh yeah you were sick last night .", "words": ["last", "sick", "were", "you"]}, {"sent": "the thing is that those were xxx and I dont want you to get sick honey .", "words": ["the", "those", "is", "to", "and", "you", "dont", "that", "get", "sick", "were"]}, {"sent": "he not sick standing up .", "words": ["up", "sick", "he", "not"]}, {"sent": "oh no well bring them over if theyre sick xxx .", "words": ["if", "them", "bring", "sick", "over"]}, {"sent": "especially when hes sick .", "words": ["sick", "when"]}, {"sent": "its okay sweetie its just that it has your germs on it because it was in your mouth she might get sick you know how youve got that little cold Erin doesnt wanna get sick because shes getting ready to go on a trip too .", "words": ["go", "mouth", "cold", "it", "on", "that", "she", "how", "was", "to", "you", "little", "too", "because", "a", "your", "get", "in", "sick", "wanna"]}, {"sent": "no squirting water on your hair wont make you sick .", "words": ["your", "you", "sick", "water", "on", "hair", "make"]}, {"sent": "slow feet quick feet well feet sick feet .", "words": ["sick", "slow"]}, {"sent": "have I got a sick throat ?", "words": ["have", "sick", "a"]}, {"sent": "your mom told me last week that you were feeling sick are you feeling better ?", "words": ["better", "your", "last", "you", "sick", "are", "me", "that", "were"]}, {"sent": "you were super oops girl was pretend that super oops girl was sick .", "words": ["was", "you", "that", "sick", "pretend", "were"]}, {"sent": "is he sick ?", "words": ["sick", "he", "is"]}, {"sent": "sick .", "words": ["sick"]}, {"sent": "sick of swimming .", "words": ["of", "sick"]}, {"sent": "shes sick shes in bad shape .", "words": ["in", "sick", "bad"]}, {"sent": "that means you you go the animals and you figure out which animal is the most sick because the animal who is the most sick is the one that needs to be seen by the doctor first .", "words": ["the", "go", "is", "to", "be", "first", "which", "and", "you", "by", "animal", "sick", "out", "who", "that", "because"]}, {"sent": "how were you sick ?", "words": ["how", "sick", "were", "you"]}, {"sent": "well you dont wanna be sick and die so .", "words": ["be", "so", "you", "and", "dont", "sick", "wanna"]}, {"sent": "well what are you doing are you still feeling sick Tinkerbell ?", "words": ["what", "are", "sick", "you"]}, {"sent": "but you aint sick .", "words": ["sick", "but", "you"]}, {"sent": "are you not sick anymore ?", "words": ["are", "sick", "not", "you"]}, {"sent": "hes already sick .", "words": ["sick"]}, {"sent": "I think I better sign you off sick because youre a bit of a problem when you go to work arent you ?", "words": ["off", "better", "think", "work", "go", "of", "to", "you", "sick", "when", "because", "a"]}, {"sent": "youre going to make yourself sick .", "words": ["to", "yourself", "make", "sick"]}, {"sent": "were you sick last night ?", "words": ["last", "sick", "were", "you"]}, {"sent": "tell Carl you was a sick little boy right ?", "words": ["was", "little", "you", "sick", "a"]}, {"sent": "but shed been out sick .", "words": ["out", "sick", "but"]}, {"sent": "you just feel sick ?", "words": ["sick", "you"]}, {"sent": "how come is she sick or tired ?", "words": ["how", "is", "tired", "sick", "she"]}, {"sent": "because he wants to find out if hes sick .", "words": ["if", "to", "sick", "he", "out", "find", "because"]}, {"sent": "seriously Thomas do you feel sick ?", "words": ["do", "sick", "you"]}, {"sent": "have you ever gone to the doctor anytime else when you were sick or anything ?", "words": ["the", "to", "you", "have", "sick", "when", "were"]}, {"sent": "well feet sick feet .", "words": ["sick"]}, {"sent": "oh no mommys sick .", "words": ["sick"]}, {"sent": "well feet sick feet .", "words": ["sick"]}, {"sent": "Thomas if you feel sick the very last thing you want is a sweetie I promise you .", "words": ["the", "if", "is", "last", "you", "sick", "a"]}, {"sent": "well because he was old and sick .", "words": ["old", "was", "and", "sick", "he", "because"]}, {"sent": "dont you get sick of playing babies ?", "words": ["of", "you", "get", "dont", "sick"]}, {"sent": "Joanne hasnt been sick .", "words": ["sick"]}, {"sent": "theyre all sick ?", "words": ["all", "sick"]}, {"sent": "arent you sick of those ?", "words": ["of", "sick", "those", "you"]}, {"sent": "see the doctors sick in the back .", "words": ["the", "back", "in", "sick", "see"]}, {"sent": "did did you do that last time you were sick ?", "words": ["do", "last", "you", "sick", "did", "that", "were"]}, {"sent": "well dont want you to be um sick eating it do I ?", "words": ["do", "to", "be", "you", "dont", "sick", "it"]}, {"sent": "yeah if your tongue is green I think youre definitely sick .", "words": ["think", "your", "if", "is", "sick", "green", "tongue"]}, {"sent": "youre gonna get sick .", "words": ["get", "sick"]}, {"sent": "when you were sick ?", "words": ["sick", "when", "were", "you"]}, {"sent": "youll be sick .", "words": ["be", "sick"]}, {"sent": "oh we hafta tell Erin about you being sick we forgot to tell Erin .", "words": ["we", "to", "you", "sick", "about"]}, {"sent": "well feet sick feet .", "words": ["sick"]}, {"sent": "are there any more sick babys ?", "words": ["any", "there", "more", "are", "sick"]}, {"sent": "you are sick .", "words": ["are", "sick", "you"]}, {"sent": "yeah Ill get sick if I eat your vitamin .", "words": ["your", "if", "get", "eat", "sick"]}, {"sent": "sound like youre being sick .", "words": ["sick", "like"]}, {"sent": "everyone was sick ?", "words": ["was", "sick"]}, {"sent": "xxx you dont get sick .", "words": ["get", "dont", "sick", "you"]}, {"sent": "three sick .", "words": ["sick"]}, {"sent": "see he must be sick .", "words": ["see", "be", "he", "sick"]}, {"sent": "tell me about one of the times you got sick .", "words": ["the", "of", "you", "sick", "about", "me"]}, {"sent": "the dog is sick ?", "words": ["the", "sick", "dog", "is"]}, {"sent": "remember you threw up when you were sick .", "words": ["up", "you", "sick", "when", "were"]}, {"sent": "xxx hell get sick ?", "words": ["get", "sick"]}, {"sent": "Sylvia uh sick sick .", "words": ["sick"]}, {"sent": "does it make him sick ?", "words": ["does", "him", "it", "sick", "make"]}, {"sent": "youre not sick .", "words": ["sick", "not"]}, {"sent": "oh I think the twins are sick .", "words": ["the", "think", "are", "sick"]}, {"sent": "were you sick ?", "words": ["sick", "were", "you"]}, {"sent": "pretend that SuperBoops girl got sick .", "words": ["pretend", "sick", "that"]}, {"sent": "Im not sick .", "words": ["sick", "not"]}, {"sent": "you wont be sick .", "words": ["be", "sick", "you"]}, {"sent": "chalk just slides right over it .", "words": ["chalk", "it", "over"]}, {"sent": "do you want another chalk ?", "words": ["another", "do", "chalk", "you"]}, {"sent": "you want some chalk ?", "words": ["some", "chalk", "you"]}, {"sent": "because you cant use the chalk on here .", "words": ["the", "chalk", "you", "here", "on", "because"]}, {"sent": "Summer do you want a piece of chalk ?", "words": ["do", "chalk", "of", "you", "a"]}, {"sent": "your sidewalk chalk .", "words": ["chalk", "your", "sidewalk"]}, {"sent": "is that the chalk that was in it ?", "words": ["the", "chalk", "is", "was", "in", "it", "that"]}, {"sent": "ask Jennifer if she wants to draw ask Jennifer if she wants to draw with the chalk .", "words": ["the", "chalk", "if", "to", "draw", "with", "she"]}, {"sent": "your chalk ?", "words": ["chalk", "your"]}, {"sent": "if you put chalk in your mouth Im going to be very cross Eleanor .", "words": ["chalk", "put", "your", "if", "to", "be", "mouth", "you", "in"]}, {"sent": "theres a chalk in there .", "words": ["chalk", "in", "there", "a"]}, {"sent": "thats a chalk .", "words": ["chalk", "a"]}, {"sent": "a black board and chalk .", "words": ["black", "chalk", "and", "a"]}, {"sent": "a pen chalk a pencil crayon whats that ?", "words": ["chalk", "crayon", "pencil", "pen", "that", "a"]}, {"sent": "are you going to draw with the chalk ?", "words": ["the", "chalk", "to", "draw", "with", "you", "are"]}, {"sent": "youre going to try to write with chalk ?", "words": ["chalk", "to", "with", "write", "try", "try to"]}, {"sent": "what else what do you make with that chalk board ?", "words": ["what", "do", "chalk", "with", "you", "that", "make"]}, {"sent": "because it willnt show up because thats chalk .", "words": ["up", "chalk", "show", "it", "because"]}, {"sent": "a chalk ?", "words": ["chalk", "a"]}, {"sent": "its chalk .", "words": ["chalk"]}, {"sent": "crayons and chalk .", "words": ["chalk", "and"]}, {"sent": "lets go into the town and buy us some white chalk .", "words": ["the", "chalk", "into", "go", "and", "us", "some", "white", "buy"]}, {"sent": "chalk boar .", "words": ["chalk"]}, {"sent": "chalk .", "words": ["chalk"]}, {"sent": "is that how you use your chalk ?", "words": ["chalk", "how", "your", "is", "you", "that"]}, {"sent": "poor Horton looked up with his face white as chalk .", "words": ["up", "chalk", "face", "with", "poor", "his", "white"]}, {"sent": "you raise your chalk after you put it away .", "words": ["chalk", "put", "your", "you", "it", "away"]}, {"sent": "you hafta do chalk on that side and crayon on the other side .", "words": ["the", "do", "chalk", "crayon", "you", "and", "other", "on", "that"]}, {"sent": "is it there to rub the chalk off or something ?", "words": ["the", "chalk", "off", "is", "to", "there", "it"]}, {"sent": "Naomi Ill take both pieces of chalk away if you dont give Jennifer one .", "words": ["chalk", "if", "of", "you", "dont", "away", "give", "take"]}, {"sent": "thats chalk .", "words": ["chalk"]}, {"sent": "you wanna you wanna draw on the chalk board ?", "words": ["the", "chalk", "draw", "you", "wanna", "on"]}, {"sent": "the chalk is in a box somewhere .", "words": ["the", "chalk", "box", "is", "in", "a"]}, {"sent": "you got chalk on your face .", "words": ["chalk", "your", "face", "you", "on"]}, {"sent": "and theres some chalk aswell .", "words": ["some", "chalk", "and"]}, {"sent": "can you find the yellow chalk John ?", "words": ["can", "the", "chalk", "you", "find", "yellow"]}, {"sent": "and Tadi you were drawing on the blackboard with some chalk .", "words": ["the", "chalk", "with", "you", "and", "on", "some", "were"]}, {"sent": "youve put some chalk in this one .", "words": ["chalk", "put", "this", "in", "some"]}, {"sent": "he can use the chalk board until hes old enough to .", "words": ["can", "the", "chalk", "old", "to", "he"]}, {"sent": "it seems like chalk ?", "words": ["chalk", "it", "like"]}, {"sent": "you got chalk on your .", "words": ["on", "chalk", "your", "you"]}, {"sent": "chalk .", "words": ["chalk"]}, {"sent": "anybody want a piece of chalk ?", "words": ["of", "chalk", "a"]}, {"sent": "poor Horton looked up with his face white as chalk .", "words": ["up", "chalk", "face", "with", "poor", "his", "white"]}, {"sent": "do you want a bit of chalk ?", "words": ["do", "chalk", "of", "you", "a"]}, {"sent": "where do you draw with chalk honey ?", "words": ["where", "do", "chalk", "draw", "with", "you"]}, {"sent": "look I got chalk .", "words": ["chalk", "look"]}, {"sent": "chalk .", "words": ["chalk"]}, {"sent": "why dont you come why dont you come here and let Mummy get you chalk out .", "words": ["chalk", "and", "here", "dont", "you", "get", "out", "why"]}, {"sent": "ask me if I wanna draw with the chalk Nomi .", "words": ["the", "chalk", "if", "draw", "with", "wanna", "me"]}, {"sent": "xxx angel you want some chalk ?", "words": ["some", "chalk", "you"]}, {"sent": "a pen chalk pencil crayon brush a feather .", "words": ["chalk", "crayon", "brush", "pencil", "pen", "a"]}, {"sent": "chalk .", "words": ["chalk"]}, {"sent": "no those are three pieces of chalk .", "words": ["of", "chalk", "are", "those"]}, {"sent": "You wanna draw on the chalk board ?", "words": ["the", "chalk", "draw", "wanna", "on"]}, {"sent": "you either use the chalk or the crayon .", "words": ["the", "chalk", "crayon", "you"]}, {"sent": "okay well I have some chalk here .", "words": ["have", "some", "chalk", "here"]}, {"sent": "it is chalk .", "words": ["chalk", "it", "is"]}, {"sent": "youre better off doing it on the board if youre using chalk .", "words": ["off", "better", "the", "chalk", "if", "it", "on"]}, {"sent": "well how would I make it with the chalk Nomi ?", "words": ["the", "chalk", "how", "with", "it", "would", "make"]}, {"sent": "thats the chalk .", "words": ["the", "chalk"]}, {"sent": "he likes to draw with chalk outside .", "words": ["chalk", "to", "outside", "draw", "with", "he"]}, {"sent": "some of its chalk .", "words": ["of", "some", "chalk"]}, {"sent": "you played with the chalk ?", "words": ["the", "chalk", "with", "you"]}, {"sent": "well you need chalk for that dont you ?", "words": ["chalk", "for", "need", "you", "dont", "that"]}, {"sent": "I dont want you to use the any chalk .", "words": ["the", "chalk", "to", "any", "you", "dont"]}, {"sent": "do you want the chalk ?", "words": ["the", "do", "chalk", "you"]}, {"sent": "its chalk .", "words": ["chalk"]}, {"sent": "what do you wanna do with the sidewalk chalk ?", "words": ["the", "what", "do", "chalk", "sidewalk", "with", "you", "wanna"]}, {"sent": "chalk ?", "words": ["chalk"]}, {"sent": "here hes holding a piece of chalk well draw a CookieMonster on his shirt CookieMonsters purple .", "words": ["chalk", "of", "shirt", "draw", "here", "his", "on", "a"]}, {"sent": "youve got chalk from school .", "words": ["chalk", "school"]}, {"sent": "where chalk ?", "words": ["where", "chalk"]}, {"sent": "putting chalk in the back of your cars .", "words": ["the", "chalk", "your", "back", "of", "in"]}, {"sent": "alright Karen you can have some piece of chalk ?", "words": ["can", "chalk", "of", "you", "have", "some"]}, {"sent": "do the sidewalk chalk ?", "words": ["the", "do", "chalk", "sidewalk"]}, {"sent": "chalk and cheese feel very small when they meet a great big hungry cat .", "words": ["chalk", "when", "hungry", "and", "they", "big", "cat", "cheese", "a"]}, {"sent": "you only use chalk on here .", "words": ["on", "chalk", "here", "you"]}, {"sent": "she said chalk .", "words": ["chalk", "she"]}, {"sent": "oh the green chalk .", "words": ["the", "chalk", "green"]}, {"sent": "you playing with the chalk ?", "words": ["the", "chalk", "with", "you"]}, {"sent": "you got chalk on your hand huh ?", "words": ["chalk", "your", "hand", "you", "on"]}, {"sent": "you won some sidewalk chalk .", "words": ["some", "chalk", "sidewalk", "you"]}, {"sent": "you wanna do some drawing on the chalk board ?", "words": ["the", "do", "chalk", "you", "wanna", "on", "some"]}, {"sent": "youve just said you want your chalk out .", "words": ["chalk", "your", "out", "you"]}, {"sent": "thats chalk .", "words": ["chalk"]}, {"sent": "well put the chalk up there .", "words": ["the", "chalk", "put", "up", "there"]}, {"sent": "he said chalk .", "words": ["chalk", "he"]}, {"sent": "Tom with a bit of chalk in him .", "words": ["chalk", "of", "with", "in", "him", "a"]}, {"sent": "what are you gonna do with the chalk ?", "words": ["the", "what", "do", "chalk", "with", "you", "are"]}, {"sent": "now you can chalk on the chalk board .", "words": ["can", "chalk", "the", "you", "on"]}, {"sent": "so you can you can get the chalk dust off .", "words": ["can", "the", "chalk", "off", "so", "you", "get"]}, {"sent": "I know its chalk but what color is it ?", "words": ["chalk", "what", "is", "it", "but"]}, {"sent": "the chalk melted ?", "words": ["the", "chalk"]}, {"sent": "you took that piece of chalk away .", "words": ["chalk", "of", "you", "away", "that"]}, {"sent": "theyre like chalk and cheese .", "words": ["chalk", "cheese", "like", "and"]}, {"sent": "and let the chalk board down .", "words": ["the", "chalk", "down", "and"]}, {"sent": "chalk .", "words": ["chalk"]}, {"sent": "you have purple all over your elbows Graeme you must have rubbed up against one of the chalk boards .", "words": ["up", "all", "the", "chalk", "your", "of", "you", "have", "over"]}, {"sent": "that might be yummy chocolate and peanutbutter .", "words": ["be", "chocolate", "that", "and"]}, {"sent": "chocolate ?", "words": ["chocolate"]}, {"sent": "what kind chocolate buttons ?", "words": ["what", "chocolate"]}, {"sent": "I love chocolate .", "words": ["chocolate", "love"]}, {"sent": "Dada likes chocolate doesnt he ?", "words": ["chocolate", "he"]}, {"sent": "so we hafta see do you like chocolate strawberry vanilla chocolatechip butter pecan banana or chocolate ?", "words": ["do", "we", "strawberry", "so", "chocolate", "vanilla", "like", "you", "banana", "butter", "see"]}, {"sent": "youre not having chocolate .", "words": ["chocolate", "not"]}, {"sent": "you ate chocolate yum .", "words": ["chocolate", "you"]}, {"sent": "and hot chocolate yeah and hot chocolate .", "words": ["hot", "chocolate", "and"]}, {"sent": "have some nice chocolate biscuits .", "words": ["have", "some", "chocolate", "nice"]}, {"sent": "oh you found your chocolate biscuit ?", "words": ["chocolate", "your", "you"]}, {"sent": "chocolate mousse ?", "words": ["chocolate"]}, {"sent": "chocolate cookies .", "words": ["chocolate"]}, {"sent": "you dont want chocolate chip pancakes ?", "words": ["dont", "chocolate", "you"]}, {"sent": "chocolate ?", "words": ["chocolate"]}, {"sent": "do you know that chocolate bar ?", "words": ["do", "chocolate", "that", "you"]}, {"sent": "is it chocolate mousse ?", "words": ["chocolate", "it", "is"]}, {"sent": "brown ice cream is chocolate .", "words": ["ice cream", "is", "brown", "chocolate", "ice"]}, {"sent": "like you know how sometimes you dont know how much something costs we would say um how much does the chocolate milk cost and then it would be all around the store and then the person who was by the chocolate milk would be able to go and see how much it costs .", "words": ["the", "go", "like", "milk", "does", "be", "store", "by", "it", "who", "then", "much", "all", "how", "was", "to", "you", "and", "dont", "would", "see", "we", "chocolate", "say", "around"]}, {"sent": "whats Carolines chocolate ?", "words": ["chocolate"]}, {"sent": "you always get chocolate donuts .", "words": ["get", "chocolate", "you"]}, {"sent": "that was a chocolate egg .", "words": ["egg", "was", "chocolate", "that", "a"]}, {"sent": "two chocolate buttons for MisterDavid .", "words": ["chocolate", "for"]}, {"sent": "or do you like chocolate chocolate chip ?", "words": ["do", "chocolate", "like", "you"]}, {"sent": "chocolate ?", "words": ["chocolate"]}, {"sent": "I know you like chocolate .", "words": ["chocolate", "like", "you"]}, {"sent": "its a chocolate .", "words": ["chocolate", "a"]}, {"sent": "I know I xxx well first you think to yourself like whats really the difference between the chocolate chip cookie and say like a banana chocolate chip muffin right ?", "words": ["the", "cookie", "think", "yourself", "muffin", "to", "first", "chocolate", "like", "say", "you", "and", "banana", "a"]}, {"sent": "actually if I was buying chocolate to melt .", "words": ["to", "was", "chocolate", "if"]}, {"sent": "chocolate chip horse .", "words": ["horse", "chocolate"]}, {"sent": "oh you bite my hand and theres no chocolate in it or were you expecting chocolate ?", "words": ["my", "hand", "chocolate", "and", "you", "in", "it", "bite", "were"]}, {"sent": "no more chocolate .", "words": ["more", "chocolate"]}, {"sent": "a chocolate chip cake ?", "words": ["chocolate", "cake", "a"]}, {"sent": "how about a piece of chocolate ?", "words": ["how", "of", "chocolate", "about", "a"]}, {"sent": "chocolate chip cookies .", "words": ["chocolate"]}, {"sent": "get your hands outof that goddamn chocolate milk .", "words": ["your", "chocolate", "get", "milk", "that"]}, {"sent": "not chocolate ?", "words": ["chocolate", "not"]}, {"sent": "she cant have chocolate .", "words": ["have", "chocolate", "she"]}, {"sent": "drinking chocolate .", "words": ["chocolate"]}, {"sent": "xxx the books are chocolate .", "words": ["the", "chocolate", "are"]}, {"sent": "I gave you a bit of Daddys chocolate .", "words": ["of", "chocolate", "a", "you"]}, {"sent": "why dont you finish your chocolate first .", "words": ["your", "chocolate", "first", "finish", "you", "dont", "why"]}, {"sent": "they have chocolate on dont they ?", "words": ["chocolate", "have", "they", "dont", "on"]}, {"sent": "this is a chocolate choux bun .", "words": ["this", "chocolate", "a", "is"]}, {"sent": "those are chocolate chip .", "words": ["chocolate", "are", "those"]}, {"sent": "who got chocolate all over Mummys face ?", "words": ["all", "face", "chocolate", "who", "over"]}, {"sent": "the chocolate tree thats very big ?", "words": ["the", "tree", "chocolate", "big"]}, {"sent": "chocolate cake .", "words": ["chocolate", "cake"]}, {"sent": "do you want some chocolate and peanut butter chips ?", "words": ["do", "chocolate", "and", "you", "butter", "peanut butter", "some"]}, {"sent": "chocolate Christmas shapes .", "words": ["chocolate"]}, {"sent": "did did you put the chocolate inside the cracker then ?", "words": ["the", "put", "inside", "chocolate", "cracker", "you", "did", "then"]}, {"sent": "and chocolate bin biscuits .", "words": ["chocolate", "and"]}, {"sent": "a dark chocolate .", "words": ["chocolate", "dark", "a"]}, {"sent": "youve got choc youve got chocolate on your nose .", "words": ["on", "chocolate", "your", "nose"]}, {"sent": "milk chocolate .", "words": ["milk", "chocolate"]}, {"sent": "well it was a WinniethePooh cake but its not chocolate cake .", "words": ["cake", "was", "chocolate", "it", "not", "but", "a"]}, {"sent": "do you see the chocolate chips ?", "words": ["the", "do", "chocolate", "you", "see"]}, {"sent": "not too much chocolate milk this year remember ?", "words": ["chocolate", "this", "too", "milk", "much", "not"]}, {"sent": "woah is that chocolate ?", "words": ["chocolate", "that", "is"]}, {"sent": "crunchy chocolate is nice .", "words": ["nice", "chocolate", "is"]}, {"sent": "this look like chocolate ?", "words": ["this", "chocolate", "like", "look"]}, {"sent": "three chocolate charmies and three orange charmies .", "words": ["chocolate", "orange", "and"]}, {"sent": "drink your chocolate .", "words": ["chocolate", "your", "drink"]}, {"sent": "can you put the chocolate on the bread ?", "words": ["can", "the", "put", "bread", "chocolate", "you", "on"]}, {"sent": "which ones got most chocolate chips in it and you choose that one .", "words": ["chocolate", "and", "you", "in", "it", "that", "which"]}, {"sent": "you turned the whole backyard into chocolate .", "words": ["the", "backyard", "into", "chocolate", "you"]}, {"sent": "with flakes and chocolate biscuits .", "words": ["chocolate", "with", "and"]}, {"sent": "look at all the chocolate cake .", "words": ["the", "all", "look", "cake", "chocolate", "at"]}, {"sent": "so weve bought some chocolate and were going to just break a few squares off .", "words": ["off", "break", "so", "to", "chocolate", "and", "some", "were", "a"]}, {"sent": "that chocolate was wasnt very nice .", "words": ["nice", "was", "chocolate", "that"]}, {"sent": "what about just plain old chocolate ?", "words": ["old", "what", "chocolate", "about"]}, {"sent": "I could tell he would want chocolate .", "words": ["could", "chocolate", "he", "would"]}, {"sent": "can I have some more hot chocolate ?", "words": ["can", "chocolate", "have", "more", "hot", "some"]}, {"sent": "and grapes and chocolate milk and cookies .", "words": ["milk", "chocolate", "grapes", "and"]}, {"sent": "you can see chocolate balls .", "words": ["can", "see", "chocolate", "you"]}, {"sent": "youve got a chocolate in your mouth .", "words": ["your", "mouth", "chocolate", "in", "a"]}, {"sent": "is that coffee or is that chocolate milk ?", "words": ["is", "chocolate", "coffee", "milk", "that"]}, {"sent": "green chocolate .", "words": ["chocolate", "green"]}, {"sent": "funny how they have chocolate for breakfast .", "words": ["how", "for", "chocolate", "have", "they"]}, {"sent": "the chocolate ones .", "words": ["the", "chocolate"]}, {"sent": "chocolate chocolate chip .", "words": ["chocolate"]}, {"sent": "wasnt it your usual chocolate chip ?", "words": ["chocolate", "it", "your"]}, {"sent": "hafta wipe the chocolate off your mouth .", "words": ["the", "off", "your", "chocolate", "mouth", "wipe"]}, {"sent": "did Daddy give you chocolate in your bed ?", "words": ["your", "chocolate", "bed", "you", "in", "did", "give"]}, {"sent": "chocolate cake .", "words": ["chocolate", "cake"]}, {"sent": "thats lovely chocolate .", "words": ["chocolate"]}, {"sent": "did you have chocolate .", "words": ["have", "chocolate", "did", "you"]}, {"sent": "we must have been hungry on the seventeenth of December because there isnt a chocolate in .", "words": ["the", "we", "of", "chocolate", "hungry", "have", "there", "in", "on", "because", "a"]}, {"sent": "you like chocolate ice cream .", "words": ["ice cream", "chocolate", "like", "you", "ice"]}, {"sent": "okay now you have good chocolate milk .", "words": ["chocolate", "you", "have", "good", "milk"]}, {"sent": "and chocolate sauce .", "words": ["chocolate", "sauce", "and"]}, {"sent": "I made you a chocolate chip cookie .", "words": ["cookie", "chocolate", "a", "you"]}, {"sent": "mm chocolate cake .", "words": ["chocolate", "cake"]}, {"sent": "give your Daddy a chocolate .", "words": ["give", "chocolate", "your", "a"]}, {"sent": "chocolate flake .", "words": ["chocolate"]}, {"sent": "chocolate button in the other .", "words": ["the", "chocolate", "other", "in", "button"]}, {"sent": "no chocolate milk .", "words": ["milk", "chocolate"]}, {"sent": "ah chocolate milkshake .", "words": ["chocolate"]}, {"sent": "do you like chocolate ?", "words": ["do", "chocolate", "like", "you"]}, {"sent": "and one more that goes here .", "words": ["here", "more", "that", "and"]}, {"sent": "do you want me to catch some more ?", "words": ["do", "to", "catch", "you", "more", "me", "some"]}, {"sent": "you wanna work on your handwriting some more or not ?", "words": ["work", "your", "you", "more", "wanna", "on", "some", "not"]}, {"sent": "if you like it I might sell some more .", "words": ["if", "like", "you", "more", "it", "some"]}, {"sent": "ah some more .", "words": ["some", "more"]}, {"sent": "bit more hair ?", "words": ["more", "hair"]}, {"sent": "I notice some days when I go down there them look more tired than the kids .", "words": ["the", "go", "them", "tired", "there", "more", "look", "some", "down", "when"]}, {"sent": "some more .", "words": ["some", "more"]}, {"sent": "oh weve got plenty more .", "words": ["more"]}, {"sent": "do one more .", "words": ["do", "more"]}, {"sent": "little more waffle ?", "words": ["more", "little"]}, {"sent": "shall we put some more turkeys ?", "words": ["some", "put", "more", "we"]}, {"sent": "no more what ?", "words": ["what", "more"]}, {"sent": "and if we put more down this side it becomes a square .", "words": ["put", "if", "we", "and", "this", "more", "it", "down", "a"]}, {"sent": "the more Paddington saw of the super market the more he liked it .", "words": ["the", "of", "more", "it", "he"]}, {"sent": "do you wanna do any more ?", "words": ["do", "any", "you", "more", "wanna"]}, {"sent": "there are several more .", "words": ["there", "more", "are"]}, {"sent": "more cheese please .", "words": ["more", "cheese"]}, {"sent": "theres more paper here look .", "words": ["here", "more", "paper", "look"]}, {"sent": "can I have some more ?", "words": ["can", "some", "have", "more"]}, {"sent": "so you need two more pieces .", "words": ["so", "more", "need", "you"]}, {"sent": "how about a couple more minutes Boo ?", "words": ["how", "more", "about", "a"]}, {"sent": "shall I pour more juice into the cup ?", "words": ["the", "cup", "into", "pour", "more", "juice"]}, {"sent": "there arent any more .", "words": ["there", "more", "any"]}, {"sent": "want some more salad ?", "words": ["some", "more"]}, {"sent": "dont get any more blankets darling .", "words": ["get", "dont", "more", "any"]}, {"sent": "okay Abe Ill get you some more paper too .", "words": ["paper", "you", "get", "more", "too", "some"]}, {"sent": "more Yobaby ?", "words": ["more"]}, {"sent": "and one more .", "words": ["more", "and"]}, {"sent": "theres one more .", "words": ["more"]}, {"sent": "little more for Nathaniel .", "words": ["more", "for", "little"]}, {"sent": "its more like a purse than a handbag .", "words": ["more", "like", "purse", "a"]}, {"sent": "some more red .", "words": ["red", "some", "more"]}, {"sent": "youre dribbling more on the floor than youre actually blowing .", "words": ["on", "the", "more"]}, {"sent": "get a little bit more .", "words": ["get", "more", "little", "a"]}, {"sent": "more what ?", "words": ["what", "more"]}, {"sent": "hm some more track there .", "words": ["some", "more", "there"]}, {"sent": "theres more than six .", "words": ["more"]}, {"sent": "next time we come therell be more toys in there .", "words": ["we", "be", "there", "more", "in"]}, {"sent": "okay one more piece of paper .", "words": ["of", "more", "paper"]}, {"sent": "you want more ?", "words": ["more", "you"]}, {"sent": "you wanna bang it a little bit more ?", "words": ["little", "you", "more", "it", "wanna", "a"]}, {"sent": "here goes one more .", "words": ["here", "more"]}, {"sent": "alright heres some more .", "words": ["some", "more"]}, {"sent": "any more eggs ?", "words": ["more", "any"]}, {"sent": "no more now .", "words": ["more"]}, {"sent": "lets do a couple more because its not quite enough .", "words": ["do", "more", "not", "because", "a"]}, {"sent": "more balloons .", "words": ["more"]}, {"sent": "you put more bread on top .", "words": ["bread", "put", "you", "more", "on"]}, {"sent": "a bit more .", "words": ["more", "a"]}, {"sent": "shes been there about ten years or more so she told her to come down .", "words": ["so", "to", "there", "more", "about", "down", "her", "she"]}, {"sent": "I suppose itll be more .", "words": ["be", "more"]}, {"sent": "there is more in the book .", "words": ["the", "is", "there", "more", "in", "book"]}, {"sent": "oh you want some more ?", "words": ["some", "more", "you"]}, {"sent": "I thought you were snitching more dough .", "words": ["more", "were", "you"]}, {"sent": "so do you think she like books more or toys more ?", "words": ["do", "think", "so", "like", "you", "more", "she"]}, {"sent": "oh you smell more pleasant .", "words": ["more", "you"]}, {"sent": "a little bit more .", "words": ["more", "little", "a"]}, {"sent": "here we go Im giving you more .", "words": ["go", "we", "you", "here", "more"]}, {"sent": "lets see if theres some more pieces .", "words": ["see", "more", "if", "some"]}, {"sent": "you find me some more blue bricks .", "words": ["you", "blue", "more", "find", "me", "some"]}, {"sent": "youve got more than me somehow .", "words": ["me", "more"]}, {"sent": "you want some more paper ?", "words": ["some", "more", "paper", "you"]}, {"sent": "have some more Tab .", "words": ["have", "some", "more"]}, {"sent": "thats right no more stuff its empty .", "words": ["more", "empty"]}, {"sent": "you wanna play some more ?", "words": ["play", "you", "more", "wanna", "some"]}, {"sent": "one more .", "words": ["more"]}, {"sent": "no more milk ?", "words": ["milk", "more"]}, {"sent": "actually Ill bring you some more of your cards .", "words": ["your", "of", "bring", "you", "more", "some"]}, {"sent": "more ?", "words": ["more"]}, {"sent": "want some more juice ?", "words": ["some", "more", "juice"]}, {"sent": "find me more in the basket .", "words": ["the", "more", "in", "basket", "find", "me"]}, {"sent": "please may I have more milk .", "words": ["have", "milk", "more"]}, {"sent": "I think this is more spaghetti look .", "words": ["think", "spaghetti", "is", "this", "more", "look"]}, {"sent": "one more ?", "words": ["more"]}, {"sent": "you color more ?", "words": ["more", "you"]}, {"sent": "lets make some more curly hair .", "words": ["some", "more", "hair", "make"]}, {"sent": "oh I was going to see if weve got some more books .", "words": ["if", "to", "was", "more", "see", "some"]}, {"sent": "I dont wanna use any more .", "words": ["dont", "more", "any", "wanna"]}, {"sent": "yeah well lets not tear it any more .", "words": ["any", "tear", "more", "it", "not"]}, {"sent": "then youve got a bit more room at the front to put the others in .", "words": ["the", "put", "to", "more", "room", "in", "at", "then", "a"]}, {"sent": "so no more botbots ?", "words": ["so", "more"]}, {"sent": "a little more .", "words": ["more", "little", "a"]}, {"sent": "is it more interesting now ?", "words": ["more", "it", "is"]}, {"sent": "theyre more like Wellington boots ?", "words": ["boots", "more", "like"]}, {"sent": "got any more ?", "words": ["more", "any"]}, {"sent": "a little vine salad which is more green than this .", "words": ["is", "little", "this", "more", "green", "which", "a"]}, {"sent": "um hes not interes he wasnt really interested in the books sometimes hes a little more interested in the books .", "words": ["the", "little", "in", "more", "he", "not", "a"]}, {"sent": "because it weighs more than you do .", "words": ["do", "you", "more", "it", "because"]}, {"sent": "do you think other than like the teething do any of the toys um fit his personality more ?", "words": ["the", "do", "think", "of", "fit", "any", "like", "other", "you", "more", "his"]}, {"sent": "you want any more bubbles ?", "words": ["bubbles", "more", "any", "you"]}, {"sent": "and I think weve only got two more music times .", "words": ["think", "more", "and"]}, {"sent": "there arent any more we can post Thomas we cant post any more darling .", "words": ["can", "we", "any", "there", "more"]}, {"sent": "weve plenty more .", "words": ["more"]}, {"sent": "is it more like a pear ?", "words": ["is", "like", "more", "it", "a"]}, {"sent": "you want more marmalade ?", "words": ["more", "you"]}, {"sent": "lets play with the slides some more .", "words": ["the", "play", "with", "more", "some"]}, {"sent": "dont like you any more .", "words": ["any", "like", "you", "dont", "more"]}, {"sent": "shall I do a little bit more .", "words": ["do", "more", "little", "a"]}, {"sent": "more pennys here .", "words": ["here", "more"]}, {"sent": "stop mimicking and eat .", "words": ["eat", "stop", "and"]}, {"sent": "Ill just stop in this petrol station .", "words": ["this", "in", "stop"]}, {"sent": "Jack climbed and climbed and didnt stop .", "words": ["stop", "and"]}, {"sent": "stop .", "words": ["stop"]}, {"sent": "Daddy hadta take the battery out to stop if ringing .", "words": ["the", "if", "to", "stop", "out", "take"]}, {"sent": "you stop him .", "words": ["him", "stop", "you"]}, {"sent": "stop getting excited .", "words": ["stop"]}, {"sent": "are you pretending Sarahs a bus stop ?", "words": ["stop", "you", "are", "bus", "a"]}, {"sent": "well let me just see if I can stop the siren working first .", "words": ["can", "the", "if", "me", "first", "stop", "see"]}, {"sent": "I ran and I said I will stop them I will .", "words": ["will", "them", "stop", "and"]}, {"sent": "the first job is to stop the oil from spreading .", "words": ["the", "is", "to", "first", "stop"]}, {"sent": "stop .", "words": ["stop"]}, {"sent": "stop him from getting my toes .", "words": ["him", "stop", "my"]}, {"sent": "now stop .", "words": ["stop"]}, {"sent": "need lotsof doors to stop the tigers getting out dont you ?", "words": ["the", "need", "to", "stop", "you", "dont", "out"]}, {"sent": "thats not the bus stop .", "words": ["the", "stop", "bus", "not"]}, {"sent": "if youre going to start doing this well hafta stop because youre just getting it everywhere now arent you ?", "words": ["if", "to", "stop", "you", "this", "it", "because"]}, {"sent": "stop .", "words": ["stop"]}, {"sent": "like your door stop .", "words": ["stop", "your", "like", "door"]}, {"sent": "driver stop .", "words": ["stop"]}, {"sent": "stop .", "words": ["stop"]}, {"sent": "Mummy stop it .", "words": ["it", "stop"]}, {"sent": "hes not going to stop is he ?", "words": ["is", "to", "stop", "he", "not"]}, {"sent": "everything had come to a stop even trains and taxi cabs .", "words": ["to", "stop", "and", "a"]}, {"sent": "stop stop stop stop .", "words": ["stop"]}, {"sent": "stop stop stop stop .", "words": ["stop"]}, {"sent": "stop Kim .", "words": ["stop"]}, {"sent": "stop it .", "words": ["it", "stop"]}, {"sent": "we hadta stop marching stop stamping our feet and when he said go we could go couldnt we ?", "words": ["we", "go", "stop", "our", "and", "he", "could", "when"]}, {"sent": "straight stop stop .", "words": ["stop"]}, {"sent": "stop stop stop .", "words": ["stop"]}, {"sent": "stop .", "words": ["stop"]}, {"sent": "if that transmitter falls down again Im gonna hafta stop transmitting .", "words": ["stop", "down", "that", "if"]}, {"sent": "stop being annoying .", "words": ["stop"]}, {"sent": "Manman stop doing that .", "words": ["that", "stop"]}, {"sent": "stop .", "words": ["stop"]}, {"sent": "stop that train I want ta get o .", "words": ["get", "that", "stop", "train"]}, {"sent": "and yet if Im waiting at the bus stop there arent any forty twos .", "words": ["the", "if", "stop", "any", "and", "there", "at", "bus"]}, {"sent": "why dont you stop the rotorblades while he climbs in ?", "words": ["the", "stop", "you", "dont", "in", "he", "why"]}, {"sent": "are you gonna stop him stinging me ?", "words": ["stop", "you", "him", "are", "me"]}, {"sent": "watch stop .", "words": ["watch", "stop"]}, {"sent": "okay stop .", "words": ["stop"]}, {"sent": "you know those little tablets I have in the summer to stop me getting hayfever .", "words": ["the", "those", "to", "stop", "little", "have", "you", "in", "me"]}, {"sent": "stop .", "words": ["stop"]}, {"sent": "I think you had better stop doing that because its going to break and its too noisy .", "words": ["better", "think", "break", "to", "stop", "noisy", "you", "and", "too", "that", "because"]}, {"sent": "that willnt stop like that .", "words": ["that", "stop", "like"]}, {"sent": "well then then stop stop be nice here .", "words": ["be", "nice", "stop", "here", "then"]}, {"sent": "stop nodding .", "words": ["stop"]}, {"sent": "oh Mama stop .", "words": ["stop"]}, {"sent": "and say stop .", "words": ["say", "stop", "and"]}, {"sent": "stop .", "words": ["stop"]}, {"sent": "heh stop xxx .", "words": ["stop"]}, {"sent": "stop it .", "words": ["it", "stop"]}, {"sent": "itll stop after Christmas Thomas .", "words": ["stop"]}, {"sent": "have you put a fence up to stop people going in and out ?", "words": ["up", "put", "to", "stop", "you", "have", "and", "in", "out", "a"]}, {"sent": "tell him im to stop .", "words": ["to", "him", "stop"]}, {"sent": "I thought that might stop you .", "words": ["that", "stop", "you"]}, {"sent": "stop lady .", "words": ["stop"]}, {"sent": "stop stop .", "words": ["stop"]}, {"sent": "I think well stop here .", "words": ["here", "think", "stop"]}, {"sent": "stop .", "words": ["stop"]}, {"sent": "stop .", "words": ["stop"]}, {"sent": "I say stop .", "words": ["say", "stop"]}, {"sent": "oh Thomas stop it .", "words": ["it", "stop"]}, {"sent": "stop it .", "words": ["it", "stop"]}, {"sent": "here Brian stop it .", "words": ["here", "it", "stop"]}, {"sent": "stop it .", "words": ["it", "stop"]}, {"sent": "stop .", "words": ["stop"]}, {"sent": "would you like to stop crying and sit at the table ?", "words": ["the", "table", "to", "stop", "like", "you", "and", "at", "would", "sit"]}, {"sent": "stop it what ?", "words": ["what", "it", "stop"]}, {"sent": "stop .", "words": ["stop"]}, {"sent": "stop running xxx walk .", "words": ["walk", "stop"]}, {"sent": "can you stop that ?", "words": ["can", "that", "stop", "you"]}, {"sent": "right are you going to stop and well get the track out of the box .", "words": ["the", "box", "of", "to", "stop", "and", "you", "get", "are", "out"]}, {"sent": "yeah thats what he going into stop thief cried Dudley as he leaped outof his disguise your bunny doll is stuffed with stolen groceries mother bunny was the thief .", "words": ["the", "what", "bunny", "your", "into", "is", "was", "stop", "with", "doll", "he", "his"]}, {"sent": "Fraser stop being silly .", "words": ["stop"]}, {"sent": "stop looking .", "words": ["stop"]}, {"sent": "stop before you spill it again .", "words": ["it", "stop", "spill", "you"]}, {"sent": "weve got to stop .", "words": ["to", "stop"]}, {"sent": "you stop being spoilt .", "words": ["stop", "you"]}, {"sent": "okay well go ahead and stop .", "words": ["stop", "and", "go"]}, {"sent": "stop being silly .", "words": ["stop"]}, {"sent": "why did he stop ?", "words": ["stop", "he", "did", "why"]}, {"sent": "stop .", "words": ["stop"]}, {"sent": "did it stop at the bus stop ?", "words": ["the", "stop", "it", "did", "at", "bus"]}, {"sent": "stop you sickening me .", "words": ["me", "stop", "you"]}, {"sent": "theres one stop just before I get to the school Ill hafta stop in for gas .", "words": ["the", "for", "to", "school", "stop", "get", "in"]}, {"sent": "will you stop playing and finish up .", "words": ["up", "stop", "finish", "and", "you", "will"]}, {"sent": "I think Ill stop .", "words": ["think", "stop"]}, {"sent": "dont you stop .", "words": ["dont", "stop", "you"]}, {"sent": "stop Lynette .", "words": ["stop"]}, {"sent": "stop .", "words": ["stop"]}, {"sent": "he couldnt stop because Troublesome was pushing him stop Troublesome .", "words": ["was", "stop", "him", "he", "because"]}, {"sent": "stop it .", "words": ["it", "stop"]}, {"sent": "once Rodney starts his hopping its never seems to stop .", "words": ["his", "to", "stop"]}, {"sent": "stop stop stop .", "words": ["stop"]}, {"sent": "it says stop and go .", "words": ["go", "it", "stop", "and"]}, {"sent": "stop .", "words": ["stop"]}, {"sent": "oh yeah I can hold your stop watch ?", "words": ["can", "watch", "your", "hold", "stop"]}, {"sent": "stop .", "words": ["stop"]}, {"sent": "youre banging into the cupboard because you like the sound of the glass .", "words": ["the", "into", "of", "like", "you", "glass", "because"]}, {"sent": "they bump into each other ?", "words": ["bump", "into", "each", "other", "they"]}, {"sent": "you ran into the chair didnt you ?", "words": ["the", "into", "chair", "you"]}, {"sent": "down into the sink ?", "words": ["the", "down", "sink", "into"]}, {"sent": "into the other room ?", "words": ["the", "room", "into", "other"]}, {"sent": "you wanna tiptoe into the green room ?", "words": ["the", "into", "you", "room", "green", "wanna"]}, {"sent": "okay its time for you to go into your bed now .", "words": ["for", "your", "into", "go", "to", "bed", "you"]}, {"sent": "go into my act .", "words": ["my", "into", "go"]}, {"sent": "and while the driver changed the wheel ThomasTheTankEngine gave all the passengers a lift into the station didnt he ?", "words": ["the", "all", "into", "and", "he", "a"]}, {"sent": "to take all the rubbish into the air and into the bin .", "words": ["the", "all", "into", "to", "and", "take"]}, {"sent": "she might not like you going into her bag .", "words": ["into", "like", "you", "she", "her", "not"]}, {"sent": "are you turning Mommys lap into a bed ?", "words": ["into", "bed", "you", "are", "a"]}, {"sent": "well I just put it in because when the shapes fall into the box Thomas they make such a big bang .", "words": ["the", "put", "box", "into", "fall", "they", "in", "it", "make", "big", "when", "because", "a"]}, {"sent": "because mummie hasta go into work .", "words": ["work", "into", "because", "go"]}, {"sent": "but as always my girls are just not into stuffed animals .", "words": ["my", "into", "are", "not", "but"]}, {"sent": "we needta talk into it .", "words": ["it", "into", "talk", "we"]}, {"sent": "nope um I think I wrote on the questionnaire shes really only gotten into toys in the past few months um shes always been more of like a help mommy kind of kid um and when she really started playing with her toys we went through a puzzle stage where all we did were puzzles for like a month and a half .", "words": ["the", "think", "help", "into", "of", "like", "more", "puzzle", "when", "where", "with", "on", "were", "she", "all", "for", "and", "a", "we", "in", "did", "her"]}, {"sent": "but when I went back to the pond again the tadpoles had turned into ?", "words": ["the", "into", "back", "to", "when", "but"]}, {"sent": "and dropping them into the tin that we store them in .", "words": ["the", "into", "we", "store", "them", "and", "in", "that"]}, {"sent": "here Ill pour into my cup .", "words": ["cup", "my", "into", "pour", "here"]}, {"sent": "and then empty all the rubbish into the truck .", "words": ["the", "all", "into", "truck", "empty", "and", "then"]}, {"sent": "youre not into creative work now are you ?", "words": ["work", "into", "you", "are", "not"]}, {"sent": "down the slide and into a bag .", "words": ["the", "slide", "into", "and", "down", "a"]}, {"sent": "now Mummys rolling the green playdough into little balls .", "words": ["the", "into", "green", "little"]}, {"sent": "and very often shed go out of the house and shed wander round and shed come inside where Mummy worked and shed come straight into the office and jump into her special chair in the office .", "words": ["the", "where", "into", "go", "of", "inside", "chair", "and", "jump", "in", "house", "out", "her"]}, {"sent": "have you bumped into the skirting board ?", "words": ["have", "the", "into", "you"]}, {"sent": "its that that its the honeysuckle isnt it that drops into my hair ?", "words": ["the", "my", "into", "it", "hair", "that"]}, {"sent": "you can go into the doll corner if you want .", "words": ["can", "the", "into", "if", "go", "doll", "you"]}, {"sent": "is she all into the dolls again xxx now that ?", "words": ["the", "all", "into", "is", "that", "she"]}, {"sent": "xxx into the mike .", "words": ["the", "into"]}, {"sent": "uh oh the tractor run into the fence .", "words": ["the", "run", "tractor", "into"]}, {"sent": "put it in her stomach right and then she went into her little house and she sat down and what did she do ?", "words": ["what", "put", "do", "into", "and", "little", "in", "it", "house", "did", "then", "down", "her", "she"]}, {"sent": "I put them away till you go out into the garden .", "words": ["the", "put", "into", "go", "them", "you", "garden", "away", "out"]}, {"sent": "can you show Manuela how you talk into a microphone ?", "words": ["can", "how", "into", "show", "you", "talk", "a"]}, {"sent": "he has gone in he has gone into the slot hasnt he ?", "words": ["the", "in", "he", "into"]}, {"sent": "did you get into trouble ?", "words": ["get", "did", "into", "you"]}, {"sent": "oil got into the bodies of fish shrimp and crabs .", "words": ["the", "into", "of", "and", "fish"]}, {"sent": "that I thought xxx by the time you get into it .", "words": ["the", "into", "you", "get", "by", "it", "that"]}, {"sent": "but I know when she goes into a bedroom .", "words": ["bedroom", "into", "when", "she", "but", "a"]}, {"sent": "why did the dragon go into the room alone ?", "words": ["the", "into", "go", "room", "did", "why"]}, {"sent": "why dont you post some more of them into the box ?", "words": ["the", "box", "into", "of", "them", "you", "dont", "more", "some", "why"]}, {"sent": "just thinking Thomas if we could make that into a a big circle .", "words": ["if", "into", "we", "big", "make", "could", "that", "a"]}, {"sent": "they climbed up into that tree .", "words": ["up", "tree", "into", "they", "that"]}, {"sent": "into your tummy ?", "words": ["your", "into", "tummy"]}, {"sent": "this is where you put the the money from the raffle thing xxx into this bag .", "words": ["where", "the", "put", "money", "into", "is", "you", "this"]}, {"sent": "he keeps bumping into things doesnt he ?", "words": ["he", "into"]}, {"sent": "kiss kiss kiss I look into your eyes .", "words": ["your", "kiss", "into", "look"]}, {"sent": "thats into quarters .", "words": ["into"]}, {"sent": "here try try to put this this end of the red one into that end of the yellow one .", "words": ["the", "red", "put", "into", "of", "to", "here", "this", "that", "yellow", "try", "try to"]}, {"sent": "you wanna go all the way downstairs into the ?", "words": ["the", "all", "into", "go", "you", "wanna"]}, {"sent": "this only to speak into .", "words": ["to", "this", "into"]}, {"sent": "dont squash it into the carpet sweetheart because you wont get it off again .", "words": ["the", "off", "into", "you", "get", "dont", "it", "because"]}, {"sent": "sweep it all into one little corner .", "words": ["all", "into", "sweep", "little", "it"]}, {"sent": "Purdie jumped into the window .", "words": ["the", "window", "into"]}, {"sent": "if I stand on your bed because I dont usually look into the neighbors garden but if I .", "words": ["the", "your", "if", "into", "stand", "bed", "dont", "garden", "but", "look", "on", "because"]}, {"sent": "because theyve flown off the fence now and gone into Linda and Brians garden havent they ?", "words": ["off", "the", "into", "and", "they", "garden", "because"]}, {"sent": "the cocoon from a bush red silk from the kite and the squirrel who crawled into my room that night it wasnt very xxx and my pockets were quite big .", "words": ["the", "red", "my", "into", "and", "big", "room", "it", "who", "that", "squirrel", "were", "a"]}, {"sent": "nearly crashed into the train ?", "words": ["the", "into", "train"]}, {"sent": "have you run into the juice with the lawnmower ?", "words": ["the", "into", "with", "you", "have", "run", "juice"]}, {"sent": "a coach and yyy Cinderellas rags into a gorgeous gown .", "words": ["into", "and", "a"]}, {"sent": "into the car .", "words": ["the", "car", "into"]}, {"sent": "and NormanLastname has fallen off the bridge into the river has he ?", "words": ["off", "the", "into", "and", "he"]}, {"sent": "and then it grew and grew and grew and grew and grew and grew and grew and grew turned into this .", "words": ["into", "and", "this", "it", "then"]}, {"sent": "it hasta fit right into the hole exactly .", "words": ["the", "fit", "it", "into"]}, {"sent": "but youre not to take it into the bathroom with you .", "words": ["the", "into", "to", "with", "you", "it", "bathroom", "take", "not", "but"]}, {"sent": "yeah youre crashing teddy into Mummy .", "words": ["into"]}, {"sent": "what else has he run into ?", "words": ["what", "he", "into", "run"]}, {"sent": "what is the bin called that you empty it into ?", "words": ["the", "what", "into", "is", "you", "empty", "it", "that"]}, {"sent": "does he try and get into the tunnel and he cant because hes an elephant ?", "words": ["the", "does", "into", "and", "get", "elephant", "an", "he", "try", "because"]}, {"sent": "shes really into hair and makeup and everything isnt she ?", "words": ["she", "hair", "into", "and"]}, {"sent": "he has gone into Manchester to look at the shops .", "words": ["the", "look", "into", "to", "he", "at"]}, {"sent": "all this going into the suitcase is it ?", "words": ["the", "all", "into", "is", "this", "it"]}, {"sent": "its crashed into my leg .", "words": ["my", "into", "leg"]}, {"sent": "dont step them into the carpet .", "words": ["the", "dont", "them", "into"]}, {"sent": "so we can put some of the fruit into the tin .", "words": ["can", "the", "put", "into", "we", "of", "so", "some"]}, {"sent": "he wont get into trouble .", "words": ["get", "he", "into"]}, {"sent": "Ill give you a ride into the middle of the stream .", "words": ["the", "into", "of", "you", "give", "ride", "a"]}, {"sent": "perhaps if somebody threw a banana into the pond they might peck at it .", "words": ["the", "if", "into", "banana", "they", "it", "at", "a"]}, {"sent": "she tried to fly up into the tree ?", "words": ["up", "the", "tree", "into", "to", "she"]}, {"sent": "you come into the ring .", "words": ["the", "into", "you"]}, {"sent": "you collected all the rubbish in your wheelbarrow with a bucket and spade and emptied it into the compost bin didnt you ?", "words": ["the", "all", "your", "into", "bucket", "with", "you", "and", "in", "it", "a"]}, {"sent": "roll that into a a ball .", "words": ["ball", "that", "into", "a"]}, {"sent": "Mark was trying to intimidate Ross into doing a setup with him .", "words": ["into", "to", "was", "with", "him", "a"]}, {"sent": "can you see that lady going into her house ?", "words": ["can", "into", "her", "you", "house", "see", "that"]}, {"sent": "and theres a dog whos also emptying some cans into the truck .", "words": ["the", "into", "truck", "and", "some", "dog", "a"]}, {"sent": "and then after they had been sorted they went into bags .", "words": ["they", "then", "into", "and"]}, {"sent": "mm well see if we mix this up then it turns into brown eventually .", "words": ["up", "if", "into", "we", "brown", "this", "it", "see", "then"]}, {"sent": "its meting into water now .", "words": ["into", "water"]}, {"sent": "he also yeah hes into throwing stuff too so .", "words": ["so", "he", "into", "too"]}, {"sent": "we can get you into that .", "words": ["can", "into", "we", "you", "get", "that"]}, {"sent": "crashed into Rusty .", "words": ["into"]}, {"sent": "xxx can you talk into the microphone ?", "words": ["can", "the", "into", "you", "talk"]}, {"sent": "you tip that into here .", "words": ["here", "that", "into", "you"]}, {"sent": "aye thats right into the beds .", "words": ["the", "into"]}, {"sent": "well just go into the kitchen and get the potatos .", "words": ["the", "into", "go", "and", "get", "kitchen"]}, {"sent": "turns into a little girl .", "words": ["into", "little", "a"]}, {"sent": "is pop your hand into the blue ink pad and do a few little marks with that .", "words": ["the", "do", "pop", "your", "into", "hand", "is", "with", "blue", "and", "little", "that", "a"]}, {"sent": "cranes can lift things high up into the air .", "words": ["can", "the", "up", "into", "high"]}, {"sent": "oh into pieces ?", "words": ["into"]}, {"sent": "you gonna put it in the garage ?", "words": ["the", "put", "garage", "you", "in", "it"]}, {"sent": "he would see her in the garage .", "words": ["the", "garage", "in", "he", "would", "see", "her"]}, {"sent": "what goes in the garage ?", "words": ["the", "what", "in", "garage"]}, {"sent": "help help my garage is on fire .", "words": ["help", "my", "is", "garage", "on"]}, {"sent": "I will tow you to the garage .", "words": ["the", "garage", "to", "you", "will"]}, {"sent": "because a garage is a building isnt it that things go inside .", "words": ["is", "garage", "go", "inside", "it", "that", "because", "a"]}, {"sent": "in that garage ?", "words": ["garage", "in", "that"]}, {"sent": "okay I saw a garage sale on the way to the store .", "words": ["the", "garage", "to", "store", "on", "a"]}, {"sent": "it went into the garage to sleep for a while didnt it ?", "words": ["the", "for", "into", "garage", "to", "sleep", "it", "a"]}, {"sent": "a garage .", "words": ["garage", "a"]}, {"sent": "oh its going in the garage then is it ?", "words": ["the", "is", "garage", "in", "it", "then"]}, {"sent": "Im going to ride on top of the garage .", "words": ["the", "of", "to", "garage", "on", "ride"]}, {"sent": "were going to have a garage sale .", "words": ["garage", "to", "have", "were", "a"]}, {"sent": "pumps at the garage .", "words": ["the", "garage", "at"]}, {"sent": "a tow truck brings broken down cars into the garage .", "words": ["the", "into", "garage", "truck", "broken", "down", "a"]}, {"sent": "shall we put a window in the garage ?", "words": ["the", "put", "we", "garage", "in", "window", "a"]}, {"sent": "oh youre gonna throw them in the garage ?", "words": ["the", "throw", "garage", "them", "in"]}, {"sent": "*SI1: theres some old tennis balls in the garage I mean in the uh basement .", "words": ["old", "the", "basement", "garage", "in", "some"]}, {"sent": "you better be careful youll break the garage down .", "words": ["the", "better", "garage", "be", "you", "careful", "down", "break"]}, {"sent": "lets play with your garage .", "words": ["your", "play", "with", "garage"]}, {"sent": "whats that on top of your garage ?", "words": ["your", "of", "garage", "on", "that"]}, {"sent": "whose garage is it ?", "words": ["garage", "it", "is"]}, {"sent": "that was in the garage ?", "words": ["the", "garage", "was", "in", "that"]}, {"sent": "its a really nice garage .", "words": ["garage", "nice", "a"]}, {"sent": "well I like the garage .", "words": ["the", "garage", "like"]}, {"sent": "its not the same Ben that has lent you the garage is it ?", "words": ["the", "is", "garage", "you", "it", "same", "that", "not"]}, {"sent": "are you taking it to the garage ?", "words": ["the", "garage", "to", "you", "it", "are"]}, {"sent": "she might have a hot line to FatherChristmass garage list .", "words": ["garage", "to", "have", "hot", "she", "a"]}, {"sent": "it opens like our garage .", "words": ["garage", "it", "like", "our"]}, {"sent": "oh they hafta go to the garage ?", "words": ["the", "go", "garage", "to", "they"]}, {"sent": "who went to the garage today ?", "words": ["the", "to", "garage", "who"]}, {"sent": "okay now I hafta go to the garage .", "words": ["the", "to", "garage", "go"]}, {"sent": "I cant make a car inside the garage .", "words": ["the", "garage", "car", "inside", "make", "a"]}, {"sent": "Im changing my mind about your garage .", "words": ["garage", "my", "your", "about"]}, {"sent": "whatre they gonna do in the garage ?", "words": ["the", "do", "garage", "they", "in"]}, {"sent": "youre making a garage so the trucks dont get him .", "words": ["the", "garage", "so", "get", "dont", "him", "a"]}, {"sent": "lets put the garage door up and get the car out .", "words": ["the", "up", "put", "garage", "car", "and", "get", "out", "door"]}, {"sent": "because he has taken the car into the garage hasnt he ?", "words": ["the", "into", "garage", "car", "he", "because"]}, {"sent": "can she go to sleep in the garage ?", "words": ["can", "the", "go", "sleep", "to", "garage", "in", "she"]}, {"sent": "thats the garage .", "words": ["the", "garage"]}, {"sent": "oh thats from your garage isnt it ?", "words": ["garage", "it", "your"]}, {"sent": "thats your par thats you parking garage thats not a new school .", "words": ["your", "garage", "school", "you", "new", "not", "a"]}, {"sent": "well I dont think we can get into the garage because we dont have the key .", "words": ["can", "the", "think", "into", "we", "garage", "get", "dont", "have", "because"]}, {"sent": "is that the garage ?", "words": ["the", "that", "garage", "is"]}, {"sent": "Dougs closing the garage .", "words": ["the", "garage"]}, {"sent": "is he coming to see them at the garage ?", "words": ["the", "is", "garage", "to", "them", "he", "at", "see"]}, {"sent": "well I think I think the car needs to go to the garage really doesnt it ?", "words": ["the", "think", "go", "garage", "car", "to", "it"]}, {"sent": "well well hafta get the garage wont we ?", "words": ["get", "the", "garage", "we"]}, {"sent": "instead of me saying oh that garage was absolutely rubbish .", "words": ["of", "was", "garage", "me", "that"]}, {"sent": "crashed his plane and he has fallen and the garage has fallen over .", "words": ["the", "garage", "and", "he", "his", "over"]}, {"sent": "useta park in the basement in the garage remember ?", "words": ["the", "basement", "garage", "in", "park"]}, {"sent": "garage .", "words": ["garage"]}, {"sent": "can she jump in the garage ?", "words": ["can", "the", "garage", "jump", "in", "she"]}, {"sent": "garage sale .", "words": ["garage"]}, {"sent": "boy this garage is getting pretty big .", "words": ["is", "garage", "this", "pretty", "big"]}, {"sent": "is that the garage ?", "words": ["the", "that", "garage", "is"]}, {"sent": "yeah and the garage door needs to be closed .", "words": ["the", "garage", "to", "be", "and", "door"]}, {"sent": "put the flat tire over here xxx put the flat tire back in there and well go over to the garage to get it patched up xxx .", "words": ["the", "up", "put", "back", "go", "garage", "to", "and", "here", "there", "in", "get", "it", "over"]}, {"sent": "and the garage ?", "words": ["the", "garage", "and"]}, {"sent": "I want to bring some of that stuff from the back yard thats in your wagon over to the garage and also the kitty pool and put it in the garage .", "words": ["the", "put", "your", "back", "of", "to", "garage", "pool", "bring", "and", "in", "kitty", "it", "want to", "over", "some", "that"]}, {"sent": "what garage ?", "words": ["garage", "what"]}, {"sent": "garage oh Im sorry .", "words": ["garage"]}, {"sent": "garage has a roof .", "words": ["garage", "roof", "a"]}, {"sent": "whats in the garage ?", "words": ["the", "in", "garage"]}, {"sent": "were you doing lotsof jobs around the garage ?", "words": ["the", "garage", "you", "around", "were"]}, {"sent": "move the garage .", "words": ["the", "garage"]}, {"sent": "we went to buy a garage but they were only very small ones and we wanted a large garage .", "words": ["we", "garage", "to", "and", "they", "were", "buy", "but", "a"]}, {"sent": "put the garage door down .", "words": ["the", "put", "garage", "down", "door"]}, {"sent": "not your garage .", "words": ["garage", "your", "not"]}, {"sent": "wheres the garage ?", "words": ["the", "garage"]}, {"sent": "didnt one go in your garage ?", "words": ["garage", "in", "your", "go"]}, {"sent": "pass me the garage over then .", "words": ["the", "garage", "me", "then", "over"]}, {"sent": "on the ramp in the garage ?", "words": ["on", "the", "in", "garage"]}, {"sent": "gone to the garage .", "words": ["the", "to", "garage"]}, {"sent": "put one in the garage xxx .", "words": ["the", "in", "put", "garage"]}, {"sent": "you can play with your garage .", "words": ["can", "your", "garage", "play", "with", "you"]}, {"sent": "and thats a garage and thats like grandads garage uhn isnt it ?", "words": ["garage", "like", "and", "it", "a"]}, {"sent": "the garage that at Christmas time becomes baby Jesuss stable .", "words": ["the", "that", "garage", "at"]}, {"sent": "because I thought uh that was PostmanPats garage yesterday morning wasnt it ?", "words": ["garage", "was", "it", "that", "because"]}, {"sent": "in the garage ?", "words": ["the", "in", "garage"]}, {"sent": "he knocked the garage door off because he forgot to open the garage door first .", "words": ["the", "off", "garage", "to", "open", "first", "he", "because", "door"]}, {"sent": "is he in the garage ?", "words": ["the", "is", "garage", "in", "he"]}, {"sent": "hes in the garage .", "words": ["the", "in", "garage"]}, {"sent": "is that the garage ?", "words": ["the", "that", "garage", "is"]}, {"sent": "there isnt a garage .", "words": ["garage", "there", "a"]}, {"sent": "are you gonna put your cars in your garage ?", "words": ["put", "your", "garage", "you", "in", "are"]}, {"sent": "in the garage .", "words": ["the", "in", "garage"]}, {"sent": "is that the garage ?", "words": ["the", "that", "garage", "is"]}, {"sent": "where where are all the cars for the carpark and the garage ?", "words": ["where", "all", "the", "for", "garage", "and", "are"]}, {"sent": "its in the garage with your with Amys pram .", "words": ["the", "your", "garage", "with", "in"]}, {"sent": "is have the houses got a garage on to put the cars in ?", "words": ["the", "put", "is", "garage", "to", "have", "in", "on", "a"]}, {"sent": "Isabelle has a garage .", "words": ["garage", "a"]}, {"sent": "when youre having your breakfast your lunch your drinks and your tea watching Teletubbies the garage is open .", "words": ["the", "your", "is", "garage", "open", "and", "when"]}, {"sent": "fireengine outof the garage please .", "words": ["the", "garage"]}, {"sent": "its not your garage is it ?", "words": ["your", "is", "garage", "it", "not"]}, {"sent": "and when youre having your bath and getting ready to go to bed at night the garage is still open .", "words": ["the", "your", "go", "garage", "to", "is", "open", "bed", "and", "at", "when"]}, {"sent": "is this the garage ?", "words": ["the", "this", "garage", "is"]}, {"sent": "they were nice high you know high chairs .", "words": ["nice", "you", "they", "high", "were"]}, {"sent": "but what would it look like from high in the sky ?", "words": ["the", "what", "sky", "like", "would", "high", "in", "it", "look", "but"]}, {"sent": "high school .", "words": ["school", "high"]}, {"sent": "way up high ?", "words": ["up", "high"]}, {"sent": "high today is only fifty eight degrees .", "words": ["is", "high"]}, {"sent": "with my high heels on .", "words": ["on", "my", "with", "high"]}, {"sent": "you wanna go high again ?", "words": ["go", "high", "wanna", "you"]}, {"sent": "sitting in my high chair big chair my chair sitting in my high chair banging my spoon .", "words": ["my", "chair", "high chair", "in", "spoon", "high", "big"]}, {"sent": "bring the high chair or give me ring when youve tried it out and if you find its hopeless Ill bring the high chair back you see .", "words": ["the", "if", "back", "see", "bring", "chair", "and", "you", "find", "high chair", "it", "out", "high", "me", "give", "when"]}, {"sent": "high five .", "words": ["high"]}, {"sent": "big fat high tower .", "words": ["big", "high"]}, {"sent": "put it up high where you cant reach it .", "words": ["up", "where", "put", "you", "it", "high"]}, {"sent": "up high .", "words": ["up", "high"]}, {"sent": "okay want me to put it up high ?", "words": ["up", "put", "to", "it", "high", "me"]}, {"sent": "what do they call the man that swings a way up high in the air ?", "words": ["the", "what", "do", "up", "they", "in", "high", "that", "a"]}, {"sent": "wet foot dry foot high foot low foot .", "words": ["wet", "foot", "dry", "high"]}, {"sent": "he jumped really high .", "words": ["he", "high"]}, {"sent": "its not high but it is fat .", "words": ["is", "it", "high", "not", "but"]}, {"sent": "here well you dont really have her pants on quite right yet either look theyre not quite up high enough over her bottom .", "words": ["up", "over", "bottom", "you", "here", "dont", "have", "high", "pants", "look", "on", "her", "not"]}, {"sent": "sing high high high .", "words": ["sing", "high"]}, {"sent": "it has flown so high but the pilot has fallen out .", "words": ["the", "so", "it", "out", "high", "but"]}, {"sent": "so its flying really high isnt it ?", "words": ["so", "it", "high"]}, {"sent": "from behind your high chair .", "words": ["your", "behind", "chair", "high chair", "high"]}, {"sent": "big chair high chair sitting in my high chair feed me soon .", "words": ["my", "feed", "chair", "high chair", "in", "high", "me", "big"]}, {"sent": "high drama .", "words": ["high"]}, {"sent": "high and low .", "words": ["and", "high"]}, {"sent": "was it very high ?", "words": ["was", "it", "high"]}, {"sent": "we hafta make xxx for the giraffe and we hafta make them high .", "words": ["the", "for", "giraffe", "we", "them", "and", "high", "make"]}, {"sent": "they sure are are they both flying who flies higher who goes up very very high Zurg or Buzz whos the highest ?", "words": ["up", "the", "they", "are", "high", "who"]}, {"sent": "Im make high tray .", "words": ["tray", "make", "high"]}, {"sent": "and he soon returned carrying two steaming cups of tea and a large plate piled high with .", "words": ["plate", "of", "with", "and", "he", "high", "a"]}, {"sent": "I dont think I can make a bridge high enough .", "words": ["can", "think", "dont", "high", "make", "a"]}, {"sent": "they just too high .", "words": ["high", "they", "too"]}, {"sent": "up above the world so high .", "words": ["up", "the", "so", "high", "above"]}, {"sent": "and snipping off any blades that were too high .", "words": ["off", "any", "and", "high", "too", "that", "were"]}, {"sent": "he knew how to climb the monkey bars and pump himself high on the swing .", "words": ["the", "how", "swing", "to", "monkey", "and", "he", "high", "on", "climb"]}, {"sent": "really high .", "words": ["high"]}, {"sent": "going high .", "words": ["high"]}, {"sent": "hes flying very high .", "words": ["high"]}, {"sent": "when he puts that on those flames are gonna go high .", "words": ["those", "when", "go", "are", "he", "high", "on", "that"]}, {"sent": "did it go really high ?", "words": ["high", "it", "did", "go"]}, {"sent": "from your high street ?", "words": ["street", "your", "high"]}, {"sent": "big strong fat high tower .", "words": ["big", "high"]}, {"sent": "way up high .", "words": ["up", "high"]}, {"sent": "can you jump high ?", "words": ["can", "jump", "high", "you"]}, {"sent": "oh over the past couple of days everything has been big high up to you hasnt it ?", "words": ["the", "up", "of", "to", "you", "it", "high", "over", "big"]}, {"sent": "wow thats pretty high .", "words": ["pretty", "high"]}, {"sent": "how about if I bounce it really high ?", "words": ["how", "if", "it", "about", "high"]}, {"sent": "a high bridge and a low bridge .", "words": ["high", "and", "a"]}, {"sent": "high fives for mommy .", "words": ["for", "high"]}, {"sent": "was it up high or low ?", "words": ["up", "was", "it", "high"]}, {"sent": "Horton stayed on that nest he held his head high and threw out his .", "words": ["head", "and", "his", "he", "out", "high", "on", "that"]}, {"sent": "dont throw it too high or too hard .", "words": ["throw", "dont", "it", "hard", "high", "too"]}, {"sent": "so I think we found the secret that youll listen to a book in a high chair .", "words": ["the", "think", "listen", "we", "so", "to", "chair", "high chair", "in", "high", "book", "that", "a"]}, {"sent": "not too high in the air .", "words": ["the", "in", "high", "too", "not"]}, {"sent": "as high as this .", "words": ["this", "high"]}, {"sent": "Harold the helicopter helps from high up in the sky .", "words": ["the", "helicopter", "up", "sky", "in", "high"]}, {"sent": "throw it high Stephen .", "words": ["it", "throw", "high"]}, {"sent": "just jumps pretty high huh ?", "words": ["pretty", "high"]}, {"sent": "it goes really high up doesnt it ?", "words": ["up", "it", "high"]}, {"sent": "whew that one jumped high .", "words": ["that", "high"]}, {"sent": "look how high his crane goes .", "words": ["high", "his", "how", "look"]}, {"sent": "yeah its too high for me .", "words": ["high", "me", "for", "too"]}, {"sent": "its the net he was holding so it probably goes up high on the picture somewhere .", "words": ["the", "up", "was", "so", "picture", "it", "he", "high", "on"]}, {"sent": "if you put it in your mouth Warren it goes away on a high shelf where you cant reach it .", "words": ["where", "put", "your", "if", "mouth", "you", "in", "it", "away", "high", "on", "a"]}, {"sent": "its going at high speed past Mummys house ?", "words": ["high", "house", "at"]}, {"sent": "but I dont think the toast would jump quite high enough to hit the light .", "words": ["the", "think", "toast", "light", "to", "high", "dont", "jump", "would", "hit", "but"]}, {"sent": "high low .", "words": ["high"]}, {"sent": "you had a very high temperature .", "words": ["high", "a", "you"]}, {"sent": "thats what I learned in high school and in camp ?", "words": ["what", "school", "and", "in", "high"]}, {"sent": "high and low .", "words": ["and", "high"]}, {"sent": "did you say high high speed ?", "words": ["high", "say", "did", "you"]}, {"sent": "and Jollytall saw Jollytall saw how tall how high up he was .", "words": ["up", "how", "was", "and", "he", "high"]}, {"sent": "high and low .", "words": ["and", "high"]}, {"sent": "are you up so high now ?", "words": ["up", "so", "you", "are", "high"]}, {"sent": "when youre way up high .", "words": ["up", "when", "high"]}, {"sent": "whats too high ?", "words": ["high", "too"]}, {"sent": "its not as high as Fraser is it ?", "words": ["it", "not", "is", "high"]}, {"sent": "we both go high in the sky .", "words": ["the", "go", "we", "sky", "in", "high"]}, {"sent": "now if I can hold them up high and you cant find them .", "words": ["can", "up", "hold", "if", "them", "find", "and", "you", "high"]}, {"sent": "wet foot dry foot high foot low foot .", "words": ["wet", "foot", "dry", "high"]}, {"sent": "can I have a high five ?", "words": ["can", "have", "high", "a"]}, {"sent": "up up high .", "words": ["up", "high"]}, {"sent": "high seats are called stools .", "words": ["are", "high"]}, {"sent": "Im building my pen high .", "words": ["high", "my", "pen"]}, {"sent": "because it was sort of a high up car wasnt it ?", "words": ["up", "of", "was", "car", "it", "high", "because", "a"]}, {"sent": "okay but first lets tell Erin about who we are visiting and then and then I will hold you up high .", "words": ["up", "hold", "we", "first", "and", "you", "high", "are", "about", "who", "will", "then", "but"]}, {"sent": "we swing up high into the trees we swing so fast we make a breeze .", "words": ["up", "the", "fast", "into", "we", "swing", "so", "high", "make", "a"]}, {"sent": "it could but its quite a high voice .", "words": ["it", "high", "could", "but", "a"]}, {"sent": "Pat Pat takes Toms model plane and starts to make it fly it soars up in the bright blue sky high up in .", "words": ["up", "the", "sky", "to", "and", "blue", "in", "it", "high", "make"]}, {"sent": "does it go too high ?", "words": ["does", "go", "it", "high", "too"]}, {"sent": "I think it only goes up that high .", "words": ["up", "think", "it", "high", "that"]}, {"sent": "high .", "words": ["high"]}, {"sent": "I shall take it off you and put it on a high shelf .", "words": ["off", "put", "and", "you", "it", "high", "on", "take", "a"]}, {"sent": "wow he jumps super high doesnt he ?", "words": ["he", "high"]}, {"sent": "is it high in the sky ?", "words": ["the", "is", "sky", "in", "it", "high"]}, {"sent": "up above the world so high .", "words": ["up", "the", "so", "high", "above"]}, {"sent": "you stack them too high .", "words": ["high", "too", "them", "you"]}, {"sent": "its not particularly high is it ?", "words": ["it", "not", "is", "high"]}, {"sent": "you put them up too high Mag .", "words": ["up", "put", "them", "you", "high", "too"]}, {"sent": "I think thats their room over there .", "words": ["think", "their", "there", "room", "over"]}, {"sent": "while we tidy up that room ?", "words": ["up", "room", "that", "we"]}, {"sent": "lets see we have room for a few more things .", "words": ["for", "we", "have", "more", "room", "see", "a"]}, {"sent": "go in the morning room .", "words": ["the", "in", "room", "go"]}, {"sent": "I thought that I saw some corn between the two couches in the living room .", "words": ["the", "living room", "in", "room", "corn", "some", "that"]}, {"sent": "this is your bed room .", "words": ["your", "is", "bed", "this", "room"]}, {"sent": "just go in the other room I said to you .", "words": ["the", "go", "to", "other", "you", "in", "room"]}, {"sent": "that room was very interesting but a little too wet .", "words": ["wet", "was", "little", "room", "too", "that", "but", "a"]}, {"sent": "there is not enough room .", "words": ["there", "room", "not", "is"]}, {"sent": "theres no room here .", "words": ["here", "room"]}, {"sent": "we havent got room for doing silly things like that .", "words": ["for", "we", "like", "room", "that"]}, {"sent": "Jenny cant cry in this room or can ?", "words": ["can", "cry", "this", "in", "room"]}, {"sent": "thats a thats a passenger in the waiting room .", "words": ["the", "in", "room", "a"]}, {"sent": "is there no room ?", "words": ["there", "room", "is"]}, {"sent": "theres the room like in our other book .", "words": ["the", "like", "our", "other", "in", "room", "book"]}, {"sent": "you give daddy a little room and well have a look for some lorries .", "words": ["for", "little", "you", "and", "have", "room", "look", "some", "give", "a"]}, {"sent": "oh well there just wasnt room enough for both of them then .", "words": ["for", "of", "them", "there", "room", "then"]}, {"sent": "you cant come in the other room .", "words": ["the", "other", "you", "in", "room"]}, {"sent": "goodnight room goodnight moon .", "words": ["room", "moon"]}, {"sent": "go to your room .", "words": ["to", "room", "your", "go"]}, {"sent": "do you wanna play in A Jwwws room ?", "words": ["do", "play", "you", "in", "room", "wanna"]}, {"sent": "theyll dry out soon enough dont go stay in the room .", "words": ["the", "go", "dry", "dont", "in", "room", "out", "stay"]}, {"sent": "why dont we put one up in your room ?", "words": ["up", "put", "your", "we", "dont", "in", "room", "why"]}, {"sent": "good night room ?", "words": ["room", "good"]}, {"sent": "why dont you why dont you go out into the other room .", "words": ["the", "into", "go", "you", "other", "dont", "room", "out", "why"]}, {"sent": "in your room .", "words": ["in", "room", "your"]}, {"sent": "and I dont think its a good idea to play with soft stuff in this room .", "words": ["think", "soft", "to", "play", "with", "and", "dont", "in", "this", "room", "good", "a"]}, {"sent": "do we have room up here ?", "words": ["up", "do", "we", "have", "here", "room"]}, {"sent": "so you have lotsof room .", "words": ["have", "so", "room", "you"]}, {"sent": "yes hes very even when we were sitting in the waiting room like he heard the door open down the hallway and looked towards it like he knew someone was coming in .", "words": ["the", "we", "open", "was", "like", "and", "in", "room", "he", "it", "down", "when", "were", "door"]}, {"sent": "how bout Gigis room ?", "words": ["how", "room"]}, {"sent": "go to the room .", "words": ["the", "to", "room", "go"]}, {"sent": "you wanna come in Jwwws room with me ?", "words": ["with", "you", "in", "room", "wanna", "me"]}, {"sent": "yeah you dont have a clock in your room .", "words": ["your", "clock", "you", "have", "dont", "in", "room", "a"]}, {"sent": "go in the living room go on .", "words": ["the", "go", "living room", "in", "room", "on"]}, {"sent": "no theres not enough room .", "words": ["room", "not"]}, {"sent": "you and Aaron go in that room and cut dat television down and sit in here .", "words": ["cut", "go", "and", "you", "here", "in", "room", "sit", "down", "that"]}, {"sent": "is this Jwwws room ?", "words": ["this", "room", "is"]}, {"sent": "because sure I had two switches Cathy one for that and one for that room .", "words": ["for", "and", "room", "that", "because"]}, {"sent": "it was a very we were sleeping at this womans farmhouse and she had an extra room and the room had no heat so she loaned me an electric blanket .", "words": ["the", "we", "was", "so", "and", "this", "an", "it", "room", "at", "blanket", "me", "were", "she", "a"]}, {"sent": "Im not sure if theres any room for it .", "words": ["for", "if", "any", "room", "it", "not"]}, {"sent": "its in Jwwws room is it ?", "words": ["it", "in", "room", "is"]}, {"sent": "sit in the dining room at the table ?", "words": ["the", "table", "in", "room", "at", "sit"]}, {"sent": "wanna play in this room for a little bit ?", "words": ["for", "play", "little", "this", "in", "room", "wanna", "a"]}, {"sent": "in living room .", "words": ["in", "room", "living room"]}, {"sent": "whos in the room ?", "words": ["the", "in", "room"]}, {"sent": "theres more room in here isnt there ?", "words": ["here", "there", "in", "more", "room"]}, {"sent": "Jwwws room messy ?", "words": ["room"]}, {"sent": "theres not really enough room is there ?", "words": ["there", "room", "not", "is"]}, {"sent": "your new attic room .", "words": ["room", "new", "your"]}, {"sent": "lets get some of these other things out of the way so you have more room free .", "words": ["the", "these", "of", "so", "other", "get", "you", "have", "more", "room", "out", "some"]}, {"sent": "you go in the front room while I hide teddy in here .", "words": ["the", "hide", "go", "you", "here", "in", "room"]}, {"sent": "running out of room now arent you ?", "words": ["of", "room", "out", "you"]}, {"sent": "you can go to your room .", "words": ["can", "your", "go", "to", "you", "room"]}, {"sent": "oh Jessica is coming into the other room with us .", "words": ["the", "into", "is", "with", "other", "us", "room"]}, {"sent": "do you wanna go in Mummys room ?", "words": ["do", "go", "you", "in", "room", "wanna"]}, {"sent": "sit in the living room .", "words": ["the", "living room", "in", "room", "sit"]}, {"sent": "in my room .", "words": ["in", "room", "my"]}, {"sent": "shall we go in the other room then .", "words": ["the", "go", "we", "other", "in", "room", "then"]}, {"sent": "this is the bedroom this is the living room .", "words": ["the", "is", "living room", "this", "room", "bedroom"]}, {"sent": "hes sitting in his room .", "words": ["his", "in", "room"]}, {"sent": "or is it Frasers room ?", "words": ["room", "it", "is"]}, {"sent": "do you have anything in your room Jerry ?", "words": ["do", "your", "you", "have", "in", "room"]}, {"sent": "yeah I I dont think you want you I dont think youre prepared to go up to your room .", "words": ["up", "think", "your", "go", "to", "you", "dont", "room"]}, {"sent": "because I wanted to make a room .", "words": ["to", "room", "make", "because", "a"]}, {"sent": "do you wanna take your blocks into Mummy and Daddys room ?", "words": ["do", "your", "into", "you", "and", "room", "wanna", "take"]}, {"sent": "she put it in the other room because there wasnt enough room with the christmas tree was there ?", "words": ["the", "put", "tree", "was", "with", "other", "there", "in", "it", "room", "she", "because"]}, {"sent": "oh you wanna go in that room ?", "words": ["go", "you", "in", "room", "wanna", "that"]}, {"sent": "do you wanna go in Mummy and Daddys room ?", "words": ["do", "go", "you", "and", "in", "room", "wanna"]}, {"sent": "and then youll have more room to do the circus .", "words": ["the", "do", "to", "and", "have", "more", "room", "then"]}, {"sent": "do you needta measure your room or the just the new house ?", "words": ["the", "do", "your", "you", "room", "house", "new"]}, {"sent": "and give you some room back there ?", "words": ["back", "you", "and", "there", "room", "some", "give"]}, {"sent": "we would have another window for that room .", "words": ["for", "we", "have", "another", "room", "would", "window", "that"]}, {"sent": "theres not much room in here thats why .", "words": ["here", "in", "room", "much", "not", "why"]}, {"sent": "okay lets go in my room show me some stuff .", "words": ["my", "go", "show", "in", "room", "me", "some"]}, {"sent": "is there room now ?", "words": ["there", "room", "is"]}, {"sent": "just not as light as the dining room .", "words": ["light", "the", "room", "not"]}, {"sent": "shed probably would think there was something funny about Spider Man and what you and Daddy set up in the living room last night .", "words": ["up", "what", "think", "the", "was", "last", "living room", "and", "you", "there", "in", "room", "about", "would"]}, {"sent": "yyy shall we take Jennifer in the living room and put her on the floor ?", "words": ["the", "put", "we", "living room", "her", "and", "in", "room", "on", "take"]}, {"sent": "theres no room .", "words": ["room"]}, {"sent": "because because Anne Marie was in dat room .", "words": ["was", "in", "room", "because"]}, {"sent": "across the room I see a tiny Adam .", "words": ["the", "tiny", "room", "see", "a"]}, {"sent": "if I dont have enough room would you be able to take one home and let him live at your house ?", "words": ["your", "if", "be", "to", "you", "have", "dont", "and", "room", "him", "at", "house", "would", "take", "home"]}, {"sent": "theres not a lot room is there ?", "words": ["is", "a lot", "there", "room", "not", "a"]}, {"sent": "I dont know if therell be enough room .", "words": ["dont", "room", "be", "if"]}, {"sent": "yeah I hafta say that I um I just wonder what people are thinking when they do like a real baby baby room theyre just gonna change it two years later you know ?", "words": ["what", "do", "when", "say", "like", "you", "they", "are", "room", "it", "that", "a"]}, {"sent": "and you can sit really on a dining room chair now but youre so low down that I think wed better stick to that chair .", "words": ["can", "better", "think", "stick", "so", "to", "chair", "you", "and", "room", "on", "sit", "down", "that", "but", "a"]}, {"sent": "babyll hafta have a bath in a minute because theres not room in the bath look .", "words": ["the", "have", "in", "room", "look", "not", "because", "a"]}, {"sent": "oh what a nice room .", "words": ["what", "room", "nice", "a"]}, {"sent": "theres no more room in my purse .", "words": ["my", "purse", "in", "more", "room"]}, {"sent": "I guess shes in the other room .", "words": ["the", "in", "room", "other"]}, {"sent": "what did we do in our room with our bathing suits ?", "words": ["what", "do", "we", "with", "our", "in", "room", "did"]}, {"sent": "but she doesnt really play with those because theyre in our play room and shes too small to really go in there so I dont she may have seen them before though .", "words": ["those", "go", "to", "so", "play", "with", "because", "them", "our", "and", "there", "in", "room", "dont", "have", "too", "she", "but"]}, {"sent": "oh are you in the staff room .", "words": ["the", "you", "in", "are", "room"]}, {"sent": "Mommy left the room .", "words": ["the", "room"]}, {"sent": "in its in your PollyPocket thing in the front room if you want it .", "words": ["the", "your", "if", "you", "in", "room", "it"]}, {"sent": "and we have um an armoire in the living room that has the knobs on .", "words": ["the", "we", "living room", "and", "have", "in", "an", "room", "on", "that"]}, {"sent": "they burglars bounded up the stairs to the top room little pandas room .", "words": ["up", "the", "to", "little", "they", "room", "stairs"]}, {"sent": "because I got my living room done .", "words": ["my", "room", "living room", "because"]}, {"sent": "oh actually you could play with stickers in the dining room while I wash up couldnt you ?", "words": ["the", "up", "play", "with", "you", "in", "room", "could", "wash"]}, {"sent": "you going to plant it on the porch or on the grass ?", "words": ["the", "to", "porch", "you", "it", "on", "plant", "grass"]}, {"sent": "you mustnt pull Mummys plant up love .", "words": ["up", "pull", "you", "love", "plant"]}, {"sent": "thats a plant .", "words": ["plant", "a"]}, {"sent": "Im afraid that theyre gonna fight see we have meat eaters and we have plant eaters theyre gonna fight .", "words": ["we", "meat", "and", "have", "see", "plant", "that"]}, {"sent": "I was looking at what else we could we could plant in the garden .", "words": ["the", "what", "plant", "we", "was", "in", "garden", "at", "could"]}, {"sent": "oo oooo thats how the plant thats how the oo oooo .", "words": ["the", "plant", "how"]}, {"sent": "to help you plant the seedlings .", "words": ["the", "help", "to", "you", "plant"]}, {"sent": "we would put the plant in the light .", "words": ["the", "put", "we", "light", "in", "would", "plant"]}, {"sent": "weve still got this pretty red plant though .", "words": ["red", "this", "plant", "pretty"]}, {"sent": "stop traumatizing that plant .", "words": ["plant", "that", "stop"]}, {"sent": "thats when that nice plant was still there .", "words": ["when", "was", "nice", "there", "plant", "that"]}, {"sent": "oh and you gave him the plant .", "words": ["the", "you", "and", "him", "plant"]}, {"sent": "weve got to some uh seeds to plant .", "words": ["to", "plant", "some"]}, {"sent": "and the plant stays the same size growing .", "words": ["the", "plant", "same", "and"]}, {"sent": "my plant ?", "words": ["plant", "my"]}, {"sent": "on the plant look theres a plant .", "words": ["the", "look", "on", "plant", "a"]}, {"sent": "get out the plant .", "words": ["get", "the", "plant", "out"]}, {"sent": "we did give some water to the plant .", "words": ["the", "plant", "we", "to", "did", "water", "some", "give"]}, {"sent": "which plant should we do first which one ?", "words": ["do", "we", "first", "plant", "which"]}, {"sent": "you cant take the tips off the plant .", "words": ["the", "off", "you", "plant", "take"]}, {"sent": "Gingers having a look at the plant that bought Mummy for MothersDay .", "words": ["the", "for", "at", "look", "plant", "that", "a"]}, {"sent": "we can put them in some plant pots couldnt we ?", "words": ["can", "put", "we", "them", "in", "some", "plant"]}, {"sent": "well I just dont know which ones are meat eaters and which ones are plant eaters .", "words": ["meat", "and", "dont", "are", "plant", "which"]}, {"sent": "the plant drank some water .", "words": ["the", "some", "plant", "water"]}, {"sent": "I think when we decide to plant some fruit pips Thomas .", "words": ["think", "we", "to", "some", "plant", "when"]}, {"sent": "thats xxx thats a plant .", "words": ["plant", "a"]}, {"sent": "cant pull that plant apart .", "words": ["plant", "that", "pull"]}, {"sent": "an eater plant on your castle yeah ?", "words": ["on", "plant", "an", "your"]}, {"sent": "get out the plant .", "words": ["get", "the", "plant", "out"]}, {"sent": "oh the plant didnt cry ?", "words": ["the", "cry", "plant"]}, {"sent": "when we plant something at home or here in school and we plant it in a regular planter that we buy in the store youll notice if you go to the store they have little holes on the bottom .", "words": ["the", "go", "school", "when", "store", "have", "it", "on", "that", "buy", "home", "bottom", "if", "to", "and", "here", "you", "little", "plant", "a", "we", "they", "in", "at"]}, {"sent": "now the now the plant has some water .", "words": ["the", "some", "plant", "water"]}, {"sent": "and you can put a plant pot there .", "words": ["can", "put", "you", "and", "there", "plant", "a"]}, {"sent": "we also took out plant to the doctor .", "words": ["the", "we", "to", "out", "plant"]}, {"sent": "whats it doing in my plant ?", "words": ["plant", "in", "it", "my"]}, {"sent": "a plant ?", "words": ["plant", "a"]}, {"sent": "and a plant .", "words": ["plant", "a", "and"]}, {"sent": "kaevit plant it .", "words": ["plant", "it"]}, {"sent": "and plant the seeds .", "words": ["the", "plant", "and"]}, {"sent": "we could plant one of the pips and have our own apple tree ?", "words": ["the", "tree", "we", "of", "and", "have", "our", "apple", "plant", "could"]}, {"sent": "ah thats thats a little plant I think .", "words": ["plant", "think", "little", "a"]}, {"sent": "it has a baby plant inside .", "words": ["plant", "it", "inside", "a"]}, {"sent": "well this is not from the plant that comes from that bean but this is from another plant and the leaves .", "words": ["the", "is", "and", "another", "this", "plant", "that", "not", "but"]}, {"sent": "got to plant it in the garden .", "words": ["the", "to", "in", "it", "garden", "plant"]}, {"sent": "we should plant your sunflower somewhere shouldnt we ?", "words": ["plant", "your", "we"]}, {"sent": "we have lots more things to plant in our garden .", "words": ["we", "to", "our", "have", "more", "in", "garden", "plant"]}, {"sent": "how do you know that pterodactyl is a plant eater ?", "words": ["do", "how", "is", "you", "plant", "that", "a"]}, {"sent": "Ma what which uh this thing we bought right that little strawberry plant or whatever its called .", "words": ["what", "we", "strawberry", "which", "little", "this", "plant", "that"]}, {"sent": "yeah thats a plant .", "words": ["plant", "a"]}, {"sent": "I think maybe that plant over there needs some more water .", "words": ["think", "there", "that", "more", "water", "some", "plant", "over"]}, {"sent": "look at all the weeds in this little plant .", "words": ["the", "all", "look", "little", "this", "in", "at", "plant"]}, {"sent": "see that plant ?", "words": ["see", "plant", "that"]}, {"sent": "Im gonna give some water to the plant .", "words": ["the", "plant", "to", "water", "some", "give"]}, {"sent": "so just the plant eaters get them .", "words": ["the", "so", "them", "get", "plant"]}, {"sent": "your Mommy forgot about this poor plant .", "words": ["your", "poor", "this", "about", "plant"]}, {"sent": "plant .", "words": ["plant"]}, {"sent": "theres a plant pot on the window ledge .", "words": ["the", "window", "on", "plant", "a"]}, {"sent": "and sow it plant it for you ?", "words": ["for", "you", "and", "it", "plant"]}, {"sent": "learn how to plant their own gardens .", "words": ["to", "how", "their", "plant"]}, {"sent": "and then a long time later a plant grew up .", "words": ["up", "plant", "long", "and", "then", "a"]}, {"sent": "Mummy said she would get some of the plant pots and clean them up .", "words": ["the", "clean", "up", "of", "them", "and", "get", "would", "some", "plant", "she"]}, {"sent": "we must plant apple because you eat so many apples in a day dont you ?", "words": ["we", "so", "you", "eat", "dont", "in", "apple", "plant", "because", "a"]}, {"sent": "and where are the plant pots on this page ?", "words": ["where", "the", "and", "this", "are", "on", "plant"]}, {"sent": "and trod on a plant .", "words": ["on", "plant", "a", "and"]}, {"sent": "oh a plant ?", "words": ["plant", "a"]}, {"sent": "okay where are you going to plant the seed .", "words": ["where", "the", "to", "you", "are", "plant"]}, {"sent": "oh you clanking one of the plant pots on Mummys tea ?", "words": ["the", "of", "you", "on", "plant"]}, {"sent": "I said dont touch that plant .", "words": ["dont", "plant", "that", "touch"]}, {"sent": "you can plant them in a garden or in pots .", "words": ["can", "them", "you", "in", "garden", "plant", "a"]}, {"sent": "can you plant grass ?", "words": ["can", "plant", "grass", "you"]}, {"sent": "and then the farmer can plant .", "words": ["the", "can", "plant", "and", "then"]}, {"sent": "and then you can plant your sunflower cant you ?", "words": ["can", "your", "then", "you", "and", "plant"]}, {"sent": "and just smiling at a plant makes that plant happy .", "words": ["happy", "and", "at", "plant", "that", "a"]}, {"sent": "what did you plant in it ?", "words": ["what", "you", "in", "it", "did", "plant"]}, {"sent": "can you say plant honey ?", "words": ["can", "plant", "say", "you"]}, {"sent": "table plant .", "words": ["plant", "table"]}, {"sent": "I plant my seeds in the row row row .", "words": ["the", "in", "plant", "my"]}, {"sent": "put them in the bin or you can plant them and then put them in the soil and plant them .", "words": ["the", "can", "put", "them", "then", "you", "and", "in", "plant"]}, {"sent": "no that your plant is growing a sunflower .", "words": ["your", "is", "plant", "that", "a"]}, {"sent": "and we can use those as little plant pots for when youre growing things like mustard cress .", "words": ["can", "for", "those", "we", "like", "little", "and", "plant", "when"]}, {"sent": "we keep saying were going to plant it .", "words": ["we", "to", "it", "plant", "were"]}, {"sent": "what did we plant in our garden ?", "words": ["what", "we", "our", "in", "garden", "did", "plant"]}, {"sent": "a tree is a giant plant food stored inside the seed nourishes the young plant as the chestnut sapling grows up .", "words": ["the", "up", "tree", "is", "inside", "food", "plant", "a"]}, {"sent": "so all the children that didnt get a turn to plant it will plant it while the other groups are in music or something didnt get a chance .", "words": ["the", "all", "so", "to", "other", "get", "in", "it", "are", "will", "plant", "that", "a"]}, {"sent": "and a plant thats great and what did Spot bring ?", "words": ["what", "bring", "and", "did", "plant", "a"]}, {"sent": "where shall we plant it ?", "words": ["where", "plant", "it", "we"]}, {"sent": "uhhuh they do look alot alike did you show Daddy your plant ?", "words": ["do", "your", "show", "you", "they", "did", "look", "plant"]}, {"sent": "yep that was because you moved your book and it knocked the plant on the floor .", "words": ["the", "your", "was", "you", "and", "on", "it", "book", "plant", "that", "because"]}, {"sent": "I suppose so if you wanna plant grass .", "words": ["if", "so", "you", "wanna", "plant", "grass"]}, {"sent": "shes giving the plant some water .", "words": ["the", "some", "plant", "water"]}, {"sent": "when an acorn falls on the ground where there is good soil and plenty of water a baby plant starts to grow .", "words": ["the", "where", "is", "of", "to", "and", "there", "an", "good", "water", "on", "plant", "when", "a"]}, {"sent": "hop hop and whos hiding behind the plant pot ?", "words": ["the", "plant", "behind", "and"]}, {"sent": "when Mummy got rid of all the plants in the spring she was going to throw them away and I decided to plant them at the bottom of the garden .", "words": ["the", "all", "bottom", "throw", "of", "was", "to", "them", "and", "in", "away", "garden", "at", "plant", "when", "she"]}, {"sent": "cotton cloth comes from the cotton plant and thats how we make clothes .", "words": ["the", "how", "we", "and", "plant", "make"]}, {"sent": "what seeds do you wanna plant tomorrow ?", "words": ["what", "do", "you", "wanna", "plant"]}, {"sent": "Daddys going to plant it isnt he ?", "words": ["to", "plant", "it", "he"]}, {"sent": "weve oughta plant that somewhere else .", "words": ["plant", "that"]}, {"sent": "plant .", "words": ["plant"]}, {"sent": "youd like your scissors ?", "words": ["scissors", "your", "like"]}, {"sent": "so I took the scissors to it myself .", "words": ["the", "so", "scissors", "to", "it", "myself"]}, {"sent": "Ill get some scissors .", "words": ["get", "some", "scissors"]}, {"sent": "yes Glenmae cut her wool thats right thats what shes doing in this picture shes using her scissors to cut the wool and shes up side down isnt she ?", "words": ["the", "what", "up", "cut", "to", "scissors", "picture", "and", "this", "in", "down", "her", "she"]}, {"sent": "doesnt cut your head with the scissors just cuts your hair with the scissors right ?", "words": ["the", "your", "cut", "head", "scissors", "with", "hair"]}, {"sent": "did you bring your scissors scissors ?", "words": ["your", "scissors", "bring", "you", "did"]}, {"sent": "theres the scissors right .", "words": ["the", "scissors"]}, {"sent": "your little baby scissors .", "words": ["scissors", "your", "little"]}, {"sent": "okay I hafta get some scissors to clip to cut it off .", "words": ["off", "cut", "to", "scissors", "get", "it", "some"]}, {"sent": "oh no no no dont play with that thats Mummys scissors .", "words": ["scissors", "play", "with", "dont", "that"]}, {"sent": "practically every pair of scissors down at the beach .", "words": ["the", "of", "scissors", "beach", "every", "at", "down"]}, {"sent": "these scissors are big Nathan theyre a little bit sharp .", "words": ["these", "scissors", "little", "are", "big", "a"]}, {"sent": "were back in full scissors mode now .", "words": ["full", "back", "scissors", "in", "were"]}, {"sent": "put your scissors down .", "words": ["scissors", "put", "down", "your"]}, {"sent": "these scissors useta be alright .", "words": ["these", "scissors", "be"]}, {"sent": "I get the scissors xxx .", "words": ["get", "the", "scissors"]}, {"sent": "how about may I borrow these scissors please ?", "words": ["scissors", "these", "how", "about"]}, {"sent": "where are your scissors darling ?", "words": ["where", "scissors", "are", "your"]}, {"sent": "scissors .", "words": ["scissors"]}, {"sent": "with the scissors .", "words": ["the", "scissors", "with"]}, {"sent": "shes got some scissors .", "words": ["some", "scissors"]}, {"sent": "wheres my scissors ?", "words": ["scissors", "my"]}, {"sent": "scissors .", "words": ["scissors"]}, {"sent": "is there another rule about running with scissors ?", "words": ["is", "scissors", "with", "another", "there", "about"]}, {"sent": "scissors .", "words": ["scissors"]}, {"sent": "you wanna use the scissors .", "words": ["the", "scissors", "wanna", "you"]}, {"sent": "I think you left your scissors didnt you behind ?", "words": ["think", "your", "scissors", "behind", "you"]}, {"sent": "we could build a seesaw if I had some cardboard and some scissors .", "words": ["if", "we", "scissors", "and", "build", "some", "could", "a"]}, {"sent": "you can use your scissors to cut out some stuff .", "words": ["can", "your", "cut", "to", "scissors", "you", "out", "some"]}, {"sent": "you know when you wanna sharpen your scissors on your knives ?", "words": ["your", "scissors", "you", "wanna", "on", "when"]}, {"sent": "theyve obviously bought some wrapping paper because when theyve got home Mummys using the scissors to cut the wrapping paper .", "words": ["the", "cut", "to", "paper", "scissors", "some", "when", "because", "home"]}, {"sent": "the scissors .", "words": ["the", "scissors"]}, {"sent": "theyre like scissors but theyre very big uhn arent they ?", "words": ["scissors", "like", "they", "big", "but"]}, {"sent": "is that where the scissors would be ?", "words": ["where", "the", "is", "be", "scissors", "would", "that"]}, {"sent": "well thats what I need the scissors for .", "words": ["the", "what", "for", "need", "scissors"]}, {"sent": "did you find my scissors ?", "words": ["my", "scissors", "you", "did", "find"]}, {"sent": "I need your scissors .", "words": ["your", "scissors", "need"]}, {"sent": "and on and on this one Thomas is using his scissors to try and cut along a straight line .", "words": ["cut", "is", "to", "scissors", "and", "his", "this", "on", "try", "a"]}, {"sent": "so now you can use the big scissors .", "words": ["can", "the", "so", "scissors", "you", "big"]}, {"sent": "Ive put the scissors back in the draw .", "words": ["the", "put", "back", "scissors", "draw", "in"]}, {"sent": "scissors all done .", "words": ["all", "scissors"]}, {"sent": "do you have a scissors ?", "words": ["do", "scissors", "you", "have", "a"]}, {"sent": "we need a pair of scissors dont we ?", "words": ["need", "we", "of", "scissors", "dont", "a"]}, {"sent": "Im gonna hafta chop it off with a pair of scissors .", "words": ["off", "of", "scissors", "with", "it", "a"]}, {"sent": "Im going to open it Mommy needs scissors to open it .", "words": ["open", "to", "scissors", "it"]}, {"sent": "lets do your hair with grandmas sharp scissors .", "words": ["do", "your", "scissors", "with", "hair"]}, {"sent": "theres scissors .", "words": ["scissors"]}, {"sent": "youre allowed to use the scissors .", "words": ["the", "to", "scissors"]}, {"sent": "youve got to be very careful with the scissors because they can cause an injury to somebody cant they ?", "words": ["the", "can", "to", "be", "scissors", "with", "they", "an", "careful", "because"]}, {"sent": "are those scissors like the lady useta cut your hair ?", "words": ["the", "those", "cut", "your", "scissors", "like", "are", "hair"]}, {"sent": "these are the red scissors but upstairs youve got some with a blue handle in your bedroom havent you ?", "words": ["the", "these", "red", "bedroom", "your", "scissors", "with", "blue", "you", "in", "are", "some", "but", "a"]}, {"sent": "is that one of the left handed scissors ?", "words": ["the", "is", "of", "scissors", "that"]}, {"sent": "scissors scissors .", "words": ["scissors"]}, {"sent": "Ill get a scissors xxx .", "words": ["get", "scissors", "a"]}, {"sent": "scissors .", "words": ["scissors"]}, {"sent": "scissors are dangerous .", "words": ["scissors", "are"]}, {"sent": "you give me the scissors and .", "words": ["the", "scissors", "you", "and", "me", "give"]}, {"sent": "scissors ?", "words": ["scissors"]}, {"sent": "you remember pap showed you how to use scissors today .", "words": ["scissors", "to", "how", "you"]}, {"sent": "I thought you were looking for a scissors .", "words": ["for", "scissors", "you", "were", "a"]}, {"sent": "do you need a scissors Adam ?", "words": ["do", "need", "scissors", "you", "a"]}, {"sent": "if you keep throwing things at me Im gonna get the scissors right now .", "words": ["the", "if", "scissors", "you", "get", "at", "me"]}, {"sent": "I could do with some scissors to cut it .", "words": ["do", "cut", "to", "scissors", "with", "it", "some", "could"]}, {"sent": "uh three yes three pairs of scissors .", "words": ["of", "scissors"]}, {"sent": "youre not using them scissors give me them .", "words": ["scissors", "them", "me", "give", "not"]}, {"sent": "she is gonna use scissors .", "words": ["scissors", "she", "is"]}, {"sent": "heres the scissors that we need to open the thing .", "words": ["the", "need", "we", "open", "to", "scissors", "need to", "that"]}, {"sent": "youll probably break the scissors .", "words": ["the", "scissors", "break"]}, {"sent": "well were going to color something and then were going to show MissRoz what you can do with your own scissors .", "words": ["can", "what", "do", "your", "show", "to", "scissors", "with", "and", "you", "then", "were"]}, {"sent": "sharp scissors isnt it ?", "words": ["scissors", "it"]}, {"sent": "a toolbox some pliers some sharp scissors ?", "words": ["some", "scissors", "a"]}, {"sent": "here are some different colors you can cut up with your scissors .", "words": ["can", "up", "your", "cut", "scissors", "with", "you", "here", "are", "some"]}, {"sent": "we need some scissors .", "words": ["some", "scissors", "need", "we"]}, {"sent": "fix it with the scissors ?", "words": ["the", "fix", "scissors", "with", "it"]}, {"sent": "scissors ?", "words": ["scissors"]}, {"sent": "whose scissors ?", "words": ["scissors"]}, {"sent": "no scissors .", "words": ["scissors"]}, {"sent": "youve got rid of all your scissors havent you ?", "words": ["all", "your", "of", "scissors", "you"]}, {"sent": "we better put these scissors away too .", "words": ["better", "put", "these", "we", "scissors", "away", "too"]}, {"sent": "ask mommy about those scissors .", "words": ["scissors", "those", "about"]}, {"sent": "okay will you show Mot the picture of the scissors ?", "words": ["the", "show", "of", "scissors", "picture", "you", "will"]}, {"sent": "would you like your little Playdoh scissors ?", "words": ["your", "scissors", "like", "little", "you", "would"]}, {"sent": "careful with the scissors you wanna stay away from xxx scissors okay ?", "words": ["the", "scissors", "with", "you", "away", "wanna", "careful", "stay"]}, {"sent": "you hafta be careful with scissors dont you ?", "words": ["be", "scissors", "with", "you", "dont", "careful"]}, {"sent": "and I needta go get the scissors .", "words": ["the", "go", "scissors", "and", "get"]}, {"sent": "those are scissors .", "words": ["scissors", "are", "those"]}, {"sent": "blue handles scissors .", "words": ["scissors", "blue"]}, {"sent": "xxx I need the scissors again .", "words": ["the", "scissors", "need"]}, {"sent": "scissors coming .", "words": ["scissors"]}, {"sent": "cut it out with scissors .", "words": ["cut", "scissors", "with", "it", "out"]}, {"sent": "I sent Derwood for the scissors right ?", "words": ["the", "scissors", "for"]}, {"sent": "we hafta have scissors to cut it out .", "words": ["cut", "we", "to", "scissors", "have", "it", "out"]}, {"sent": "its a scissors .", "words": ["scissors", "a"]}, {"sent": "you used my scissors ?", "words": ["scissors", "my", "you"]}, {"sent": "and Ill get you some scissors .", "words": ["scissors", "you", "get", "and", "some"]}, {"sent": "thats right they do that on purpose so you dont lose it Ill take it off with a little scissors and Ill give it to you .", "words": ["off", "do", "give", "so", "scissors", "to", "with", "you", "little", "they", "dont", "it", "and", "on", "take", "that", "a"]}, {"sent": "and you mustnt touch them because you can hurt yourself with scissors .", "words": ["can", "yourself", "hurt", "scissors", "them", "with", "touch", "you", "and", "because"]}, {"sent": "where are the scissors ?", "words": ["where", "the", "scissors", "are"]}, {"sent": "wheres the scissors ?", "words": ["the", "scissors"]}, {"sent": "lets put it away then .", "words": ["put", "it", "then", "away"]}, {"sent": "away .", "words": ["away"]}, {"sent": "we needta run away .", "words": ["run", "away", "we"]}, {"sent": "you hafta do it right away .", "words": ["do", "it", "away", "you"]}, {"sent": "okay you put away the blocks and Ill go get you some apple juice .", "words": ["the", "put", "go", "and", "you", "get", "away", "apple", "some", "juice"]}, {"sent": "before you walk away William .", "words": ["away", "walk", "you"]}, {"sent": "put the pencils away .", "words": ["the", "put", "away"]}, {"sent": "Ill put the red scissors away now .", "words": ["the", "red", "put", "scissors", "away"]}, {"sent": "you going with Daddy to throw it away ?", "words": ["throw", "to", "with", "you", "it", "away"]}, {"sent": "are we are we tidying the train set away ?", "words": ["the", "train", "we", "are", "away"]}, {"sent": "if we put this away Ill find something else .", "words": ["put", "if", "we", "this", "away", "find"]}, {"sent": "went away Shem what happened to our car ?", "words": ["what", "to", "car", "our", "away"]}, {"sent": "no animals are gonna run away .", "words": ["run", "are", "away"]}, {"sent": "thats all Mummys birthday cards put away for another year .", "words": ["all", "put", "for", "another", "away"]}, {"sent": "Ill take the guitar away from you .", "words": ["the", "away", "take", "you"]}, {"sent": "we can just throw those away .", "words": ["can", "those", "throw", "we", "away"]}, {"sent": "is it away ?", "words": ["it", "away", "is"]}, {"sent": "dont throw it away .", "words": ["away", "dont", "it", "throw"]}, {"sent": "are we gonna put away your toys first ?", "words": ["put", "your", "we", "first", "are", "away"]}, {"sent": "he had a little toy that rolled away from him and he sat there wondering whether it would be worth going to get or not .", "words": ["be", "to", "little", "and", "there", "toy", "away", "he", "him", "it", "would", "get", "that", "not", "a"]}, {"sent": "lets move away from the steps .", "words": ["the", "away"]}, {"sent": "put the box away now out the way .", "words": ["the", "put", "box", "away", "out"]}, {"sent": "xxx thing away .", "words": ["away"]}, {"sent": "its flown away .", "words": ["away"]}, {"sent": "get away from here .", "words": ["get", "away", "here"]}, {"sent": "can you get that one and give it to me and well put it away ?", "words": ["can", "put", "to", "and", "get", "you", "it", "away", "me", "give", "that"]}, {"sent": "youre not running away .", "words": ["away", "not"]}, {"sent": "put your shapes away .", "words": ["put", "away", "your"]}, {"sent": "let me put this shoe away .", "words": ["put", "shoe", "this", "away", "me"]}, {"sent": "he went away ?", "words": ["away", "he"]}, {"sent": "well youll hafta take at least a pear away .", "words": ["away", "take", "a", "at"]}, {"sent": "I wanted to I just wanna put this away .", "words": ["put", "to", "this", "away", "wanna"]}, {"sent": "hm that egg just keep getting away from you doesnt it ?", "words": ["egg", "you", "it", "away", "that"]}, {"sent": "its not Ill hafta put it away .", "words": ["away", "put", "it", "not"]}, {"sent": "I think he I think he ran away .", "words": ["think", "away", "he"]}, {"sent": "are you gonna put some of the jigsawpuzzles away ?", "words": ["the", "put", "of", "you", "are", "away", "some"]}, {"sent": "I guess I got carried away Toot said .", "words": ["away"]}, {"sent": "he moved away from you because you know why ?", "words": ["because", "you", "away", "he", "why"]}, {"sent": "away yeah .", "words": ["away"]}, {"sent": "youre gonna put it away ?", "words": ["put", "it", "away"]}, {"sent": "go away .", "words": ["away", "go"]}, {"sent": "yeah let me get my sewing machine put away .", "words": ["my", "put", "get", "away", "me"]}, {"sent": "put the fire engine away .", "words": ["the", "put", "away"]}, {"sent": "you you put you put your bobbins away .", "words": ["put", "away", "your", "you"]}, {"sent": "it went very far away .", "words": ["it", "away"]}, {"sent": "go and put them away on the bookcase .", "words": ["the", "put", "go", "them", "and", "away", "on"]}, {"sent": "oh no I gotta run away from the tractor .", "words": ["the", "run", "away", "tractor"]}, {"sent": "weve only got three minutes left so we can put this away .", "words": ["can", "put", "we", "so", "this", "away"]}, {"sent": "I will put your slide away and you wont be able to use it any more if I ever see you doing that again .", "words": ["put", "slide", "your", "if", "be", "to", "any", "and", "you", "more", "away", "it", "will", "see", "that"]}, {"sent": "shall we put it away now ?", "words": ["put", "it", "away", "we"]}, {"sent": "the red bag got away he said .", "words": ["the", "red", "he", "away"]}, {"sent": "take it away Kip .", "words": ["it", "take", "away"]}, {"sent": "she put the crayons away .", "words": ["the", "put", "away", "she"]}, {"sent": "you gonna put them away or dyou want me to put them away ?", "words": ["put", "to", "them", "you", "away", "me"]}, {"sent": "Im just gonna kind of stand away and .", "words": ["of", "stand", "away", "and"]}, {"sent": "they ran away ?", "words": ["they", "away"]}, {"sent": "yeah hes asked it to go away because theyre trying to tidy up .", "words": ["up", "go", "to", "it", "away", "because"]}, {"sent": "go away .", "words": ["away", "go"]}, {"sent": "perhaps we needta put the kitchen away another day .", "words": ["the", "put", "we", "another", "away", "kitchen"]}, {"sent": "it will blow fly away didnt I ?", "words": ["will", "away", "it", "blow"]}, {"sent": "put that away .", "words": ["put", "away", "that"]}, {"sent": "we were away Sylvia for a week .", "words": ["for", "we", "away", "were", "a"]}, {"sent": "Ive poured yours away now .", "words": ["away"]}, {"sent": "is daddy away now ?", "words": ["away", "is"]}, {"sent": "well put that nasty fire away .", "words": ["put", "away", "that"]}, {"sent": "shall I put the rabbit away ?", "words": ["the", "put", "away"]}, {"sent": "Im away .", "words": ["away"]}, {"sent": "well Ill put it away .", "words": ["put", "it", "away"]}, {"sent": "okay were gonna keep licking that were gonna put it away .", "words": ["put", "it", "away", "that", "were"]}, {"sent": "you can still get away with the summery things .", "words": ["can", "the", "with", "you", "get", "away"]}, {"sent": "well it went away went way over there past the stopsign .", "words": ["the", "there", "it", "away", "over"]}, {"sent": "put it away .", "words": ["put", "it", "away"]}, {"sent": "dont do that dinosaur or well put you away .", "words": ["do", "put", "you", "dont", "away", "that"]}, {"sent": "throw it away .", "words": ["away", "it", "throw"]}, {"sent": "you put the toys away ?", "words": ["the", "put", "away", "you"]}, {"sent": "no you took it away .", "words": ["it", "away", "you"]}, {"sent": "lets lets move it away from the microphone .", "words": ["the", "it", "away"]}, {"sent": "they throwed it away .", "words": ["they", "it", "away"]}, {"sent": "little pumpkin scoots away so fast little pumpkins grandma .", "words": ["fast", "so", "little", "away", "pumpkin"]}, {"sent": "Ill take the book away .", "words": ["the", "book", "away", "take"]}, {"sent": "is she putting away clothes ?", "words": ["away", "she", "is"]}, {"sent": "ah well put him away .", "words": ["put", "away", "him"]}, {"sent": "you want you wanna throw that away should we bring that to the garbage can ?", "words": ["the", "can", "throw", "we", "to", "bring", "you", "away", "wanna", "that", "garbage"]}, {"sent": "Im getting away from that meat eater .", "words": ["away", "that", "meat"]}, {"sent": "you were talking away and Im sitting here wondering should I xxx ?", "words": ["and", "here", "you", "away", "were"]}, {"sent": "can you make the bunny go away ?", "words": ["can", "the", "bunny", "go", "you", "away", "make"]}, {"sent": "are you putting everything away ?", "words": ["are", "away", "you"]}, {"sent": "then he shut up the things with the box with the hook and the cat went away with a sad kind of look .", "words": ["up", "the", "box", "of", "with", "and", "cat", "away", "he", "look", "sad", "then", "a"]}, {"sent": "put them back in your hat and you take them away .", "words": ["put", "your", "back", "them", "and", "you", "in", "away", "hat", "take"]}, {"sent": "because if I went away when Anna came to leave youd be all by yourself in the house .", "words": ["the", "all", "yourself", "if", "to", "be", "by", "in", "away", "house", "when", "because"]}, {"sent": "that all your bottles are gonna be thrown away .", "words": ["all", "your", "be", "are", "away", "that"]}, {"sent": "away from the dough .", "words": ["the", "away"]}, {"sent": "see it rolled away huh ?", "words": ["see", "it", "away"]}, {"sent": "aw well if theyre going to fight I think we should put all this away its not nice fighting all the time .", "words": ["the", "all", "think", "put", "if", "we", "to", "nice", "this", "away", "not"]}, {"sent": "hes driving away .", "words": ["away"]}, {"sent": "well why dont we put these toys away and have a break ?", "words": ["these", "put", "we", "and", "have", "dont", "away", "break", "why", "a"]}, {"sent": "you jump and Ill put them away .", "words": ["put", "them", "and", "you", "jump", "away"]}, {"sent": "Im not throwing it away Im putting it away .", "words": ["away", "it", "not"]}, {"sent": "putting the things away .", "words": ["the", "away"]}, {"sent": "go away bear .", "words": ["bear", "away", "go"]}, {"sent": "theres another butterfly over there .", "words": ["another", "there", "over", "butterfly"]}, {"sent": "turn your pen over turn your pen over .", "words": ["over", "your", "pen"]}, {"sent": "come over here Alexander .", "words": ["here", "over"]}, {"sent": "whos that over there ?", "words": ["over", "there", "that"]}, {"sent": "you wanna take that down you can bring that over .", "words": ["can", "bring", "you", "wanna", "over", "take", "down", "that"]}, {"sent": "dont throw yourself all over my neck .", "words": ["all", "my", "yourself", "throw", "dont", "over"]}, {"sent": "the xxx over there .", "words": ["the", "there", "over"]}, {"sent": "and over the weekend one or two big lorrys stay for the weekend while the drivers have a rest .", "words": ["the", "for", "and", "have", "over", "stay", "big", "a"]}, {"sent": "she put a piece of paper over her comb then she hummed through its teeth .", "words": ["put", "of", "paper", "her", "comb", "then", "over", "she", "a"]}, {"sent": "dont take the top of there because thats all the those cotton buds that weve just nicely put away and if the top comes off theyll be all over the floor now wont they ?", "words": ["the", "all", "put", "off", "those", "if", "of", "be", "and", "dont", "there", "that", "away", "they", "over", "take", "because"]}, {"sent": "come over here Summer .", "words": ["here", "over"]}, {"sent": "Leslie came over last night while you were asleep and left them out .", "words": ["last", "them", "you", "and", "asleep", "out", "over", "were"]}, {"sent": "tip me over and pour me out .", "words": ["pour", "and", "out", "me", "over"]}, {"sent": "she went right over there to the breakdown truck .", "words": ["the", "to", "truck", "there", "over", "she"]}, {"sent": "if you sit here nicely and have a nice big wee without getting it all over everywhere maybe I will .", "words": ["all", "if", "nice", "will", "and", "here", "you", "have", "it", "over", "sit", "big", "a"]}, {"sent": "Im un sit over here .", "words": ["here", "sit", "over"]}, {"sent": "because hes come to this park and he can see it over there .", "words": ["can", "to", "and", "this", "there", "park", "he", "it", "see", "over", "because"]}, {"sent": "and it looks like there is a space like that over here a triangle space .", "words": ["is", "like", "and", "here", "there", "it", "over", "that", "a"]}, {"sent": "over ?", "words": ["over"]}, {"sent": "hes not turned over the pages .", "words": ["the", "over", "not"]}, {"sent": "no come over here .", "words": ["here", "over"]}, {"sent": "no you do it over here .", "words": ["do", "you", "here", "it", "over"]}, {"sent": "xxx over there ?", "words": ["there", "over"]}, {"sent": "wed better go over there quick .", "words": ["better", "over", "there", "go"]}, {"sent": "oh wow and look over here .", "words": ["here", "look", "over", "and"]}, {"sent": "let me build it first then you can come crawl over and try to knock it down .", "words": ["can", "try", "to", "first", "you", "and", "it", "build", "knock", "me", "then", "down", "over", "try to"]}, {"sent": "xxx no get over there .", "words": ["get", "there", "over"]}, {"sent": "over there .", "words": ["there", "over"]}, {"sent": "Misterfox is over there somewhere .", "words": ["there", "over", "is"]}, {"sent": "this mans got a bag over his shoulder .", "words": ["shoulder", "this", "his", "over", "a"]}, {"sent": "turn it over lets see .", "words": ["see", "it", "over"]}, {"sent": "you knocked your drink over .", "words": ["over", "your", "drink", "you"]}, {"sent": "its not nice to push people over on the snow is it ?", "words": ["the", "push", "is", "to", "nice", "snow", "it", "on", "over", "not"]}, {"sent": "uh this one matches over there see ?", "words": ["see", "this", "over", "there"]}, {"sent": "you would have spilt it all over the table .", "words": ["the", "all", "table", "you", "have", "it", "would", "over"]}, {"sent": "were falling over one another .", "words": ["over", "were", "another"]}, {"sent": "oh youre gonna crawl over and get it ?", "words": ["get", "it", "over", "and"]}, {"sent": "you know your little voices travel all over the room and you are very very loud .", "words": ["the", "all", "your", "little", "you", "and", "room", "are", "over", "loud"]}, {"sent": "here come over here .", "words": ["here", "over"]}, {"sent": "you gotta be over there right over here .", "words": ["be", "you", "here", "there", "over"]}, {"sent": "you mean way over there ?", "words": ["there", "over", "you"]}, {"sent": "wanna sit over here son ?", "words": ["here", "sit", "over", "wanna"]}, {"sent": "over here for now .", "words": ["here", "for", "over"]}, {"sent": "I am sitting but I wanna take it over to .", "words": ["to", "it", "am", "wanna", "over", "take", "but"]}, {"sent": "so we can have our house over here .", "words": ["can", "we", "so", "our", "have", "here", "house", "over"]}, {"sent": "scootch over .", "words": ["over"]}, {"sent": "move it over a little bit .", "words": ["it", "over", "little", "a"]}, {"sent": "mhm do you see that jack over there ?", "words": ["do", "you", "there", "over", "see", "that"]}, {"sent": "this is the man with the suit and this is the man whos got a blue bag over his shoulder .", "words": ["the", "is", "with", "and", "blue", "this", "shoulder", "his", "over", "a"]}, {"sent": "up over the bummie .", "words": ["up", "the", "over"]}, {"sent": "oh now I hafta put you over here you are such a little silly .", "words": ["put", "you", "here", "little", "are", "over", "a"]}, {"sent": "over here .", "words": ["here", "over"]}, {"sent": "is it not over there ?", "words": ["is", "there", "it", "over", "not"]}, {"sent": "this one should be over .", "words": ["this", "be", "over"]}, {"sent": "well take them over there just the blocks okay .", "words": ["the", "them", "there", "take", "over"]}, {"sent": "on this page maybe over by mommys hand .", "words": ["hand", "by", "this", "on", "over"]}, {"sent": "over here Kalie .", "words": ["here", "over"]}, {"sent": "she couldnt get over it at all .", "words": ["all", "get", "it", "at", "over", "she"]}, {"sent": "I was just going to wipe the surfaces over now .", "words": ["the", "to", "was", "wipe", "over"]}, {"sent": "yeah you go over here at the outside .", "words": ["the", "go", "outside", "you", "here", "at", "over"]}, {"sent": "pass it over here and maybe youll sit still uh ?", "words": ["and", "here", "it", "sit", "over"]}, {"sent": "come back over here and play with .", "words": ["back", "play", "with", "and", "here", "over"]}, {"sent": "who was over here for supper last night ?", "words": ["for", "was", "last", "here", "who", "over"]}, {"sent": "Thomas come over here .", "words": ["here", "over"]}, {"sent": "come sit over here .", "words": ["here", "sit", "over"]}, {"sent": "well go over there in a minute okay .", "words": ["go", "there", "in", "over", "a"]}, {"sent": "how come you threw that spoon over there ?", "words": ["how", "you", "there", "that", "spoon", "over"]}, {"sent": "no over here look over to this picture .", "words": ["to", "picture", "here", "this", "look", "over"]}, {"sent": "what do you have over there ?", "words": ["what", "do", "you", "have", "there", "over"]}, {"sent": "bring your train over here .", "words": ["your", "train", "bring", "here", "over"]}, {"sent": "oh the crumbs are going all over the floor arent they ?", "words": ["the", "all", "they", "are", "over"]}, {"sent": "somebody has got a dustbin over there .", "words": ["there", "over", "a"]}, {"sent": "over your socks .", "words": ["your", "over"]}, {"sent": "op dont you tip over .", "words": ["dont", "over", "you"]}, {"sent": "when I went over .", "words": ["over", "when"]}, {"sent": "need a really long piece to go over the top .", "words": ["the", "need", "go", "to", "long", "over", "a"]}, {"sent": "put it over my toe .", "words": ["put", "my", "toe", "it", "over"]}, {"sent": "I dont think weve gone over these books these particular ones with her .", "words": ["these", "think", "with", "her", "dont", "over"]}, {"sent": "you were safe when you crossed over ?", "words": ["over", "were", "when", "you"]}, {"sent": "you knocked it over .", "words": ["it", "over", "you"]}, {"sent": "he fell over .", "words": ["over", "he"]}, {"sent": "you go over there then .", "words": ["go", "you", "there", "then", "over"]}, {"sent": "wanna see whats over here ?", "words": ["here", "see", "over", "wanna"]}, {"sent": "all over again .", "words": ["all", "over"]}, {"sent": "over the ocean and oh what a trip .", "words": ["the", "what", "and", "over", "a"]}, {"sent": "you can sleep over when Mommy goes away .", "words": ["can", "when", "sleep", "you", "away", "over"]}, {"sent": "the one over near the door there .", "words": ["the", "there", "over", "door"]}, {"sent": "is it over here with the big trucks ?", "words": ["the", "is", "with", "here", "big", "it", "over"]}, {"sent": "your potties over there pet .", "words": ["over", "there", "your"]}, {"sent": "and put that wire clip over there like that .", "words": ["put", "like", "and", "there", "that", "over"]}, {"sent": "youre gonna knock my car over ?", "words": ["car", "my", "knock", "over"]}, {"sent": "ah whats that over there ?", "words": ["over", "there", "that"]}, {"sent": "you want me to find out if they wanna come over ?", "words": ["if", "to", "you", "they", "out", "find", "wanna", "me", "over"]}, {"sent": "thats why you dont throw things all over the place because it makes a mess and then if you step on them like you did you hurt yourself .", "words": ["the", "yourself", "throw", "like", "it", "on", "then", "why", "all", "if", "hurt", "you", "and", "dont", "because", "a", "them", "did", "over"]}, {"sent": "sit over this way .", "words": ["this", "sit", "over"]}, {"sent": "maybe you could put a smock over that or Graemes shirt will be covered with spaghetti .", "words": ["put", "spaghetti", "shirt", "be", "with", "you", "that", "will", "could", "over", "a"]}, {"sent": "and you come over and buy something .", "words": ["over", "buy", "you", "and"]}, {"sent": "fall over again .", "words": ["over", "fall"]}, {"sent": "I think maybe you had so much ginger youre falling over .", "words": ["think", "so", "much", "you", "over"]}, {"sent": "find your coat hat and scarf .", "words": ["your", "scarf", "and", "find", "hat", "coat"]}, {"sent": "Im not gonna tell you fold up your coat no more Kim .", "words": ["up", "your", "not", "you", "more", "coat"]}, {"sent": "a lovely cat shes got a lovely coat hasnt she ?", "words": ["cat", "she", "coat", "a"]}, {"sent": "mommy coat .", "words": ["coat"]}, {"sent": "here give us that coat xxx .", "words": ["here", "us", "give", "that", "coat"]}, {"sent": "that ladys got a green coat on .", "words": ["green", "on", "that", "coat", "a"]}, {"sent": "are you not a bit warm with your coat on Thomas and the waist coat ?", "words": ["the", "your", "with", "coat", "you", "and", "are", "on", "not", "a"]}, {"sent": "Michael dis is the coat you gonna wear .", "words": ["the", "coat", "you", "is"]}, {"sent": "okay gimme your coat now .", "words": ["your", "coat"]}, {"sent": "Daddys coat ?", "words": ["coat"]}, {"sent": "thats a coat .", "words": ["coat", "a"]}, {"sent": "where do you wanna put your coat ?", "words": ["where", "do", "put", "your", "you", "wanna", "coat"]}, {"sent": "I think Peggy be wearing a white or plaid coat something .", "words": ["think", "be", "white", "coat", "a"]}, {"sent": "put his coat on .", "words": ["his", "put", "on", "coat"]}, {"sent": "shes wearing an orange coat .", "words": ["an", "orange", "coat"]}, {"sent": "you lost your coat .", "words": ["your", "coat", "you"]}, {"sent": "and a coat .", "words": ["coat", "a", "and"]}, {"sent": "twenty five pounds for a winter coat .", "words": ["for", "coat", "a"]}, {"sent": "thats the lady with the green coat on isnt it ?", "words": ["the", "with", "it", "green", "on", "coat"]}, {"sent": "theres a coat .", "words": ["coat", "a"]}, {"sent": "lift up your coat .", "words": ["up", "your", "coat"]}, {"sent": "a coat very good .", "words": ["coat", "good", "a"]}, {"sent": "left my coat there aswell .", "words": ["there", "my", "coat"]}, {"sent": "yes that is coat .", "words": ["that", "coat", "is"]}, {"sent": "a coat .", "words": ["coat", "a"]}, {"sent": "maybe we should make a rain coat for Koala baby koala Baby Kate Koala .", "words": ["for", "rain", "we", "make", "coat", "a"]}, {"sent": "well weve lost the button of your coat this morning somewhere havent we ?", "words": ["the", "your", "we", "of", "this", "button", "coat"]}, {"sent": "look at the coat .", "words": ["the", "at", "coat", "look"]}, {"sent": "because I just wanna put your coat on .", "words": ["put", "your", "wanna", "on", "coat", "because"]}, {"sent": "you wanna get your coat on and go for a walk with Mom ?", "words": ["for", "your", "go", "with", "you", "get", "and", "wanna", "on", "walk", "coat", "a"]}, {"sent": "coat hanger .", "words": ["coat"]}, {"sent": "its not a coat .", "words": ["coat", "not", "a"]}, {"sent": "come on and let Daddy take your coat off .", "words": ["off", "your", "and", "on", "take", "coat"]}, {"sent": "put his coat on .", "words": ["his", "put", "on", "coat"]}, {"sent": "would you like to try isnt that a nice coat ?", "words": ["to", "nice", "like", "you", "that", "would", "try", "coat", "a"]}, {"sent": "thats called a quilted coat a rag coat .", "words": ["coat", "a"]}, {"sent": "well whats all this mess on the front of your ah coat right there ?", "words": ["the", "all", "your", "of", "this", "there", "on", "coat"]}, {"sent": "and Mrs xxx mended this coat up .", "words": ["up", "this", "coat", "and"]}, {"sent": "this ladys got a green coat on .", "words": ["this", "green", "on", "coat", "a"]}, {"sent": "would you like to feel the wooly coat on the sheep ?", "words": ["the", "to", "like", "you", "sheep", "would", "on", "coat"]}, {"sent": "what colors your coat ?", "words": ["what", "your", "coat"]}, {"sent": "and she has her big coat so hes trying to get her in and shes so enervated shes like rr .", "words": ["so", "to", "her", "like", "and", "get", "in", "she", "big", "coat"]}, {"sent": "hang up your coat .", "words": ["up", "your", "coat"]}, {"sent": "you may get your coat .", "words": ["get", "your", "coat", "you"]}, {"sent": "we bought you a new coat didnt we ?", "words": ["we", "you", "new", "coat", "a"]}, {"sent": "wee coat that was always xxx .", "words": ["was", "that", "coat"]}, {"sent": "okay waitll I get my coat on .", "words": ["get", "my", "on", "coat"]}, {"sent": "dont take coat off .", "words": ["off", "dont", "take", "coat"]}, {"sent": "you going for the coat again ?", "words": ["the", "for", "coat", "you"]}, {"sent": "xxx here xxx take your coat and put it in the closet .", "words": ["the", "put", "your", "closet", "and", "here", "in", "it", "take", "coat"]}, {"sent": "and that way you wont hafta take your coat off .", "words": ["off", "your", "you", "and", "that", "take", "coat"]}, {"sent": "bring his coat ?", "words": ["his", "coat", "bring"]}, {"sent": "a coat and red boots .", "words": ["red", "and", "boots", "coat", "a"]}, {"sent": "put the coat up here where it goes .", "words": ["the", "up", "put", "where", "here", "it", "coat"]}, {"sent": "coat .", "words": ["coat"]}, {"sent": "a green coat .", "words": ["coat", "green", "a"]}, {"sent": "would he be alright like about where to put his lunchbox and where to put his coat and making sure his coat and changing his shoes and all that sorta stuff and going to the toilet by himself and all that ?", "words": ["where", "all", "put", "the", "be", "to", "like", "and", "by", "he", "about", "would", "his", "that", "coat"]}, {"sent": "put your coat and cap on Kim .", "words": ["put", "your", "and", "on", "coat"]}, {"sent": "he has got a hood on his coat .", "words": ["his", "he", "on", "coat", "a"]}, {"sent": "what I did last Saturday was I got another coat on .", "words": ["what", "was", "last", "another", "did", "on", "coat"]}, {"sent": "was it a long or a short coat ?", "words": ["was", "long", "it", "coat", "a"]}, {"sent": "get your coat .", "words": ["get", "your", "coat"]}, {"sent": "Mommys coat ?", "words": ["coat"]}, {"sent": "Mummys putting one or two of these things on coat hangers look .", "words": ["these", "of", "look", "on", "coat"]}, {"sent": "and theres a blue coat .", "words": ["blue", "coat", "a", "and"]}, {"sent": "take your coat off ?", "words": ["your", "take", "coat", "off"]}, {"sent": "Pat PostmanPat has got a coat on hasnt he ?", "words": ["on", "he", "coat", "a"]}, {"sent": "okay let Mama put the coat on .", "words": ["the", "put", "on", "coat"]}, {"sent": "Mommys shoes and Mandys coat .", "words": ["coat", "and"]}, {"sent": "did you pull it off the coat rack ?", "words": ["off", "the", "pull", "you", "it", "did", "coat"]}, {"sent": "go and get your coat and keep your Panda .", "words": ["your", "go", "and", "get", "coat"]}, {"sent": "I only want them for my my little boy Thomass coat .", "words": ["my", "for", "them", "little", "coat"]}, {"sent": "and your new shoes and your new coat .", "words": ["new", "your", "coat", "and"]}, {"sent": "put your coat on .", "words": ["on", "put", "your", "coat"]}, {"sent": "oh is that a duvet coat ?", "words": ["that", "coat", "a", "is"]}, {"sent": "zip your coat .", "words": ["your", "coat"]}, {"sent": "I dont think dolly has a coat .", "words": ["dont", "think", "coat", "a"]}, {"sent": "youve still got your coat on havent you ?", "words": ["on", "your", "coat", "you"]}, {"sent": "so this time we thought wed get a different color to go with your brown coat and then we can leave the blue ones in your blue coat .", "words": ["can", "the", "your", "we", "go", "so", "to", "brown", "with", "and", "get", "this", "blue", "in", "then", "coat", "a"]}, {"sent": "xxx wretched coat .", "words": ["coat"]}, {"sent": "that is his coat .", "words": ["his", "that", "coat", "is"]}, {"sent": "when she arrived home she took her coat off and hung it on the coat peg .", "words": ["off", "the", "when", "and", "it", "she", "on", "her", "coat", "home"]}, {"sent": "lets get your coat and then well play with those .", "words": ["your", "those", "play", "with", "and", "get", "then", "coat"]}, {"sent": "the one with the green coat on .", "words": ["the", "with", "green", "on", "coat"]}, {"sent": "are you going to a coat or anything ?", "words": ["to", "you", "are", "coat", "a"]}, {"sent": "this coat is the cleanest I think you should wear it .", "words": ["the", "think", "is", "you", "this", "it", "coat"]}, {"sent": "Im keeping my coat on .", "words": ["on", "my", "coat"]}, {"sent": "where has the coat gone ?", "words": ["where", "the", "coat"]}, {"sent": "the one with the green coat thats right .", "words": ["the", "with", "coat", "green"]}, {"sent": "coat .", "words": ["coat"]}, {"sent": "this is the lady with the green coat on .", "words": ["the", "is", "with", "this", "green", "on", "coat"]}, {"sent": "what hides from the lions has a black and white striped coat .", "words": ["the", "what", "and", "black", "white", "coat", "a"]}, {"sent": "why doesnt she have any coat on ?", "words": ["any", "have", "she", "on", "coat", "why"]}, {"sent": "blue coat oops see thats a jacket .", "words": ["coat", "blue", "see", "jacket", "a"]}, {"sent": "I know but do you know whose coat dis is ?", "words": ["do", "is", "you", "coat", "but"]}, {"sent": "are you busy playing with some coat hangers ?", "words": ["with", "you", "are", "some", "coat"]}, {"sent": "mummies coat .", "words": ["coat"]}, {"sent": "right and a coat .", "words": ["coat", "a", "and"]}, {"sent": "his coat ?", "words": ["his", "coat"]}, {"sent": "and whose black car ?", "words": ["black", "car", "and"]}, {"sent": "well the crayon is black .", "words": ["the", "black", "crayon", "is"]}, {"sent": "we could do with a black one there couldnt we ?", "words": ["do", "we", "with", "black", "there", "could", "a"]}, {"sent": "its red and black isnt it ?", "words": ["red", "it", "black", "and"]}, {"sent": "a soccer balls white and black and the little black shapes are pentagons .", "words": ["the", "and", "black", "little", "are", "white", "a"]}, {"sent": "blue sky and a big black cloud with lots of rain .", "words": ["rain", "sky", "of", "with", "and", "blue", "black", "cloud", "big", "a"]}, {"sent": "the black ones .", "words": ["the", "black"]}, {"sent": "do you remember he was having a fight with the black and white cat wasnt he ?", "words": ["the", "do", "was", "with", "you", "black", "and", "cat", "he", "white", "a"]}, {"sent": "and a black .", "words": ["black", "a", "and"]}, {"sent": "because hes black he goes really really brown and that .", "words": ["brown", "and", "black", "he", "that", "because"]}, {"sent": "thats black .", "words": ["black"]}, {"sent": "its a big black roof .", "words": ["black", "big", "roof", "a"]}, {"sent": "no just black thanks the way it is .", "words": ["black", "the", "it", "is"]}, {"sent": "this is going to get very hot and so were not going to touch this black part at all okay ?", "words": ["all", "is", "to", "so", "touch", "and", "get", "this", "hot", "black", "at", "were", "not"]}, {"sent": "but you can have black sheep .", "words": ["can", "you", "have", "black", "sheep", "but"]}, {"sent": "there was a black monkey ?", "words": ["was", "monkey", "black", "there", "a"]}, {"sent": "black .", "words": ["black"]}, {"sent": "the animal eats the black thing ?", "words": ["the", "animal", "black"]}, {"sent": "I must say I am surprised that little black engine can pull that big train arent you ?", "words": ["can", "train", "say", "little", "black", "pull", "big", "you", "am", "that"]}, {"sent": "a black cord .", "words": ["black", "a"]}, {"sent": "one man has wavy black hair streaked with white like lightning .", "words": ["hair", "with", "like", "black", "white"]}, {"sent": "GrandmaBealbys pussy cat is black and white isnt it ?", "words": ["is", "and", "black", "cat", "it", "white"]}, {"sent": "a black kitty yeah .", "words": ["black", "kitty", "a"]}, {"sent": "he has black eyes .", "words": ["black", "he"]}, {"sent": "see if theres a black plum .", "words": ["black", "see", "if", "a"]}, {"sent": "black and white .", "words": ["black", "white", "and"]}, {"sent": "you like black .", "words": ["black", "like", "you"]}, {"sent": "got a black and blue mark .", "words": ["black", "blue", "and", "a"]}, {"sent": "one pitch black .", "words": ["black"]}, {"sent": "have they gathered all the black bags up into a pile ?", "words": ["the", "all", "up", "into", "have", "they", "black", "a"]}, {"sent": "can you see that pandabears black and white ?", "words": ["can", "you", "black", "and", "see", "white", "that"]}, {"sent": "black .", "words": ["black"]}, {"sent": "yours is a pink and black sheep isnt it ?", "words": ["is", "and", "black", "it", "sheep", "a"]}, {"sent": "I cant work out which is black and which is green .", "words": ["work", "is", "and", "black", "out", "green", "which"]}, {"sent": "no thats thats why I think alotof sports especially on the east coast black children rarely learn to ice skate .", "words": ["the", "think", "to", "ice", "skate", "black", "on", "why"]}, {"sent": "these black ones ?", "words": ["black", "these"]}, {"sent": "black black black .", "words": ["black"]}, {"sent": "theyve all got a strip of black and thats called a magnet .", "words": ["all", "of", "and", "black", "a"]}, {"sent": "I dont like black Ste .", "words": ["black", "dont", "like"]}, {"sent": "and it wasnt but there was a little girls white shoe and a little boys it was either black or blue shoe .", "words": ["was", "and", "little", "there", "shoe", "it", "black", "blue", "white", "but", "a"]}, {"sent": "oh theres a black one .", "words": ["black", "a"]}, {"sent": "press harder and itll be black .", "words": ["black", "be", "and"]}, {"sent": "let me put that black one there .", "words": ["put", "black", "there", "me", "that"]}, {"sent": "you have the red ones and Ill have the black ones .", "words": ["the", "red", "and", "have", "you", "black"]}, {"sent": "black .", "words": ["black"]}, {"sent": "uh do you think its time to screw these two black bits back on ?", "words": ["do", "think", "these", "back", "to", "you", "black", "on"]}, {"sent": "that umbrellas got black spots like the dog .", "words": ["the", "like", "black", "that", "dog"]}, {"sent": "the red and black .", "words": ["the", "red", "black", "and"]}, {"sent": "Linda and Brians black and white cat .", "words": ["black", "cat", "white", "and"]}, {"sent": "Pooh was pretending to be a small black cloud .", "words": ["to", "was", "be", "black", "cloud", "a"]}, {"sent": "you mean the long black hair ?", "words": ["the", "long", "you", "black", "hair"]}, {"sent": "what things are black ?", "words": ["black", "what", "are"]}, {"sent": "that black .", "words": ["black", "that"]}, {"sent": "miss Mary Mack Mack sat on a doon black black black with silver buttons buttons buttons all down her back back back .", "words": ["all", "back", "with", "black", "on", "down", "her", "a"]}, {"sent": "a black one ?", "words": ["black", "a"]}, {"sent": "little black baby .", "words": ["black", "little"]}, {"sent": "uh I dont know black paint .", "words": ["black", "dont", "paint"]}, {"sent": "its nice to have black hair isnt it ?", "words": ["to", "nice", "have", "black", "it", "hair"]}, {"sent": "putting black sacks through that we can use next week .", "words": ["black", "can", "that", "we"]}, {"sent": "and a black tongue .", "words": ["black", "tongue", "a", "and"]}, {"sent": "should we give the flying head some really black eyes here ?", "words": ["the", "we", "head", "black", "here", "some", "give"]}, {"sent": "that is black .", "words": ["black", "that", "is"]}, {"sent": "show Mommy where the black car is .", "words": ["the", "where", "is", "show", "car", "black"]}, {"sent": "well Donald and Douglas look at sort of greyie black color but James is nice and bright .", "words": ["is", "of", "nice", "and", "black", "at", "look", "but"]}, {"sent": "its usually a very very dark grey not actually black .", "words": ["black", "dark", "not", "a"]}, {"sent": "so one has black tea and the other has herbal .", "words": ["the", "so", "other", "black", "and"]}, {"sent": "say black .", "words": ["black", "say"]}, {"sent": "where is your black crayon at ?", "words": ["where", "your", "is", "crayon", "black", "at"]}, {"sent": "black ?", "words": ["black"]}, {"sent": "remember black ?", "words": ["black"]}, {"sent": "can you see anything else thats black ?", "words": ["can", "see", "black", "you"]}, {"sent": "and what are you going to draw with black ?", "words": ["what", "to", "draw", "with", "you", "and", "black", "are"]}, {"sent": "um keep it on these black things .", "words": ["on", "these", "it", "black"]}, {"sent": "Annas wearing black .", "words": ["black"]}, {"sent": "and the stone is almost black .", "words": ["the", "is", "and", "black", "stone"]}, {"sent": "the black one .", "words": ["the", "black"]}, {"sent": "yeah black eyed Susans .", "words": ["black"]}, {"sent": "one pitch black .", "words": ["black"]}, {"sent": "theres a black lid missing with the black PlayDoh .", "words": ["black", "the", "with", "a"]}, {"sent": "she is in black .", "words": ["black", "in", "she", "is"]}, {"sent": "show Mommy where the black .", "words": ["show", "the", "black", "where"]}, {"sent": "got a black hat .", "words": ["black", "hat", "a"]}, {"sent": "does he have a black hat on his head ?", "words": ["does", "head", "have", "black", "on", "his", "he", "hat", "a"]}, {"sent": "black .", "words": ["black"]}, {"sent": "and the black and white check one has a squeaker .", "words": ["the", "and", "black", "white", "a"]}, {"sent": "mines black already .", "words": ["black"]}, {"sent": "a bit rounder and very very black .", "words": ["black", "and", "a"]}, {"sent": "its funny how the two women have got black hair .", "words": ["the", "how", "have", "black", "hair"]}, {"sent": "I think that one is black .", "words": ["black", "think", "that", "is"]}, {"sent": "dont do your butterfly black whatever you do because theyre too pretty to be black .", "words": ["butterfly", "do", "your", "to", "be", "you", "black", "dont", "pretty", "too", "because"]}, {"sent": "what is black and white .", "words": ["what", "is", "and", "black", "white"]}, {"sent": "wheres the black horsie ?", "words": ["the", "black"]}, {"sent": "thats like on your cat your kitty has black spots and white spots ?", "words": ["your", "like", "and", "black", "cat", "kitty", "on", "white"]}, {"sent": "this fishy is black .", "words": ["black", "this", "is"]}, {"sent": "we sang baa baa black sheep .", "words": ["black", "sheep", "we"]}, {"sent": "a black and a gray .", "words": ["black", "and", "a"]}, {"sent": "this is the black board .", "words": ["the", "this", "black", "is"]}, {"sent": "it says Pooh was rolling in the mud until he was black all over .", "words": ["the", "all", "was", "black", "in", "it", "he", "over"]}, {"sent": "we get black ants sometimes .", "words": ["get", "black", "we"]}, {"sent": "who do we know thats black and white ?", "words": ["do", "we", "and", "black", "who", "white"]}, {"sent": "can you bring the photos through darling ?", "words": ["can", "the", "bring", "you"]}, {"sent": "I might even pop down and bring Kathryn some day .", "words": ["pop", "bring", "and", "some", "down"]}, {"sent": "cards some cards that I bring ?", "words": ["some", "that", "bring"]}, {"sent": "aw I didnt bring the tissues very close .", "words": ["the", "close", "bring"]}, {"sent": "can you bring the photo photo frame ?", "words": ["can", "the", "bring", "you"]}, {"sent": "bring bring that over here .", "words": ["over", "here", "that", "bring"]}, {"sent": "bring the paper through there .", "words": ["the", "there", "paper", "bring"]}, {"sent": "we can bring Prudence down yeah and then we gotta go .", "words": ["can", "we", "go", "bring", "and", "then", "down"]}, {"sent": "Caroline didnt bring that .", "words": ["that", "bring"]}, {"sent": "right bring it out now .", "words": ["it", "bring", "out"]}, {"sent": "you bring the bag of shopping over .", "words": ["the", "of", "bring", "you", "over"]}, {"sent": "shall we bring the car ?", "words": ["the", "car", "bring", "we"]}, {"sent": "bring the baby to bed .", "words": ["the", "to", "bring", "bed"]}, {"sent": "then well bring them in .", "words": ["in", "then", "them", "bring"]}, {"sent": "do you want me to bring your jigsawpuzzles in here ?", "words": ["do", "your", "to", "bring", "you", "here", "in", "me"]}, {"sent": "can you bring her ?", "words": ["can", "her", "bring", "you"]}, {"sent": "go put it away and bring the box to Mommy .", "words": ["the", "put", "box", "go", "to", "bring", "and", "it", "away"]}, {"sent": "bring it to daddy daddy can read tiny tiger .", "words": ["can", "tiny", "to", "bring", "it", "read", "tiger"]}, {"sent": "you going to bring it here ?", "words": ["to", "bring", "you", "here", "it"]}, {"sent": "shall I bring bring her out ?", "words": ["her", "bring", "out"]}, {"sent": "oh okay well Patsy and I forgot to bring our car with us .", "words": ["to", "car", "with", "bring", "our", "and", "us"]}, {"sent": "well bring it out here and Ill look .", "words": ["bring", "and", "here", "it", "out", "look"]}, {"sent": "bring a book over .", "words": ["book", "over", "bring", "a"]}, {"sent": "can you bring me the towel ?", "words": ["can", "the", "bring", "you", "towel", "me"]}, {"sent": "well shall I bring them in so we can look at them ?", "words": ["can", "we", "so", "them", "bring", "in", "at", "look"]}, {"sent": "well xxx Mummy bring a cup of tea .", "words": ["cup", "of", "bring", "a"]}, {"sent": "do not bring this to the nursery school .", "words": ["the", "do", "to", "school", "bring", "this", "not"]}, {"sent": "they can come out to the side listen Todd how can we bring this train and train next to closer to eachother ?", "words": ["can", "the", "how", "listen", "we", "train", "to", "bring", "and", "they", "this", "out"]}, {"sent": "bring it over here by the .", "words": ["the", "bring", "by", "here", "it", "over"]}, {"sent": "can you bring it over here Abe ?", "words": ["can", "bring", "you", "here", "it", "over"]}, {"sent": "what did he bring for christmas ?", "words": ["what", "for", "bring", "he", "did"]}, {"sent": "they bring things like waiters do at a restaurant ?", "words": ["do", "bring", "like", "they", "at", "a"]}, {"sent": "xxx bring it over to xxx to me ?", "words": ["to", "bring", "it", "me", "over"]}, {"sent": "bring the letters back to me .", "words": ["the", "back", "to", "bring", "me"]}, {"sent": "he told me to bring that uh .", "words": ["to", "bring", "he", "me", "that"]}, {"sent": "bring me the red pen when youre done .", "words": ["the", "red", "bring", "pen", "me", "when"]}, {"sent": "Dimitra useta bring one just the same didnt she ?", "words": ["the", "same", "bring", "she"]}, {"sent": "bring it Loi bring it to her .", "words": ["to", "it", "her", "bring"]}, {"sent": "can I just bring you down for a second ?", "words": ["can", "for", "bring", "you", "down", "a"]}, {"sent": "then Ill bring it .", "words": ["then", "it", "bring"]}, {"sent": "oh you know what we hafta do djou wanna help me pick out some teacups to bring to Daisys party ?", "words": ["what", "do", "help", "we", "pick", "to", "bring", "you", "party", "wanna", "out", "me", "some"]}, {"sent": "well Ill maybe bring it next time .", "words": ["it", "bring"]}, {"sent": "bring over here .", "words": ["here", "over", "bring"]}, {"sent": "bring flowers .", "words": ["bring"]}, {"sent": "bring daddy ?", "words": ["bring"]}, {"sent": "lets bring him with us okay ?", "words": ["us", "him", "with", "bring"]}, {"sent": "and grandpa said dont get any dessert for dinner tonight because were gonna bring something special .", "words": ["for", "any", "bring", "and", "get", "dont", "were", "because"]}, {"sent": "and my brother bring his friens up here he leave em out too .", "words": ["up", "my", "bring", "and", "here", "he", "out", "too", "his"]}, {"sent": "and then Ill bring it back sometimes .", "words": ["back", "bring", "and", "it", "then"]}, {"sent": "bring your chair here with Mama .", "words": ["your", "with", "bring", "chair", "here"]}, {"sent": "now we bring her arms down .", "words": ["down", "her", "bring", "we"]}, {"sent": "did you bring bread to everybody in school ?", "words": ["bread", "to", "school", "bring", "you", "in", "did"]}, {"sent": "you going to bring them down one at a time two at a time like that ?", "words": ["to", "them", "bring", "like", "you", "at", "down", "that", "a"]}, {"sent": "did you bring that little pocket knife with you ?", "words": ["knife", "with", "bring", "little", "you", "did", "that"]}, {"sent": "Renees gonna pick you up at school and bring you home .", "words": ["up", "pick", "school", "bring", "you", "and", "at", "home"]}, {"sent": "okay youre gonna bring the Barneys downstairs .", "words": ["the", "bring"]}, {"sent": "but I thought Id bring it down today to show it to Michelle and Nicola .", "words": ["show", "to", "bring", "and", "it", "down", "but"]}, {"sent": "well then you bring it outside .", "words": ["outside", "bring", "you", "it", "then"]}, {"sent": "you get a book and bring it to me and Ill read it to you .", "words": ["to", "bring", "and", "get", "you", "it", "read", "book", "me", "a"]}, {"sent": "bring the paper first .", "words": ["the", "paper", "first", "bring"]}, {"sent": "you gonna bring your doggie ?", "words": ["your", "bring", "you"]}, {"sent": "shall we bring your teaset in here and then we can you can put it on the plate .", "words": ["can", "the", "put", "your", "plate", "we", "bring", "and", "here", "you", "in", "it", "on", "then"]}, {"sent": "you can bring the clock .", "words": ["can", "the", "clock", "bring", "you"]}, {"sent": "what are ya gonna bring me ?", "words": ["what", "me", "are", "bring"]}, {"sent": "can you bring it here ?", "words": ["can", "bring", "you", "here", "it"]}, {"sent": "shall I bring them every week ?", "words": ["every", "them", "bring"]}, {"sent": "wait until mummys got the er Amys bottle and then well you can go and sleep in your new bed and Ill Ill bring a story up and we can read a story .", "words": ["the", "go", "bed", "read", "can", "then", "bottle", "up", "sleep", "bring", "and", "you", "new", "a", "wait", "your", "we", "in", "story"]}, {"sent": "well bring em over here .", "words": ["here", "over", "bring"]}, {"sent": "what did he bring her ?", "words": ["what", "bring", "he", "did", "her"]}, {"sent": "xxx Stephanie bring this up like this .", "words": ["up", "this", "bring", "like"]}, {"sent": "um hell give it um sometimes if you tell him give the baby a hug hell youknow bring it to him .", "words": ["the", "if", "to", "bring", "you", "him", "it", "hug", "give", "a"]}, {"sent": "bring it to Mummy .", "words": ["to", "it", "bring"]}, {"sent": "we forgot to bring the wee people to get on the train .", "words": ["the", "we", "train", "to", "bring", "get", "on"]}, {"sent": "dont bring the cover .", "words": ["the", "dont", "bring", "cover"]}, {"sent": "youll hafta remind daddy to bring one home .", "words": ["to", "bring", "home"]}, {"sent": "I wanted ta bring some but I was rushing and I didnt wanna bring the whole thing an I left the house at ten .", "words": ["the", "was", "bring", "and", "an", "house", "wanna", "at", "some", "but"]}, {"sent": "pull those over any more that you wanna bring out to the front .", "words": ["the", "those", "to", "any", "bring", "pull", "you", "more", "that", "wanna", "out", "over"]}, {"sent": "if weve got visitors if weve got a house full of visitors and they bring little children then I always hide your tricycle anyway because the tricycles fine for you to whiz up and down with inside when theres just me and you or me and you and Daddy but when theres a houseful of people there isnt room for the tricycle .", "words": ["the", "fine", "of", "me", "when", "with", "there", "then", "down", "up", "for", "if", "full", "hide", "to", "bring", "and", "little", "you", "tricycle", "because", "a", "your", "inside", "they", "room", "house", "but"]}, {"sent": "alright Ill bring it down .", "words": ["down", "it", "bring"]}, {"sent": "oh why dont you bring bunny here ?", "words": ["bunny", "bring", "you", "here", "dont", "why"]}, {"sent": "bring Mommy a book .", "words": ["book", "bring", "a"]}, {"sent": "okay well bring my cat home thank you .", "words": ["my", "bring", "you", "cat", "home"]}, {"sent": "because well just bring him down when hes sleepy Mom .", "words": ["sleepy", "bring", "him", "down", "when", "because"]}, {"sent": "can you bring me the grapes please .", "words": ["can", "the", "bring", "you", "grapes", "me"]}, {"sent": "you hafta bring it to Mommy .", "words": ["to", "it", "bring", "you"]}, {"sent": "she doesnt want you to feel rejected incase she has intentions no I want you to bring me some where ?", "words": ["where", "to", "bring", "you", "me", "she", "some"]}, {"sent": "bring one for Jennifer too no ?", "words": ["for", "bring", "too"]}, {"sent": "bring your people .", "words": ["your", "bring"]}, {"sent": "whoa go over there Daddys gonna bring it over here .", "words": ["go", "bring", "here", "there", "it", "over"]}, {"sent": "you wanna bring that toy ?", "words": ["bring", "you", "toy", "wanna", "that"]}, {"sent": "oh yeah bring some milk yeah .", "words": ["milk", "some", "bring"]}, {"sent": "and then bring out both hands to show me .", "words": ["show", "to", "bring", "and", "out", "me", "then"]}, {"sent": "bring it in here .", "words": ["here", "in", "it", "bring"]}, {"sent": "you go back in there and Ill bring it .", "words": ["back", "go", "bring", "you", "and", "there", "in", "it"]}, {"sent": "well I hafta bring them home again .", "words": ["them", "bring", "home"]}, {"sent": "bring the horsie here .", "words": ["the", "here", "bring"]}, {"sent": "I I was just trying to bring them in today to change her um the toys she was playing with just to kinda show her other things .", "words": ["the", "show", "to", "was", "them", "bring", "with", "other", "in", "her", "she"]}, {"sent": "Mommy will bring it down for you .", "words": ["for", "bring", "you", "it", "will", "down"]}, {"sent": "its to clean the kitchen floor .", "words": ["clean", "to", "kitchen", "the"]}, {"sent": "its in the kitchen .", "words": ["the", "in", "kitchen"]}, {"sent": "shall we make it the kitchen ?", "words": ["the", "we", "it", "kitchen", "make"]}, {"sent": "heres their kitchen .", "words": ["their", "kitchen"]}, {"sent": "and um she with the kitchen stuff shell put stuff in her pots and pans and mix it up and shell say shes making me breakfast or lunch or dinner .", "words": ["the", "up", "put", "with", "say", "and", "in", "it", "kitchen", "me", "her", "she"]}, {"sent": "lets go in the kitchen and make some jelly .", "words": ["the", "jelly", "go", "and", "in", "kitchen", "some", "make"]}, {"sent": "so it seemed like um he was really interested in playing with the kitchen huh ?", "words": ["the", "was", "so", "with", "like", "in", "it", "he", "kitchen"]}, {"sent": "we have a play kitchen .", "words": ["we", "play", "have", "kitchen", "a"]}, {"sent": "there were all the things in the kitchen .", "words": ["the", "all", "there", "in", "kitchen", "were"]}, {"sent": "well Ill go in the kitchen but you mustnt be rough .", "words": ["the", "go", "be", "you", "in", "kitchen", "but"]}, {"sent": "youll go in the kitchen ?", "words": ["the", "in", "kitchen", "go"]}, {"sent": "look at the kitchen .", "words": ["the", "at", "kitchen", "look"]}, {"sent": "the kitchen .", "words": ["the", "kitchen"]}, {"sent": "thankyou for letting me clean the kitchen floor .", "words": ["clean", "the", "for", "kitchen", "me"]}, {"sent": "maybe you did not say kitchen .", "words": ["say", "you", "did", "kitchen", "not"]}, {"sent": "youre going to brush a wall in the kitchen ?", "words": ["the", "brush", "to", "in", "kitchen", "a"]}, {"sent": "youll hafta go back in the kitchen .", "words": ["the", "back", "go", "in", "kitchen"]}, {"sent": "youre in the kitchen .", "words": ["the", "in", "kitchen"]}, {"sent": "yeah these are both the kitchen .", "words": ["the", "these", "are", "kitchen"]}, {"sent": "hes traveling the kitchen by basket .", "words": ["the", "basket", "kitchen", "by"]}, {"sent": "I mean in the kitchen .", "words": ["the", "in", "kitchen"]}, {"sent": "underneath the kitchen window .", "words": ["the", "window", "kitchen"]}, {"sent": "okay pick up the chair and put it in the kitchen .", "words": ["the", "up", "put", "pick", "chair", "and", "in", "it", "kitchen"]}, {"sent": "are you going to hide in the kitchen ?", "words": ["the", "hide", "to", "you", "in", "are", "kitchen"]}, {"sent": "Ive been out in the kitchen .", "words": ["the", "in", "kitchen", "out"]}, {"sent": "chair here for the kitchen table .", "words": ["the", "for", "table", "chair", "here", "kitchen"]}, {"sent": "okay can you stay in the kitchen with me please ?", "words": ["can", "the", "me", "with", "you", "in", "kitchen", "stay"]}, {"sent": "and Im trying to make a stab at our horrendously messy kitchen .", "words": ["to", "and", "our", "kitchen", "at", "make", "a"]}, {"sent": "uhhuh thats a kitchen sink .", "words": ["sink", "kitchen", "a"]}, {"sent": "mm yeah he has a kitchen at home so he knows .", "words": ["so", "he", "kitchen", "at", "a", "home"]}, {"sent": "is Daddy in the kitchen ?", "words": ["the", "in", "kitchen", "is"]}, {"sent": "we stand them on the shelf in the kitchen dont we ?", "words": ["the", "we", "stand", "them", "dont", "in", "kitchen", "on"]}, {"sent": "no what do you cook at the kitchen .", "words": ["the", "what", "do", "you", "cook", "kitchen", "at"]}, {"sent": "I gobbled water straight out of the handy kitchen sink .", "words": ["the", "water", "of", "sink", "kitchen", "out"]}, {"sent": "and youve the big table and all in your kitchen .", "words": ["the", "all", "your", "table", "and", "in", "kitchen", "big"]}, {"sent": "wheres the kitchen stuff ?", "words": ["the", "kitchen"]}, {"sent": "shes ready now to cook in your kitchen .", "words": ["your", "to", "cook", "in", "kitchen"]}, {"sent": "who else goes in the kitchen cooking ?", "words": ["the", "in", "kitchen", "who"]}, {"sent": "in the kitchen .", "words": ["the", "in", "kitchen"]}, {"sent": "ahhah got a kitchen full of goodies .", "words": ["of", "full", "kitchen", "a"]}, {"sent": "Im going to put it in the kitchen so you can have it when you want it .", "words": ["the", "can", "put", "to", "so", "you", "have", "in", "it", "kitchen", "when"]}, {"sent": "she must have gone to the kitchen to sort containers for Mommy .", "words": ["the", "for", "to", "have", "kitchen", "she"]}, {"sent": "oh the kitchen .", "words": ["the", "kitchen"]}, {"sent": "just round the corner on the kitchen stool .", "words": ["the", "on", "kitchen"]}, {"sent": "are you implying my kitchen looked messy ?", "words": ["my", "are", "kitchen", "you"]}, {"sent": "see here is the kitchen .", "words": ["the", "is", "here", "kitchen", "see"]}, {"sent": "he was in the kitchen last time .", "words": ["the", "was", "last", "in", "he", "kitchen"]}, {"sent": "the kitchen flew away ?", "words": ["the", "away", "kitchen"]}, {"sent": "its cleaner for the kitchen .", "words": ["the", "for", "kitchen"]}, {"sent": "its here in the kitchen .", "words": ["here", "the", "in", "kitchen"]}, {"sent": "his mother is at home in the kitchen ?", "words": ["the", "is", "in", "kitchen", "at", "his", "home"]}, {"sent": "he did not go in the kitchen .", "words": ["the", "go", "in", "he", "did", "kitchen", "not"]}, {"sent": "yep all different words for all the different things in the kitchen .", "words": ["the", "all", "for", "in", "kitchen"]}, {"sent": "well just I was just gonna walk him in the kitchen .", "words": ["the", "was", "him", "in", "kitchen", "walk"]}, {"sent": "we have a little kitchen .", "words": ["we", "little", "have", "kitchen", "a"]}, {"sent": "do you wanna find a plate from your kitchen ?", "words": ["do", "plate", "your", "you", "wanna", "find", "kitchen", "a"]}, {"sent": "yeah she plays with the kitchen .", "words": ["the", "with", "she", "kitchen"]}, {"sent": "whats Blue doing in the kitchen ?", "words": ["the", "in", "kitchen"]}, {"sent": "are there any utensils that shell play with like on the kitchen floor at home or something ?", "words": ["the", "play", "any", "with", "like", "there", "are", "kitchen", "at", "on", "that", "home"]}, {"sent": "go and put it on the kitchen table for me .", "words": ["the", "put", "for", "table", "go", "and", "it", "kitchen", "on", "me"]}, {"sent": "oh look at the kitchen .", "words": ["the", "at", "kitchen", "look"]}, {"sent": "Megan look at all the kitchen .", "words": ["the", "all", "look", "kitchen", "at"]}, {"sent": "Ive just been washing up at the kitchen sink .", "words": ["up", "the", "sink", "kitchen", "at"]}, {"sent": "its a tortoise under the kitchen sink not Danny .", "words": ["the", "under", "sink", "kitchen", "not", "a"]}, {"sent": "take them back to the kitchen .", "words": ["the", "back", "to", "them", "kitchen", "take"]}, {"sent": "you wanna go in the kitchen with mommy ?", "words": ["the", "go", "with", "you", "in", "wanna", "kitchen"]}, {"sent": "heres the kitchen .", "words": ["the", "kitchen"]}, {"sent": "this is the one from the kitchen .", "words": ["the", "this", "kitchen", "is"]}, {"sent": "the trash can does go in the kitchen .", "words": ["the", "does", "can", "go", "trash", "in", "kitchen"]}, {"sent": "theyre in the kitchen arent they ?", "words": ["the", "they", "in", "kitchen"]}, {"sent": "oh what did you put in the kitchen ?", "words": ["the", "what", "put", "you", "in", "did", "kitchen"]}, {"sent": "Dan was polishing the wheels on the fire engine and Anthony was upstairs in the kitchen looking for a snack .", "words": ["the", "for", "was", "and", "in", "kitchen", "on", "a"]}, {"sent": "theres a kitchen .", "words": ["kitchen", "a"]}, {"sent": "well you know to be more stable you should go on the kitchen floor .", "words": ["the", "go", "to", "be", "you", "more", "kitchen", "on"]}, {"sent": "and Im looking in the kitchen and I dont see them .", "words": ["the", "them", "and", "dont", "in", "kitchen", "see"]}, {"sent": "in the kitchen .", "words": ["the", "in", "kitchen"]}, {"sent": "on the kitchen floor ?", "words": ["on", "the", "kitchen"]}, {"sent": "wow look theyre in the kitchen .", "words": ["the", "in", "kitchen", "look"]}, {"sent": "I told you not to put a wooden spoon on your head and you come back from kitchen with a sieve on your head .", "words": ["put", "your", "back", "to", "head", "with", "you", "and", "spoon", "kitchen", "on", "not", "a"]}, {"sent": "hm thats your kitchen .", "words": ["your", "kitchen"]}, {"sent": "we and we have measuring spoons in the kitchen and sippycups for sure .", "words": ["the", "for", "we", "and", "have", "in", "kitchen"]}, {"sent": "that is a kitchen tissue .", "words": ["tissue", "is", "kitchen", "that", "a"]}, {"sent": "I hadta bring it out in the living room so I could mop the kitchen floor .", "words": ["the", "so", "living room", "bring", "in", "it", "room", "out", "mop", "kitchen", "could"]}, {"sent": "Patsy wants you to put the kitchen back together .", "words": ["the", "put", "back", "to", "you", "kitchen"]}, {"sent": "okay go in the kitchen .", "words": ["the", "in", "kitchen", "go"]}, {"sent": "right xxx the kitchen xxx have some breakfast .", "words": ["the", "some", "have", "kitchen"]}, {"sent": "is it a kitchen ?", "words": ["it", "kitchen", "a", "is"]}, {"sent": "here is our kitchen .", "words": ["here", "our", "kitchen", "is"]}, {"sent": "Ive put it away in the kitchen darling .", "words": ["the", "put", "in", "it", "away", "kitchen"]}, {"sent": "that goes in the kitchen .", "words": ["the", "in", "that", "kitchen"]}, {"sent": "its the sound of my slippers sticking to the kitchen floor .", "words": ["the", "my", "of", "to", "kitchen"]}, {"sent": "its Lynettes turn again because um she went in the kitchen and she didnt get a chance to hear it .", "words": ["the", "to", "and", "get", "hear", "in", "it", "kitchen", "she", "because", "a"]}, {"sent": "alright I mean youll close the one over to the kitchen right at least ?", "words": ["the", "to", "kitchen", "at", "close", "over"]}, {"sent": "well we have our table in the kitchen but wheres the other part of the kitchen ?", "words": ["the", "table", "we", "of", "our", "have", "other", "in", "kitchen", "but"]}, {"sent": "help Daddy push that highchair back into the kitchen .", "words": ["the", "help", "push", "into", "back", "kitchen", "that"]}, {"sent": "yes in the kitchen .", "words": ["the", "in", "kitchen"]}, {"sent": "its in the kitchen .", "words": ["the", "in", "kitchen"]}, {"sent": "I thought you were gonna help me in the kitchen .", "words": ["the", "help", "you", "in", "kitchen", "me", "were"]}, {"sent": "do you wanna read your snow book honey ?", "words": ["do", "your", "snow", "you", "read", "wanna", "book"]}, {"sent": "Thomas what about dashing through the snow .", "words": ["the", "what", "snow", "about"]}, {"sent": "a great big blade and I can use it for pushing snow outof the way .", "words": ["can", "the", "for", "snow", "and", "it", "big", "a"]}, {"sent": "you threw some snow like Joshua ?", "words": ["some", "snow", "like", "you"]}, {"sent": "and Teletubbies did a program where Po was playing in the snow and it was so funny because it teletubbies came on in the morning about ten oclock and Po was playing in the snow .", "words": ["where", "the", "was", "so", "snow", "and", "in", "it", "did", "about", "on", "because", "a"]}, {"sent": "yeah the puppy is walking in the snow right ?", "words": ["the", "is", "snow", "in", "puppy"]}, {"sent": "we hafta wait for the snow .", "words": ["the", "wait", "for", "we", "snow"]}, {"sent": "where did Miffy see the snow ?", "words": ["where", "the", "snow", "did", "see"]}, {"sent": "you wanna go to the snow scene ?", "words": ["the", "go", "to", "snow", "you", "wanna"]}, {"sent": "did you tell Daddy what we were building in the snow yesterday ?", "words": ["the", "what", "we", "snow", "you", "in", "did", "were"]}, {"sent": "oh no snow .", "words": ["snow"]}, {"sent": "it looks like snow .", "words": ["it", "like", "snow"]}, {"sent": "mommy hadta carry you from corner to corner because they didnt remove the snow .", "words": ["the", "to", "snow", "you", "carry", "they", "because"]}, {"sent": "snow snow nothing but snow .", "words": ["snow", "but"]}, {"sent": "thats you in the snow .", "words": ["the", "in", "snow", "you"]}, {"sent": "snow scene .", "words": ["snow"]}, {"sent": "are they in the snow ?", "words": ["the", "snow", "they", "in", "are"]}, {"sent": "the snow tubby looks to me as if its a big huge .", "words": ["the", "if", "to", "snow", "me", "big", "a"]}, {"sent": "and then came the winter the snow and sleet and icicles hung from .", "words": ["the", "then", "snow", "and"]}, {"sent": "thats right snow is white .", "words": ["white", "snow", "is"]}, {"sent": "is Frosty made out of snow ?", "words": ["of", "snow", "out", "is"]}, {"sent": "when dis is snow .", "words": ["snow", "when", "is"]}, {"sent": "you can walk in the snow ?", "words": ["can", "the", "snow", "you", "in", "walk"]}, {"sent": "snow .", "words": ["snow"]}, {"sent": "you do want it to snow ?", "words": ["do", "to", "snow", "you", "it"]}, {"sent": "thats snow .", "words": ["snow"]}, {"sent": "kill snow spots with popguns ?", "words": ["snow", "with"]}, {"sent": "when Santa walked out to his sleigh his boots squeaked in the snow .", "words": ["the", "to", "snow", "boots", "in", "out", "his", "when"]}, {"sent": "and look at all the snow on the ground .", "words": ["the", "all", "snow", "and", "at", "look", "on"]}, {"sent": "Bella likes eating snow .", "words": ["snow"]}, {"sent": "yeah see all the snow coming down can you make it snow ?", "words": ["the", "all", "can", "snow", "you", "it", "see", "down", "make"]}, {"sent": "mhm snow at the window huh ?", "words": ["the", "window", "snow", "at"]}, {"sent": "snow bear has furry feet .", "words": ["snow", "bear"]}, {"sent": "walking through the snow .", "words": ["the", "snow"]}, {"sent": "snow .", "words": ["snow"]}, {"sent": "snow .", "words": ["snow"]}, {"sent": "that was a big job too look it shes carrying this big lump of snow .", "words": ["look", "of", "was", "snow", "this", "that", "it", "too", "big", "a"]}, {"sent": "snow .", "words": ["snow"]}, {"sent": "its gonna snow .", "words": ["snow"]}, {"sent": "snow .", "words": ["snow"]}, {"sent": "you saw one part of it sticking out of the snow huh ?", "words": ["the", "of", "snow", "you", "it", "out"]}, {"sent": "soon its gonna snow hard .", "words": ["snow", "hard"]}, {"sent": "Spot and Steve rolled out into the snow .", "words": ["the", "into", "snow", "and", "out"]}, {"sent": "do you like snow ?", "words": ["do", "snow", "like", "you"]}, {"sent": "and what did the xxx the snow fall on ?", "words": ["the", "what", "snow", "fall", "and", "did", "on"]}, {"sent": "in the snow ?", "words": ["the", "in", "snow"]}, {"sent": "do you like to go out and play in the snow ?", "words": ["the", "do", "go", "to", "play", "snow", "like", "you", "and", "in", "out"]}, {"sent": "hes walking in the snow ?", "words": ["the", "in", "snow"]}, {"sent": "you didnt go for a walk in the snow ?", "words": ["the", "for", "go", "snow", "you", "in", "walk", "a"]}, {"sent": "it doesnt really snow in the summer .", "words": ["the", "in", "it", "snow"]}, {"sent": "the snow man gonna come down .", "words": ["the", "down", "snow"]}, {"sent": "you just wanna watch the snow ?", "words": ["the", "watch", "snow", "you", "wanna"]}, {"sent": "make it clean up the snow .", "words": ["clean", "the", "up", "snow", "it", "make"]}, {"sent": "it does look like snow doesnt it ?", "words": ["does", "snow", "like", "it", "look"]}, {"sent": "George likes to read about the snow .", "words": ["the", "to", "snow", "read", "about"]}, {"sent": "and some places there may be snow on the ground for several months .", "words": ["the", "for", "be", "snow", "and", "there", "on", "some"]}, {"sent": "well hafta get your snow suit and zipper fixed somehow .", "words": ["your", "snow", "and", "get", "zipper"]}, {"sent": "so when the snow goes there the birds come back here ?", "words": ["the", "back", "so", "snow", "here", "there", "when"]}, {"sent": "the trees are blowing and theres snow on them .", "words": ["the", "snow", "them", "and", "are", "on"]}, {"sent": "Thomas isnt very happy with the snow is he ?", "words": ["the", "happy", "is", "snow", "with", "he"]}, {"sent": "oh youve been moving the snow .", "words": ["the", "snow"]}, {"sent": "crisp snow lay on the ground and covered the red roofs .", "words": ["the", "red", "snow", "and", "on"]}, {"sent": "they took off across the powdery snow Belle laughed happily as the beast guided the sleigh through the forest path .", "words": ["off", "they", "snow", "the"]}, {"sent": "shoveling up all this snow ?", "words": ["up", "all", "snow", "this"]}, {"sent": "snow snow xxx .", "words": ["snow"]}, {"sent": "shall Mummy make it snow ?", "words": ["it", "make", "snow"]}, {"sent": "and I suddenly looked out and saw the snow and thought .", "words": ["the", "snow", "out", "and"]}, {"sent": "you lay down where in the snow ?", "words": ["where", "the", "snow", "you", "in", "down"]}, {"sent": "the one when Thomas went to buy to get the Christmas tree ready for the carol concert at the station and he got stuck in the snow ?", "words": ["the", "tree", "for", "to", "snow", "and", "get", "in", "he", "stuck", "at", "when", "buy"]}, {"sent": "snow on it yeah .", "words": ["on", "it", "snow"]}, {"sent": "I see some snow you see the snow ?", "words": ["the", "snow", "you", "see", "some"]}, {"sent": "and youve just pointed out the snow havent you ?", "words": ["the", "snow", "you", "and", "out"]}, {"sent": "its long curved tusks enabled it to find food by clearing away snow and ice .", "words": ["to", "snow", "long", "by", "and", "food", "it", "away", "find", "ice"]}, {"sent": "and you said snow for the first time and then you rushed to the patio window and it was snowing .", "words": ["the", "for", "to", "was", "snow", "first", "you", "and", "window", "it", "then"]}, {"sent": "snow ?", "words": ["snow"]}, {"sent": "do you wanna go out and play in the snow ?", "words": ["the", "do", "go", "play", "snow", "you", "and", "in", "out", "wanna"]}, {"sent": "and off he went making tracks in new snow .", "words": ["off", "snow", "and", "in", "he", "new"]}, {"sent": "which one of these do you use in the snow ?", "words": ["the", "these", "do", "of", "snow", "you", "in", "which"]}, {"sent": "a stick that was just right for smacking a snow covered tree .", "words": ["stick", "for", "tree", "was", "snow", "that", "a"]}, {"sent": "cant see anything now theres so much snow .", "words": ["see", "snow", "so", "much"]}, {"sent": "wheres the snow man ?", "words": ["the", "snow"]}, {"sent": "well I think that uh usually what happens is theres so many other things that I hafta do that I dont get a chance to enjoy the snow .", "words": ["the", "what", "think", "do", "is", "so", "to", "snow", "other", "get", "dont", "that", "a"]}, {"sent": "you want him to wear winter like snow gloves ?", "words": ["to", "snow", "like", "you", "him", "gloves"]}, {"sent": "all the people walking in the snow ?", "words": ["the", "all", "in", "snow"]}, {"sent": "what well do is is sit here and then you can have some snow outside .", "words": ["can", "what", "do", "is", "outside", "snow", "then", "and", "here", "you", "have", "some", "sit"]}, {"sent": "have you shown Teddy the snow ?", "words": ["have", "the", "snow", "you"]}, {"sent": "its when its cold and when it cold and it could be snow and its almost Christmas .", "words": ["be", "snow", "and", "it", "could", "when", "cold"]}, {"sent": "snow storm ?", "words": ["snow"]}, {"sent": "hes catching a snow a piece of snow .", "words": ["of", "snow", "a"]}, {"sent": "stay tuned for snow .", "words": ["stay", "for", "snow"]}, {"sent": "theres too much snow .", "words": ["snow", "much", "too"]}, {"sent": "after tea they sat in the big purple chair by the window and waited for snow .", "words": ["the", "for", "snow", "chair", "by", "and", "they", "in", "window", "big"]}, {"sent": "I dont think we had enough snow last year to make a snowman .", "words": ["think", "we", "to", "last", "snow", "dont", "snowman", "make", "a"]}, {"sent": "well I cant see snow Thomas .", "words": ["see", "snow"]}, {"sent": "yeah oh are you xxx snow white ?", "words": ["white", "are", "snow", "you"]}, {"sent": "I hope somebody has cleared the snow from the roads Thomas .", "words": ["the", "snow"]}, {"sent": "I can make some ice ice house some some playyard of ice of snow with my little sister .", "words": ["can", "my", "of", "snow", "with", "little", "house", "ice", "some", "make"]}, {"sent": "do you like the snow ?", "words": ["the", "do", "snow", "like", "you"]}, {"sent": "do you see the snow Lily ?", "words": ["the", "do", "snow", "you", "see"]}, {"sent": "okay you havent had that in a long time .", "words": ["long", "you", "in", "that", "a"]}, {"sent": "thats taking an awful long time .", "words": ["an", "long"]}, {"sent": "I tell ya shes uh theyre so surprising at this age because literally she is never she is by my side all day long and I .", "words": ["all", "my", "is", "so", "long", "by", "and", "this", "at", "she", "because"]}, {"sent": "do you remember those long lollies on the sticks ?", "words": ["the", "do", "those", "long", "you", "on"]}, {"sent": "I know I I know you do I know I know youve been waiting for this day for a long time .", "words": ["do", "for", "long", "you", "this", "a"]}, {"sent": "that long ?", "words": ["that", "long"]}, {"sent": "is it a long trunk ?", "words": ["long", "it", "a", "is"]}, {"sent": "long long necks .", "words": ["long"]}, {"sent": "xxx we have a very long sort out .", "words": ["we", "long", "have", "out", "a"]}, {"sent": "youve gotten to be a long tall girl .", "words": ["to", "be", "long", "a"]}, {"sent": "that was a long time ago .", "words": ["was", "that", "long", "a"]}, {"sent": "a long long song .", "words": ["long", "a"]}, {"sent": "yeah wont be long .", "words": ["be", "long"]}, {"sent": "thats really long isnt it ?", "words": ["it", "long"]}, {"sent": "youve been talking about her for a long time havent you ?", "words": ["for", "long", "you", "about", "her", "a"]}, {"sent": "why the why are we suddenly out of all of your long underwear .", "words": ["the", "all", "your", "we", "of", "long", "are", "out", "why"]}, {"sent": "and it makes you feel better in the long run .", "words": ["the", "better", "long", "you", "and", "run", "in", "it"]}, {"sent": "and um will a long long time .", "words": ["will", "long", "a", "and"]}, {"sent": "thats a long time ago .", "words": ["long", "a"]}, {"sent": "whoa thats a long airplane ride .", "words": ["airplane", "ride", "long", "a"]}, {"sent": "whys he a long time ?", "words": ["he", "long", "a"]}, {"sent": "thats a long time but first clean up your give your room a quick clean up because Matties coming tomorrow .", "words": ["clean", "up", "your", "first", "because", "long", "room", "give", "but", "a"]}, {"sent": "theyve taken a long time have they had jelly aswell ?", "words": ["jelly", "long", "have", "they", "a"]}, {"sent": "a long gun .", "words": ["long", "a"]}, {"sent": "wow he has a long arm .", "words": ["arm", "he", "long", "a"]}, {"sent": "because were not having dinner for a long time .", "words": ["for", "long", "were", "not", "because", "a"]}, {"sent": "the kettle boiled not so long ago so water from the kettle and we needta measure this out with these measuring spoons .", "words": ["the", "these", "we", "so", "with", "long", "and", "this", "out", "water", "not"]}, {"sent": "it says it was an old house that had been empty for a long time but someone saw a young boy playing in the garden this morning .", "words": ["old", "the", "for", "was", "long", "empty", "this", "an", "it", "house", "in", "garden", "that", "but", "a"]}, {"sent": "as long as she gets there .", "words": ["there", "she", "long"]}, {"sent": "as long as I can just come back and turn the thing off and every thing .", "words": ["can", "the", "off", "back", "long", "and", "every"]}, {"sent": "its a long snake that Liz isnt it ?", "words": ["it", "that", "long", "a"]}, {"sent": "I had a nice long nap .", "words": ["nice", "long", "a"]}, {"sent": "long yellow thing with a peel on the outside .", "words": ["the", "outside", "with", "long", "on", "yellow", "a"]}, {"sent": "youve not been up very long .", "words": ["up", "not", "long"]}, {"sent": "has it got a big long neck ?", "words": ["it", "big", "long", "a"]}, {"sent": "xxx I havent seen you so long .", "words": ["so", "long", "you"]}, {"sent": "you see its quite exciting getting out the toys that youve not had for a long time isnt it ?", "words": ["the", "for", "long", "you", "it", "out", "see", "that", "not", "a"]}, {"sent": "long time from now before we got that owl .", "words": ["owl", "that", "long", "we"]}, {"sent": "the wee girl before her had long hair and all the way down her back .", "words": ["the", "all", "back", "down", "long", "and", "hair", "her"]}, {"sent": "ooh you have a long necked .", "words": ["have", "long", "a", "you"]}, {"sent": "no you wanna stay in there for a very long time ?", "words": ["for", "long", "you", "there", "in", "wanna", "stay", "a"]}, {"sent": "it wont take long .", "words": ["it", "take", "long"]}, {"sent": "nice long ones like yours .", "words": ["nice", "like", "long"]}, {"sent": "I think we had better go for a nice long walk this afternoon somewhere dont you ?", "words": ["better", "think", "for", "go", "we", "nice", "long", "you", "this", "dont", "walk", "a"]}, {"sent": "it doesnt seem long since you were too tiny to bath in this bath .", "words": ["tiny", "to", "long", "you", "this", "in", "it", "too", "were"]}, {"sent": "I wanted to xxx only do the games so it wont really take long .", "words": ["the", "do", "to", "so", "long", "it", "take"]}, {"sent": "didnt you have a nice long sleep in the car ?", "words": ["the", "sleep", "car", "nice", "long", "you", "have", "in", "a"]}, {"sent": "how long did they hafta stay in jail ?", "words": ["how", "long", "they", "in", "did", "stay"]}, {"sent": "and how long it takes in the mornings .", "words": ["the", "how", "long", "and", "in", "it"]}, {"sent": "thats a long way to go for yoga .", "words": ["for", "go", "to", "long", "a"]}, {"sent": "he was born not long before we went away .", "words": ["we", "was", "long", "away", "he", "not"]}, {"sent": "but I like it long .", "words": ["long", "it", "like", "but"]}, {"sent": "it must have been there a long time .", "words": ["long", "have", "there", "it", "a"]}, {"sent": "ohmygoodness we havent seen him in a long time .", "words": ["we", "long", "in", "him", "a"]}, {"sent": "we havent been to Baltimore actually in a good long while .", "words": ["we", "to", "long", "in", "good", "a"]}, {"sent": "heres the one with the long tail .", "words": ["the", "with", "long"]}, {"sent": "but not for long because the sugar cubes fell out .", "words": ["the", "for", "long", "but", "out", "not", "because"]}, {"sent": "youre making it a little bit too long .", "words": ["long", "little", "it", "too", "a"]}, {"sent": "thats a long word isnt it ?", "words": ["it", "long", "a"]}, {"sent": "you got a long arm too ?", "words": ["long", "you", "arm", "too", "a"]}, {"sent": "its a bit its a bit long until your birthday though .", "words": ["your", "long", "a"]}, {"sent": "is that what took you so long to say ?", "words": ["what", "is", "so", "to", "say", "long", "you", "that"]}, {"sent": "I havent read this book in a long time .", "words": ["long", "this", "in", "read", "book", "a"]}, {"sent": "like a long time and then a short time .", "words": ["like", "long", "and", "then", "a"]}, {"sent": "its too long .", "words": ["long", "too"]}, {"sent": "we did that a long time ago didnt we ?", "words": ["we", "long", "did", "that", "a"]}, {"sent": "theres a big long line .", "words": ["big", "long", "a"]}, {"sent": "how long do we leave those in do you think ?", "words": ["do", "how", "think", "those", "we", "long", "you", "in"]}, {"sent": "not a long time .", "words": ["not", "long", "a"]}, {"sent": "thats very long .", "words": ["long"]}, {"sent": "we havent read that in a long time .", "words": ["we", "long", "in", "read", "that", "a"]}, {"sent": "wow thats a long ladder int it ?", "words": ["it", "ladder", "long", "a"]}, {"sent": "who else has a long neck ?", "words": ["long", "a", "who"]}, {"sent": "not very long .", "words": ["not", "long"]}, {"sent": "whatre you making with the long one ?", "words": ["the", "with", "long", "you"]}, {"sent": "oop look at this guy with his long neck .", "words": ["look", "with", "long", "this", "at", "his"]}, {"sent": "if you blow it gently it will go high up a long long way .", "words": ["up", "if", "go", "long", "you", "it", "high", "will", "blow", "a"]}, {"sent": "it is quite a long way actually isnt it there ?", "words": ["is", "long", "there", "it", "a"]}, {"sent": "yeah you know what I think I see that one with the black long hair right ?", "words": ["the", "what", "think", "with", "long", "you", "black", "see", "hair", "that"]}, {"sent": "well have a long wait Thomas because hes not coming home till about six seven oclock this evening .", "words": ["wait", "long", "have", "this", "about", "home", "not", "because", "a"]}, {"sent": "he wont be long now .", "words": ["be", "he", "long"]}, {"sent": "no no it got very long .", "words": ["it", "long"]}, {"sent": "thats gonna be really noisy for a really long time .", "words": ["for", "be", "noisy", "long", "a"]}, {"sent": "no it doesnt take too long .", "words": ["it", "take", "long", "too"]}, {"sent": "for a long time .", "words": ["for", "long", "a"]}, {"sent": "weve been enjoying cooking for a very long time .", "words": ["for", "long", "a"]}, {"sent": "yeah as long as youre here and you dont leave it open .", "words": ["open", "long", "you", "here", "dont", "and", "it"]}, {"sent": "no its not a tub its on a big long stick its on like a pencil .", "words": ["stick", "like", "long", "pencil", "on", "big", "not", "a"]}, {"sent": "he thought about that for a long time .", "words": ["for", "long", "he", "about", "that", "a"]}, {"sent": "have you got a long long neck Fraser ?", "words": ["have", "long", "a", "you"]}, {"sent": "and now its a long way over that bridge so theyre having another rest .", "words": ["so", "long", "and", "another", "over", "that", "a"]}, {"sent": "one of the largest snakes is the giant anaconda it can grow more than thirty feet long .", "words": ["the", "can", "is", "of", "long", "more", "it"]}, {"sent": "and then one small line and another long line .", "words": ["another", "then", "long", "and"]}, {"sent": "I dont know when I made this label it was a long time ago .", "words": ["was", "long", "dont", "this", "it", "when", "a"]}, {"sent": "you know lovely long whiskers ?", "words": ["long", "you"]}, {"sent": "its a Daddy long legs Thomas .", "words": ["long", "a"]}, {"sent": "right like when it was long before right ?", "words": ["was", "like", "long", "it", "when"]}, {"sent": "it sure is a long book .", "words": ["is", "long", "it", "book", "a"]}, {"sent": "there were all those big long scars on his leg werent there ?", "words": ["all", "those", "long", "his", "there", "leg", "on", "big", "were"]}, {"sent": "hes just run over everybody at the school .", "words": ["the", "school", "run", "at", "over"]}, {"sent": "xxx run in here and put some water on it and run right out .", "words": ["put", "and", "here", "run", "in", "it", "out", "water", "on", "some"]}, {"sent": "run run run run run run run .", "words": ["run"]}, {"sent": "run a run .", "words": ["run", "a"]}, {"sent": "you better run .", "words": ["better", "run", "you"]}, {"sent": "alright listen Im just gonna run and get the laundry Ill be right back while you make me a picture for when I come back up .", "words": ["the", "up", "for", "when", "listen", "back", "be", "picture", "and", "get", "run", "you", "me", "make", "a"]}, {"sent": "yeah she run well no but she runs that childrens place .", "words": ["run", "that", "she", "but"]}, {"sent": "how can he run ?", "words": ["can", "run", "how", "he"]}, {"sent": "and tigers can run very fast .", "words": ["can", "run", "fast", "and"]}, {"sent": "I know but your dad said when you come in that um he wants you to take a bath so well let it run for a few minutes okay ?", "words": ["for", "your", "to", "so", "you", "run", "in", "that", "he", "it", "take", "when", "but", "a"]}, {"sent": "I think she would run away wouldnt she ?", "words": ["think", "run", "away", "would", "she"]}, {"sent": "being last to run or trot on gravel path or grassy plot you find out what is wet and likewise what is not .", "words": ["what", "wet", "is", "to", "last", "you", "and", "run", "out", "find", "on", "not"]}, {"sent": "I have run away from a little old man ?", "words": ["old", "little", "have", "run", "away", "a"]}, {"sent": "pop the oven door burst open the fudge had exploded run run for your lives .", "words": ["the", "pop", "for", "your", "open", "run", "oven", "door"]}, {"sent": "and how to run up trees .", "words": ["up", "how", "to", "and", "run"]}, {"sent": "gonna go out and run around ?", "words": ["go", "and", "run", "out", "around"]}, {"sent": "shes an old cat but she can run very quickly though cant she ?", "words": ["old", "can", "cat", "an", "run", "she", "but"]}, {"sent": "Nannas been run over oh dear you dont have very nice dreams if you keep dreaming about people being run over .", "words": ["if", "nice", "you", "have", "dont", "run", "about", "over"]}, {"sent": "yeah at home a lot of time she cries if shes put down on the floor because maybe she thinks Im gonna go run off and do something else .", "words": ["the", "off", "put", "do", "if", "go", "of", "a lot", "because", "and", "run", "at", "on", "down", "she", "a", "home"]}, {"sent": "run up and go teetee .", "words": ["up", "run", "go", "and"]}, {"sent": "got run over .", "words": ["run", "over"]}, {"sent": "if she thought you were going to hurt her I think shed run away like she useta .", "words": ["think", "if", "hurt", "to", "her", "like", "you", "run", "away", "were", "she"]}, {"sent": "that was a bit silly of him to run over the road without looking wasnt it ?", "words": ["the", "of", "was", "to", "run", "him", "it", "over", "that", "a"]}, {"sent": "a run a run a run a run a run a run run .", "words": ["run", "a"]}, {"sent": "okay Im gonna run out of these very soon .", "words": ["of", "run", "these", "out"]}, {"sent": "and you run round .", "words": ["run", "you", "and"]}, {"sent": "they both ran very fast but Paul won because he could run .", "words": ["fast", "they", "run", "he", "but", "could", "because"]}, {"sent": "and I think Purdie wants to be your friend because she doesnt run away when you come down does she ?", "words": ["does", "think", "your", "to", "be", "and", "you", "run", "away", "down", "when", "she", "because"]}, {"sent": "because grandmas too tired to run .", "words": ["to", "tired", "run", "too", "because"]}, {"sent": "hes going to run over him .", "words": ["to", "him", "over", "run"]}, {"sent": "we can run around .", "words": ["can", "run", "around", "we"]}, {"sent": "how do you know that tape doesnt run out by now ?", "words": ["do", "how", "tape", "you", "by", "run", "out", "that"]}, {"sent": "run away .", "words": ["run", "away"]}, {"sent": "they can run they can jump they can slide they can swing they can dig they can climb they can talk they can sing .", "words": ["can", "slide", "swing", "sing", "they", "jump", "run", "talk", "climb"]}, {"sent": "no because run xxx like this .", "words": ["run", "like", "because", "this"]}, {"sent": "well I dont think my little boy would run across the rails .", "words": ["the", "think", "my", "little", "dont", "run", "would"]}, {"sent": "you can see children on the road playing a football match and youre going to run over them .", "words": ["can", "the", "to", "them", "you", "and", "run", "on", "see", "over", "a"]}, {"sent": "but Max let the ants run into his pocket instead of into his ant farm .", "words": ["the", "into", "of", "run", "ant", "his", "but"]}, {"sent": "to run over ?", "words": ["to", "over", "run"]}, {"sent": "you did run off didnt you ?", "words": ["off", "run", "did", "you"]}, {"sent": "big enough for her to run up .", "words": ["up", "for", "to", "run", "big", "her"]}, {"sent": "Ill run and Ill run .", "words": ["run", "and"]}, {"sent": "they both ran very fast but Paul won because he could run .", "words": ["fast", "they", "run", "he", "but", "could", "because"]}, {"sent": "whos going to run up the tree ?", "words": ["up", "the", "tree", "to", "run"]}, {"sent": "oh its run teddy down .", "words": ["run", "down"]}, {"sent": "how do you make him run ?", "words": ["do", "how", "you", "run", "him", "make"]}, {"sent": "did you throw it did he run and get it ?", "words": ["throw", "you", "and", "run", "get", "it", "he", "did"]}, {"sent": "youre not supposed to run things on the wall .", "words": ["the", "to", "run", "on", "not"]}, {"sent": "run run as fast as you can .", "words": ["can", "run", "fast", "you"]}, {"sent": "eventually she can run no farther and she falls to the ground in tears oh .", "words": ["can", "the", "to", "and", "run", "in", "she"]}, {"sent": "to hold him onto there so he cant run off uhn hm ?", "words": ["off", "hold", "to", "so", "there", "him", "run", "he"]}, {"sent": "Im going to run out in the garden and get some onions and tomatoes .", "words": ["the", "to", "and", "get", "run", "in", "garden", "out", "some"]}, {"sent": "is he run run run run .", "words": ["run", "he", "is"]}, {"sent": "weve run out of talcum powder havent we ?", "words": ["of", "run", "out", "we"]}, {"sent": "you better run .", "words": ["better", "run", "you"]}, {"sent": "so its saying they couldnt run nearly as fast he could .", "words": ["fast", "so", "they", "run", "he", "could"]}, {"sent": "I think theyve run away .", "words": ["run", "think", "away"]}, {"sent": "lemme run some .", "words": ["run", "some"]}, {"sent": "dont run over that rhino .", "words": ["dont", "that", "over", "run"]}, {"sent": "you just run to the wall like a nice little man .", "words": ["the", "to", "nice", "like", "you", "little", "run", "a"]}, {"sent": "they both ran very fast but Paul won because he could run .", "words": ["fast", "they", "run", "he", "but", "could", "because"]}, {"sent": "well she helps run a wee charity shop in the DonegallRoad .", "words": ["the", "run", "in", "she", "a"]}, {"sent": "are you supposed to run after him now ?", "words": ["to", "you", "run", "him", "are"]}, {"sent": "she loves to just like run in the tent and in and out around the backyard .", "words": ["the", "backyard", "to", "like", "and", "run", "in", "out", "around", "she"]}, {"sent": "well thats a good place for them at night but during the day when theres when theres sunlight and farmers are around and the dogs and cats and everyones around its safe to have the chickens come out and peck in the ground and run around .", "words": ["the", "for", "around", "to", "them", "and", "have", "run", "in", "are", "good", "at", "out", "when", "but", "a"]}, {"sent": "Daddys gonna go run the water .", "words": ["the", "run", "water", "go"]}, {"sent": "see they didnt run away today .", "words": ["see", "away", "they", "run"]}, {"sent": "they wont run away .", "words": ["they", "away", "run"]}, {"sent": "I think I hafta jump over the little person and run around the big person and then lay down on the bench .", "words": ["the", "bench", "think", "little", "and", "jump", "run", "big", "around", "on", "then", "down", "over"]}, {"sent": "I told you not to run after her .", "words": ["to", "you", "run", "her", "not"]}, {"sent": "run ?", "words": ["run"]}, {"sent": "you run out .", "words": ["run", "out", "you"]}, {"sent": "the mouse run up the clock ?", "words": ["the", "up", "clock", "run", "mouse"]}, {"sent": "run run run run .", "words": ["run"]}, {"sent": "you know something we dont run with things in our mouth .", "words": ["we", "mouth", "with", "you", "our", "dont", "run", "in"]}, {"sent": "he hadta run for the bus today didnt he ?", "words": ["the", "for", "run", "he", "bus"]}, {"sent": "hes not going to get run over uhn is he ?", "words": ["is", "to", "get", "run", "he", "over", "not"]}, {"sent": "do you like to run ?", "words": ["do", "to", "like", "you", "run"]}, {"sent": "can you make him run fast ?", "words": ["can", "fast", "you", "run", "him", "make"]}, {"sent": "we better run fast if we see any ghosts huh ?", "words": ["better", "fast", "if", "we", "any", "run", "see"]}, {"sent": "we hadta run quickly .", "words": ["run", "we"]}, {"sent": "run and run .", "words": ["run", "and"]}, {"sent": "you ran around when it was time to run around and when it wasnt the time to run around you were brilliant .", "words": ["the", "was", "to", "you", "and", "run", "it", "around", "when", "were"]}, {"sent": "yeah cats know how to run down trees but people have a problem sometimes .", "words": ["how", "to", "have", "run", "down", "but", "a"]}, {"sent": "hes gonna run very quickly but the train will try and stop him .", "words": ["the", "train", "stop", "and", "run", "him", "will", "try", "but"]}, {"sent": "I run across the flower bed whats inside I scratch my head .", "words": ["the", "my", "flower", "inside", "head", "bed", "run"]}, {"sent": "has Rosa run away again ?", "words": ["run", "away"]}, {"sent": "what timers nearly run out ?", "words": ["what", "out", "run"]}, {"sent": "okay I didnt run the play session last time so can you tell me which toys she was most interested in last time as well ?", "words": ["the", "can", "so", "last", "play", "was", "you", "run", "in", "me", "which", "she"]}, {"sent": "that pens trying to run away .", "words": ["to", "away", "that", "run"]}, {"sent": "you wriggle and run away dont you ?", "words": ["and", "you", "dont", "run", "away"]}, {"sent": "Naima you ready to run around ?", "words": ["to", "around", "run", "you"]}, {"sent": "dont run .", "words": ["dont", "run"]}, {"sent": "did Elsa run away ?", "words": ["run", "away", "did"]}, {"sent": "are you going to run away from him in that picture ?", "words": ["to", "picture", "you", "run", "him", "are", "away", "in", "that"]}, {"sent": "hed run outof wellys wouldnt he ?", "words": ["run", "he"]}, {"sent": "you just hafta run onto the ball and run down the wing and score a goal .", "words": ["the", "and", "ball", "run", "you", "down", "a"]}, {"sent": "Laura was sitting beside us .", "words": ["was", "beside", "us"]}, {"sent": "let us get some more juice .", "words": ["get", "us", "more", "some", "juice"]}, {"sent": "okay let us play with the book .", "words": ["the", "play", "with", "us", "book"]}, {"sent": "well lets see what theyre telling us .", "words": ["see", "what", "us"]}, {"sent": "can you tell us what he did .", "words": ["can", "what", "you", "us", "he", "did"]}, {"sent": "okay let us put these away .", "words": ["us", "put", "away", "these"]}, {"sent": "okay let us read .", "words": ["us", "read"]}, {"sent": "Im sorry that we hafta wait xxx but we hafta wait until Missus Allen can give it to us .", "words": ["can", "wait", "we", "to", "us", "it", "give", "that", "but"]}, {"sent": "well what were gonna do is leave it down so it can film us .", "words": ["can", "what", "do", "is", "so", "us", "it", "down", "were"]}, {"sent": "let us make a red picture .", "words": ["red", "picture", "us", "make", "a"]}, {"sent": "and she stays right in front of us to protect us .", "words": ["of", "to", "and", "us", "in", "she"]}, {"sent": "let us see .", "words": ["see", "us"]}, {"sent": "let us see what else is inside .", "words": ["what", "is", "inside", "us", "see"]}, {"sent": "let us do our routine .", "words": ["us", "do", "our"]}, {"sent": "Eleanor do you wanna pretend to make us a coffee then ?", "words": ["do", "to", "then", "you", "us", "coffee", "wanna", "pretend", "make", "a"]}, {"sent": "let us see what we do have .", "words": ["what", "do", "we", "have", "us", "see"]}, {"sent": "comeon let us color together .", "words": ["us"]}, {"sent": "tell us what you think .", "words": ["us", "what", "think", "you"]}, {"sent": "if she thinks shes gonna wake us up .", "words": ["up", "if", "us", "wake", "she"]}, {"sent": "hey give us the gun .", "words": ["the", "us", "give"]}, {"sent": "just lets us play .", "words": ["us", "play"]}, {"sent": "okay let us put him to sleep .", "words": ["put", "sleep", "to", "us", "him"]}, {"sent": "so it can see us better .", "words": ["can", "better", "so", "us", "it", "see"]}, {"sent": "whos making us a cup of tea ?", "words": ["cup", "us", "of", "a"]}, {"sent": "say Gus come here and play with us .", "words": ["play", "say", "with", "and", "here", "us"]}, {"sent": "because they go to a class earlier than us and usually we talk to them as theyre coming out and were going in .", "words": ["go", "we", "to", "them", "and", "they", "us", "in", "talk", "out", "were", "because", "a"]}, {"sent": "they could also be similar to like youknow keys that she tries to take from us that I dont let her have .", "words": ["to", "be", "her", "like", "have", "they", "us", "dont", "take", "could", "keys", "that", "she"]}, {"sent": "weve been to music group and Sue gave us a lift in her car didnt she ?", "words": ["to", "car", "and", "us", "in", "her", "she", "a"]}, {"sent": "let us do it all again .", "words": ["all", "us", "do", "it"]}, {"sent": "show us the rabbit .", "words": ["show", "us", "the"]}, {"sent": "but just behind us the estate going up the hill they have their collections on Tuesdays .", "words": ["the", "up", "their", "behind", "have", "us", "they", "on", "but"]}, {"sent": "come one you want ta help us put the toys away ?", "words": ["the", "help", "put", "you", "us", "away"]}, {"sent": "let us see what is in here .", "words": ["what", "is", "here", "us", "in", "see"]}, {"sent": "are you going to sing to us ?", "words": ["to", "you", "sing", "us", "are"]}, {"sent": "youve just told us it was downstairs .", "words": ["was", "us", "it"]}, {"sent": "you sit Ill sit here you can move in there so youre in the middle and you can tell us the story .", "words": ["can", "the", "story", "so", "you", "here", "there", "in", "and", "us", "sit"]}, {"sent": "and bring it with us .", "words": ["with", "bring", "and", "us", "it"]}, {"sent": "let us see .", "words": ["see", "us"]}, {"sent": "did anyone bought this for us ?", "words": ["this", "for", "did", "us"]}, {"sent": "and the two of us was sitting xxx .", "words": ["the", "of", "was", "and", "us"]}, {"sent": "and invite them to come to our house and see us .", "words": ["to", "them", "our", "and", "us", "house", "see"]}, {"sent": "let us go outside .", "words": ["us", "outside", "go"]}, {"sent": "could you tell us something besides that ?", "words": ["us", "could", "that", "you"]}, {"sent": "you told us this morning you didnt like bikes .", "words": ["us", "like", "this", "you"]}, {"sent": "and Bethanys Mummy told us this morning that Bethany talks about you all the time .", "words": ["the", "all", "and", "you", "us", "this", "about", "that"]}, {"sent": "and to give us some tapes .", "words": ["to", "and", "us", "some", "give"]}, {"sent": "probably HopalongBill will go with us no doubt .", "words": ["will", "us", "with", "go"]}, {"sent": "now let us try RaggedyAnn .", "words": ["us", "try"]}, {"sent": "now let us push this back on the plate .", "words": ["the", "push", "plate", "back", "us", "this", "on"]}, {"sent": "can you make one up for us now ?", "words": ["can", "up", "for", "you", "us", "make"]}, {"sent": "so she needs to find out how much it costs to tell us .", "words": ["how", "to", "so", "us", "it", "out", "find", "much", "she"]}, {"sent": "and feeding us icecream .", "words": ["us", "and"]}, {"sent": "so yeah they were ashes and they put them on his head and then they said a blessing and then why do they even put ashes on your head what is it remind us of ?", "words": ["what", "put", "of", "us", "it", "on", "then", "were", "why", "is", "so", "head", "and", "a", "do", "your", "them", "they", "his"]}, {"sent": "we could go and ask mum and dad to help us .", "words": ["help", "go", "we", "to", "and", "us", "could"]}, {"sent": "okay Kim you like to tell us xxx ?", "words": ["to", "like", "us", "you"]}, {"sent": "let us see what else .", "words": ["see", "us", "what"]}, {"sent": "well let us put these things back in to the box and Ill tell you what were going to do next okay ?", "words": ["the", "these", "put", "what", "box", "do", "back", "to", "and", "you", "us", "in", "were"]}, {"sent": "because it wasnt until I was half way through talking to these people that you decided that you decided that maybe it would be worth us asking them to do all the work .", "words": ["the", "these", "do", "all", "work", "to", "was", "be", "them", "you", "us", "it", "would", "that", "because"]}, {"sent": "do you think you could get us some kind of a tray to put all our gift shop items on ?", "words": ["do", "think", "put", "all", "of", "to", "you", "get", "us", "our", "on", "some", "could", "tray", "a"]}, {"sent": "let us see you do it .", "words": ["do", "you", "us", "it", "see"]}, {"sent": "cant remember the names of the others but hes the one that seems to serve us most doesnt he ?", "words": ["the", "of", "to", "us", "he", "that", "but"]}, {"sent": "because he cant eat with us he has someplace else to go .", "words": ["go", "to", "with", "eat", "us", "he", "because"]}, {"sent": "Im gonna go make us a salad .", "words": ["us", "make", "a", "go"]}, {"sent": "let us see what we can make .", "words": ["can", "what", "we", "us", "see", "make"]}, {"sent": "Jennys coming with us uhn isnt she ?", "words": ["us", "with", "she"]}, {"sent": "is he bringing us a pizza ?", "words": ["is", "pizza", "us", "he", "a"]}, {"sent": "let us open the door .", "words": ["open", "us", "door", "the"]}, {"sent": "let us look at the next page .", "words": ["the", "us", "at", "look"]}, {"sent": "let us sit down so we can write .", "words": ["can", "we", "so", "write", "us", "sit", "down"]}, {"sent": "watching us .", "words": ["us"]}, {"sent": "hey let us sing our song .", "words": ["sing", "us", "our"]}, {"sent": "and he said to us why do you sit there like that ?", "words": ["do", "to", "like", "and", "you", "us", "there", "he", "sit", "that", "why"]}, {"sent": "watch us go .", "words": ["us", "watch", "go"]}, {"sent": "Im gonna take the typewriter to Utica with us .", "words": ["the", "to", "with", "us", "take"]}, {"sent": "because f someone could just walk in that part and find us here .", "words": ["and", "here", "us", "in", "that", "find", "could", "walk", "because"]}, {"sent": "spell thursday foah us .", "words": ["us"]}, {"sent": "let us see your head .", "words": ["see", "us", "head", "your"]}, {"sent": "come on let us color .", "words": ["on", "us"]}, {"sent": "let us pretend okay .", "words": ["us", "pretend"]}, {"sent": "someone sent us a letter in a bottle .", "words": ["us", "in", "bottle", "a"]}, {"sent": "let us see what is in here .", "words": ["what", "is", "here", "us", "in", "see"]}, {"sent": "let us count .", "words": ["us"]}, {"sent": "do you think Rachel wants us to like play with the cafe or do you think she wants us to mess it up ?", "words": ["the", "do", "think", "up", "to", "play", "with", "like", "you", "us", "it", "she"]}, {"sent": "us them .", "words": ["us", "them"]}, {"sent": "look at that guy walking down there hes looking at us see him ?", "words": ["look", "there", "us", "him", "at", "see", "down", "that"]}, {"sent": "comeon let us throw the ball some more .", "words": ["the", "throw", "ball", "us", "more", "some"]}, {"sent": "let us create an animal .", "words": ["us", "an", "animal"]}, {"sent": "tells us tells us .", "words": ["us"]}, {"sent": "the couch is a little bit too small for us to sit in .", "words": ["the", "for", "is", "to", "couch", "little", "us", "in", "too", "sit", "a"]}, {"sent": "she alta also was telling us that the ceiling of grand central used to be so dirty that it was black .", "words": ["the", "of", "was", "to", "be", "so", "black", "us", "it", "dirty", "that", "she"]}, {"sent": "now Ill get us some toast to eat .", "words": ["toast", "to", "get", "us", "eat", "some"]}, {"sent": "let us put these back on .", "words": ["these", "put", "back", "us", "on"]}, {"sent": "is he hiding somewhere from us do you think ?", "words": ["do", "think", "is", "you", "us", "he"]}, {"sent": "might help warm us up .", "words": ["up", "us", "help"]}, {"sent": "are you sure you dont want us to help you find Gordon ?", "words": ["help", "to", "you", "dont", "us", "are", "find"]}, {"sent": "no one can get us in our house .", "words": ["can", "our", "get", "us", "in", "house"]}, {"sent": "let us hear the duck .", "words": ["the", "us", "duck", "hear"]}, {"sent": "we were brushing them off and Brian came back from the opticians and came to talk to us didnt he ?", "words": ["off", "the", "back", "we", "to", "them", "and", "us", "he", "talk", "were"]}, {"sent": "okay you read us the story now .", "words": ["the", "you", "us", "read", "story"]}, {"sent": "put the tape in the other room the electric companys on .", "words": ["the", "put", "tape", "other", "in", "room", "on"]}, {"sent": "and what happened to Daddy the other day when he was coming home from work a bit too fast .", "words": ["the", "what", "fast", "work", "to", "was", "other", "and", "he", "too", "when", "a", "home"]}, {"sent": "the other table ?", "words": ["the", "table", "other"]}, {"sent": "baby can go at the other end of the settee .", "words": ["can", "the", "go", "of", "other", "at"]}, {"sent": "wherere the other balls you have the green one and the orange one ?", "words": ["the", "you", "have", "other", "and", "orange", "green"]}, {"sent": "oh hes got to come back for the other one now hasnt he ?", "words": ["the", "for", "back", "to", "other", "he"]}, {"sent": "look xxx the other window would you like to go look out the other window ?", "words": ["the", "look", "go", "to", "like", "other", "you", "out", "would", "window"]}, {"sent": "I went to the doctors office the other day and I hadta wait three hours to see him .", "words": ["the", "wait", "to", "and", "other", "him", "see"]}, {"sent": "here is the other car .", "words": ["the", "is", "car", "other", "here"]}, {"sent": "what other color do you want ?", "words": ["what", "do", "you", "other"]}, {"sent": "ya wanna put other animals in ?", "words": ["in", "put", "wanna", "other"]}, {"sent": "he wants to go back to the other animals .", "words": ["the", "back", "go", "to", "other", "he"]}, {"sent": "because other children are gonna wear it .", "words": ["it", "are", "because", "other"]}, {"sent": "well you can put them in the other purse perhaps .", "words": ["can", "the", "put", "purse", "them", "you", "other", "in"]}, {"sent": "your other browns .", "words": ["your", "other"]}, {"sent": "now Thomas embarrassed me the other day .", "words": ["the", "me", "other"]}, {"sent": "when you ate it for breakfast the other day that was the last one .", "words": ["the", "for", "was", "last", "you", "other", "that", "it", "when"]}, {"sent": "whats the other one ?", "words": ["the", "other"]}, {"sent": "gimme your other hand .", "words": ["your", "hand", "other"]}, {"sent": "well I dont know where the other nuts are John .", "words": ["where", "the", "other", "dont", "are"]}, {"sent": "theres some other pictures in here .", "words": ["here", "some", "in", "other"]}, {"sent": "I saw in the book the other day there was a story about somebody who wasnt sitting nicely .", "words": ["the", "was", "other", "there", "in", "about", "who", "book", "story", "a"]}, {"sent": "it seems asif almost every other country spell their uh .", "words": ["their", "it", "every", "other"]}, {"sent": "what other songs ?", "words": ["what", "other"]}, {"sent": "he we started out with hi uh the babydoll sitting with me so that he would get useta something other than him being with me .", "words": ["the", "we", "so", "with", "other", "get", "him", "he", "out", "would", "me", "that"]}, {"sent": "he has got above his bed in hospital he has got a string that runs from one side of the room to the other .", "words": ["the", "of", "to", "bed", "other", "his", "in", "room", "he", "above", "that", "a"]}, {"sent": "its uh we bought that the other day .", "words": ["the", "other", "that", "we"]}, {"sent": "hes spent a lot of time lately taking two things and banging them against each other .", "words": ["each", "of", "a lot", "them", "and", "other", "a"]}, {"sent": "we went to look at some didnt we the other day .", "words": ["the", "we", "to", "other", "at", "look", "some"]}, {"sent": "wherere the other ones ?", "words": ["the", "other"]}, {"sent": "shall we look at the other box too ?", "words": ["the", "box", "we", "other", "at", "look", "too"]}, {"sent": "do you know what he was doing the other day ?", "words": ["the", "do", "what", "was", "you", "other", "he"]}, {"sent": "one other bit missing I think .", "words": ["think", "other"]}, {"sent": "you hafta put them next to each other dont you ?", "words": ["put", "each", "to", "them", "you", "other", "dont"]}, {"sent": "yes whats your other name ?", "words": ["your", "other"]}, {"sent": "you have no idea how many other babies have put that in their mouths .", "words": ["how", "put", "their", "other", "have", "you", "in", "that"]}, {"sent": "are there some other rabbits ?", "words": ["there", "are", "other", "some"]}, {"sent": "Imean the next time I come Ill Ill sure you can take him into the other room even and play with him or just give you your .", "words": ["the", "can", "your", "into", "play", "with", "you", "other", "and", "him", "room", "give", "take"]}, {"sent": "this oh heres the other one .", "words": ["the", "this", "other"]}, {"sent": "would you like to have a look at some other animals ?", "words": ["look", "to", "like", "you", "have", "other", "at", "would", "some", "a"]}, {"sent": "the other time you said go .", "words": ["the", "go", "you", "other"]}, {"sent": "my other Edward Scissorhand ?", "words": ["my", "other"]}, {"sent": "other side .", "words": ["other"]}, {"sent": "turn it the other way .", "words": ["the", "it", "other"]}, {"sent": "this goes the other way up .", "words": ["the", "this", "up", "other"]}, {"sent": "oh one other thing .", "words": ["other"]}, {"sent": "you hafta push it from one side to the other .", "words": ["the", "push", "to", "you", "other", "it"]}, {"sent": "where are the other keys ?", "words": ["where", "the", "other", "are", "keys"]}, {"sent": "what other one ?", "words": ["what", "other"]}, {"sent": "what other dollie ?", "words": ["what", "other"]}, {"sent": "well hafta make another bridge with some of those other bricks wont we ?", "words": ["those", "we", "of", "with", "other", "another", "some", "make"]}, {"sent": "two fingers in one and your thumb in the other .", "words": ["the", "your", "other", "and", "in"]}, {"sent": "and taro root and winter melon and baby corn cried the other villagers .", "words": ["the", "other", "and", "corn", "melon"]}, {"sent": "youre blowing them because the other day when we went for a walk we saw some dandelions .", "words": ["the", "for", "we", "them", "other", "walk", "some", "when", "because", "a"]}, {"sent": "what other colors are on the chair ?", "words": ["the", "what", "chair", "other", "are", "on"]}, {"sent": "and whats the other one ?", "words": ["the", "other", "and"]}, {"sent": "you go in the other room sweety .", "words": ["the", "go", "other", "you", "in", "room"]}, {"sent": "do you remember the other day we had a bag through .", "words": ["the", "do", "we", "other", "you", "a"]}, {"sent": "you know I was in the um bookstore the other day .", "words": ["the", "was", "other", "you", "in"]}, {"sent": "squeeze the other end .", "words": ["the", "other"]}, {"sent": "wow and let see what other pictures they have here .", "words": ["what", "other", "and", "they", "have", "here", "see"]}, {"sent": "go in the other room .", "words": ["the", "go", "other", "in", "room"]}, {"sent": "hafta look at the other ones .", "words": ["the", "at", "other", "look"]}, {"sent": "other way .", "words": ["other"]}, {"sent": "now shall we put the other one quite near the top ?", "words": ["the", "put", "other", "we"]}, {"sent": "whats the other one ?", "words": ["the", "other"]}, {"sent": "maybe hes helping that other dragon build that sandcastle so hes just holding it for a minute .", "words": ["for", "so", "other", "it", "build", "that", "a"]}, {"sent": "oh heres the other one .", "words": ["the", "other"]}, {"sent": "I know some other things do too princesses princes .", "words": ["too", "some", "do", "other"]}, {"sent": "other towers right ?", "words": ["other"]}, {"sent": "wheres the other one ?", "words": ["the", "other"]}, {"sent": "other other .", "words": ["other"]}, {"sent": "what other what other kind what animal does a zebra look like ?", "words": ["does", "what", "zebra", "like", "other", "animal", "look", "a"]}, {"sent": "are you looking for other shapes ?", "words": ["other", "are", "for", "you"]}, {"sent": "wheres wheres the other bridge one ?", "words": ["the", "other"]}, {"sent": "do the other color .", "words": ["the", "do", "other"]}, {"sent": "what other kind of toys would you like to play with today ?", "words": ["what", "of", "to", "play", "with", "like", "you", "other", "would"]}, {"sent": "what other toys are there ?", "words": ["what", "are", "there", "other"]}, {"sent": "one other kind of thing ?", "words": ["of", "other"]}, {"sent": "the other way Thomas .", "words": ["the", "other"]}, {"sent": "wheres the other shape ?", "words": ["the", "other"]}, {"sent": "no not in there in the other bag Adam .", "words": ["the", "other", "there", "in", "not"]}, {"sent": "wheres the other one like this Eric ?", "words": ["the", "this", "like", "other"]}, {"sent": "the other ones a nice bold one isnt it ?", "words": ["the", "nice", "other", "it", "a"]}, {"sent": "what other verses did we sing last week ?", "words": ["what", "we", "last", "other", "sing", "did"]}, {"sent": "did other people brush their hair with this mirror ?", "words": ["their", "brush", "with", "other", "this", "did", "hair"]}, {"sent": "tell Melissa where ya went the other night .", "words": ["where", "the", "other"]}, {"sent": "I know some other animal that likes to eat eucalyptus .", "words": ["to", "other", "eat", "animal", "some", "that"]}, {"sent": "other than these eggs .", "words": ["these", "other"]}, {"sent": "the moon yeah I think the other songs gonna the next songs gonna come on .", "words": ["the", "think", "moon", "other", "on"]}, {"sent": "how about we find the other three ?", "words": ["the", "how", "we", "other", "about", "find"]}, {"sent": "are the horsies talking to each other ?", "words": ["the", "each", "to", "other", "are"]}, {"sent": "what other shapes do you know ?", "words": ["what", "do", "you", "other"]}, {"sent": "theres just too much other stuff for her to look at .", "words": ["for", "look", "to", "much", "other", "at", "too", "her"]}, {"sent": "well I was looking for the other piece of the puppy dog did you have it ?", "words": ["the", "for", "of", "was", "other", "you", "have", "it", "puppy", "did", "dog"]}, {"sent": "there wasnt any other side to the mountain .", "words": ["the", "to", "any", "other", "there"]}, {"sent": "what other record do you like then ?", "words": ["what", "do", "like", "you", "other", "then"]}, {"sent": "heres the other green one .", "words": ["the", "green", "other"]}, {"sent": "with all your other junk .", "words": ["your", "all", "with", "other"]}, {"sent": "mhm whatre you gonna buy ?", "words": ["buy", "you"]}, {"sent": "its time to go and buy some new clothes for you .", "words": ["for", "go", "to", "and", "you", "some", "new", "buy"]}, {"sent": "whats she gone to buy ?", "words": ["to", "buy", "she"]}, {"sent": "what else do you think we hafta buy ?", "words": ["what", "do", "think", "we", "you", "buy"]}, {"sent": "I hafta buy more than one .", "words": ["more", "buy"]}, {"sent": "buy clothes or something else .", "words": ["buy"]}, {"sent": "you buy me one .", "words": ["me", "buy", "you"]}, {"sent": "were going to buy her a bed .", "words": ["to", "bed", "her", "buy", "were", "a"]}, {"sent": "Eleanor what did we buy her ?", "words": ["what", "we", "did", "her", "buy"]}, {"sent": "I think thats thats the one that Im going to buy .", "words": ["the", "think", "to", "that", "buy"]}, {"sent": "yes I would buy things in there too .", "words": ["would", "there", "in", "too", "buy"]}, {"sent": "you can buy hula hoops in any shop .", "words": ["can", "any", "you", "in", "buy"]}, {"sent": "when the kids come home from school Mommy give you give Donna some money to buy you a popsicle and buy Donna a popsicle .", "words": ["the", "money", "school", "to", "you", "and", "some", "give", "popsicle", "when", "buy", "a", "home"]}, {"sent": "what would you buy your friend for a present ?", "words": ["what", "for", "your", "present", "you", "would", "buy", "a"]}, {"sent": "but my father buy a whole lot and he buy it and squeeze it and make for my sister every morning .", "words": ["my", "for", "and", "it", "he", "every", "make", "buy", "but", "a"]}, {"sent": "you cant buy anymore .", "words": ["buy", "you"]}, {"sent": "I dont think ya can I dont know buy clothes for her or not .", "words": ["can", "think", "for", "not", "dont", "her", "buy"]}, {"sent": "oh xxx maybe maybe when we came home from school well buy that .", "words": ["we", "school", "that", "when", "buy", "home"]}, {"sent": "buy some yyy .", "words": ["some", "buy"]}, {"sent": "Im gonna buy some .", "words": ["some", "buy"]}, {"sent": "you didnt buy that .", "words": ["that", "buy", "you"]}, {"sent": "where else can we buy Smarties ?", "words": ["where", "can", "buy", "we"]}, {"sent": "that one theres two there you get little twin babies if you buy this one .", "words": ["if", "little", "get", "there", "you", "this", "that", "buy"]}, {"sent": "besides are you gonna go shopping with xxx and buy the eggs ?", "words": ["the", "go", "with", "and", "you", "are", "buy"]}, {"sent": "the green ones are salt and vinegar and the blue ones are salt and vinegar but the other firm that we buy from the ones are salt and vinegar and the green ones are cheese and onion .", "words": ["the", "we", "and", "blue", "other", "salt", "are", "green", "cheese", "that", "buy", "but"]}, {"sent": "Mummy doesnt buy those .", "words": ["those", "buy"]}, {"sent": "are they gonna come in here and buy the xxx .", "words": ["the", "and", "here", "they", "in", "are", "buy"]}, {"sent": "I said wed buy you some slippers when we were there .", "words": ["we", "you", "there", "some", "when", "buy", "were"]}, {"sent": "I would like to buy some uh ah cookies .", "words": ["to", "like", "would", "some", "buy"]}, {"sent": "youll hafta buy some more xxx for Christmas .", "words": ["some", "more", "for", "buy"]}, {"sent": "I wonder if you could buy mince pies in the shop all year round .", "words": ["the", "all", "if", "you", "in", "could", "buy"]}, {"sent": "did you buy it ?", "words": ["it", "buy", "did", "you"]}, {"sent": "what would we buy from the bakers Thomas ?", "words": ["the", "what", "we", "would", "buy"]}, {"sent": "what did Unclexxx buy you ?", "words": ["what", "buy", "did", "you"]}, {"sent": "people buy food in a cafeteria just like .", "words": ["like", "in", "food", "buy", "a"]}, {"sent": "whos going to buy one ?", "words": ["to", "buy"]}, {"sent": "but when we buy sweeties from the sweetie shop they come in a white bag .", "words": ["the", "we", "they", "in", "white", "when", "buy", "but", "a"]}, {"sent": "what can you what can you buy at the store ?", "words": ["can", "what", "the", "store", "you", "at", "buy"]}, {"sent": "these are the only comics that that Im allowed to buy .", "words": ["the", "these", "to", "are", "that", "buy"]}, {"sent": "buy some more ?", "words": ["some", "more", "buy"]}, {"sent": "well we could MoreDadaw could buy some more or we could buy some more .", "words": ["we", "more", "some", "could", "buy"]}, {"sent": "to buy something .", "words": ["to", "buy"]}, {"sent": "are you gonna buy the things and put the things in a bag for me ?", "words": ["the", "put", "for", "and", "you", "in", "are", "me", "buy", "a"]}, {"sent": "we can leave if you leave your pram here we can go to the shop and buy you some sweeties with your pocket money .", "words": ["can", "the", "money", "your", "if", "we", "go", "to", "with", "you", "here", "and", "some", "buy"]}, {"sent": "what did we buy ?", "words": ["what", "buy", "did", "we"]}, {"sent": "because xxx nice xxx buy it or MulberryCrush .", "words": ["it", "nice", "buy", "because"]}, {"sent": "do you think you could buy all those things for her ?", "words": ["do", "think", "all", "those", "for", "you", "could", "her", "buy"]}, {"sent": "what do I wanna buy ?", "words": ["what", "do", "buy", "wanna"]}, {"sent": "but weve got to buy some more pieces to complete the track .", "words": ["the", "to", "more", "some", "buy", "but"]}, {"sent": "would you like to buy excuse me would you like to buy some vegetables ?", "words": ["to", "like", "you", "would", "me", "buy", "some"]}, {"sent": "and what well do is buy an engine shed as well .", "words": ["what", "do", "is", "and", "an", "buy"]}, {"sent": "Ill come back and buy something else .", "words": ["buy", "back", "and"]}, {"sent": "and when I get the money I will buy you something for you .", "words": ["the", "money", "for", "and", "get", "you", "will", "when", "buy"]}, {"sent": "youre going to buy everything are you ?", "words": ["to", "are", "buy", "you"]}, {"sent": "and what did you buy in Stockport ?", "words": ["what", "you", "and", "in", "did", "buy"]}, {"sent": "once in a while you buy potato chips ?", "words": ["you", "in", "buy", "a", "potato"]}, {"sent": "looks like were gonna needta go buy some more formula .", "words": ["go", "like", "more", "some", "were", "buy"]}, {"sent": "we didnt hafta buy it but we have bought you one .", "words": ["we", "you", "have", "it", "buy", "but"]}, {"sent": "but these magazines we buy in the shop .", "words": ["the", "these", "we", "in", "buy", "but"]}, {"sent": "and buy some comics .", "words": ["some", "buy", "and"]}, {"sent": "in boxes because we can go into a lot of shops and buy sweeties cant we ?", "words": ["can", "into", "go", "we", "of", "a lot", "and", "in", "buy", "because", "a"]}, {"sent": "whatre we going to buy ?", "words": ["to", "buy", "we"]}, {"sent": "or have I won the loaf of bread I wanna buy ?", "words": ["the", "bread", "of", "have", "wanna", "buy"]}, {"sent": "well I would like to buy him .", "words": ["to", "like", "him", "would", "buy"]}, {"sent": "I wanna buy something too .", "words": ["buy", "wanna", "too"]}, {"sent": "I think when we go out this afternoon we must go out to the shop and buy some more bread .", "words": ["the", "think", "bread", "go", "we", "to", "and", "this", "more", "out", "some", "when", "buy"]}, {"sent": "well buy some more at the AandP .", "words": ["the", "more", "at", "some", "buy"]}, {"sent": "thats where I would buy a movie .", "words": ["where", "buy", "a", "would"]}, {"sent": "xxx to buy to buy .", "words": ["to", "buy"]}, {"sent": "we dont buy HP worms .", "words": ["dont", "buy", "we"]}, {"sent": "were gonna hafta buy some more I think .", "words": ["think", "more", "some", "were", "buy"]}, {"sent": "dont you need the money to buy stuff for your store ?", "words": ["the", "money", "for", "need", "your", "to", "store", "you", "dont", "buy"]}, {"sent": "I think Id like to buy some coffee .", "words": ["think", "to", "like", "coffee", "some", "buy"]}, {"sent": "you can go and buy stamps .", "words": ["can", "go", "and", "you", "buy"]}, {"sent": "tell me what you wanna buy .", "words": ["what", "you", "wanna", "me", "buy"]}, {"sent": "why should I buy something ?", "words": ["buy", "why"]}, {"sent": "were going to one where we can buy some leggings .", "words": ["where", "can", "we", "to", "some", "were", "buy"]}, {"sent": "what do I wanna buy ?", "words": ["what", "do", "buy", "wanna"]}, {"sent": "are you going to go and buy some apples at the greengrocers ?", "words": ["the", "go", "to", "and", "you", "are", "at", "some", "buy"]}, {"sent": "that right and then Im gonna buy some diet candy .", "words": ["and", "some", "then", "candy", "that", "buy"]}, {"sent": "go to the shops and you buy the food and then you take it round to other people .", "words": ["the", "go", "to", "you", "and", "other", "food", "it", "then", "take", "buy"]}, {"sent": "or you wanna buy a sneakers ?", "words": ["a", "buy", "wanna", "you"]}, {"sent": "what would you like to buy ?", "words": ["what", "to", "like", "you", "would", "buy"]}, {"sent": "you dont think he should buy food ?", "words": ["think", "you", "dont", "food", "he", "buy"]}, {"sent": "and then Ill buy some things from you .", "words": ["you", "and", "some", "then", "buy"]}, {"sent": "while youre at the store buy me a blueberry fish okay ?", "words": ["the", "store", "fish", "at", "me", "buy", "a"]}, {"sent": "hes going to buy a new door today .", "words": ["to", "new", "buy", "a", "door"]}, {"sent": "which shop are you going to buy your sugar .", "words": ["your", "to", "you", "are", "which", "buy"]}, {"sent": "what else did we buy this morning ?", "words": ["what", "we", "this", "did", "buy"]}, {"sent": "what would you like to buy today ?", "words": ["what", "to", "like", "you", "would", "buy"]}, {"sent": "youre going to buy some from the shop are you ?", "words": ["the", "to", "you", "are", "some", "buy"]}, {"sent": "whatre you going to buy this time ?", "words": ["to", "buy", "this", "you"]}, {"sent": "we can buy those pumpkins in there and that farmers going to come and buy some now .", "words": ["can", "those", "we", "to", "and", "there", "in", "some", "that", "buy"]}, {"sent": "what did we buy ?", "words": ["what", "buy", "did", "we"]}, {"sent": "can I buy a cake and some cheese for Anne ?", "words": ["can", "for", "cake", "and", "some", "cheese", "buy", "a"]}, {"sent": "Kims going to buy something .", "words": ["to", "buy"]}, {"sent": "but he didnt buy it so .", "words": ["so", "it", "he", "buy", "but"]}, {"sent": "what did he buy you ?", "words": ["what", "you", "he", "did", "buy"]}, {"sent": "why did the tree fall down ?", "words": ["the", "tree", "fall", "did", "down", "why"]}, {"sent": "why dont you wait to ride on the big bus to school ?", "words": ["the", "wait", "to", "school", "you", "dont", "big", "on", "ride", "bus", "why"]}, {"sent": "why ?", "words": ["why"]}, {"sent": "why are you being so picky ?", "words": ["so", "are", "why", "you"]}, {"sent": "did you bang your head or why did you hafta go to hospital ?", "words": ["your", "go", "to", "head", "you", "did", "why"]}, {"sent": "why not ?", "words": ["not", "why"]}, {"sent": "why ?", "words": ["why"]}, {"sent": "why are you throwing the dogs ?", "words": ["the", "are", "why", "you"]}, {"sent": "why dont you like it like that ?", "words": ["like", "you", "dont", "it", "that", "why"]}, {"sent": "so thats why I go to the store tomorrow .", "words": ["the", "go", "to", "so", "store", "why"]}, {"sent": "why poor Daddy ?", "words": ["why", "poor"]}, {"sent": "why do you think hed be running after the car ?", "words": ["the", "do", "think", "be", "car", "you", "why"]}, {"sent": "but you know what you didnt even see the mean spiders because why ?", "words": ["the", "what", "because", "you", "see", "why", "but"]}, {"sent": "why ?", "words": ["why"]}, {"sent": "why doesnt he go in with the other horsie ?", "words": ["the", "go", "with", "other", "in", "he", "why"]}, {"sent": "thats why there are plastic bottles .", "words": ["there", "are", "why"]}, {"sent": "why not ?", "words": ["not", "why"]}, {"sent": "thats why hes having a meltdown he wants to go out on the s w i n g .", "words": ["the", "i", "go", "to", "he", "out", "on", "why", "a"]}, {"sent": "why ?", "words": ["why"]}, {"sent": "why would he shoot somebody with the gun ?", "words": ["the", "with", "he", "would", "why"]}, {"sent": "why do you think they left it open ?", "words": ["do", "think", "open", "you", "they", "it", "why"]}, {"sent": "my hairs blue why xxx my hairs blue ?", "words": ["my", "why", "blue"]}, {"sent": "why ?", "words": ["why"]}, {"sent": "why why did he scratch you ?", "words": ["he", "did", "why", "you"]}, {"sent": "why not ?", "words": ["not", "why"]}, {"sent": "why do you wanna do that ?", "words": ["do", "you", "wanna", "that", "why"]}, {"sent": "why ?", "words": ["why"]}, {"sent": "why not ?", "words": ["not", "why"]}, {"sent": "why dont you take the box out ?", "words": ["the", "box", "you", "dont", "out", "take", "why"]}, {"sent": "why ?", "words": ["why"]}, {"sent": "why not ?", "words": ["not", "why"]}, {"sent": "well why dont you go and open the door for us so .", "words": ["the", "for", "go", "open", "so", "and", "you", "dont", "us", "why", "door"]}, {"sent": "well why would he be cross ?", "words": ["be", "he", "why", "would"]}, {"sent": "why are you talking about Sues house ?", "words": ["you", "are", "house", "about", "why"]}, {"sent": "well why have you come to see me PostmanPat ?", "words": ["to", "me", "you", "have", "see", "why"]}, {"sent": "why ?", "words": ["why"]}, {"sent": "and uh why didnt he get on the ball and just sorta tell the guy he was coming down .", "words": ["the", "was", "and", "get", "ball", "he", "on", "down", "why"]}, {"sent": "why ?", "words": ["why"]}, {"sent": "why dont you tidy them away ?", "words": ["them", "you", "dont", "away", "why"]}, {"sent": "Cal why dont you make him talk ?", "words": ["you", "dont", "him", "talk", "make", "why"]}, {"sent": "thats why its called a beanbag .", "words": ["why", "a"]}, {"sent": "why isnt he going in the train ?", "words": ["the", "train", "in", "he", "why"]}, {"sent": "why arent you doing it ?", "words": ["it", "why", "you"]}, {"sent": "why would you not like it ?", "words": ["like", "you", "it", "would", "not", "why"]}, {"sent": "why did you bang your head .", "words": ["your", "head", "you", "did", "why"]}, {"sent": "I dont really know why .", "words": ["dont", "why"]}, {"sent": "why dont you go play with the blocks and you can have the cookie later .", "words": ["the", "can", "cookie", "go", "play", "with", "you", "and", "dont", "have", "why"]}, {"sent": "why dont you take this piece off first ?", "words": ["off", "first", "you", "dont", "this", "take", "why"]}, {"sent": "why ?", "words": ["why"]}, {"sent": "why ?", "words": ["why"]}, {"sent": "why ?", "words": ["why"]}, {"sent": "why ?", "words": ["why"]}, {"sent": "Nomi why dont you put the hanger down while you boo .", "words": ["the", "put", "you", "dont", "down", "why"]}, {"sent": "what well they why why did they want him to stop ?", "words": ["what", "to", "stop", "they", "him", "did", "why"]}, {"sent": "why why cant I swim there ?", "words": ["there", "swim", "why"]}, {"sent": "why do you think theyre chasing the moo ?", "words": ["the", "do", "think", "you", "why"]}, {"sent": "wha why is he um a happy monkey ?", "words": ["happy", "is", "monkey", "he", "why", "a"]}, {"sent": "why dont you pull right here ?", "words": ["pull", "here", "dont", "you", "why"]}, {"sent": "why ?", "words": ["why"]}, {"sent": "the tape thats why I did at the end .", "words": ["the", "tape", "did", "at", "why"]}, {"sent": "then why are you giving yourself some ?", "words": ["yourself", "you", "are", "some", "then", "why"]}, {"sent": "trying to think why .", "words": ["to", "think", "why"]}, {"sent": "okay now why dont you finish xxx .", "words": ["dont", "finish", "why", "you"]}, {"sent": "well I think thats why I said ironing .", "words": ["think", "why"]}, {"sent": "why ?", "words": ["why"]}, {"sent": "why ?", "words": ["why"]}, {"sent": "why are you taking your socks off ?", "words": ["off", "your", "you", "are", "why"]}, {"sent": "why did the settee cross the road ?", "words": ["the", "did", "why"]}, {"sent": "thats why you hafta be careful with because you can break it .", "words": ["can", "be", "with", "because", "you", "it", "careful", "break", "why"]}, {"sent": "why are you telling Dwww to shut up ?", "words": ["up", "to", "you", "are", "why"]}, {"sent": "why is that ?", "words": ["that", "why", "is"]}, {"sent": "why do you want me to throw her in the bin ?", "words": ["the", "do", "throw", "to", "you", "in", "me", "her", "why"]}, {"sent": "because why ?", "words": ["why", "because"]}, {"sent": "its sad why is it sad ?", "words": ["sad", "it", "why", "is"]}, {"sent": "why dont you give him a big kiss ?", "words": ["you", "dont", "him", "kiss", "give", "big", "why", "a"]}, {"sent": "why dont you stand up ?", "words": ["up", "stand", "you", "dont", "why"]}, {"sent": "why not ?", "words": ["not", "why"]}, {"sent": "thats why we hafta keep washing our hands .", "words": ["our", "why", "we"]}, {"sent": "why dont you look at your books ?", "words": ["your", "you", "dont", "at", "look", "why"]}, {"sent": "why is Coco upside down ?", "words": ["down", "why", "is"]}, {"sent": "why are you banging it ?", "words": ["it", "are", "why", "you"]}, {"sent": "why do you say Ursulas words ?", "words": ["do", "say", "why", "you"]}, {"sent": "why are you not eating fruit Thomas ?", "words": ["are", "not", "why", "you"]}, {"sent": "why ?", "words": ["why"]}, {"sent": "why did the tree fall down ?", "words": ["the", "tree", "fall", "did", "down", "why"]}, {"sent": "well if you cant get them out why dont you tip them on the floor ?", "words": ["the", "if", "them", "you", "get", "dont", "out", "on", "why"]}, {"sent": "do you know why ?", "words": ["do", "why", "you"]}, {"sent": "wash your hands why ?", "words": ["your", "wash", "why"]}, {"sent": "well why dont you get ?", "words": ["get", "dont", "why", "you"]}, {"sent": "why ?", "words": ["why"]}, {"sent": "why ?", "words": ["why"]}, {"sent": "why was he sad ?", "words": ["was", "sad", "he", "why"]}, {"sent": "why dont you try to eat as much as you can ?", "words": ["can", "to", "much", "you", "eat", "dont", "try", "try to", "why"]}, {"sent": "why not ?", "words": ["not", "why"]}, {"sent": "why dont you come and help me build ?", "words": ["help", "and", "you", "dont", "build", "me", "why"]}, {"sent": "see this is why .", "words": ["see", "why", "this", "is"]}, {"sent": "oh why is it going to the Doctors ?", "words": ["the", "is", "to", "it", "why"]}, {"sent": "why ?", "words": ["why"]}, {"sent": "do you know why ?", "words": ["do", "why", "you"]}, {"sent": "I know why hes gone quiet .", "words": ["quiet", "why"]}, {"sent": "lets see where Josephs is .", "words": ["where", "see", "is"]}, {"sent": "where is this .", "words": ["where", "this", "is"]}, {"sent": "where are the babies ?", "words": ["where", "the", "are"]}, {"sent": "where did we cut your bangs ?", "words": ["where", "your", "cut", "we", "did"]}, {"sent": "where did she push you ?", "words": ["where", "push", "you", "did", "she"]}, {"sent": "where do you go on a Wednesday ?", "words": ["where", "do", "go", "you", "on", "a"]}, {"sent": "but Rainbow stayed where he was .", "words": ["where", "was", "he", "but"]}, {"sent": "where is his stomach ?", "words": ["where", "his", "is"]}, {"sent": "I dont know where the squirrels are .", "words": ["where", "dont", "are", "the"]}, {"sent": "you tell me where it is .", "words": ["where", "is", "you", "it", "me"]}, {"sent": "where is she ?", "words": ["where", "she", "is"]}, {"sent": "is that where that goes right there ?", "words": ["where", "there", "that", "is"]}, {"sent": "where does the sink go ?", "words": ["where", "does", "the", "go", "sink"]}, {"sent": "where ?", "words": ["where"]}, {"sent": "where did the dog go Marjorie ?", "words": ["where", "the", "go", "did", "dog"]}, {"sent": "where together they began a grand tour of the entire art museum .", "words": ["where", "the", "of", "they", "a"]}, {"sent": "where is your foot ?", "words": ["where", "foot", "your", "is"]}, {"sent": "you wanna put it where it goes ?", "words": ["where", "put", "you", "it", "wanna"]}, {"sent": "but uh is that where they go ?", "words": ["where", "is", "go", "they", "that", "but"]}, {"sent": "where do you live ?", "words": ["where", "do", "you"]}, {"sent": "I dont know where oh theres the cat hiding behind the number nine .", "words": ["where", "the", "behind", "dont", "cat"]}, {"sent": "where that came from .", "words": ["where", "that"]}, {"sent": "thats where the the steam comes from .", "words": ["where", "the"]}, {"sent": "well where do I sit ?", "words": ["where", "do", "sit"]}, {"sent": "where do you think where do you think this planes flying to look .", "words": ["where", "do", "think", "to", "you", "this", "look"]}, {"sent": "this is where youre naughty on this page arent you ?", "words": ["where", "is", "you", "this", "on", "naughty"]}, {"sent": "where are you going ?", "words": ["where", "are", "you"]}, {"sent": "I dont want you to go where the doggie goes peepee .", "words": ["where", "the", "go", "to", "you", "dont"]}, {"sent": "where does it go .", "words": ["where", "does", "it", "go"]}, {"sent": "well let me see daddy where does it seem to hurt ?", "words": ["where", "does", "hurt", "to", "me", "it", "see"]}, {"sent": "hey Brian where your fat tummy .", "words": ["where", "your", "tummy"]}, {"sent": "where do you live ?", "words": ["where", "do", "you"]}, {"sent": "and where did I say the station was ?", "words": ["where", "the", "was", "say", "and", "did"]}, {"sent": "where ?", "words": ["where"]}, {"sent": "yeah now dyou know where this is ?", "words": ["where", "this", "is"]}, {"sent": "where are you Alex ?", "words": ["where", "are", "you"]}, {"sent": "um did she show any interest in any of the animals or where there any that you thought .", "words": ["the", "where", "show", "of", "any", "you", "there", "in", "did", "that", "she"]}, {"sent": "what did it where did it .", "words": ["where", "what", "it", "did"]}, {"sent": "yeah but wheres the train whistle where did it go ?", "words": ["the", "where", "train", "go", "it", "did", "but"]}, {"sent": "do you know where it is ?", "words": ["where", "do", "is", "you", "it"]}, {"sent": "where shall we go for this walk ?", "words": ["where", "for", "go", "we", "this", "walk"]}, {"sent": "where your nose at ?", "words": ["where", "at", "your", "nose"]}, {"sent": "where ?", "words": ["where"]}, {"sent": "where shall we put it ?", "words": ["where", "put", "it", "we"]}, {"sent": "where did he take you ?", "words": ["where", "you", "he", "did", "take"]}, {"sent": "where ?", "words": ["where"]}, {"sent": "do you know my where my round um tub is ?", "words": ["where", "do", "my", "is", "you"]}, {"sent": "down where ?", "words": ["where", "down"]}, {"sent": "where is yellow like a banana ?", "words": ["where", "is", "like", "banana", "yellow", "a"]}, {"sent": "tell Anna where youre going tonight .", "words": ["where"]}, {"sent": "where is daddy Fra ?", "words": ["where", "is"]}, {"sent": "where is it ?", "words": ["where", "it", "is"]}, {"sent": "where was the fork Sarah .", "words": ["where", "was", "fork", "the"]}, {"sent": "where did your Mummy go then ?", "words": ["where", "your", "go", "did", "then"]}, {"sent": "so where are you going tomorrow ?", "words": ["where", "so", "are", "you"]}, {"sent": "where has she gone ?", "words": ["where", "she"]}, {"sent": "now where did I put ?", "words": ["where", "put", "did"]}, {"sent": "where would you walk ?", "words": ["where", "walk", "you", "would"]}, {"sent": "youre gonna go where ?", "words": ["where", "go"]}, {"sent": "where did it go ?", "words": ["where", "it", "did", "go"]}, {"sent": "now where should we keep the light you know because theyre kinda spread out ?", "words": ["where", "the", "we", "light", "you", "out", "because"]}, {"sent": "where is it ?", "words": ["where", "it", "is"]}, {"sent": "honey why dont you show Manuela where you live ?", "words": ["where", "show", "you", "dont", "why"]}, {"sent": "where do these go ?", "words": ["where", "do", "these", "go"]}, {"sent": "mom wants to know where the baby is .", "words": ["where", "to", "is", "the"]}, {"sent": "where are you going for your helicopter ?", "words": ["where", "helicopter", "for", "your", "you", "are"]}, {"sent": "where are they going ?", "words": ["where", "they", "are"]}, {"sent": "know where what ?", "words": ["where", "what"]}, {"sent": "down where ?", "words": ["where", "down"]}, {"sent": "where does he go ?", "words": ["where", "does", "he", "go"]}, {"sent": "so where does it go on him ?", "words": ["where", "does", "go", "so", "him", "it", "on"]}, {"sent": "put it where ?", "words": ["where", "put", "it"]}, {"sent": "where are you Ethan ?", "words": ["where", "are", "you"]}, {"sent": "where are you going Tallulah ?", "words": ["where", "are", "you"]}, {"sent": "where does he sit ?", "words": ["where", "does", "sit", "he"]}, {"sent": "where are the other cupcakes ?", "words": ["where", "the", "are", "other"]}, {"sent": "where ?", "words": ["where"]}, {"sent": "well listen um do you have a um where do you get our key chains from ?", "words": ["where", "do", "listen", "you", "have", "get", "our", "a"]}, {"sent": "where ?", "words": ["where"]}, {"sent": "at where ?", "words": ["where", "at"]}, {"sent": "thats where he lives ?", "words": ["where", "he"]}, {"sent": "but where is TinyPig ?", "words": ["where", "but", "is"]}, {"sent": "where is it now ?", "words": ["where", "it", "is"]}, {"sent": "where is their home ?", "words": ["where", "home", "their", "is"]}, {"sent": "where does that one go ?", "words": ["where", "does", "that", "go"]}, {"sent": "where they sit right here in these little boxes .", "words": ["where", "these", "little", "here", "they", "in", "sit"]}, {"sent": "where ?", "words": ["where"]}, {"sent": "where do you go with him ?", "words": ["where", "do", "go", "with", "you", "him"]}, {"sent": "where is your tongue ?", "words": ["where", "your", "tongue", "is"]}, {"sent": "where we going ?", "words": ["where", "we"]}, {"sent": "sometimes you get a little accident where you spill ?", "words": ["where", "little", "get", "you", "spill", "a"]}, {"sent": "where is it ?", "words": ["where", "it", "is"]}, {"sent": "where did he put his ball ?", "words": ["where", "put", "ball", "he", "did", "his"]}, {"sent": "right where ?", "words": ["where"]}, {"sent": "frog where are you ?", "words": ["where", "are", "you", "frog"]}, {"sent": "see where the slot is ?", "words": ["where", "see", "is", "the"]}, {"sent": "but they realize where they are when they hear the church bells .", "words": ["where", "the", "church", "hear", "they", "are", "when", "but"]}, {"sent": "where is Gals eyes ?", "words": ["where", "is"]}, {"sent": "where where do you think that goes ?", "words": ["where", "do", "think", "you", "that"]}, {"sent": "where is the tv ?", "words": ["where", "the", "tv", "is"]}, {"sent": "orange jump suit .", "words": ["jump", "orange"]}, {"sent": "but thats orange sweetie .", "words": ["orange", "but"]}, {"sent": "well find the orange one then .", "words": ["the", "then", "orange", "find"]}, {"sent": "do you want some of your orange juice ?", "words": ["do", "your", "of", "you", "orange", "some", "juice"]}, {"sent": "you dont eat orange peel Thomas .", "words": ["eat", "dont", "orange", "you"]}, {"sent": "you mean orange juice ?", "words": ["orange", "juice", "you"]}, {"sent": "no orange okay ?", "words": ["orange"]}, {"sent": "orange light .", "words": ["light", "orange"]}, {"sent": "youre playing with your train and youve just wheeled your yellow train youve just wheeled it past the orange havent you ?", "words": ["the", "your", "train", "with", "and", "you", "it", "orange", "yellow"]}, {"sent": "its pretend orange squash .", "words": ["pretend", "orange"]}, {"sent": "orange .", "words": ["orange"]}, {"sent": "there are two shades of green orange and red .", "words": ["red", "of", "and", "there", "are", "orange", "green"]}, {"sent": "were are the other orange ones ?", "words": ["the", "other", "are", "orange", "were"]}, {"sent": "this is orange orange .", "words": ["this", "orange", "is"]}, {"sent": "say orange .", "words": ["orange", "say"]}, {"sent": "with those orange and purple letters that are on the truck .", "words": ["the", "those", "with", "truck", "and", "orange", "are", "on", "that"]}, {"sent": "an orange digger ?", "words": ["an", "orange"]}, {"sent": "its actually orange yellow .", "words": ["yellow", "orange"]}, {"sent": "you just needta get the orange teddy in the orange basket .", "words": ["the", "you", "get", "in", "orange", "basket"]}, {"sent": "and Im going to blick this orange juice .", "words": ["to", "and", "this", "orange", "juice"]}, {"sent": "its an orange .", "words": ["an", "orange"]}, {"sent": "maybe some orange hair .", "words": ["some", "hair", "orange"]}, {"sent": "yes that is orange right .", "words": ["orange", "that", "is"]}, {"sent": "what is the orange gonna be ?", "words": ["the", "what", "is", "be", "orange"]}, {"sent": "that is an orange tree .", "words": ["tree", "is", "an", "orange", "that"]}, {"sent": "theres an orange one .", "words": ["an", "orange"]}, {"sent": "oh you mean orange paint on the tablecloth ?", "words": ["the", "paint", "you", "orange", "on"]}, {"sent": "its an orange ?", "words": ["an", "orange"]}, {"sent": "he got the orange one here too .", "words": ["the", "here", "orange", "he", "too"]}, {"sent": "then you might like a little orange .", "words": ["like", "little", "you", "orange", "then", "a"]}, {"sent": "orange dish .", "words": ["dish", "orange"]}, {"sent": "how do you make orange ?", "words": ["do", "how", "you", "orange", "make"]}, {"sent": "can you find an orange ?", "words": ["can", "you", "an", "orange", "find"]}, {"sent": "you can have an orange tractor .", "words": ["can", "you", "have", "an", "orange", "tractor"]}, {"sent": "Charlotte had green paint and orange paint on her face .", "words": ["paint", "face", "and", "orange", "green", "on", "her"]}, {"sent": "the orange color ?", "words": ["the", "orange"]}, {"sent": "here here is orange .", "words": ["here", "orange", "is"]}, {"sent": "it was an orange a small orange cement mixer wasnt it ?", "words": ["was", "an", "it", "orange", "a"]}, {"sent": "pink green orange yellow .", "words": ["yellow", "orange", "green"]}, {"sent": "see if you can make the eyes show with your orange pen .", "words": ["can", "the", "your", "if", "show", "with", "you", "orange", "pen", "see", "make"]}, {"sent": "hes got a black hat orange hair .", "words": ["black", "orange", "hat", "hair", "a"]}, {"sent": "orange .", "words": ["orange"]}, {"sent": "orange .", "words": ["orange"]}, {"sent": "orange ?", "words": ["orange"]}, {"sent": "not orange blue .", "words": ["orange", "not", "blue"]}, {"sent": "why have you put the orange on the .", "words": ["the", "put", "you", "have", "orange", "on", "why"]}, {"sent": "it was orange you had this morning .", "words": ["was", "you", "this", "it", "orange"]}, {"sent": "you always shake orange juice .", "words": ["orange", "juice", "shake", "you"]}, {"sent": "and what is that orange shape ?", "words": ["what", "is", "and", "orange", "that"]}, {"sent": "get the orange one .", "words": ["get", "the", "orange"]}, {"sent": "Lalas ball is orange though isnt it ?", "words": ["ball", "it", "orange", "is"]}, {"sent": "pink and orange .", "words": ["orange", "and"]}, {"sent": "it was a big orange lorry wasnt it ?", "words": ["was", "it", "orange", "big", "a"]}, {"sent": "it is orange .", "words": ["it", "orange", "is"]}, {"sent": "orange okay .", "words": ["orange"]}, {"sent": "orange .", "words": ["orange"]}, {"sent": "I can hear the orange juice squeaking as youre sucking it through the beaker .", "words": ["can", "the", "hear", "orange", "it", "juice"]}, {"sent": "orange juice ?", "words": ["orange", "juice"]}, {"sent": "orange chocolate .", "words": ["chocolate", "orange"]}, {"sent": "blue sky orange sun yellow moon yellow stars .", "words": ["moon", "sky", "sun", "blue", "orange", "yellow"]}, {"sent": "orange .", "words": ["orange"]}, {"sent": "purple green purple green yellow white orange or pink ?", "words": ["yellow", "white", "green", "orange"]}, {"sent": "lets see if we can see an orange bit .", "words": ["can", "if", "we", "an", "orange", "see"]}, {"sent": "thats MrsNoah wearing an orange dress .", "words": ["an", "orange", "dress"]}, {"sent": "oh I bought you orange juice its down the beach .", "words": ["the", "juice", "you", "beach", "orange", "down"]}, {"sent": "just orange .", "words": ["orange"]}, {"sent": "how bout orange ?", "words": ["how", "orange"]}, {"sent": "and orange .", "words": ["orange", "and"]}, {"sent": "an orange triangle .", "words": ["an", "orange"]}, {"sent": "the green and orange light are working are they ?", "words": ["the", "light", "and", "they", "orange", "are", "green"]}, {"sent": "orange clay .", "words": ["orange"]}, {"sent": "we use some more orange juice and we make more lollys .", "words": ["we", "and", "more", "orange", "some", "juice", "make"]}, {"sent": "thats an orange .", "words": ["an", "orange"]}, {"sent": "orange green egg .", "words": ["orange", "egg", "green"]}, {"sent": "what else is orange ?", "words": ["what", "orange", "is"]}, {"sent": "because hes orange isnt he ?", "words": ["orange", "he", "because"]}, {"sent": "orange trousers .", "words": ["orange"]}, {"sent": "I think its the beaker of orange isnt it ?", "words": ["the", "think", "of", "orange", "it"]}, {"sent": "Ill have the orange one to cut in to color in the bird .", "words": ["the", "cut", "to", "bird", "have", "in", "orange"]}, {"sent": "youre looking at a cover of a magazine and it has got a picture of three girls or ladys all eating ice lollies orange ice lollies .", "words": ["all", "of", "picture", "and", "it", "orange", "cover", "at", "ice", "a"]}, {"sent": "it was a nice orange wasnt it ?", "words": ["was", "nice", "it", "orange", "a"]}, {"sent": "its sort of orange .", "words": ["of", "orange"]}, {"sent": "an aeroplane and a I think its highly likely that the apples rather than orange and the lemon and the pineapple they will be flown in a helicopter or an aeroplane .", "words": ["the", "helicopter", "think", "be", "and", "they", "an", "orange", "in", "will", "that", "a"]}, {"sent": "now we have some more orange lilies .", "words": ["we", "have", "more", "orange", "some"]}, {"sent": "some orange juice .", "words": ["some", "orange", "juice"]}, {"sent": "and we have a big orange and a small orange .", "words": ["we", "and", "have", "orange", "big", "a"]}, {"sent": "an orange one damaged .", "words": ["an", "orange"]}, {"sent": "orange .", "words": ["orange"]}, {"sent": "Becky the orange .", "words": ["the", "orange"]}, {"sent": "do you want orange ?", "words": ["do", "orange", "you"]}, {"sent": "I see orange and red and beige .", "words": ["red", "see", "orange", "and"]}, {"sent": "a carrots orange .", "words": ["orange", "carrots", "a"]}, {"sent": "some orange juice .", "words": ["some", "orange", "juice"]}, {"sent": "this orange dinosaur that Im playing with ha orange PlayDoh .", "words": ["this", "orange", "that", "with"]}, {"sent": "it isnt the orange juice out of the carton Thomas .", "words": ["the", "of", "it", "orange", "out", "juice"]}, {"sent": "orange paint and green PlayDoh .", "words": ["orange", "paint", "green", "and"]}, {"sent": "I dont have the orange .", "words": ["have", "dont", "orange", "the"]}, {"sent": "put the peaches are in the this orange red .", "words": ["the", "red", "put", "this", "in", "are", "orange"]}, {"sent": "make the carrot orange ?", "words": ["the", "orange", "make"]}, {"sent": "may I have some more orange juice please ?", "words": ["have", "more", "orange", "some", "juice"]}, {"sent": "dont see any bread in here .", "words": ["bread", "any", "here", "dont", "in", "see"]}, {"sent": "right some of these people dont have any room for their chairs .", "words": ["these", "for", "their", "of", "any", "have", "dont", "room", "some"]}, {"sent": "oh you dont want any help ?", "words": ["dont", "help", "any", "you"]}, {"sent": "so are there any other Imean besides the tools any other toys that she has at home that we didnt have here that she usually plays with ?", "words": ["the", "we", "so", "any", "with", "other", "have", "there", "here", "are", "at", "that", "she", "home"]}, {"sent": "you dont wanna see any of these books or you just wanna do that at bed time .", "words": ["these", "do", "of", "any", "bed", "you", "dont", "wanna", "at", "see", "that"]}, {"sent": "no there isnt any purple .", "words": ["there", "any"]}, {"sent": "any more green blocks ?", "words": ["more", "any", "green"]}, {"sent": "do you see any candy ?", "words": ["do", "any", "you", "see", "candy"]}, {"sent": "Ill do any anything for xxx Ive eaten snails .", "words": ["do", "for", "any"]}, {"sent": "I dont want any crisps on my sandwich .", "words": ["my", "any", "dont", "on", "sandwich"]}, {"sent": "um so you said you dont have any dolls at home but it sounds like you have books that you do at um sleeping time .", "words": ["do", "so", "any", "like", "you", "have", "dont", "it", "at", "that", "but", "home"]}, {"sent": "we havent got any paper .", "words": ["paper", "any", "we"]}, {"sent": "I dont know if youll remember any of these .", "words": ["these", "if", "of", "any", "dont"]}, {"sent": "we hafta finish getting dressed before we can play any more .", "words": ["can", "we", "play", "any", "finish", "more"]}, {"sent": "I dont think you need any blanket .", "words": ["think", "need", "any", "you", "dont", "blanket"]}, {"sent": "do you want any more ?", "words": ["do", "more", "any", "you"]}, {"sent": "I dont see any cars .", "words": ["see", "dont", "any"]}, {"sent": "have you got any middle income or professional families in Harlem ?", "words": ["have", "in", "any", "you"]}, {"sent": "is there any fierce animals in there ?", "words": ["there", "in", "any", "is"]}, {"sent": "and we got out of the car and she didnt make any attempt to move .", "words": ["the", "we", "of", "car", "to", "any", "and", "out", "make", "she"]}, {"sent": "I do it or you dont get any .", "words": ["do", "any", "you", "get", "dont", "it"]}, {"sent": "are you gonna play with any of your toys ?", "words": ["your", "of", "play", "any", "with", "you", "are"]}, {"sent": "you wont get any more in .", "words": ["any", "you", "get", "in", "more"]}, {"sent": "dont sit on any of those .", "words": ["those", "of", "any", "dont", "on", "sit"]}, {"sent": "does he want any milk in his tea ?", "words": ["does", "any", "his", "in", "he", "milk"]}, {"sent": "I didnt hear any music .", "words": ["any", "hear"]}, {"sent": "any more ?", "words": ["more", "any"]}, {"sent": "any mention of Jemima ?", "words": ["of", "any"]}, {"sent": "he wasnt interested in any of the stuffed animals .", "words": ["the", "of", "any", "in", "he"]}, {"sent": "do you know any of them ?", "words": ["do", "of", "them", "any", "you"]}, {"sent": "youre not having any crisps .", "words": ["any", "not"]}, {"sent": "dont you wanna make any people ?", "words": ["any", "you", "dont", "wanna", "make"]}, {"sent": "we havent got any Bandaids .", "words": ["any", "we"]}, {"sent": "it doesnt go on any of the pens in that pot sweetheart .", "words": ["the", "go", "of", "any", "in", "it", "on", "that"]}, {"sent": "I was never any good at singing songs .", "words": ["was", "any", "good", "at"]}, {"sent": "we dont want any more fire engines do we ?", "words": ["do", "we", "any", "dont", "more"]}, {"sent": "and we dont wanna give them any more or theyll get sick .", "words": ["we", "them", "any", "and", "get", "dont", "more", "sick", "wanna", "give"]}, {"sent": "did Karen read you any storys today ?", "words": ["any", "read", "did", "you"]}, {"sent": "theres hardly any um tips left on it .", "words": ["on", "it", "any"]}, {"sent": "there isnt any sugar on it .", "words": ["on", "there", "it", "any"]}, {"sent": "and alotof the animals didnt have any water to drink .", "words": ["the", "drink", "to", "any", "and", "have", "water"]}, {"sent": "did you get any eggs for Easter ?", "words": ["for", "any", "you", "get", "did"]}, {"sent": "any color .", "words": ["any"]}, {"sent": "you wont eat any supper .", "words": ["eat", "any", "you"]}, {"sent": "did you get any more Christmas cards ?", "words": ["any", "you", "get", "more", "did"]}, {"sent": "yeah that actually doesnt really have any words .", "words": ["have", "any", "that"]}, {"sent": "oh I dont know if we have any apple juice .", "words": ["if", "we", "any", "have", "dont", "apple", "juice"]}, {"sent": "Fluffy you wont be getting any presents for your birthday will you ?", "words": ["for", "your", "be", "any", "you", "will"]}, {"sent": "any more washing up ?", "words": ["up", "more", "any"]}, {"sent": "I dont know if we have any bean bags .", "words": ["if", "we", "any", "have", "dont"]}, {"sent": "will I find any blouses on the grabby boat ?", "words": ["the", "any", "on", "find", "will", "boat"]}, {"sent": "you xxx eat any more crackers .", "words": ["eat", "more", "any", "you"]}, {"sent": "I dont think there are any more boats in here .", "words": ["think", "any", "here", "dont", "there", "are", "more", "in"]}, {"sent": "no Sarah doesnt have any teeth at all .", "words": ["have", "all", "any", "at"]}, {"sent": "are we shall we shall we empty this out and see if theres any more in there ?", "words": ["if", "we", "any", "empty", "and", "this", "more", "are", "in", "there", "out", "see"]}, {"sent": "oh we dont have any bubbles right now .", "words": ["we", "any", "have", "dont", "bubbles"]}, {"sent": "glad he is cause I dont want any leftovers .", "words": ["any", "dont", "he", "is"]}, {"sent": "I dont think theres any hornets .", "words": ["dont", "think", "any"]}, {"sent": "we dont really need any more pink .", "words": ["need", "we", "any", "dont", "more"]}, {"sent": "we never had any .", "words": ["any", "we"]}, {"sent": "I dont think hes any flavor at all .", "words": ["all", "think", "any", "dont", "at"]}, {"sent": "we dont have any more mustard ?", "words": ["we", "any", "have", "dont", "more"]}, {"sent": "we dont hafta buy any .", "words": ["dont", "any", "buy", "we"]}, {"sent": "are there any more ants anywhere around here ?", "words": ["any", "here", "there", "more", "are", "around"]}, {"sent": "soon there was hardly any lightening the thunder was a low rumble from far away and the rain stopped .", "words": ["the", "rain", "was", "any", "and", "there", "away", "a"]}, {"sent": "okay dont get any more .", "words": ["get", "dont", "more", "any"]}, {"sent": "we dont have any left over sausages anyway do we ?", "words": ["do", "we", "any", "have", "dont", "over"]}, {"sent": "I havent got any .", "words": ["any"]}, {"sent": "there werent any babies ?", "words": ["there", "any"]}, {"sent": "row the boat and well see if she catches any fish .", "words": ["the", "if", "any", "and", "fish", "see", "boat", "she"]}, {"sent": "we dont have any car .", "words": ["we", "car", "any", "have", "dont"]}, {"sent": "do you think theyll catch any ?", "words": ["do", "think", "catch", "any", "you"]}, {"sent": "you feeling any better ?", "words": ["better", "any", "you"]}, {"sent": "youve come to see if she has got any sausage rolls for lunch havent you ?", "words": ["for", "if", "to", "any", "you", "see", "she"]}, {"sent": "and um any specific ones that he .", "words": ["he", "that", "any", "and"]}, {"sent": "I havent heard any a word from him either so I dont know how how hes doing .", "words": ["how", "so", "any", "dont", "him", "a"]}, {"sent": "do you like any of these other books ?", "words": ["do", "these", "of", "any", "like", "you", "other"]}, {"sent": "yup he doesnt have it any more .", "words": ["any", "have", "more", "it", "he"]}, {"sent": "you havent got any water .", "words": ["water", "any", "you"]}, {"sent": "if you put any more money in your mouth its got to go away .", "words": ["money", "put", "your", "if", "go", "to", "mouth", "any", "you", "more", "in", "away"]}, {"sent": "I dont think weve got any chocolate darling .", "words": ["dont", "think", "chocolate", "any"]}, {"sent": "and Im look whats dodedah dodedah means cookie yyy Im like oh so she just like all week shes been making stuff up so if it doesnt make any sense to you Ill try and point out that it makes no sense to me too .", "words": ["any", "like", "me", "it", "that", "she", "up", "cookie", "all", "if", "so", "to", "and", "you", "too", "try", "make", "out", "look"]}, {"sent": "I cant see any blue trucks .", "words": ["see", "any", "blue"]}, {"sent": "or we dont have any of the plastic foods .", "words": ["the", "we", "of", "any", "have", "dont"]}, {"sent": "you ate five hamburgers and the baby didnt get any .", "words": ["the", "any", "and", "get", "you"]}, {"sent": "oooo shes gonna get something and you wont get any .", "words": ["get", "any", "you", "and"]}, {"sent": "do you have any questions ?", "words": ["have", "do", "any", "you"]}, {"sent": "no you dont need any binkies .", "words": ["any", "dont", "need", "you"]}, {"sent": "and were there any ?", "words": ["any", "there", "were", "and"]}, {"sent": "well I wont do any brick building then .", "words": ["do", "then", "any"]}, {"sent": "you cant even read that stone any more .", "words": ["any", "you", "more", "read", "stone", "that"]}, {"sent": "is Mummy any good ?", "words": ["any", "good", "is"]}, {"sent": "Im just gonna look and see if we have any other books about birds too .", "words": ["if", "we", "any", "and", "have", "other", "about", "look", "see", "too"]}, {"sent": "we havent any like watched any of the like really baby ones .", "words": ["the", "we", "of", "any", "like"]}, {"sent": "there arent any more .", "words": ["there", "more", "any"]}, {"sent": "wow I didnt know can we find any that are alive in this museum .", "words": ["can", "we", "any", "this", "in", "are", "find", "that"]}, {"sent": "do you have any candlesticks here ?", "words": ["do", "any", "you", "have", "here"]}, {"sent": "a cup ?", "words": ["cup", "a"]}, {"sent": "now you have a red cup .", "words": ["red", "cup", "you", "have", "a"]}, {"sent": "well well just have a cup of tea in the kitchen .", "words": ["cup", "the", "of", "have", "in", "kitchen", "a"]}, {"sent": "or a cup of tea ?", "words": ["cup", "of", "a"]}, {"sent": "no I didnt ask for a cup .", "words": ["cup", "for", "a"]}, {"sent": "a nest mostly just looks like a little cup or a bowl .", "words": ["cup", "bowl", "like", "little", "a"]}, {"sent": "oh youre bringing me the cup .", "words": ["the", "me", "cup"]}, {"sent": "are you taking the cup and saucer into the kitchen ?", "words": ["the", "cup", "into", "and", "you", "are", "kitchen"]}, {"sent": "oh we can put the cup on there .", "words": ["can", "the", "put", "cup", "we", "there", "on"]}, {"sent": "you have to leave them in some sort of a bowl or a cup .", "words": ["bowl", "cup", "of", "to", "them", "you", "have", "in", "some", "a"]}, {"sent": "and how is the purple cup different from the yellow cup ?", "words": ["the", "cup", "how", "is", "and", "yellow"]}, {"sent": "Eve is my cup in the table ?", "words": ["cup", "the", "my", "table", "is", "in"]}, {"sent": "hurry up with that cup .", "words": ["up", "cup", "hurry", "with", "that"]}, {"sent": "when Pip got scalded with that hot cup of coffee she screamed for her Daddy didnt she ?", "words": ["cup", "for", "when", "of", "with", "her", "hot", "coffee", "that", "she"]}, {"sent": "very nice cup xxx .", "words": ["cup", "nice"]}, {"sent": "mm cup of tea ?", "words": ["cup", "of"]}, {"sent": "Im just going to drink that cup of tea .", "words": ["cup", "drink", "of", "to", "that"]}, {"sent": "perhaps youd like to come for a cup of tea or something later .", "words": ["cup", "for", "of", "to", "like", "a"]}, {"sent": "pizza in a cup .", "words": ["cup", "in", "pizza", "a"]}, {"sent": "a cup of tea ?", "words": ["cup", "of", "a"]}, {"sent": "I want you to eat with your cup there .", "words": ["cup", "your", "to", "with", "you", "eat", "there"]}, {"sent": "Ill give ya a cup of tea .", "words": ["cup", "of", "give", "a"]}, {"sent": "it is a cup .", "words": ["cup", "it", "a", "is"]}, {"sent": "thats a cup isnt it ?", "words": ["cup", "it", "a"]}, {"sent": "its like your cup .", "words": ["cup", "your", "like"]}, {"sent": "going to have a cup of tea ?", "words": ["cup", "of", "to", "have", "a"]}, {"sent": "Im going for a cup of tea didnt he ?", "words": ["cup", "for", "of", "he", "a"]}, {"sent": "Jessicas cup ?", "words": ["cup"]}, {"sent": "did you find a cup ?", "words": ["cup", "you", "did", "find", "a"]}, {"sent": "pass me your cup please .", "words": ["cup", "me", "your"]}, {"sent": "heres one cup .", "words": ["cup"]}, {"sent": "right Im going to pour a cup of tea and then its time to get the mince pies out Tom .", "words": ["cup", "the", "of", "to", "pour", "and", "get", "out", "then", "a"]}, {"sent": "can you say cup car block truck ?", "words": ["can", "cup", "block", "car", "say", "truck", "you"]}, {"sent": "is it an egg cup or is it a cup ?", "words": ["cup", "egg", "is", "an", "it", "a"]}, {"sent": "where is the cup ?", "words": ["where", "the", "cup", "is"]}, {"sent": "I think she jumped in her cup .", "words": ["cup", "think", "in", "her", "she"]}, {"sent": "you want wine in your cup ?", "words": ["cup", "in", "your", "you"]}, {"sent": "would you like one made in your little WinniethePooh cup ?", "words": ["cup", "your", "like", "you", "little", "in", "would"]}, {"sent": "Im putting on the cup .", "words": ["on", "the", "cup"]}, {"sent": "you want my cup of tea ?", "words": ["cup", "of", "my", "you"]}, {"sent": "like she knows the cup youre sposta drink out of it .", "words": ["the", "cup", "drink", "of", "like", "it", "out", "she"]}, {"sent": "the short now the short wide cup gets filled all the way up and the tall narrow cup gets filled all the way up .", "words": ["the", "all", "up", "cup", "and"]}, {"sent": "what is this cup ?", "words": ["cup", "what", "this", "is"]}, {"sent": "you want some water in your little cup ?", "words": ["cup", "your", "you", "little", "in", "water", "some"]}, {"sent": "oh be careful with that cup .", "words": ["cup", "be", "with", "careful", "that"]}, {"sent": "another cup .", "words": ["another", "cup"]}, {"sent": "teacher said dont dont get your own cup .", "words": ["get", "dont", "your", "cup"]}, {"sent": "you want a cup ?", "words": ["cup", "a", "you"]}, {"sent": "little straining cup .", "words": ["cup", "little"]}, {"sent": "thats a cup a drink .", "words": ["cup", "drink", "a"]}, {"sent": "did you trip over the chair and drop the cup ?", "words": ["the", "cup", "chair", "you", "and", "did", "drop", "over"]}, {"sent": "aw thats a sippy cup .", "words": ["cup", "a"]}, {"sent": "having a cup of tea ?", "words": ["cup", "of", "a"]}, {"sent": "get the cup get the cup .", "words": ["get", "the", "cup"]}, {"sent": "can I have a cup of tea ?", "words": ["can", "cup", "of", "have", "a"]}, {"sent": "do you know what I would love with this cup of tea ?", "words": ["cup", "do", "what", "of", "with", "you", "this", "love", "would"]}, {"sent": "I can put this in a cup .", "words": ["can", "cup", "put", "this", "in", "a"]}, {"sent": "by the fruit cup he means all different fruits in one fruit cup .", "words": ["the", "all", "cup", "by", "in", "he"]}, {"sent": "a cup .", "words": ["cup", "a"]}, {"sent": "no my cup .", "words": ["cup", "my"]}, {"sent": "you cant have my cup .", "words": ["have", "my", "cup", "you"]}, {"sent": "get your cup and drink .", "words": ["cup", "your", "drink", "and", "get"]}, {"sent": "can we each have a cup ?", "words": ["can", "cup", "each", "we", "have", "a"]}, {"sent": "those arent apples on that cup .", "words": ["on", "that", "those", "cup"]}, {"sent": "yeah they go together youre right a cup and a bottle .", "words": ["cup", "go", "and", "they", "bottle", "a"]}, {"sent": "theyre eating tea from a cup ?", "words": ["cup", "a"]}, {"sent": "what is inside the cup ?", "words": ["the", "what", "cup", "is", "inside"]}, {"sent": "you got a cup ?", "words": ["cup", "a", "you"]}, {"sent": "mummies going to make me a wee cup of tea .", "words": ["cup", "of", "to", "me", "make", "a"]}, {"sent": "can you find me a cup ?", "words": ["can", "cup", "you", "find", "me", "a"]}, {"sent": "and um sometimes she drinks from a sippy cup .", "words": ["cup", "she", "a", "and"]}, {"sent": "but today because its in different cup so I shook the juice xxx and it made a mess on the floor .", "words": ["cup", "the", "so", "and", "in", "it", "but", "on", "juice", "because", "a"]}, {"sent": "dyou want it in the cup or do you want the straw ?", "words": ["the", "do", "cup", "you", "in", "it"]}, {"sent": "Mummys just putting the kettle on to make another cup of tea .", "words": ["the", "cup", "of", "to", "another", "on", "make"]}, {"sent": "do you want your cup of tea ?", "words": ["cup", "do", "your", "of", "you"]}, {"sent": "he wants another cup another cup another cup another cup of Rosies tea .", "words": ["another", "of", "he", "cup"]}, {"sent": "I better pour my cup of tea hadnt I ?", "words": ["cup", "better", "my", "of", "pour"]}, {"sent": "oh look at that are you gonna get the cup in the pitcher ?", "words": ["the", "cup", "look", "you", "get", "in", "are", "at", "that"]}, {"sent": "cup ?", "words": ["cup"]}, {"sent": "this a big cup .", "words": ["cup", "this", "big", "a"]}, {"sent": "whats in the cup ?", "words": ["the", "in", "cup"]}, {"sent": "Im just going to get my cup of coffee Thomas .", "words": ["cup", "my", "of", "to", "get", "coffee"]}, {"sent": "or in her sippy cup .", "words": ["cup", "in", "her"]}, {"sent": "there must be other good toys in there maybe there is another cup in there .", "words": ["cup", "is", "be", "other", "another", "there", "in", "good"]}, {"sent": "Mommy do you wanna see whats in my cup ?", "words": ["cup", "do", "my", "you", "in", "wanna", "see"]}, {"sent": "theres a plastic cup on the table look .", "words": ["cup", "the", "table", "look", "on", "a"]}, {"sent": "and put it in the cup ?", "words": ["the", "cup", "put", "and", "in", "it"]}, {"sent": "this is a cup of tea .", "words": ["cup", "is", "of", "this", "a"]}, {"sent": "the cup has a top .", "words": ["the", "cup", "a"]}, {"sent": "juice in your cup .", "words": ["cup", "in", "juice", "your"]}, {"sent": "do you like a cup of tea ?", "words": ["cup", "do", "of", "like", "you", "a"]}, {"sent": "right Im going to have a cup of coffee now .", "words": ["cup", "of", "to", "have", "coffee", "a"]}, {"sent": "Mummys just pouring herself a cup of tea .", "words": ["cup", "of", "a"]}, {"sent": "oh you gonna get the cup ?", "words": ["get", "the", "cup", "you"]}, {"sent": "yes no you dont eat out the cup .", "words": ["the", "cup", "you", "eat", "dont", "out"]}, {"sent": "do you want the cup again ?", "words": ["the", "do", "cup", "you"]}, {"sent": "what is it is it a cup ?", "words": ["cup", "what", "is", "it", "a"]}, {"sent": "all these toys and you wanna play with a cup ?", "words": ["cup", "all", "these", "play", "with", "you", "and", "wanna", "a"]}, {"sent": "a nice cup of tea .", "words": ["cup", "of", "nice", "a"]}, {"sent": "can you say cup ?", "words": ["can", "say", "cup", "you"]}, {"sent": "but if you see him will you call me back ?", "words": ["if", "back", "me", "you", "him", "will", "see", "but"]}, {"sent": "will you look after me other piggy ?", "words": ["other", "you", "look", "will", "me"]}, {"sent": "what will they do in Malta ?", "words": ["what", "do", "they", "in", "will"]}, {"sent": "will I try and make the pig walk ?", "words": ["the", "try", "and", "pig", "walk", "will", "make"]}, {"sent": "wait Ill get you juice and because you will want dee thing huh ?", "words": ["wait", "and", "get", "you", "will", "juice", "because"]}, {"sent": "will you put all the stones back in that one .", "words": ["the", "all", "put", "back", "you", "in", "will", "that"]}, {"sent": "then I will disappear immediately .", "words": ["will", "then"]}, {"sent": "will that one fit ?", "words": ["will", "fit", "that"]}, {"sent": "she will I know .", "words": ["will", "she"]}, {"sent": "mommy will take you home .", "words": ["will", "home", "take", "you"]}, {"sent": "that will never do will it ?", "words": ["will", "do", "it", "that"]}, {"sent": "maybe Peter will talk to Nana and see if uncle Bills there .", "words": ["if", "to", "and", "there", "talk", "will", "see"]}, {"sent": "do you think that will go in it ?", "words": ["do", "think", "go", "you", "in", "it", "will", "that"]}, {"sent": "will he not fit in ?", "words": ["fit", "in", "he", "will", "not"]}, {"sent": "it I think it will fall soon .", "words": ["will", "think", "it", "fall"]}, {"sent": "Mummy will go and get it .", "words": ["go", "and", "get", "it", "will"]}, {"sent": "will you put the puzzle together huh ?", "words": ["the", "put", "you", "puzzle", "will"]}, {"sent": "will you carry that for me ?", "words": ["for", "you", "carry", "will", "me", "that"]}, {"sent": "those ones will go together .", "words": ["will", "those", "go"]}, {"sent": "I think the medicine will start working very soon and he will feel better very soon .", "words": ["the", "better", "think", "medicine", "and", "he", "will"]}, {"sent": "only one dress will stay on at one time .", "words": ["on", "at", "will", "stay", "dress"]}, {"sent": "will you put the baked beans tin on the train ?", "words": ["the", "put", "train", "you", "on", "beans", "will"]}, {"sent": "no it will be time to take it off soon .", "words": ["off", "be", "to", "it", "will", "take"]}, {"sent": "well that will not interfere at all .", "words": ["all", "at", "will", "that", "not"]}, {"sent": "I dont think it will .", "words": ["will", "dont", "think", "it"]}, {"sent": "and if you are a little boy I will become your mother and catch you in my arms .", "words": ["my", "your", "if", "catch", "you", "and", "little", "in", "are", "will", "a"]}, {"sent": "thats awhile okay well um if I can I will try and bring it in later .", "words": ["can", "if", "bring", "and", "in", "it", "will", "try"]}, {"sent": "do you think it will hurt her ?", "words": ["do", "think", "hurt", "you", "it", "will", "her"]}, {"sent": "and then youve moved so that you will knock it over .", "words": ["over", "so", "you", "and", "knock", "it", "will", "then", "that"]}, {"sent": "you will be so full you will pop .", "words": ["pop", "full", "be", "so", "you", "will"]}, {"sent": "will daddy drop them off on his way ?", "words": ["off", "them", "his", "on", "drop", "will"]}, {"sent": "hey Ill bet it will go in here .", "words": ["go", "here", "in", "it", "will"]}, {"sent": "she will kind of sit .", "words": ["will", "of", "sit", "she"]}, {"sent": "when will I bring the toys eh Eve ?", "words": ["will", "the", "when", "bring"]}, {"sent": "Jeannine will just think oh thats naughty Thomas again again .", "words": ["will", "think", "naughty"]}, {"sent": "what will he do with it ?", "words": ["what", "do", "with", "it", "he", "will"]}, {"sent": "yes it will .", "words": ["will", "it"]}, {"sent": "will you make the star ?", "words": ["the", "star", "you", "will", "make"]}, {"sent": "your beak or your snuffle will work just as well .", "words": ["will", "work", "your"]}, {"sent": "will you do that ?", "words": ["will", "do", "that", "you"]}, {"sent": "you will when it gets to be winter .", "words": ["to", "be", "you", "it", "will", "when"]}, {"sent": "if we fill in all the gaps theres no way it will fall .", "words": ["the", "all", "if", "we", "fall", "in", "it", "will"]}, {"sent": "and the sun will melt the snow man okay ?", "words": ["the", "sun", "snow", "and", "will"]}, {"sent": "tiger will eat all the smarties .", "words": ["the", "all", "eat", "tiger", "will"]}, {"sent": "he will fit .", "words": ["will", "fit", "he"]}, {"sent": "will that make me better ?", "words": ["better", "that", "will", "me", "make"]}, {"sent": "maybe Phillipe will play with you if Summer wont .", "words": ["if", "play", "with", "you", "will"]}, {"sent": "poor Spot will have nothing to eat now will he ?", "words": ["to", "poor", "have", "eat", "he", "will"]}, {"sent": "oh she will .", "words": ["will", "she"]}, {"sent": "fresh air will give you a boost she said .", "words": ["you", "will", "give", "she", "a"]}, {"sent": "well then it doesnt come out it will come out the other side .", "words": ["the", "other", "it", "out", "will", "then"]}, {"sent": "yes I know you will .", "words": ["will", "you"]}, {"sent": "Daddy will be home later .", "words": ["will", "be", "home"]}, {"sent": "will hafta put some good tape on the arm .", "words": ["the", "put", "tape", "on", "good", "arm", "will", "some"]}, {"sent": "I will be right back .", "words": ["will", "be", "back"]}, {"sent": "and then Eleanor will give you some at home .", "words": ["then", "you", "and", "at", "will", "some", "give", "home"]}, {"sent": "trouble is if Mummy lets you sleep much longer you wont sleep tonight will you ?", "words": ["if", "is", "sleep", "you", "will", "much"]}, {"sent": "do you think do you think the shoe will fit me ?", "words": ["the", "do", "think", "fit", "shoe", "you", "will", "me"]}, {"sent": "will he not ?", "words": ["will", "he", "not"]}, {"sent": "will hafta get a butter knife to get it out .", "words": ["to", "knife", "get", "it", "butter", "out", "will", "a"]}, {"sent": "and Riordan will be there with his Mummy .", "words": ["be", "with", "and", "his", "there", "will"]}, {"sent": "will you what ?", "words": ["will", "what", "you"]}, {"sent": "dont forget to sit on your potty if you needta go to the toilet will you ?", "words": ["the", "your", "if", "go", "to", "will", "you", "dont", "potty", "on", "sit"]}, {"sent": "and Ray will be every so busy wont he ?", "words": ["be", "so", "and", "he", "every", "will"]}, {"sent": "shell drop her bowl and it will break .", "words": ["bowl", "break", "and", "it", "drop", "will", "her"]}, {"sent": "you think he will .", "words": ["will", "think", "he", "you"]}, {"sent": "I will but not right now .", "words": ["will", "not", "but"]}, {"sent": "Im sure it will be .", "words": ["will", "be", "it"]}, {"sent": "but the pear will go in a box and when you want a snack this afternoon out it will come again .", "words": ["the", "box", "go", "and", "you", "this", "in", "it", "out", "will", "when", "but", "a"]}, {"sent": "what will they have a drink of ?", "words": ["what", "drink", "of", "have", "they", "will", "a"]}, {"sent": "for if I do my mother will say .", "words": ["do", "my", "for", "if", "say", "will"]}, {"sent": "and then it will be gone .", "words": ["be", "and", "it", "will", "then"]}, {"sent": "say horsie will you give me a smartie .", "words": ["say", "you", "will", "me", "give", "a"]}, {"sent": "will you ?", "words": ["will", "you"]}, {"sent": "what will happen to them ?", "words": ["will", "what", "to", "them"]}, {"sent": "what color will the costume be ?", "words": ["will", "what", "be", "the"]}, {"sent": "okay Mommy will get you a piece Ill bring it up .", "words": ["up", "bring", "you", "get", "it", "will", "a"]}, {"sent": "I will pick up the hook you will see something new .", "words": ["the", "up", "pick", "you", "will", "see", "new"]}, {"sent": "I think I will also borrow the record because I like marching bands and parades I can play it at home on our record player wow what fun that will be .", "words": ["the", "can", "think", "what", "be", "play", "like", "and", "our", "on", "it", "at", "will", "that", "because", "home"]}, {"sent": "the cat screeching and he will have heard .", "words": ["the", "and", "have", "cat", "he", "will"]}, {"sent": "and your stomach will enjoy it .", "words": ["will", "it", "your", "and"]}, {"sent": "I will .", "words": ["will"]}, {"sent": "whereas xxx and I will be first .", "words": ["will", "be", "first", "and"]}, {"sent": "will you do it for me in school ?", "words": ["do", "for", "school", "you", "in", "it", "will", "me"]}, {"sent": "yeah he will uhn wont he ?", "words": ["will", "he"]}, {"sent": "and everything will soon be alright .", "words": ["will", "be", "and"]}, {"sent": "if you shake it the lid will come off and the chocolate will go everywhere .", "words": ["the", "off", "if", "go", "chocolate", "shake", "you", "and", "it", "will"]}, {"sent": "if youre happy and know it and your face will really show it .", "words": ["your", "if", "happy", "face", "show", "and", "it", "will"]}, {"sent": "Alex wait a minute you have to put some in here you cant keep all of them in the bowl the bowl will be too heavy .", "words": ["the", "all", "wait", "put", "bowl", "of", "to", "be", "them", "you", "have", "here", "in", "too", "will", "some", "heavy", "a"]}, {"sent": "but she also will bring new books over .", "words": ["bring", "will", "new", "over", "she", "but"]}, {"sent": "will we have a look ?", "words": ["we", "have", "look", "will", "a"]}, {"sent": "feet will be warm .", "words": ["will", "be"]}, {"sent": "Daddy will eat those ones .", "words": ["will", "those", "eat"]}, {"sent": "I dont think she will .", "words": ["will", "dont", "think", "she"]}, {"sent": "it will be easier for you .", "words": ["for", "be", "you", "it", "will"]}, {"sent": "will we use this as a blanket or is it too hot ?", "words": ["blanket", "we", "is", "this", "hot", "it", "too", "will", "a"]}, {"sent": "if youre happy and you know it then your life will surely show it .", "words": ["your", "if", "happy", "show", "you", "and", "it", "will", "then"]}, {"sent": "will you eat that grape ?", "words": ["will", "that", "eat", "you"]}, {"sent": "can you say hi and Jessica will know its for her ?", "words": ["can", "for", "say", "and", "you", "will", "her"]}, {"sent": "but by that time things will be a lot different .", "words": ["a lot", "be", "by", "will", "that", "but", "a"]}, {"sent": "ya hear it ?", "words": ["it", "hear"]}, {"sent": "I cant hear you .", "words": ["you", "hear"]}, {"sent": "Kent cant hear you in there .", "words": ["there", "in", "you", "hear"]}, {"sent": "hear the rain come down the gutter ?", "words": ["the", "down", "rain", "hear"]}, {"sent": "let me hear your stomach .", "words": ["me", "your", "hear"]}, {"sent": "what did you hear about pizzeroni ?", "words": ["what", "hear", "you", "did", "about"]}, {"sent": "Id like to hear a whistling butterfly .", "words": ["butterfly", "to", "like", "hear", "a"]}, {"sent": "I shouldnt try and get it out of you because Ill probably hear it too much one of these days .", "words": ["these", "of", "much", "and", "get", "you", "hear", "it", "out", "too", "try", "because"]}, {"sent": "I can hear you .", "words": ["can", "you", "hear"]}, {"sent": "you can hear the music ?", "words": ["can", "the", "hear", "you"]}, {"sent": "when I hear please Ill do anything .", "words": ["do", "when", "hear"]}, {"sent": "I cant hear .", "words": ["hear"]}, {"sent": "I cant hear you because all those kids are going out .", "words": ["all", "those", "you", "hear", "are", "out", "because"]}, {"sent": "here is my handle hear is my spout .", "words": ["here", "my", "hear", "is"]}, {"sent": "can you hear the rubbing the soap onto your hand ?", "words": ["can", "the", "your", "hand", "soap", "hear", "you"]}, {"sent": "mm oh she did hear her voice .", "words": ["did", "her", "she", "hear"]}, {"sent": "you could hear me right .", "words": ["me", "could", "hear", "you"]}, {"sent": "what do you wanna hear .", "words": ["what", "do", "hear", "you", "wanna"]}, {"sent": "do you wanna hear a wee bit of this ?", "words": ["do", "of", "hear", "you", "this", "wanna", "a"]}, {"sent": "he cant hear you .", "words": ["he", "you", "hear"]}, {"sent": "I hear that .", "words": ["that", "hear"]}, {"sent": "I cant hear you .", "words": ["you", "hear"]}, {"sent": "you dont wanna hear it either .", "words": ["hear", "you", "dont", "it", "wanna"]}, {"sent": "what can you hear ?", "words": ["can", "what", "hear", "you"]}, {"sent": "and you know Ive just driven all the way here to come and say hello and hear about your school because .", "words": ["the", "all", "your", "to", "school", "say", "you", "here", "and", "hear", "about", "because"]}, {"sent": "if you dont hear it go click it is not on and it will come right back off again when you put it in the box .", "words": ["off", "the", "put", "box", "if", "back", "go", "is", "will", "hear", "you", "dont", "and", "it", "in", "on", "when", "not"]}, {"sent": "I could hear a noise .", "words": ["could", "a", "hear"]}, {"sent": "oh you can hear it without putting it really close to your ear Thomas .", "words": ["can", "your", "to", "hear", "you", "it", "ear", "close"]}, {"sent": "oh you hear that water ?", "words": ["water", "hear", "that", "you"]}, {"sent": "its just nice to hear them going properly .", "words": ["to", "nice", "them", "hear"]}, {"sent": "they wanna hear a story about the sky .", "words": ["the", "sky", "hear", "they", "wanna", "about", "story", "a"]}, {"sent": "You should not hear from her fusses but if she does wake up shell holler a lot .", "words": ["up", "does", "if", "a lot", "hear", "she", "wake", "her", "not", "but", "a"]}, {"sent": "she turned the direction so she could hear better .", "words": ["the", "better", "so", "hear", "could", "she"]}, {"sent": "I can hear her voice .", "words": ["can", "her", "hear"]}, {"sent": "if its about what hah what I went to today I dont wanna hear it .", "words": ["what", "if", "to", "hear", "dont", "it", "about", "wanna"]}, {"sent": "I would love to hear about it .", "words": ["to", "would", "hear", "it", "about", "love"]}, {"sent": "did you ever hear one ?", "words": ["hear", "did", "you"]}, {"sent": "then talk with me and well hear my voice .", "words": ["my", "with", "and", "hear", "talk", "me", "then"]}, {"sent": "can you hear that ?", "words": ["can", "hear", "that", "you"]}, {"sent": "let me hear it .", "words": ["me", "it", "hear"]}, {"sent": "hear that fire engine ?", "words": ["that", "hear"]}, {"sent": "and I think I can hear UncleBrian with the hoover on now the vacuum cleaner .", "words": ["can", "the", "think", "with", "hear", "and", "vacuum", "on"]}, {"sent": "lets hear you say ta .", "words": ["say", "you", "hear"]}, {"sent": "let me hear .", "words": ["me", "hear"]}, {"sent": "youll probably hear that now .", "words": ["that", "hear"]}, {"sent": "because we gotta hear were trying a put what you hafta say down .", "words": ["what", "put", "we", "say", "hear", "you", "down", "were", "because", "a"]}, {"sent": "let me hear .", "words": ["me", "hear"]}, {"sent": "did you hear me ?", "words": ["me", "hear", "did", "you"]}, {"sent": "hear the noise .", "words": ["the", "hear"]}, {"sent": "because you can hear a big noise ?", "words": ["can", "hear", "you", "big", "because", "a"]}, {"sent": "I cant hear you .", "words": ["you", "hear"]}, {"sent": "do I hear forty pounds ?", "words": ["do", "hear"]}, {"sent": "I cant hear you .", "words": ["you", "hear"]}, {"sent": "did you hear it ?", "words": ["hear", "it", "did", "you"]}, {"sent": "lets hear your nose squeak .", "words": ["nose", "your", "hear"]}, {"sent": "I dont wanna hear the noise .", "words": ["the", "dont", "wanna", "hear"]}, {"sent": "I cant hear myself talking and I cant hear you talking .", "words": ["you", "myself", "and", "hear"]}, {"sent": "but the butterfly hadnt hear a single word .", "words": ["the", "butterfly", "hear", "but", "a"]}, {"sent": "did you hear that knock knock knock ?", "words": ["hear", "you", "knock", "did", "that"]}, {"sent": "I can hear a funny noise outside .", "words": ["can", "outside", "a", "hear"]}, {"sent": "can you hear it ?", "words": ["can", "hear", "it", "you"]}, {"sent": "I can hear some kind of animal .", "words": ["can", "of", "hear", "animal", "some"]}, {"sent": "can you hear anything John ?", "words": ["can", "hear", "you"]}, {"sent": "oh hear it is this is the special part where the sound goes in so you can tell him in there .", "words": ["the", "where", "can", "is", "so", "hear", "you", "this", "in", "it", "him", "there"]}, {"sent": "if you can hear both of us .", "words": ["can", "if", "of", "hear", "you", "us"]}, {"sent": "I hear you down here .", "words": ["here", "down", "you", "hear"]}, {"sent": "so that we can hear you so that Jeannine can hear you talking .", "words": ["can", "we", "so", "hear", "you", "that"]}, {"sent": "I said let me hear it .", "words": ["me", "it", "hear"]}, {"sent": "well hear more weather in a minute .", "words": ["in", "more", "a", "hear"]}, {"sent": "lets hear .", "words": ["hear"]}, {"sent": "its hard ta hear over that nonsense .", "words": ["that", "over", "hard", "hear"]}, {"sent": "do you hear the birds ?", "words": ["the", "do", "hear", "you"]}, {"sent": "did you hear that ?", "words": ["hear", "that", "did", "you"]}, {"sent": "what can you hear .", "words": ["can", "what", "hear", "you"]}, {"sent": "do you hear somebody ?", "words": ["do", "hear", "you"]}, {"sent": "can you hear this iron making a noise ?", "words": ["can", "hear", "you", "this", "a"]}, {"sent": "I could hear that .", "words": ["could", "that", "hear"]}, {"sent": "say something and Ill let you hear it .", "words": ["say", "hear", "you", "and", "it"]}, {"sent": "do you hear ?", "words": ["do", "hear", "you"]}, {"sent": "okay lets hear it .", "words": ["it", "hear"]}, {"sent": "I hear somebody going I hear somebody going hu ?", "words": ["hear"]}, {"sent": "I cant hear you .", "words": ["you", "hear"]}, {"sent": "Im just going to hear about your mummie .", "words": ["to", "your", "about", "hear"]}, {"sent": "I hear you coming .", "words": ["you", "hear"]}, {"sent": "did you hear about the zoo in New York city in central park .", "words": ["the", "zoo", "hear", "you", "in", "park", "did", "about"]}, {"sent": "you better not let Daddy hear you say that .", "words": ["better", "say", "hear", "you", "that", "not"]}, {"sent": "do you hear an ice cream truck Tim ?", "words": ["do", "ice cream", "truck", "hear", "you", "an", "ice"]}, {"sent": "xxx to hear you talking more .", "words": ["to", "more", "you", "hear"]}, {"sent": "and then she can hear this noise going boing boing boing .", "words": ["can", "hear", "and", "this", "then", "she"]}, {"sent": "you should tell Jessica again who Heidi is because Jessica didnt really hear .", "words": ["is", "hear", "you", "who", "because"]}, {"sent": "dont put your hands in it you hear ?", "words": ["put", "your", "you", "hear", "dont", "in", "it"]}, {"sent": "she still cant hear .", "words": ["she", "hear"]}, {"sent": "if your a good boy and come here you can hear them when were done .", "words": ["can", "your", "if", "them", "and", "here", "you", "hear", "good", "when", "were", "a"]}, {"sent": "can you hear yourself ?", "words": ["can", "hear", "yourself", "you"]}, {"sent": "you want ta hear that ?", "words": ["hear", "that", "you"]}, {"sent": "Im sorry I didnt hear you .", "words": ["you", "hear"]}, {"sent": "I cant hear you xxx .", "words": ["you", "hear"]}, {"sent": "ah you can hear more through that .", "words": ["can", "hear", "you", "more", "that"]}, {"sent": "what can you hear ?", "words": ["can", "what", "hear", "you"]}, {"sent": "can you hear it ?", "words": ["can", "hear", "it", "you"]}, {"sent": "whats the story ?", "words": ["the", "story"]}, {"sent": "or shall we read another story .", "words": ["another", "story", "read", "we"]}, {"sent": "no more story right now ?", "words": ["more", "story"]}, {"sent": "you tell me this story .", "words": ["me", "story", "this", "you"]}, {"sent": "and what was the story of the movie ?", "words": ["the", "what", "of", "was", "and", "story"]}, {"sent": "what was the story about ?", "words": ["the", "what", "was", "about", "story"]}, {"sent": "and make an ending for the story ?", "words": ["the", "for", "and", "an", "story", "make"]}, {"sent": "do you remember the story ?", "words": ["the", "do", "story", "you"]}, {"sent": "thats my story .", "words": ["my", "story"]}, {"sent": "story makers come out at night ?", "words": ["story", "out", "at"]}, {"sent": "Im gonna start the story and I want you to finish it okay ?", "words": ["the", "to", "finish", "you", "and", "it", "story"]}, {"sent": "do you wanna listen to the story ?", "words": ["the", "do", "listen", "to", "you", "wanna", "story"]}, {"sent": "Maisy writes a story .", "words": ["story", "a"]}, {"sent": "well what about everybody else I hafta have theyre all gonna come up and tell me the story .", "words": ["up", "what", "all", "the", "and", "have", "about", "me", "story"]}, {"sent": "last night when I was getting you to sleep I said shall we have a story ?", "words": ["we", "sleep", "to", "was", "last", "you", "have", "story", "when", "a"]}, {"sent": "and then I need you to get undressed or we wont have time to read our story tonight .", "words": ["story", "need", "we", "to", "you", "and", "get", "have", "our", "read", "then"]}, {"sent": "I think you should tell her a little story .", "words": ["think", "little", "you", "story", "her", "a"]}, {"sent": "but in this story the children are away .", "words": ["the", "this", "in", "are", "away", "story", "but"]}, {"sent": "can you tell me the story ?", "words": ["can", "the", "you", "me", "story"]}, {"sent": "would you like to read me the story about the three goats ?", "words": ["the", "to", "like", "you", "read", "about", "would", "me", "story"]}, {"sent": "lets finish this story first .", "words": ["this", "story", "first", "finish"]}, {"sent": "or do you want to get a new story ?", "words": ["do", "to", "you", "get", "want to", "story", "new", "a"]}, {"sent": "I have the story for that .", "words": ["the", "for", "have", "story", "that"]}, {"sent": "and what was the story about the daffodils Thomas .", "words": ["the", "what", "was", "and", "about", "story"]}, {"sent": "come here lets finish reading the story shell get it .", "words": ["the", "finish", "here", "get", "it", "story"]}, {"sent": "think you can tell the story ?", "words": ["can", "the", "think", "you", "story"]}, {"sent": "are you going to read the story to me too ?", "words": ["the", "to", "you", "are", "read", "too", "me", "story"]}, {"sent": "whats your favorite story Fraser ?", "words": ["story", "your"]}, {"sent": "tell her the story you just told me .", "words": ["the", "you", "me", "story", "her"]}, {"sent": "this is the the story that we know about isnt it ?", "words": ["the", "is", "we", "this", "it", "about", "story", "that"]}, {"sent": "in this retelling I took xxx took the traditional form of the stone soup story and set it in China .", "words": ["the", "of", "and", "this", "in", "it", "stone", "soup", "story"]}, {"sent": "read her some story ?", "words": ["some", "her", "read", "story"]}, {"sent": "after Julius was born it was a different story Lily took her things back she pinched his tail and she yelled insulting comments into his crib .", "words": ["into", "back", "was", "crib", "and", "it", "his", "story", "her", "she", "a"]}, {"sent": "youre gonna tell me the story whats going on in the story William ?", "words": ["the", "in", "on", "me", "story"]}, {"sent": "what is that story all about ?", "words": ["all", "what", "is", "about", "story", "that"]}, {"sent": "find a good story you can tell all your guests .", "words": ["can", "all", "your", "you", "good", "find", "story", "a"]}, {"sent": "there is no supermarket story in this one Sweetie Pie .", "words": ["is", "there", "in", "this", "story"]}, {"sent": "you wanna read the story ?", "words": ["the", "you", "read", "wanna", "story"]}, {"sent": "Thomas I dont know whether Daddy heard this story but .", "words": ["but", "dont", "story", "this"]}, {"sent": "you can tell us a story .", "words": ["can", "you", "us", "story", "a"]}, {"sent": "wanna go to bed without a story ?", "words": ["go", "to", "bed", "wanna", "story", "a"]}, {"sent": "you wanna try tell the story ?", "words": ["the", "you", "wanna", "story", "try"]}, {"sent": "and you said you couldnt tell me a story before you told one of the best stories .", "words": ["the", "of", "you", "and", "me", "story", "a"]}, {"sent": "theyre in the story too .", "words": ["the", "in", "story", "too"]}, {"sent": "should we start off with that story ?", "words": ["off", "we", "with", "story", "that"]}, {"sent": "and this is another story about the town and the country .", "words": ["the", "is", "and", "another", "this", "about", "story"]}, {"sent": "do you want me to read you that story now ?", "words": ["do", "to", "you", "read", "me", "story", "that"]}, {"sent": "you dont want a bedtime story with your milk ?", "words": ["your", "with", "you", "dont", "milk", "story", "a"]}, {"sent": "youd hafta tell me a story .", "words": ["me", "story", "a"]}, {"sent": "do you wanna answer some questions about this story ?", "words": ["do", "you", "this", "about", "wanna", "some", "story"]}, {"sent": "you didnt like that story ?", "words": ["story", "that", "like", "you"]}, {"sent": "perhaps daddyll read the whole story for your bedtime story .", "words": ["the", "for", "your", "read", "story"]}, {"sent": "are you going to listen to a story ?", "words": ["listen", "to", "you", "are", "story", "a"]}, {"sent": "do you remember that story ?", "words": ["do", "story", "that", "you"]}, {"sent": "and funnily enough I think the story was read on television a little while ago .", "words": ["the", "think", "was", "and", "little", "read", "on", "story", "a"]}, {"sent": "and you can you can play with you new story book .", "words": ["can", "play", "with", "you", "and", "book", "story", "new"]}, {"sent": "okay so tell me the story of what happens .", "words": ["the", "what", "of", "so", "me", "story"]}, {"sent": "because its nearly bedtime shall we have a story down here instead of in bed ?", "words": ["we", "of", "bed", "have", "here", "in", "story", "down", "because", "a"]}, {"sent": "you want me to tell a story ?", "words": ["to", "you", "me", "story", "a"]}, {"sent": "is this the end of your story ?", "words": ["the", "your", "is", "of", "this", "story"]}, {"sent": "do you like that story ?", "words": ["do", "like", "you", "story", "that"]}, {"sent": "shall we read this Christmas story ?", "words": ["this", "story", "read", "we"]}, {"sent": "that was a good story can you come up with something else ?", "words": ["can", "up", "was", "with", "you", "good", "story", "that", "a"]}, {"sent": "tell story .", "words": ["story"]}, {"sent": "about the story ?", "words": ["the", "story", "about"]}, {"sent": "who taught you how to be such a good story teller ?", "words": ["how", "to", "be", "you", "good", "who", "story", "a"]}, {"sent": "one one minute til story time Lara .", "words": ["story"]}, {"sent": "do you wanna read the story ?", "words": ["the", "do", "you", "read", "wanna", "story"]}, {"sent": "when he was reading you the story ?", "words": ["the", "was", "you", "he", "story", "when"]}, {"sent": "we want a story .", "words": ["story", "a", "we"]}, {"sent": "like he knows when things are funny we went to a story hour the other day at the library and .", "words": ["the", "we", "to", "like", "other", "and", "are", "he", "at", "story", "when", "a"]}, {"sent": "this is our story .", "words": ["our", "this", "story", "is"]}, {"sent": "its a long story and I cant explain it to you .", "words": ["to", "long", "and", "you", "it", "story", "a"]}, {"sent": "it asked me specially just as we were coming over her to see if Sarah would tell would tell a story .", "words": ["if", "we", "see", "to", "her", "it", "would", "me", "story", "over", "were", "a"]}, {"sent": "Mary wants to tell the story about the diapers .", "words": ["the", "to", "story", "about"]}, {"sent": "tell me about the story .", "words": ["the", "me", "story", "about"]}, {"sent": "and the story about the dandelions is quite different .", "words": ["the", "is", "and", "about", "story"]}, {"sent": "and what happens in the game is that Jilly makes up a story about what shes playing with .", "words": ["the", "what", "game", "up", "is", "with", "and", "in", "about", "story", "that", "a"]}, {"sent": "what happens in the story ?", "words": ["the", "what", "in", "story"]}, {"sent": "its a little early for your story time .", "words": ["for", "your", "little", "story", "a"]}, {"sent": "what story sweetie ?", "words": ["what", "story"]}, {"sent": "we can read a story .", "words": ["can", "we", "read", "story", "a"]}, {"sent": "what story do you wanna night ?", "words": ["what", "do", "you", "wanna", "story"]}, {"sent": "I think youre making up a big story .", "words": ["up", "think", "story", "big", "a"]}, {"sent": "like the teddy bear in that story .", "words": ["the", "like", "bear", "in", "story", "that"]}, {"sent": "I bet the birds would like that story .", "words": ["the", "like", "would", "story", "that"]}, {"sent": "and tell the story of The Red Balloon .", "words": ["the", "of", "story", "and"]}, {"sent": "its called little white puffy cloud a story book .", "words": ["little", "cloud", "book", "story", "white", "a"]}, {"sent": "who wants to hear a story ?", "words": ["to", "hear", "who", "story", "a"]}, {"sent": "are you gonna listen to the story or not ?", "words": ["the", "listen", "to", "you", "are", "story", "not"]}, {"sent": "what is this story about ?", "words": ["what", "is", "this", "about", "story"]}, {"sent": "thats a cute story .", "words": ["cute", "story", "a"]}, {"sent": "tell us a story .", "words": ["us", "story", "a"]}, {"sent": "okay how does the story go ?", "words": ["the", "does", "how", "go", "story"]}, {"sent": "ahhah whats he reading a bed time story .", "words": ["story", "he", "bed", "a"]}, {"sent": "he wont let me read the story .", "words": ["the", "he", "read", "me", "story"]}, {"sent": "can I start a story and then you can finish it ?", "words": ["can", "then", "finish", "and", "you", "it", "story", "a"]}, {"sent": "you tell me the story .", "words": ["the", "me", "story", "you"]}, {"sent": "me a story darling .", "words": ["me", "story", "a"]}, {"sent": "thats green .", "words": ["green"]}, {"sent": "I know its green .", "words": ["green"]}, {"sent": "xxx you know my big green scarf dont I wear a big green scarf ?", "words": ["my", "scarf", "you", "dont", "green", "big", "a"]}, {"sent": "green .", "words": ["green"]}, {"sent": "Kaleb whos the little green engine ?", "words": ["the", "green", "little"]}, {"sent": "here is the green one .", "words": ["here", "the", "green", "is"]}, {"sent": "then Ill take green .", "words": ["then", "take", "green"]}, {"sent": "what is green ?", "words": ["what", "green", "is"]}, {"sent": "the green button this one .", "words": ["the", "this", "button", "green"]}, {"sent": "or who likes green beans ?", "words": ["green beans", "beans", "green", "who"]}, {"sent": "and did you color it in green or was this Nicole ?", "words": ["was", "you", "and", "this", "in", "it", "did", "green"]}, {"sent": "okay but it needs to be green that is a green pencil but I need to have a a right green .", "words": ["need", "is", "need to", "to", "be", "pencil", "have", "it", "green", "that", "but", "a"]}, {"sent": "green shirt .", "words": ["shirt", "green"]}, {"sent": "well youve lotsof different shades of green havent you ?", "words": ["of", "green", "you"]}, {"sent": "that ones green .", "words": ["that", "green"]}, {"sent": "no not blue green .", "words": ["not", "green", "blue"]}, {"sent": "oh no whered the green car go ?", "words": ["the", "car", "green", "go"]}, {"sent": "green .", "words": ["green"]}, {"sent": "theres a green one .", "words": ["green", "a"]}, {"sent": "wheres that green thing ?", "words": ["that", "green"]}, {"sent": "green .", "words": ["green"]}, {"sent": "no green .", "words": ["green"]}, {"sent": "and this is a gocart with green wheels .", "words": ["is", "with", "and", "this", "green", "a"]}, {"sent": "wheres green ?", "words": ["green"]}, {"sent": "thats not green .", "words": ["not", "green"]}, {"sent": "theyre green grapes .", "words": ["grapes", "green"]}, {"sent": "weve got green spots .", "words": ["green"]}, {"sent": "youre coloring him in green arent you ?", "words": ["in", "him", "green", "you"]}, {"sent": "was there something green that was reflecting on them ?", "words": ["was", "them", "there", "green", "on", "that"]}, {"sent": "green .", "words": ["green"]}, {"sent": "the helicopters red blue yellow and green .", "words": ["the", "red", "and", "blue", "green", "yellow"]}, {"sent": "its a green net bag .", "words": ["green", "a"]}, {"sent": "Kentucky is called the Bluegrass state and was nicknamed for its blue green grass .", "words": ["the", "for", "is", "was", "and", "blue", "green", "grass"]}, {"sent": "yes its green .", "words": ["green"]}, {"sent": "are they green ?", "words": ["they", "are", "green"]}, {"sent": "could be a very dark green .", "words": ["be", "green", "dark", "could", "a"]}, {"sent": "the green car .", "words": ["the", "car", "green"]}, {"sent": "its a nice green boat though isnt it ?", "words": ["nice", "it", "green", "boat", "a"]}, {"sent": "what are you gonna say to you Patrick green ?", "words": ["what", "to", "say", "you", "are", "green"]}, {"sent": "put the green on .", "words": ["the", "put", "on", "green"]}, {"sent": "thanks said the duck who got out of the muck down by the deep green marsh .", "words": ["the", "of", "by", "out", "who", "duck", "green", "down"]}, {"sent": "what does green tell us ?", "words": ["what", "does", "green", "us"]}, {"sent": "its the green truck isnt it ?", "words": ["the", "it", "truck", "green"]}, {"sent": "blue orange green .", "words": ["orange", "green", "blue"]}, {"sent": "a lime green and a jade green .", "words": ["and", "green", "a"]}, {"sent": "you want the green to do someum ?", "words": ["the", "do", "to", "you", "green"]}, {"sent": "you want dark green ?", "words": ["dark", "green", "you"]}, {"sent": "by the big red barn in the great green field .", "words": ["the", "red", "by", "in", "green", "big"]}, {"sent": "thats a long green one isnt it ?", "words": ["it", "green", "long", "a"]}, {"sent": "yours is green .", "words": ["green", "is"]}, {"sent": "green \u7684 \u4e2d\u6587 \u662f \u4ec0\u4e48 ?", "words": ["green"]}, {"sent": "youre getting green on your hand .", "words": ["on", "your", "green", "hand"]}, {"sent": "green grass .", "words": ["grass", "green"]}, {"sent": "it was green like that was it ?", "words": ["was", "like", "it", "green", "that"]}, {"sent": "when Honey Bunny was fast asleep pj painted her face bright green .", "words": ["fast", "face", "was", "her", "asleep", "green", "when"]}, {"sent": "the green towel .", "words": ["the", "towel", "green"]}, {"sent": "you press the green go circle .", "words": ["the", "go", "green", "you"]}, {"sent": "I go to green I didnt go very far .", "words": ["to", "green", "go"]}, {"sent": "how bout a green ?", "words": ["how", "green", "a"]}, {"sent": "are the green lights still on Keisha ?", "words": ["the", "on", "are", "green"]}, {"sent": "how many green lights ?", "words": ["how", "green"]}, {"sent": "maybe open the green one ?", "words": ["open", "the", "green"]}, {"sent": "so it is a yellowy green isnt it ?", "words": ["is", "so", "it", "green", "a"]}, {"sent": "it says over in the green grass theres an orange .", "words": ["the", "in", "it", "an", "orange", "green", "over", "grass"]}, {"sent": "the green bag .", "words": ["the", "green"]}, {"sent": "its the green turtle .", "words": ["the", "turtle", "green"]}, {"sent": "and theres a green steamroller .", "words": ["green", "a", "and"]}, {"sent": "a green one .", "words": ["green", "a"]}, {"sent": "a green Smartie and the yellow Smartie .", "words": ["the", "and", "green", "yellow", "a"]}, {"sent": "green bits xxx .", "words": ["green"]}, {"sent": "I dont think there is another green one is there ?", "words": ["think", "is", "another", "dont", "there", "green"]}, {"sent": "gimme the green green the screen .", "words": ["the", "green"]}, {"sent": "and green grapes .", "words": ["grapes", "green", "and"]}, {"sent": "green means .", "words": ["green"]}, {"sent": "this is a small green sieve .", "words": ["this", "green", "a", "is"]}, {"sent": "and your mountains green .", "words": ["your", "green", "and"]}, {"sent": "I do not like green eggs and ham .", "words": ["do", "like", "and", "green", "not"]}, {"sent": "I wonder if the green .", "words": ["the", "if", "green"]}, {"sent": "do you know why it was green ?", "words": ["do", "was", "you", "it", "green", "why"]}, {"sent": "green .", "words": ["green"]}, {"sent": "shall we change it to green ?", "words": ["to", "it", "green", "we"]}, {"sent": "a green mermaid .", "words": ["green", "a"]}, {"sent": "green stripy trousers .", "words": ["green"]}, {"sent": "green .", "words": ["green"]}, {"sent": "with green leaves on it ?", "words": ["on", "it", "with", "green"]}, {"sent": "you hafta color the frog in green darling .", "words": ["the", "frog", "you", "in", "green"]}, {"sent": "got green .", "words": ["green"]}, {"sent": "green beans ?", "words": ["green beans", "beans", "green"]}, {"sent": "hat is green isnt it ?", "words": ["hat", "it", "green", "is"]}, {"sent": "what do you think to make with the green one ?", "words": ["the", "what", "do", "think", "to", "with", "you", "green", "make"]}, {"sent": "but what would you like after green and blue ?", "words": ["what", "like", "you", "and", "blue", "green", "would", "but"]}, {"sent": "the one with the green coat on .", "words": ["the", "with", "green", "on", "coat"]}, {"sent": "green .", "words": ["green"]}, {"sent": "thats green and yellow right right .", "words": ["yellow", "green", "and"]}, {"sent": "and this is a car with green wheels .", "words": ["is", "car", "with", "and", "this", "green", "a"]}, {"sent": "you need a green one okay .", "words": ["need", "green", "a", "you"]}, {"sent": "green .", "words": ["green"]}, {"sent": "green .", "words": ["green"]}, {"sent": "in the great green room there was a telephone .", "words": ["the", "was", "there", "in", "room", "green", "telephone", "a"]}, {"sent": "red green purple pink violet or grey or orange ?", "words": ["red", "orange", "green"]}, {"sent": "are we missing some pieces ?", "words": ["some", "are", "we"]}, {"sent": "do you want some more potato ?", "words": ["do", "you", "more", "some", "potato"]}, {"sent": "and he tips his head over the font and gets this hand with some water from the bowl and puts .", "words": ["the", "bowl", "hand", "head", "with", "and", "this", "he", "water", "his", "some", "over"]}, {"sent": "Id like some tomato soup and some macaroni cheese please .", "words": ["like", "and", "soup", "some", "cheese"]}, {"sent": "shes got shes got some more beans on her brown bread .", "words": ["bread", "brown", "more", "beans", "on", "some", "her"]}, {"sent": "wanna look at some books ?", "words": ["some", "at", "wanna", "look"]}, {"sent": "and then Ill blow on it when its done so that you can have some .", "words": ["can", "so", "and", "you", "have", "that", "it", "on", "some", "then", "when", "blow"]}, {"sent": "shop lady can I have some smarties and some peas ?", "words": ["can", "peas", "and", "have", "some"]}, {"sent": "the game were going to play now is a game with some puppets .", "words": ["the", "game", "is", "to", "play", "with", "some", "were", "a"]}, {"sent": "to buy some nuts .", "words": ["to", "buy", "some"]}, {"sent": "some beans .", "words": ["some", "beans"]}, {"sent": "here some Martinellis sparkling apple cider .", "words": ["here", "some", "apple"]}, {"sent": "heres some more of the puzzle Megan that fell out .", "words": ["the", "of", "more", "puzzle", "out", "some", "that"]}, {"sent": "we could do with some sheep or some donkeys and some cows .", "words": ["do", "we", "with", "and", "sheep", "some", "could"]}, {"sent": "shall I go and get some paper ?", "words": ["go", "paper", "and", "get", "some"]}, {"sent": "Ill grab some paper upstairs .", "words": ["some", "paper"]}, {"sent": "have they been running across some fields .", "words": ["have", "they", "some"]}, {"sent": "can I have some of those right there ?", "words": ["can", "those", "of", "have", "there", "some"]}, {"sent": "do you want some paper ?", "words": ["do", "paper", "some", "you"]}, {"sent": "sing some more .", "words": ["sing", "some", "more"]}, {"sent": "some lambs .", "words": ["some"]}, {"sent": "you wanna color some more ?", "words": ["some", "more", "wanna", "you"]}, {"sent": "the hungry caterpillar eats some of those .", "words": ["the", "those", "of", "hungry", "some"]}, {"sent": "there are some buss picking up passengers .", "words": ["up", "there", "are", "some"]}, {"sent": "well Ill go run some other errands and come back .", "words": ["back", "go", "other", "and", "run", "some"]}, {"sent": "you sat on Daddys knee and had some strawberries for tea .", "words": ["knee", "for", "and", "you", "on", "some"]}, {"sent": "put some of your Vicks .", "words": ["of", "some", "put", "your"]}, {"sent": "we could plant some pips from the middle of the fruit couldnt we ?", "words": ["the", "we", "of", "some", "plant", "could"]}, {"sent": "shall we put some spreading cheese .", "words": ["some", "put", "cheese", "we"]}, {"sent": "and when we go in here we can put some lotion on .", "words": ["can", "put", "go", "we", "and", "here", "in", "on", "some", "when"]}, {"sent": "oh she spit up some milk .", "words": ["up", "some", "she", "milk"]}, {"sent": "make some sandwiches .", "words": ["some", "make"]}, {"sent": "make have some cereal .", "words": ["have", "some", "make", "cereal"]}, {"sent": "maybe we should get you some spoons .", "words": ["get", "some", "you", "we"]}, {"sent": "some there .", "words": ["some", "there"]}, {"sent": "some nights .", "words": ["some"]}, {"sent": "or drink some .", "words": ["some", "drink"]}, {"sent": "you wanna read another book or do you wanna play with this some toys ?", "words": ["do", "play", "with", "you", "another", "this", "read", "wanna", "book", "some"]}, {"sent": "I need some a the green take some a those .", "words": ["the", "need", "those", "green", "some", "take", "a"]}, {"sent": "some of them went .", "words": ["of", "some", "them"]}, {"sent": "do you want us some other place now or ?", "words": ["do", "other", "you", "us", "some"]}, {"sent": "well Ill do some myself then okay ?", "words": ["do", "then", "myself", "some"]}, {"sent": "didjou see some borage ?", "words": ["see", "some"]}, {"sent": "Mummys still washing some pots isnt she ?", "words": ["some", "she"]}, {"sent": "put some shoes on ?", "words": ["on", "some", "put"]}, {"sent": "so what did we do some of the time when we were home with grandma and grandpa .", "words": ["the", "what", "do", "we", "of", "so", "with", "and", "did", "some", "when", "were", "home"]}, {"sent": "I think well have some lunch .", "words": ["have", "some", "think"]}, {"sent": "have some more toast if you want it love .", "words": ["if", "toast", "you", "have", "more", "it", "love", "some"]}, {"sent": "well well just hafta build some walls and make a roof outof something else wont we ?", "words": ["we", "and", "build", "roof", "some", "make", "a"]}, {"sent": "the mans holding some pink wool now isnt he ?", "words": ["the", "some", "he"]}, {"sent": "some cars ?", "words": ["some"]}, {"sent": "or some red socks or some blue socks .", "words": ["red", "some", "blue"]}, {"sent": "its just some cheese from the fridge .", "words": ["the", "some", "cheese"]}, {"sent": "I can answer why when you ask me some things but .", "words": ["can", "you", "but", "me", "when", "why", "some"]}, {"sent": "wanna pour some in your juice ?", "words": ["your", "pour", "in", "wanna", "some", "juice"]}, {"sent": "and come and ask a grownup for some juice or water or soy milk .", "words": ["for", "and", "water", "milk", "some", "juice", "a"]}, {"sent": "can you draw some other pictures ?", "words": ["can", "draw", "other", "you", "some"]}, {"sent": "I have some books I have some cute tshirts .", "words": ["have", "some", "cute"]}, {"sent": "we need some a little wand to make bubbles .", "words": ["need", "we", "bubbles", "to", "little", "some", "make", "a"]}, {"sent": "you dont want some milk ?", "words": ["milk", "dont", "some", "you"]}, {"sent": "we might hafta go and get some more you know .", "words": ["go", "we", "and", "get", "you", "more", "some"]}, {"sent": "I think that their other ones arent ripe but you could have some strawberry the other ones are hard .", "words": ["the", "think", "their", "strawberry", "other", "you", "have", "are", "hard", "some", "could", "that", "but"]}, {"sent": "okay well well get you some water relax my little friend .", "words": ["my", "you", "get", "little", "water", "some"]}, {"sent": "would you like some of this ?", "words": ["of", "like", "you", "this", "would", "some"]}, {"sent": "well I was actually wondering about getting some jelly babies and jelly tots to go on top of these buns when weve made them .", "words": ["jelly", "these", "go", "of", "was", "to", "them", "and", "about", "on", "some", "when"]}, {"sent": "you have some friends there ?", "words": ["have", "some", "there", "you"]}, {"sent": "get some more toys out ?", "words": ["get", "some", "more", "out"]}, {"sent": "and theres some clothes on the floor as well .", "words": ["on", "some", "the", "and"]}, {"sent": "this is the man whos holding some milkbottles .", "words": ["the", "this", "some", "is"]}, {"sent": "some socks .", "words": ["some"]}, {"sent": "have some ?", "words": ["have", "some"]}, {"sent": "can we put some books away ?", "words": ["can", "put", "we", "away", "some"]}, {"sent": "some spaghetti for you .", "words": ["some", "for", "you", "spaghetti"]}, {"sent": "theyll be feeling some of them will be feeling quite miserable because theyll be away from home for the first time .", "words": ["the", "for", "of", "be", "first", "them", "away", "will", "some", "because", "home"]}, {"sent": "why dont you bring some fish .", "words": ["bring", "you", "dont", "fish", "some", "why"]}, {"sent": "I will climb into the pasture and after a few words with the gentle cow I will be back with some delicious milk .", "words": ["the", "milk", "into", "back", "be", "with", "and", "will", "some", "gentle", "climb", "cow", "a"]}, {"sent": "hes flirting with some girl .", "words": ["some", "with"]}, {"sent": "you better pour me some more please .", "words": ["better", "pour", "you", "more", "me", "some"]}, {"sent": "you sit here and you can have some dinner .", "words": ["can", "and", "here", "you", "have", "some", "sit"]}, {"sent": "lets pour some more .", "words": ["some", "more", "pour"]}, {"sent": "whoa so now well give him some lotsof grass so hell have plenty to eat .", "words": ["so", "to", "have", "eat", "him", "some", "give", "grass"]}, {"sent": "would you like some coke ?", "words": ["like", "you", "coke", "would", "some"]}, {"sent": "have you got some boots ?", "words": ["have", "some", "boots", "you"]}, {"sent": "some sugar beans ?", "words": ["some", "beans"]}, {"sent": "it could use some fixing .", "words": ["some", "could", "it"]}, {"sent": "its got a watering can a teapot a cake and some bread on it .", "words": ["can", "bread", "cake", "and", "it", "on", "some", "a"]}, {"sent": "want some apple juice ?", "words": ["apple", "some", "juice"]}, {"sent": "lets put some more juice in there .", "words": ["put", "there", "more", "in", "some", "juice"]}, {"sent": "are you going to do some repair work on it ?", "words": ["do", "work", "to", "you", "are", "it", "on", "some"]}, {"sent": "would you like some apple ?", "words": ["like", "you", "would", "apple", "some"]}, {"sent": "you want some in there .", "words": ["some", "in", "there", "you"]}, {"sent": "youknow he gave me some little pink pills that were about the size of a penny .", "words": ["the", "a", "of", "little", "penny", "he", "about", "me", "that", "were", "some"]}, {"sent": "some more milk .", "words": ["milk", "some", "more"]}, {"sent": "do you want some dinner Nomi ?", "words": ["do", "some", "you"]}, {"sent": "do you wanna read some books ?", "words": ["do", "you", "read", "wanna", "some"]}, {"sent": "mhm theres some here and some more tapes there .", "words": ["and", "here", "there", "more", "some"]}, {"sent": "well we better give her some then .", "words": ["better", "we", "then", "some", "give", "her"]}, {"sent": "can I have some cake too ?", "words": ["can", "cake", "have", "too", "some"]}, {"sent": "would you like some more pasta ?", "words": ["like", "you", "more", "would", "some"]}, {"sent": "lets see if we can take the camera with us into the other room .", "words": ["can", "the", "camera", "if", "into", "we", "with", "other", "us", "room", "see", "take"]}, {"sent": "oh wait a second you hafta wait until she comes back in so she can stop stop the camera .", "words": ["can", "the", "wait", "camera", "back", "so", "stop", "you", "in", "she", "a"]}, {"sent": "I see your potty in the camera .", "words": ["the", "your", "camera", "in", "potty", "see"]}, {"sent": "my camera .", "words": ["my", "camera"]}, {"sent": "shes watching the camera ?", "words": ["the", "camera"]}, {"sent": "youre not meant to be looking at that camera .", "words": ["camera", "to", "be", "at", "that", "not"]}, {"sent": "you work for candid camera .", "words": ["camera", "for", "work", "you"]}, {"sent": "yeah were gonna just ignore that camera okay ?", "words": ["camera", "that", "were"]}, {"sent": "do you know who that is behind the camera ?", "words": ["the", "do", "camera", "is", "behind", "you", "who", "that"]}, {"sent": "do do do do do do do the camera doesnt need a baby .", "words": ["the", "do", "camera", "need", "a"]}, {"sent": "camera .", "words": ["camera"]}, {"sent": "we asked to take the camera in .", "words": ["the", "camera", "we", "to", "in", "take"]}, {"sent": "the camera made a big noise huh ?", "words": ["the", "big", "camera", "a"]}, {"sent": "lets pretend lets pretend that this is a camera .", "words": ["camera", "is", "this", "pretend", "that", "a"]}, {"sent": "thats a television camera truck .", "words": ["truck", "camera", "a"]}, {"sent": "u uh thats no thats no camera .", "words": ["camera"]}, {"sent": "do you remember the gentleman coming round with the camera ?", "words": ["the", "do", "camera", "with", "you"]}, {"sent": "let me see get the camera over there .", "words": ["the", "camera", "me", "get", "there", "see", "over"]}, {"sent": "camera .", "words": ["camera"]}, {"sent": "can you wave at the camera ?", "words": ["can", "the", "camera", "you", "at"]}, {"sent": "thats a camera .", "words": ["camera", "a"]}, {"sent": "how can it be a camera ?", "words": ["can", "how", "camera", "be", "it", "a"]}, {"sent": "thats for the camera yeah no that goes in the bag put it back in the bag .", "words": ["the", "put", "for", "camera", "back", "in", "it", "that"]}, {"sent": "okay I will hafta move the camera in a few minutes .", "words": ["the", "camera", "in", "will", "a"]}, {"sent": "it shows how we are moving Grandmas camera isnt moving so she has to fix it .", "words": ["how", "camera", "we", "so", "to", "fix", "it", "are", "she"]}, {"sent": "you wanna see the camera ?", "words": ["the", "camera", "you", "wanna", "see"]}, {"sent": "they wouldnt have pictures because they wouldnt have a camera and it wasnt planned that he would fall .", "words": ["camera", "fall", "and", "have", "they", "it", "he", "would", "that", "because", "a"]}, {"sent": "yeah therere letters on the camera .", "words": ["on", "the", "camera"]}, {"sent": "Ill move the camera so it can see you eating your cereal .", "words": ["the", "can", "camera", "your", "so", "you", "it", "see", "cereal"]}, {"sent": "camera .", "words": ["camera"]}, {"sent": "this is like a camera .", "words": ["camera", "is", "like", "this", "a"]}, {"sent": "camera .", "words": ["camera"]}, {"sent": "get on the camera .", "words": ["get", "the", "on", "camera"]}, {"sent": "its just the camera its making noise isnt it ?", "words": ["the", "it", "camera"]}, {"sent": "I got my name on my camera .", "words": ["on", "my", "camera"]}, {"sent": "okay well then dont touch the camera .", "words": ["the", "camera", "touch", "dont", "then"]}, {"sent": "camera .", "words": ["camera"]}, {"sent": "say cheese again at the camera .", "words": ["the", "camera", "say", "at", "cheese"]}, {"sent": "you wanna say hello to the camera ?", "words": ["the", "camera", "to", "say", "you", "wanna"]}, {"sent": "Roy and Tom both fixed the camera .", "words": ["the", "camera", "and"]}, {"sent": "*AD2: hey ya cant come near that camera over there honey .", "words": ["over", "camera", "that", "there"]}, {"sent": "smile for the camera Roman .", "words": ["the", "for", "camera", "smile"]}, {"sent": "oh its the other camera that does that Eleanor .", "words": ["the", "does", "camera", "other", "that"]}, {"sent": "thats the camera .", "words": ["the", "camera"]}, {"sent": "you going to take a picture of the camera with the camera ?", "words": ["the", "camera", "of", "to", "picture", "with", "you", "take", "a"]}, {"sent": "hello camera .", "words": ["camera"]}, {"sent": "wheres the camera ?", "words": ["the", "camera"]}, {"sent": "camera .", "words": ["camera"]}, {"sent": "look at that camera .", "words": ["camera", "at", "that", "look"]}, {"sent": "can you wave to the camera ?", "words": ["can", "the", "camera", "to", "you"]}, {"sent": "I noticed you had um she was running over to the camera and so you were giving her uh youknow you said some no and she stopped .", "words": ["the", "camera", "to", "was", "so", "her", "you", "and", "some", "over", "she", "were"]}, {"sent": "you see the camera .", "words": ["the", "see", "camera", "you"]}, {"sent": "do you remember what was weird about that camera ?", "words": ["do", "what", "camera", "was", "you", "about", "that"]}, {"sent": "he hasta be facing the camera .", "words": ["the", "be", "he", "camera"]}, {"sent": "she can move the camera so she follows you .", "words": ["can", "the", "camera", "so", "you", "she"]}, {"sent": "heres a camera .", "words": ["camera", "a"]}, {"sent": "usually youre noisy in front of the camera dancing and singing .", "words": ["the", "camera", "of", "noisy", "and", "in"]}, {"sent": "and put the big camera on your tummy and theyll say ah he has been eating green jelly babies .", "words": ["the", "jelly", "put", "camera", "your", "say", "and", "he", "green", "tummy", "on", "big"]}, {"sent": "yeah thats a camera .", "words": ["camera", "a"]}, {"sent": "but there was something really strange about that camera anyway .", "words": ["camera", "was", "there", "about", "that", "but"]}, {"sent": "we dont wanna play with the camera .", "words": ["the", "camera", "we", "play", "with", "dont", "wanna"]}, {"sent": "camera .", "words": ["camera"]}, {"sent": "is that a camera like UncleRonny had at your birthdayparty ?", "words": ["camera", "your", "is", "like", "at", "that", "a"]}, {"sent": "hi camera .", "words": ["camera"]}, {"sent": "thats a camera .", "words": ["camera", "a"]}, {"sent": "wish I could find the camera .", "words": ["the", "camera", "wish", "find", "could"]}, {"sent": "you might wanna move the camera .", "words": ["the", "camera", "wanna", "you"]}, {"sent": "that is a tv camera .", "words": ["camera", "is", "tv", "that", "a"]}, {"sent": "camera can you say camera ?", "words": ["can", "camera", "say", "you"]}, {"sent": "camera and chips ?", "words": ["camera", "and"]}, {"sent": "you go talk in front of the camera okay ?", "words": ["the", "camera", "go", "of", "you", "in", "talk"]}, {"sent": "you know how Mommy and Dad and we go camping and Daddy gets the camera out and we take pictures of you guys riding your bikes and things ?", "words": ["the", "how", "camera", "your", "go", "we", "of", "and", "you", "out", "take"]}, {"sent": "in front of the invisible camera .", "words": ["of", "the", "in", "camera"]}, {"sent": "um he got a camera for his birthday .", "words": ["for", "camera", "he", "his", "a"]}, {"sent": "a camera .", "words": ["camera", "a"]}, {"sent": "it is a camera .", "words": ["it", "camera", "a", "is"]}, {"sent": "you see the camera .", "words": ["the", "see", "camera", "you"]}, {"sent": "or a camera .", "words": ["camera", "a"]}, {"sent": "when you speak it goes into the camera .", "words": ["the", "camera", "into", "you", "it", "when"]}, {"sent": "its a good job this isnt on camera .", "words": ["camera", "this", "good", "on", "a"]}, {"sent": "Id love to have had a video camera .", "words": ["camera", "to", "have", "love", "a"]}, {"sent": "well Laura because Im so dumb Im trying to figure out the camera .", "words": ["the", "camera", "to", "so", "out", "because"]}, {"sent": "camera take a picture of you .", "words": ["camera", "of", "picture", "you", "take", "a"]}, {"sent": "their stuck yeah they belong on the camera .", "words": ["the", "camera", "their", "they", "stuck", "on"]}, {"sent": "and fit the camera .", "words": ["the", "fit", "camera", "and"]}, {"sent": "camera .", "words": ["camera"]}, {"sent": "you know whatever he does Ill kinda move the camera to keep track of him and uh you know it itll just be whatevers convenient .", "words": ["the", "does", "camera", "of", "to", "be", "you", "and", "him", "it", "he"]}, {"sent": "I think hes a bit scared of this big video camera .", "words": ["think", "camera", "of", "scared", "this", "big", "a"]}, {"sent": "I got a camera I got .", "words": ["camera", "a"]}, {"sent": "wheres the camera thats the camera right there .", "words": ["the", "there", "camera"]}, {"sent": "heehee would you like to sing a song for the camera ?", "words": ["the", "for", "camera", "to", "like", "you", "sing", "would", "a"]}, {"sent": "its for giving electricity to the camera .", "words": ["the", "to", "for", "camera"]}, {"sent": "oh youre looking for something want me to move the camera ?", "words": ["the", "for", "camera", "to", "me"]}, {"sent": "its a camera .", "words": ["camera", "a"]}, {"sent": "I see a camera .", "words": ["see", "camera", "a"]}, {"sent": "let me get my camera .", "words": ["get", "me", "my", "camera"]}, {"sent": "camera .", "words": ["camera"]}, {"sent": "you hafta go play in the camera and then youll be on .", "words": ["the", "camera", "go", "be", "play", "you", "and", "in", "on", "then"]}, {"sent": "thats a camera .", "words": ["camera", "a"]}, {"sent": "and whats the tiger doing Fraser .", "words": ["the", "tiger", "and"]}, {"sent": "or the tiger .", "words": ["the", "tiger"]}, {"sent": "wheres the baby tiger ?", "words": ["the", "tiger"]}, {"sent": "a tiger .", "words": ["tiger", "a"]}, {"sent": "she can have tiger .", "words": ["can", "have", "she", "tiger"]}, {"sent": "mm the tiger .", "words": ["the", "tiger"]}, {"sent": "tiger do like dis .", "words": ["do", "like", "tiger"]}, {"sent": "and the tiger ones green .", "words": ["the", "green", "tiger", "and"]}, {"sent": "what does tiger say ?", "words": ["what", "does", "say", "tiger"]}, {"sent": "is Mummy tiger walking round ?", "words": ["tiger", "is"]}, {"sent": "lets give it to the tiger .", "words": ["the", "to", "it", "tiger", "give"]}, {"sent": "is it the tiger ?", "words": ["the", "it", "tiger", "is"]}, {"sent": "if horsie says tiger is naughty then tiger says no tiger is not naughty .", "words": ["if", "is", "not", "tiger", "then", "naughty"]}, {"sent": "tiger is stupid .", "words": ["tiger", "is"]}, {"sent": "if horsie says tiger is naughty then tiger says no tiger is not naughty .", "words": ["if", "is", "not", "tiger", "then", "naughty"]}, {"sent": "tiger .", "words": ["tiger"]}, {"sent": "youve got the fish out and youve got the tiger out .", "words": ["the", "and", "tiger", "out", "fish"]}, {"sent": "tiger .", "words": ["tiger"]}, {"sent": "no tiger is not naughty is he ?", "words": ["is", "not", "he", "tiger", "naughty"]}, {"sent": "tiger .", "words": ["tiger"]}, {"sent": "shall we put baby tiger in ?", "words": ["in", "put", "tiger", "we"]}, {"sent": "thats a little tiger baby I think isnt it ?", "words": ["think", "little", "it", "tiger", "a"]}, {"sent": "look this is a horsie and this is a tiger .", "words": ["is", "and", "this", "tiger", "look", "a"]}, {"sent": "piece of wood thats for your tiger can you move it ?", "words": ["can", "for", "your", "of", "you", "it", "tiger"]}, {"sent": "tiger do you like smarties ?", "words": ["do", "like", "tiger", "you"]}, {"sent": "better put the tiger in there because hes a bit grrr .", "words": ["the", "better", "put", "there", "in", "tiger", "because", "a"]}, {"sent": "tiger has a smartie .", "words": ["tiger", "a"]}, {"sent": "penguin and giraffe and little tiger and little whale and monkey and horsie and big tiger all need birthday cake please .", "words": ["all", "giraffe", "need", "cake", "monkey", "penguin", "little", "and", "tiger", "big"]}, {"sent": "tiger says no tiger is not naughty .", "words": ["naughty", "not", "tiger", "is"]}, {"sent": "I love that tiger .", "words": ["tiger", "that", "love"]}, {"sent": "horsie says tiger is naughty .", "words": ["naughty", "tiger", "is"]}, {"sent": "and a tiger .", "words": ["tiger", "a", "and"]}, {"sent": "if horsie says tiger is naughty then tiger says no tiger is not naughty .", "words": ["if", "is", "not", "tiger", "then", "naughty"]}, {"sent": "you could do an operation on tiger .", "words": ["do", "you", "an", "tiger", "on", "could"]}, {"sent": "now were going to play with horse and tiger .", "words": ["to", "play", "with", "and", "tiger", "horse", "were"]}, {"sent": "horsie says tiger has a big mouth .", "words": ["mouth", "big", "tiger", "a"]}, {"sent": "the elephant says I think its a tiger .", "words": ["the", "think", "elephant", "tiger", "a"]}, {"sent": "where Annas baby tiger does .", "words": ["where", "does", "tiger"]}, {"sent": "let me find me a tiger .", "words": ["me", "tiger", "a", "find"]}, {"sent": "because there was a tiger at Disneyland ?", "words": ["was", "there", "tiger", "at", "because", "a"]}, {"sent": "tiger hungry ?", "words": ["hungry", "tiger"]}, {"sent": "tiger .", "words": ["tiger"]}, {"sent": "one day this little boy saw a big tiger .", "words": ["little", "this", "tiger", "big", "a"]}, {"sent": "watch what horse and tiger are doing and listen to what they say .", "words": ["what", "watch", "listen", "to", "say", "and", "they", "are", "tiger", "horse"]}, {"sent": "tiger says doesnt he ?", "words": ["he", "tiger"]}, {"sent": "tiger is stupid .", "words": ["tiger", "is"]}, {"sent": "if horse says tiger has a big mouth then tiger says hasnt he ?", "words": ["if", "mouth", "he", "tiger", "horse", "then", "big", "a"]}, {"sent": "what does tiger say ?", "words": ["what", "does", "say", "tiger"]}, {"sent": "here comes the tiger .", "words": ["here", "the", "tiger"]}, {"sent": "pardon me tiger .", "words": ["me", "tiger"]}, {"sent": "whats the tiger going to eat ?", "words": ["the", "to", "eat", "tiger"]}, {"sent": "and then tiger says no tiger is not naughty .", "words": ["is", "naughty", "and", "tiger", "then", "not"]}, {"sent": "tiger will eat all the smarties .", "words": ["the", "all", "eat", "tiger", "will"]}, {"sent": "tiger say tiger .", "words": ["say", "tiger"]}, {"sent": "you are tiger now and horsie says tiger is naughty .", "words": ["is", "and", "you", "are", "tiger", "naughty"]}, {"sent": "if horsie says tiger is naughty then tiger says no tiger is not naughty .", "words": ["if", "is", "not", "tiger", "then", "naughty"]}, {"sent": "thats a tiger .", "words": ["tiger", "a"]}, {"sent": "and what does the tiger do ?", "words": ["the", "what", "does", "do", "and", "tiger"]}, {"sent": "now you be tiger again .", "words": ["be", "tiger", "you"]}, {"sent": "tiger is frightened .", "words": ["tiger", "is"]}, {"sent": "are you biting me tiger ?", "words": ["me", "are", "tiger", "you"]}, {"sent": "tiger should be quiet .", "words": ["be", "quiet", "tiger"]}, {"sent": "if horsie says tiger is naughty then tiger says no tiger is not naughty .", "words": ["if", "is", "not", "tiger", "then", "naughty"]}, {"sent": "a lion or a tiger ?", "words": ["lion", "tiger", "a"]}, {"sent": "tiger is naughty .", "words": ["naughty", "tiger", "is"]}, {"sent": "hes a nice stripy tiger isnt he ?", "words": ["nice", "he", "tiger", "a"]}, {"sent": "tiger .", "words": ["tiger"]}, {"sent": "meow yeow goes the tiger tomcat .", "words": ["the", "tiger"]}, {"sent": "its the baby tiger .", "words": ["the", "tiger"]}, {"sent": "the flying tiger a flying elephant .", "words": ["the", "elephant", "tiger", "a"]}, {"sent": "tiger ?", "words": ["tiger"]}, {"sent": "tiger good boy .", "words": ["good", "tiger"]}, {"sent": "the elephant and the hippo and the tiger .", "words": ["the", "tiger", "and", "elephant"]}, {"sent": "Im a tiger .", "words": ["tiger", "a"]}, {"sent": "tiger has a smartie .", "words": ["tiger", "a"]}, {"sent": "you saw the tiger though .", "words": ["the", "tiger", "you"]}, {"sent": "horsie says tiger is hungry .", "words": ["hungry", "tiger", "is"]}, {"sent": "is it a baby tiger ?", "words": ["tiger", "it", "a", "is"]}, {"sent": "a tiger .", "words": ["tiger", "a"]}, {"sent": "this is an exotic train with a baby tiger on the roof .", "words": ["the", "train", "is", "with", "this", "an", "roof", "tiger", "on", "a"]}, {"sent": "tiger has a smartie .", "words": ["tiger", "a"]}, {"sent": "and tiger says no Tiger doesnt like school .", "words": ["school", "like", "tiger", "and"]}, {"sent": "make something for the tiger ?", "words": ["the", "for", "make", "tiger"]}, {"sent": "tiger .", "words": ["tiger"]}, {"sent": "does a tiger say roar ?", "words": ["does", "say", "tiger", "a"]}, {"sent": "tiger .", "words": ["tiger"]}, {"sent": "think its for the tiger food ?", "words": ["the", "think", "for", "food", "tiger"]}, {"sent": "does Nina have a tiger ?", "words": ["have", "does", "tiger", "a"]}, {"sent": "and the bunny and the tiger and the panda .", "words": ["the", "bunny", "tiger", "and"]}, {"sent": "this time there was a tiger .", "words": ["was", "this", "there", "tiger", "a"]}, {"sent": "this tiger ?", "words": ["this", "tiger"]}, {"sent": "and tiger says no tiger doesnt like school .", "words": ["school", "like", "tiger", "and"]}, {"sent": "hes a naughty tiger .", "words": ["naughty", "tiger", "a"]}, {"sent": "and horsie says now tiger has a big mouth what does tiger say ?", "words": ["what", "does", "mouth", "say", "and", "tiger", "big", "a"]}, {"sent": "the little lion little tiger wants to go on there .", "words": ["the", "go", "to", "little", "there", "tiger", "on", "lion"]}, {"sent": "sometimes horsie and tiger talk in a different way .", "words": ["and", "in", "talk", "tiger", "a"]}, {"sent": "and the tiger .", "words": ["the", "tiger", "and"]}, {"sent": "baby tiger got you .", "words": ["tiger", "you"]}, {"sent": "its a mummy daddy tiger is it ?", "words": ["it", "is", "tiger", "a"]}, {"sent": "put the money in the bank .", "words": ["the", "in", "put", "money"]}, {"sent": "look at all this money .", "words": ["money", "all", "look", "this", "at"]}, {"sent": "it looks like money .", "words": ["money", "it", "like"]}, {"sent": "dont you see that money ?", "words": ["money", "you", "dont", "see", "that"]}, {"sent": "look at all that money .", "words": ["money", "all", "look", "at", "that"]}, {"sent": "that rhymes money in your bunny .", "words": ["money", "bunny", "your", "in", "that"]}, {"sent": "bring monkey to the zoo and then I you hafta pay me some money to get into the zoo .", "words": ["the", "zoo", "money", "into", "to", "monkey", "bring", "and", "you", "get", "me", "then", "some"]}, {"sent": "so they had to spend money .", "words": ["to", "so", "they", "money"]}, {"sent": "you put the money in a pile .", "words": ["the", "money", "put", "you", "in", "a"]}, {"sent": "theres not money in here as well is there ?", "words": ["money", "is", "here", "there", "in", "not"]}, {"sent": "is that your worldly money ?", "words": ["your", "money", "that", "is"]}, {"sent": "that must have been your school money Michelle .", "words": ["money", "your", "school", "have", "that"]}, {"sent": "more money ?", "words": ["money", "more"]}, {"sent": "what could we do if we had a bunch of money ?", "words": ["money", "what", "do", "if", "we", "of", "could", "a"]}, {"sent": "what if he starts chasing you because they really want that money ?", "words": ["money", "what", "if", "you", "they", "he", "that", "because"]}, {"sent": "theres no money in here .", "words": ["money", "in", "here"]}, {"sent": "are you getting some more money out of Mummys purse ?", "words": ["money", "purse", "of", "you", "more", "are", "out", "some"]}, {"sent": "I mean anything over seventy cents you know I always say yeah gimme my money back .", "words": ["money", "my", "back", "say", "you", "over"]}, {"sent": "what have I done to deserve all that money ?", "words": ["money", "what", "all", "to", "have", "that"]}, {"sent": "youre gonna put the money in ?", "words": ["the", "in", "put", "money"]}, {"sent": "but sometimes people get money .", "words": ["get", "but", "money"]}, {"sent": "and perhaps we could play a game with the money .", "words": ["the", "money", "game", "we", "play", "with", "and", "could", "a"]}, {"sent": "put the money in the till .", "words": ["the", "in", "put", "money"]}, {"sent": "I got my money .", "words": ["money", "my"]}, {"sent": "we needta sort some money out later on .", "words": ["money", "we", "out", "on", "some"]}, {"sent": "I got the money .", "words": ["the", "money"]}, {"sent": "I wish this was real money .", "words": ["was", "this", "money", "wish"]}, {"sent": "and Mummy usually gives them some money .", "words": ["money", "some", "them", "and"]}, {"sent": "real money real .", "words": ["money"]}, {"sent": "mummie your mummie has plenty of money to buy you Adidas and a ManUnited top and all .", "words": ["money", "all", "your", "of", "to", "you", "and", "buy", "a"]}, {"sent": "she wanted more money ?", "words": ["money", "more", "she"]}, {"sent": "how much money ya got ?", "words": ["money", "how", "much"]}, {"sent": "heres my money okay thank you bye .", "words": ["money", "my", "you"]}, {"sent": "yeah youve got to put the money in the till .", "words": ["the", "money", "put", "to", "in"]}, {"sent": "I gotta put the money in the cash register .", "words": ["the", "in", "put", "money"]}, {"sent": "you throwing your money away now are you ?", "words": ["money", "your", "you", "away", "are"]}, {"sent": "Ive got some money here .", "words": ["money", "some", "here"]}, {"sent": "I dont think Paul can have any money .", "words": ["can", "money", "think", "any", "have", "dont"]}, {"sent": "yeah and thats to put the money in .", "words": ["the", "money", "put", "to", "and", "in"]}, {"sent": "I havent got the money to keep going and buying new things .", "words": ["the", "money", "to", "and", "new"]}, {"sent": "uh I xxx say ice skating it involves money now I mean we went to the rink .", "words": ["money", "the", "we", "to", "say", "it", "ice"]}, {"sent": "Carl its fixed I know wheres my rent money .", "words": ["money", "my"]}, {"sent": "cold money ?", "words": ["money", "cold"]}, {"sent": "you should bring your pocket money added Jonathan .", "words": ["money", "your", "bring", "you"]}, {"sent": "youve got alotof money there .", "words": ["money", "there"]}, {"sent": "put your money in the till darling .", "words": ["money", "the", "put", "your", "in"]}, {"sent": "xxx money that goes directly into the savings .", "words": ["money", "the", "that", "into"]}, {"sent": "and youd better get some money .", "words": ["money", "better", "and", "get", "some"]}, {"sent": "money .", "words": ["money"]}, {"sent": "and you just give them a little bit extra money .", "words": ["money", "them", "you", "and", "little", "give", "a"]}, {"sent": "lets see what this how much money were gonna spend Joe .", "words": ["money", "what", "how", "this", "see", "much", "were"]}, {"sent": "can I have some money ?", "words": ["can", "some", "have", "money"]}, {"sent": "there willnt be any money left for mummie and daddy to buy their food with will there the way you carry on .", "words": ["money", "the", "for", "their", "be", "to", "any", "with", "and", "you", "there", "food", "carry", "on", "will", "buy"]}, {"sent": "do you wanchur money back ?", "words": ["money", "do", "back", "you"]}, {"sent": "you hafta lend me some money .", "words": ["money", "me", "some", "you"]}, {"sent": "Im raising money for a new sports pavilion .", "words": ["money", "new", "for", "a"]}, {"sent": "and any money thats made it goes to poorly animals .", "words": ["money", "to", "any", "and", "it"]}, {"sent": "think the money goes in there ?", "words": ["the", "money", "think", "there", "in"]}, {"sent": "oh well I thought Id count the paper money .", "words": ["the", "paper", "money"]}, {"sent": "money .", "words": ["money"]}, {"sent": "lets see if I have any more money .", "words": ["money", "if", "any", "have", "more", "see"]}, {"sent": "you put all the money back in the right spots .", "words": ["the", "all", "put", "money", "back", "you", "in"]}, {"sent": "have you not got any money ?", "words": ["money", "any", "you", "have", "not"]}, {"sent": "see if we can sell it in the art world for lotsof money .", "words": ["can", "the", "money", "for", "if", "we", "in", "it", "see"]}, {"sent": "oh youre giving me money for buying .", "words": ["money", "me", "for"]}, {"sent": "thats not very much money .", "words": ["money", "much", "not"]}, {"sent": "put your money in the till .", "words": ["money", "the", "put", "your", "in"]}, {"sent": "so they hadta get their money back .", "words": ["money", "their", "back", "so", "get", "they"]}, {"sent": "I think they have one more thats worth quite a bit of money and that was the xxx one .", "words": ["money", "the", "think", "of", "was", "and", "have", "they", "more", "that", "a"]}, {"sent": "dont put any money down there for goodness sake .", "words": ["money", "put", "for", "any", "dont", "there", "down"]}, {"sent": "would you like some money for the monkey ?", "words": ["money", "the", "for", "monkey", "like", "you", "would", "some"]}, {"sent": "let me just put this money away before it all falls down xxx .", "words": ["money", "all", "put", "this", "away", "it", "me", "down"]}, {"sent": "well Ill give you some money anyway .", "words": ["money", "some", "give", "you"]}, {"sent": "theres no money behind the rocking chair ?", "words": ["money", "the", "rocking chair", "behind", "chair"]}, {"sent": "I want some money .", "words": ["money", "some"]}, {"sent": "is that is there money in there ?", "words": ["money", "is", "there", "in", "that"]}, {"sent": "youve got money .", "words": ["money"]}, {"sent": "weve weve money to get in the bank have we ?", "words": ["money", "the", "we", "to", "get", "have", "in"]}, {"sent": "lets lets have pretend money .", "words": ["have", "pretend", "money"]}, {"sent": "do you want some money ?", "words": ["money", "do", "some", "you"]}, {"sent": "I hafta give you money because Ive bought it .", "words": ["money", "you", "it", "give", "because"]}, {"sent": "have you got enough money to buy everything ?", "words": ["money", "to", "you", "have", "buy"]}, {"sent": "he sells the rest to grocer cat in exchange for money .", "words": ["the", "money", "for", "to", "cat", "in", "he"]}, {"sent": "can you put the money back in the box for me ?", "words": ["can", "the", "put", "money", "box", "for", "back", "you", "in", "me"]}, {"sent": "oh are we gonna make some money ?", "words": ["money", "we", "are", "some", "make"]}, {"sent": "weve got some money .", "words": ["money", "some"]}, {"sent": "now were going to fill the purse up with money .", "words": ["the", "up", "money", "purse", "to", "with", "were"]}, {"sent": "Ill tell you where you can put some money .", "words": ["where", "can", "put", "money", "you", "some"]}, {"sent": "you just gave me money for it .", "words": ["money", "for", "you", "it", "me"]}, {"sent": "do you needta put some money in ?", "words": ["money", "do", "put", "you", "in", "some"]}, {"sent": "they cost alotof money those and I xxx .", "words": ["money", "they", "those", "and"]}, {"sent": "you gonna make me some money when you about eight ?", "words": ["money", "when", "you", "about", "me", "make", "some"]}, {"sent": "this is really dirty money .", "words": ["money", "this", "dirty", "is"]}, {"sent": "now its a postbox with money in it .", "words": ["money", "with", "in", "it", "a"]}, {"sent": "money .", "words": ["money"]}, {"sent": "your money right .", "words": ["money", "your"]}, {"sent": "oh there is money in there .", "words": ["money", "there", "in", "is"]}, {"sent": "and you need money for sweets and toys and Mummy hasnt got paid yet .", "words": ["money", "for", "need", "you", "and"]}, {"sent": "what kind of money was it Ross ?", "words": ["money", "what", "of", "was", "it"]}, {"sent": "how much money do you have ?", "words": ["money", "do", "how", "you", "have", "much"]}, {"sent": "well little girls and little boys .", "words": ["and", "little"]}, {"sent": "he looks like a naughty little schoolboy doesnt he ?", "words": ["like", "little", "he", "naughty", "a"]}, {"sent": "a little a little plastic seat that raises you up a little bit higher .", "words": ["up", "you", "little", "that", "a"]}, {"sent": "it looks like his guys got a little helmet .", "words": ["like", "little", "it", "his", "a"]}, {"sent": "thats a little boy .", "words": ["little", "a"]}, {"sent": "right its a very tiny deer a little baby deer .", "words": ["tiny", "deer", "little", "a"]}, {"sent": "it looks a little bit like a kittycat .", "words": ["it", "like", "little", "a"]}, {"sent": "did you see this little puppet ?", "words": ["little", "you", "this", "did", "see"]}, {"sent": "huh my little pumpkin .", "words": ["my", "pumpkin", "little"]}, {"sent": "ah a little furry .", "words": ["little", "a"]}, {"sent": "would you like to walk around a little ?", "words": ["around", "to", "like", "you", "little", "would", "walk", "a"]}, {"sent": "is that the little .", "words": ["the", "that", "little", "is"]}, {"sent": "cheeky little horror .", "words": ["little"]}, {"sent": "lets wash your little nose .", "words": ["your", "nose", "wash", "little"]}, {"sent": "just a little longer .", "words": ["little", "a"]}, {"sent": "you can go ahead and let her play a little bit while I ask you some questions .", "words": ["can", "go", "play", "and", "you", "little", "some", "her", "a"]}, {"sent": "thankyou little boy .", "words": ["little"]}, {"sent": "were missing the little girl and the little boy .", "words": ["the", "were", "and", "little"]}, {"sent": "did you say you wanted a little stool to stand on ?", "words": ["to", "stand", "say", "little", "you", "did", "on", "a"]}, {"sent": "oh I think you said theres a little sore place on my hand .", "words": ["think", "my", "hand", "little", "you", "on", "a"]}, {"sent": "can I have a look at that little picture ?", "words": ["can", "picture", "little", "have", "at", "look", "that", "a"]}, {"sent": "he knows little sushi .", "words": ["he", "little"]}, {"sent": "come on little baby .", "words": ["on", "little"]}, {"sent": "hi little girl .", "words": ["little"]}, {"sent": "this little part of the of the Cleopatra coverlet is what I call a bell .", "words": ["the", "what", "is", "of", "little", "this", "a"]}, {"sent": "and little bedside table .", "words": ["table", "little", "and"]}, {"sent": "theres two little ones .", "words": ["little"]}, {"sent": "it might be a little hot .", "words": ["be", "little", "hot", "it", "a"]}, {"sent": "this little bird here .", "words": ["here", "this", "bird", "little"]}, {"sent": "youre wearing your little purple purse .", "words": ["your", "purse", "little"]}, {"sent": "you like these little whistles dont you ?", "words": ["these", "like", "little", "you", "dont"]}, {"sent": "the little cups ?", "words": ["the", "little"]}, {"sent": "are you going to give one of your nice little books to school when she leaves ?", "words": ["your", "of", "to", "school", "nice", "you", "little", "are", "give", "when", "she"]}, {"sent": "say see you in a little bit .", "words": ["say", "you", "little", "in", "see", "a"]}, {"sent": "hes turning a little red because hes working so hard at trying to go fast .", "words": ["red", "fast", "go", "so", "to", "little", "hard", "at", "because", "a"]}, {"sent": "theres a little xxx of blue tack there .", "words": ["of", "blue", "little", "there", "a"]}, {"sent": "it was a little bit too soft yesterday .", "words": ["soft", "was", "little", "it", "too", "a"]}, {"sent": "and a little piece of bread .", "words": ["bread", "of", "little", "and", "a"]}, {"sent": "I saw a little baby pony yesterday .", "words": ["pony", "little", "a"]}, {"sent": "xxx looks like he got stuck a little on the bottom there we go .", "words": ["the", "bottom", "we", "go", "like", "little", "there", "stuck", "he", "on", "a"]}, {"sent": "is that a little heavy there for you Lily ?", "words": ["for", "is", "little", "you", "there", "heavy", "that", "a"]}, {"sent": "a little rest now ?", "words": ["little", "a"]}, {"sent": "and you can put the little man in .", "words": ["can", "the", "put", "you", "and", "little", "in"]}, {"sent": "spider theres Poppy shes in the laundry basket its a little stinky in here .", "words": ["the", "little", "here", "in", "basket", "a"]}, {"sent": "when you say a big Tesco truck are you thinking of the very big trucks that deliver to Tesco or are you thinking of the little van that comes round if we order food over Daddys computer ?", "words": ["the", "when", "if", "we", "of", "to", "say", "truck", "you", "little", "that", "are", "food", "over", "big", "a"]}, {"sent": "which I hafta take them back with me when we go but you can play with them for a little while with your mother .", "words": ["can", "for", "your", "when", "back", "we", "go", "play", "with", "which", "them", "you", "little", "me", "take", "but", "a"]}, {"sent": "into a little plastic uh paper bag ?", "words": ["paper", "into", "little", "a"]}, {"sent": "Ive washed up the little blue bowl that you had your cornflakes in so Ill put the apple in that .", "words": ["up", "the", "bowl", "put", "your", "so", "blue", "little", "you", "in", "apple", "that"]}, {"sent": "is it little ?", "words": ["it", "little", "is"]}, {"sent": "are you being little boy ?", "words": ["are", "little", "you"]}, {"sent": "well may I have a little piece of PlayDoh ?", "words": ["have", "of", "little", "a"]}, {"sent": "and where the sand was wet there were little foot marks of a very little person .", "words": ["where", "the", "wet", "of", "was", "and", "little", "there", "foot", "were", "a"]}, {"sent": "the big pool or the little pool ?", "words": ["the", "big", "pool", "little"]}, {"sent": "well get you a little knife .", "words": ["knife", "little", "get", "you", "a"]}, {"sent": "this little check skirt .", "words": ["this", "little"]}, {"sent": "and heres another little .", "words": ["another", "little", "and"]}, {"sent": "because the the little arrow is pointing at the nought isnt it ?", "words": ["the", "is", "little", "it", "at", "because"]}, {"sent": "peekaboo its little you .", "words": ["you", "little"]}, {"sent": "were going to be getting all these little characters out .", "words": ["all", "these", "to", "be", "little", "out", "were"]}, {"sent": "my little babys growing up arent you ?", "words": ["up", "my", "you", "little"]}, {"sent": "but I think she tries to do little thing that she does get frustrated easily at times .", "words": ["do", "think", "does", "to", "little", "get", "at", "that", "she", "but"]}, {"sent": "thats his is she little or big ?", "words": ["is", "little", "his", "big", "she"]}, {"sent": "naughty little boys .", "words": ["naughty", "little"]}, {"sent": "this is a little Japanese song .", "words": ["this", "little", "a", "is"]}, {"sent": "has a funny little snout .", "words": ["little", "a"]}, {"sent": "if you are a gardener and find me said the little bunny I will be a bird and fly away from you .", "words": ["the", "bunny", "if", "be", "bird", "and", "you", "little", "are", "away", "find", "will", "me", "a"]}, {"sent": "xxx the little boys riding his horsie .", "words": ["the", "his", "little"]}, {"sent": "Im a little too big to get in that farm can you go for me ?", "words": ["can", "for", "go", "to", "little", "get", "you", "in", "that", "too", "me", "big", "a"]}, {"sent": "I think its a little sticker isnt it ?", "words": ["think", "it", "little", "a"]}, {"sent": "that little horsie goes in the trailer doesnt he ?", "words": ["the", "little", "in", "he", "that"]}, {"sent": "and its got a little bag you see .", "words": ["you", "little", "and", "see", "a"]}, {"sent": "its a little too noisy for right now .", "words": ["for", "noisy", "little", "too", "a"]}, {"sent": "this little piggy .", "words": ["this", "little"]}, {"sent": "two to ten with these little girls is not my idea of fun .", "words": ["these", "my", "is", "of", "to", "with", "little", "not"]}, {"sent": "play food and pot little pots and stuff and theres a little kitchen so .", "words": ["so", "play", "little", "and", "food", "kitchen", "a"]}, {"sent": "its a little .", "words": ["little", "a"]}, {"sent": "you think thats silly dont you my little friend .", "words": ["think", "my", "little", "you", "dont"]}, {"sent": "large and little ?", "words": ["little", "and"]}, {"sent": "I gotta do one more little wipie okay ?", "words": ["do", "more", "little"]}, {"sent": "a little bed .", "words": ["bed", "little", "a"]}, {"sent": "there might be a little bit of egg shell in there .", "words": ["egg", "of", "be", "little", "there", "in", "a"]}, {"sent": "its a little its a little hill of dirt .", "words": ["of", "little", "a"]}, {"sent": "the little boy sees his balloon over a wall .", "words": ["the", "little", "balloon", "his", "over", "a"]}, {"sent": "when we weigh up when we add the egg we wanna pop a little drop of flour in the mix as well .", "words": ["up", "the", "pop", "egg", "we", "of", "little", "in", "drop", "wanna", "when", "a"]}, {"sent": "yeah shes got a lot xxx little word shapes xxx .", "words": ["a lot", "little", "a"]}, {"sent": "that is a little holly bush .", "words": ["little", "that", "a", "is"]}, {"sent": "but youre only a little boy Thomas .", "words": ["little", "but", "a"]}, {"sent": "who has a collection of little Williams ?", "words": ["of", "little", "a", "who"]}, {"sent": "Santa is reading a letter from a little boy and its obviously saying please may I have a toy giraffe for Christmas .", "words": ["for", "giraffe", "is", "and", "have", "little", "toy", "a"]}, {"sent": "oh those are little kids on there ?", "words": ["those", "little", "there", "are", "on"]}, {"sent": "shall we straighten them up a little bit ?", "words": ["up", "we", "them", "little", "a"]}, {"sent": "well hes probably a little bit like earlier on on the paper Thomas looked cross .", "words": ["the", "paper", "like", "little", "on", "a"]}, {"sent": "no little Freddie .", "words": ["little"]}, {"sent": "theres a little egg .", "words": ["egg", "little", "a"]}, {"sent": "you want another little piece of track ?", "words": ["another", "of", "little", "you"]}, {"sent": "can you see his little light xxx to his helmet ?", "words": ["can", "light", "to", "little", "you", "his", "see"]}, {"sent": "see this little mouse .", "words": ["see", "mouse", "this", "little"]}, {"sent": "Imean I guess she did seem like a little bit interested for like a minute but that was about it .", "words": ["for", "was", "like", "little", "it", "did", "about", "that", "she", "but", "a"]}, {"sent": "thats the little car that you ride round in at um toddler group isnt it ?", "words": ["the", "car", "ride", "you", "little", "in", "it", "at", "that"]}, {"sent": "youre home now Thomas .", "words": ["home"]}, {"sent": "Grandma was not angry at all she was so thrilled she played the musical blue bird earrings and wore the vampire teeth all the way home .", "words": ["the", "all", "was", "not", "so", "bird", "blue", "and", "at", "she", "home"]}, {"sent": "and we said wed better go home and do some taping .", "words": ["better", "do", "go", "we", "and", "some", "home"]}, {"sent": "after we came home ?", "words": ["home", "we"]}, {"sent": "which I honestly he um at home he doesnt take toys and bang them together because his toys make noise so if was banging them .", "words": ["if", "so", "was", "them", "which", "and", "make", "he", "at", "his", "take", "because", "home"]}, {"sent": "does he have any of those animals at home ?", "words": ["does", "those", "of", "any", "have", "he", "at", "home"]}, {"sent": "xxx shes probably at home .", "words": ["home", "at"]}, {"sent": "well she l walked down the stairs and she lost her slipper and she went back home but who found the slipper ?", "words": ["the", "back", "and", "but", "who", "stairs", "down", "her", "she", "slipper", "home"]}, {"sent": "you came home carrying something .", "words": ["home", "you"]}, {"sent": "and the bunny goes home to his mommy and the babies .", "words": ["the", "bunny", "to", "and", "his", "home"]}, {"sent": "but we ate that when we got home didnt we ?", "words": ["when", "we", "that", "but", "home"]}, {"sent": "because everybodys rushing home from work .", "words": ["work", "because", "home"]}, {"sent": "yes your baby can home with you .", "words": ["can", "your", "with", "you", "home"]}, {"sent": "does he ever try to feed the doll at home or .", "words": ["the", "does", "to", "doll", "at", "he", "feed", "try", "try to", "home"]}, {"sent": "I hit you when I got you home .", "words": ["home", "hit", "when", "you"]}, {"sent": "did you bring it home ?", "words": ["bring", "you", "it", "did", "home"]}, {"sent": "but yeah she enjoys playing with the food items at home .", "words": ["the", "with", "food", "at", "she", "but", "home"]}, {"sent": "oh I see a book we have at home .", "words": ["home", "we", "have", "at", "book", "see", "a"]}, {"sent": "um we dont have any pets we have tons of stuffed animals so he plays with those a lot at home .", "words": ["those", "we", "of", "so", "a lot", "any", "with", "have", "dont", "he", "at", "home", "a"]}, {"sent": "what does he what else does he do at home ?", "words": ["what", "do", "does", "he", "at", "home"]}, {"sent": "cancel the wedding send everybody home .", "words": ["the", "home"]}, {"sent": "I know whenever your fathers not home we have patatas patatas patatas .", "words": ["your", "we", "have", "not", "home"]}, {"sent": "because we dont have the play animals at home .", "words": ["the", "we", "play", "have", "dont", "at", "because", "home"]}, {"sent": "I mean you said that you do have pretend food at home .", "words": ["do", "you", "have", "food", "at", "pretend", "that", "home"]}, {"sent": "the dog or the cat will go home .", "words": ["the", "go", "cat", "will", "dog", "home"]}, {"sent": "what of these toys does she have at home ?", "words": ["what", "does", "these", "of", "have", "at", "she", "home"]}, {"sent": "and then we came home .", "words": ["home", "then", "we", "and"]}, {"sent": "would you like to come home with me ?", "words": ["to", "with", "like", "you", "would", "me", "home"]}, {"sent": "ladybird ladybird fly away home .", "words": ["away", "home"]}, {"sent": "that happens to me every time Toot told him I love to go off on a trip and then when the time comes I love to go back home .", "words": ["off", "the", "when", "back", "go", "home", "to", "and", "him", "every", "love", "on", "me", "then", "that", "a"]}, {"sent": "I will when he comes home .", "words": ["will", "he", "when", "home"]}, {"sent": "well Id put it in a cup so you can take it home .", "words": ["cup", "can", "put", "so", "you", "in", "it", "home", "take", "a"]}, {"sent": "like the kind you have at home .", "words": ["the", "like", "you", "have", "at", "home"]}, {"sent": "wheres the rest of my home ?", "words": ["the", "of", "my", "home"]}, {"sent": "and then you said he has a doll at home what kinds of things does he like to do with the doll ?", "words": ["the", "what", "does", "do", "of", "to", "with", "like", "doll", "you", "and", "he", "at", "home", "then", "a"]}, {"sent": "all the way home .", "words": ["the", "all", "home"]}, {"sent": "okay which one shall I take home first ?", "words": ["take", "first", "which", "home"]}, {"sent": "shell probably get home before it comes on .", "words": ["get", "on", "it", "home"]}, {"sent": "just you know make yourself at home .", "words": ["yourself", "you", "at", "make", "home"]}, {"sent": "do you think Mel liked your home ?", "words": ["do", "think", "your", "you", "home"]}, {"sent": "so um at home theres a little play piano .", "words": ["so", "play", "little", "at", "a", "home"]}, {"sent": "we werent expecting him home were we ?", "words": ["home", "him", "were", "we"]}, {"sent": "yeah but when she gets home from school shes going to take the car .", "words": ["the", "school", "to", "car", "take", "when", "she", "but", "home"]}, {"sent": "home were gonna have a new home .", "words": ["have", "home", "new", "were", "a"]}, {"sent": "he is back home ?", "words": ["home", "he", "back", "is"]}, {"sent": "I dont on Sunday she came home from Sunday school .", "words": ["school", "dont", "on", "she", "home"]}, {"sent": "daddys working but when hes home .", "words": ["when", "but", "home"]}, {"sent": "uh I think theres only like one book we have at home that has a kangaroo in it .", "words": ["think", "we", "like", "have", "in", "it", "at", "book", "that", "a", "home"]}, {"sent": "so this is how she usually plays at home or ?", "words": ["how", "is", "so", "this", "at", "she", "home"]}, {"sent": "when Daddy comes home from the chip shop hell be cross with her wont he ?", "words": ["the", "be", "with", "her", "he", "when", "home"]}, {"sent": "and does she is she interested at home ?", "words": ["does", "is", "and", "at", "she", "home"]}, {"sent": "we have a play kitchen at home so shes useta playing with kitchens but it doesnt look like this one we dont have any of the same foods that you have we dont have a little .", "words": ["the", "of", "any", "like", "same", "with", "have", "it", "that", "home", "so", "play", "you", "little", "dont", "kitchen", "a", "we", "this", "at", "look", "but"]}, {"sent": "lets go home Pippin .", "words": ["home", "go"]}, {"sent": "did grandpa Ma went in another plane and she was gonna fly all the way home .", "words": ["the", "all", "was", "and", "another", "in", "did", "she", "home"]}, {"sent": "home .", "words": ["home"]}, {"sent": "the car will drive them home .", "words": ["the", "car", "them", "drive", "will", "home"]}, {"sent": "and one little piggy went weeweewee all the way home .", "words": ["the", "all", "little", "and", "home"]}, {"sent": "has he hadta sail all the way home ?", "words": ["the", "all", "he", "home"]}, {"sent": "this is the party bag that you came home with .", "words": ["the", "is", "with", "party", "you", "this", "that", "home"]}, {"sent": "wheres their home ?", "words": ["their", "home"]}, {"sent": "we get them when theyre a little green and then they ripen at home dont they ?", "words": ["we", "them", "little", "get", "and", "they", "dont", "green", "at", "home", "then", "when", "a"]}, {"sent": "and Daddy came home for lunch didnt he ?", "words": ["home", "for", "he", "and"]}, {"sent": "shes familar with the baby doll because we have one at home .", "words": ["the", "we", "with", "doll", "have", "at", "because", "home"]}, {"sent": "and your trousers were a little wet when we came home .", "words": ["wet", "your", "when", "we", "little", "and", "home", "were", "a"]}, {"sent": "now are you going to make their home ?", "words": ["their", "to", "you", "are", "make", "home"]}, {"sent": "at home Lily pretended to be Mister Slinger .", "words": ["to", "home", "be", "at"]}, {"sent": "we need blocks for you at home huh ?", "words": ["for", "need", "we", "you", "at", "home"]}, {"sent": "well I suppose if theyre successful with their fishing Thomas and they catch any fish they could take the fish home for their supper couldnt they ?", "words": ["the", "for", "if", "their", "catch", "with", "any", "and", "they", "fish", "could", "take", "home"]}, {"sent": "whos she home with ?", "words": ["with", "she", "home"]}, {"sent": "yeah but she can pick one or you can pick one for the ride home .", "words": ["can", "the", "for", "pick", "you", "ride", "she", "but", "home"]}, {"sent": "when Jwww comes home from playschool .", "words": ["when", "home"]}, {"sent": "and then what happens when you get home ?", "words": ["what", "you", "and", "get", "then", "when", "home"]}, {"sent": "does she have that at home ?", "words": ["does", "have", "at", "that", "she", "home"]}, {"sent": "she loves cheese at home .", "words": ["home", "cheese", "she", "at"]}, {"sent": "oh this looks just like the one at home .", "words": ["the", "like", "this", "at", "home"]}, {"sent": "stay at home ?", "words": ["stay", "home", "at"]}, {"sent": "and when he came home yesterday he saw that his brother .", "words": ["when", "and", "he", "his", "that", "home"]}, {"sent": "now they ready to go home but there are no trains .", "words": ["go", "to", "they", "there", "are", "but", "home"]}, {"sent": "have you got a racing car at home ?", "words": ["car", "you", "have", "at", "home", "a"]}, {"sent": "it happened at home ?", "words": ["home", "it", "at"]}, {"sent": "we ate breakfast at home today .", "words": ["home", "at", "we"]}, {"sent": "and what dyou get when you come home ?", "words": ["what", "you", "get", "and", "when", "home"]}, {"sent": "hell get back hell be back at home in a minute .", "words": ["back", "be", "get", "in", "at", "a", "home"]}, {"sent": "is that what they say in at home ?", "words": ["what", "is", "say", "they", "in", "at", "that", "home"]}, {"sent": "we dont have this toy at home .", "words": ["we", "have", "dont", "toy", "this", "at", "home"]}, {"sent": "clap handies till daddy gets home .", "words": ["clap", "home"]}, {"sent": "a couple of books at home about Fraggles .", "words": ["of", "about", "at", "a", "home"]}, {"sent": "we eat peas at home .", "words": ["we", "peas", "eat", "at", "home"]}, {"sent": "have you got one of those games at home ?", "words": ["those", "of", "you", "have", "at", "home"]}, {"sent": "would you like Mummy to use the shower at home ?", "words": ["the", "shower", "to", "like", "you", "at", "would", "home"]}, {"sent": "in fifteen minutes Im going home .", "words": ["in", "home"]}, {"sent": "I told you boys and girls that I wanted you to take them home and have natural light .", "words": ["light", "to", "them", "and", "you", "have", "that", "take", "home"]}, {"sent": "and he called in on the way home .", "words": ["the", "and", "in", "he", "on", "home"]}, {"sent": "this little piggie stayed home .", "words": ["this", "home", "little"]}, {"sent": "um hm let me call my wife and see if I forgot anything everything because I think were all set to go home .", "words": ["all", "my", "think", "if", "go", "to", "me", "and", "see", "were", "because", "home"]}, {"sent": "when we get home .", "words": ["get", "home", "when", "we"]}, {"sent": "so Im gonna go home .", "words": ["so", "home", "go"]}, {"sent": "this little piggy stayed at home .", "words": ["this", "at", "home", "little"]}, {"sent": "I think I am tired now .", "words": ["think", "am", "tired"]}, {"sent": "are you tired honey ?", "words": ["are", "tired", "you"]}, {"sent": "are you tired ?", "words": ["are", "tired", "you"]}, {"sent": "are you tired ?", "words": ["are", "tired", "you"]}, {"sent": "shes pretty tired .", "words": ["pretty", "tired"]}, {"sent": "I dont know why youre not tired .", "words": ["dont", "not", "why", "tired"]}, {"sent": "were you tired today ?", "words": ["tired", "were", "you"]}, {"sent": "are you getting tired ?", "words": ["are", "tired", "you"]}, {"sent": "she tired I think .", "words": ["think", "she", "tired"]}, {"sent": "Mommys tired .", "words": ["tired"]}, {"sent": "are you tired ?", "words": ["are", "tired", "you"]}, {"sent": "beautiful Laras tired .", "words": ["tired"]}, {"sent": "are you tired ?", "words": ["are", "tired", "you"]}, {"sent": "are you tired ?", "words": ["are", "tired", "you"]}, {"sent": "are you tired as well ?", "words": ["are", "tired", "you"]}, {"sent": "Mummys a bit tired today .", "words": ["tired", "a"]}, {"sent": "Im tired Liz .", "words": ["tired"]}, {"sent": "yes is Dougall tired ?", "words": ["tired", "is"]}, {"sent": "are you tired ?", "words": ["are", "tired", "you"]}, {"sent": "oh my dear god am I tired .", "words": ["my", "am", "tired"]}, {"sent": "tired xxx .", "words": ["tired"]}, {"sent": "and his legs feel tired most of the time .", "words": ["the", "of", "tired", "and", "his"]}, {"sent": "are you tired ?", "words": ["are", "tired", "you"]}, {"sent": "are you tired ?", "words": ["are", "tired", "you"]}, {"sent": "are you tired ?", "words": ["are", "tired", "you"]}, {"sent": "you were a tired boy .", "words": ["tired", "were", "a", "you"]}, {"sent": "is Thomas tired ?", "words": ["tired", "is"]}, {"sent": "he looks tired doesnt he ?", "words": ["he", "tired"]}, {"sent": "oh theyre so tired they havta take a nap .", "words": ["so", "tired", "they", "take", "a"]}, {"sent": "are you tired ?", "words": ["are", "tired", "you"]}, {"sent": "he is very tired .", "words": ["tired", "he", "is"]}, {"sent": "I think you were very tired and you were wondering along cuddling your blanket and you lay on the floor next to Purdie and you said nightnight Bow .", "words": ["the", "think", "your", "to", "tired", "and", "you", "on", "blanket", "were"]}, {"sent": "hes tired now isnt he ?", "words": ["he", "tired"]}, {"sent": "hes tired um .", "words": ["tired"]}, {"sent": "youre tired ?", "words": ["tired"]}, {"sent": "tired children .", "words": ["tired"]}, {"sent": "I think youre very tired actually .", "words": ["think", "tired"]}, {"sent": "he looks tired .", "words": ["he", "tired"]}, {"sent": "are you tired ?", "words": ["are", "tired", "you"]}, {"sent": "xxx tired .", "words": ["tired"]}, {"sent": "are you tired ?", "words": ["are", "tired", "you"]}, {"sent": "Mamas tired .", "words": ["tired"]}, {"sent": "are you tired ?", "words": ["are", "tired", "you"]}, {"sent": "aw youre tired its okay its alright little sweetheart .", "words": ["little", "tired"]}, {"sent": "do Mamas eyes look tired ?", "words": ["do", "tired", "look"]}, {"sent": "she is tired ?", "words": ["tired", "she", "is"]}, {"sent": "that was enough for toad tired and sore he crept back under his stone .", "words": ["for", "back", "was", "and", "tired", "under", "he", "stone", "his", "that"]}, {"sent": "iun mommy I Im tired of eating .", "words": ["of", "tired"]}, {"sent": "mummys tired .", "words": ["tired"]}, {"sent": "because Im a bit tired .", "words": ["tired", "because", "a"]}, {"sent": "are you tired boy ?", "words": ["are", "tired", "you"]}, {"sent": "very tired .", "words": ["tired"]}, {"sent": "too tired Eleanor .", "words": ["tired", "too"]}, {"sent": "yeah that was good she was actually trying to read to you a little bit the other story but I think she just got tired of it but .", "words": ["the", "think", "of", "to", "was", "you", "little", "other", "tired", "it", "read", "good", "story", "that", "she", "but", "a"]}, {"sent": "whos tired ?", "words": ["tired"]}, {"sent": "youre tired out arent you ?", "words": ["you", "out", "tired"]}, {"sent": "whyre you getting tired of cutting ?", "words": ["of", "tired", "you"]}, {"sent": "theyre very tired .", "words": ["tired"]}, {"sent": "oh well lets go and have a drink first weve got tired legs .", "words": ["drink", "go", "first", "and", "have", "tired", "a"]}, {"sent": "I think youre tired .", "words": ["think", "tired"]}, {"sent": "shes tired .", "words": ["tired"]}, {"sent": "she may be tired I thought she was beginning to get a little bit tired .", "words": ["was", "be", "to", "tired", "get", "little", "she", "a"]}, {"sent": "and youre really tired from the day ?", "words": ["the", "tired", "and"]}, {"sent": "mummy is tired .", "words": ["tired", "is"]}, {"sent": "Daddy is tired of piggies .", "words": ["of", "tired", "is"]}, {"sent": "you sound very tired doesnt he you sound very very tired .", "words": ["tired", "he", "you"]}, {"sent": "are you tired ?", "words": ["are", "tired", "you"]}, {"sent": "getting tired .", "words": ["tired"]}, {"sent": "Im not too tired .", "words": ["tired", "not", "too"]}, {"sent": "are you tired ?", "words": ["are", "tired", "you"]}, {"sent": "are you tired ?", "words": ["are", "tired", "you"]}, {"sent": "who is tired ?", "words": ["tired", "is", "who"]}, {"sent": "Im tired now .", "words": ["tired"]}, {"sent": "yes Mummy is quite tired Thomas .", "words": ["tired", "is"]}, {"sent": "you a tired boy ?", "words": ["tired", "a", "you"]}, {"sent": "a tired snake and a crocodile ?", "words": ["tired", "and", "a"]}, {"sent": "does he get tired of bread and jam ?", "words": ["does", "bread", "of", "tired", "get", "and", "he"]}, {"sent": "shes so tired .", "words": ["so", "tired"]}, {"sent": "you were telling Granddad that he was tired and that he needed a little rest and telling him to go to sleep .", "words": ["go", "sleep", "was", "to", "you", "tired", "and", "little", "him", "he", "that", "were", "a"]}, {"sent": "thats tired me out .", "words": ["me", "out", "tired"]}, {"sent": "are you tired ?", "words": ["are", "tired", "you"]}, {"sent": "youre tired ?", "words": ["tired"]}, {"sent": "and you were tired .", "words": ["tired", "were", "you", "and"]}, {"sent": "xxx decidedly tired .", "words": ["tired"]}, {"sent": "and rabbit got tired again .", "words": ["tired", "and"]}, {"sent": "ah theyre all tired them arent they ?", "words": ["all", "them", "they", "tired"]}, {"sent": "Im sorry youre so tired .", "words": ["so", "tired"]}, {"sent": "they were as hungry as they were bruised and tired .", "words": ["hungry", "tired", "and", "they", "were"]}, {"sent": "Billy tired ?", "words": ["tired"]}, {"sent": "but after pulling the trolley so much TinkeyWinkey was very tired .", "words": ["the", "so", "was", "tired", "much", "but"]}, {"sent": "no I think hes kinda tired out so they put him in the carriage I guess .", "words": ["the", "think", "put", "so", "tired", "they", "him", "in", "out"]}, {"sent": "is A is Fraser tired ?", "words": ["tired", "is"]}, {"sent": "are you getting tired ?", "words": ["are", "tired", "you"]}, {"sent": "are you really that tired ?", "words": ["are", "that", "tired", "you"]}, {"sent": "this is something she never gets tired of .", "words": ["is", "of", "tired", "this", "she"]}, {"sent": "are you tired are you a tired boy ?", "words": ["are", "tired", "a", "you"]}, {"sent": "if youre not tired .", "words": ["not", "if", "tired"]}, {"sent": "now the truck is really tired .", "words": ["the", "tired", "truck", "is"]}, {"sent": "were they very tired ?", "words": ["they", "were", "tired"]}, {"sent": "are you tired ?", "words": ["are", "tired", "you"]}, {"sent": "the baby needs to go to bed now okay ?", "words": ["the", "to", "bed", "go"]}, {"sent": "yeah can you see a bed anywhere ?", "words": ["can", "bed", "you", "see", "a"]}, {"sent": "is baby gonna try and sleep in the bed ?", "words": ["the", "is", "sleep", "bed", "and", "in", "try"]}, {"sent": "man took Michaels bed .", "words": ["bed"]}, {"sent": "you dont make your bed .", "words": ["your", "bed", "you", "dont", "make"]}, {"sent": "he ran into dads bedroom and then the cat said it is good that your dad has the right kind of bed .", "words": ["the", "your", "into", "is", "of", "bed", "and", "cat", "it", "he", "good", "then", "that", "bedroom"]}, {"sent": "I think the machines must be all going to bed .", "words": ["the", "all", "think", "be", "to", "bed"]}, {"sent": "you dont wont wanna go in a pink bed do you ?", "words": ["do", "go", "bed", "you", "dont", "in", "wanna", "a"]}, {"sent": "when grandma went to bed ?", "words": ["to", "when", "bed"]}, {"sent": "whose bed ?", "words": ["bed"]}, {"sent": "its not time for bed though is it ?", "words": ["for", "is", "bed", "it", "not"]}, {"sent": "shall we put the man in the bed ?", "words": ["the", "put", "we", "bed", "in"]}, {"sent": "you know what you can put a bed in here .", "words": ["can", "what", "put", "bed", "you", "here", "in", "a"]}, {"sent": "then when its over would you find a book and Ill read it and you go to bed .", "words": ["when", "go", "to", "bed", "find", "you", "and", "it", "read", "would", "book", "then", "over", "a"]}, {"sent": "you wanna go to bed ?", "words": ["go", "to", "bed", "you", "wanna"]}, {"sent": "Daddy took Andy to bed and when he came to see where you were you were in bed asleep .", "words": ["where", "to", "bed", "and", "you", "asleep", "in", "he", "see", "when", "were"]}, {"sent": "do you think well need the bed ?", "words": ["the", "do", "think", "need", "bed", "you"]}, {"sent": "and then it says Dipsy is supposed to be in bed .", "words": ["is", "to", "be", "bed", "and", "in", "it", "then"]}, {"sent": "do you wanna go to bed ?", "words": ["do", "go", "to", "bed", "you", "wanna"]}, {"sent": "you can go jump on the bed once and then go pick out your striped shirt .", "words": ["can", "the", "your", "go", "pick", "shirt", "bed", "you", "and", "jump", "out", "on", "then"]}, {"sent": "one more time and then Nathaniel goes to bed .", "words": ["to", "bed", "and", "more", "then"]}, {"sent": "a hospital bed .", "words": ["bed", "a"]}, {"sent": "theres one bed here one bed here and one bed here .", "words": ["here", "bed", "and"]}, {"sent": "no cant we go to bed Daddy ?", "words": ["go", "to", "bed", "we"]}, {"sent": "or maybe hes getting ready for bed .", "words": ["for", "bed"]}, {"sent": "hes yawning up in his bed .", "words": ["up", "his", "in", "bed"]}, {"sent": "um its in your bed .", "words": ["in", "your", "bed"]}, {"sent": "they have a pretty big bed .", "words": ["bed", "have", "they", "pretty", "big", "a"]}, {"sent": "and then we tried to convince him he really wasnt a guest we said he was just a friend but he thought he was really the guest so he should have slept in that other bed right ?", "words": ["the", "we", "to", "was", "so", "bed", "and", "have", "other", "him", "in", "he", "then", "that", "but", "a"]}, {"sent": "nappie change for bed .", "words": ["for", "bed"]}, {"sent": "why arent you ready to go to bed ?", "words": ["go", "to", "bed", "you", "why"]}, {"sent": "put the bed inside the house .", "words": ["the", "put", "inside", "bed", "house"]}, {"sent": "yeah but wheres Frasers bed ?", "words": ["bed", "but"]}, {"sent": "well you dont hafta go to bed yet .", "words": ["go", "to", "bed", "you", "dont"]}, {"sent": "you nearly ready for bed ?", "words": ["for", "bed", "you"]}, {"sent": "what did they do before they went to bed ?", "words": ["what", "do", "to", "bed", "they", "did"]}, {"sent": "if you want a story read youre going to hafta get into bed now .", "words": ["if", "into", "to", "bed", "you", "get", "read", "story", "a"]}, {"sent": "well how about I bring him a purple doggy bed ?", "words": ["how", "bring", "bed", "him", "about", "a"]}, {"sent": "but this morning when you got out of bed who did you get in the bath with ?", "words": ["the", "of", "with", "bed", "you", "get", "this", "in", "did", "out", "who", "when", "but"]}, {"sent": "by your bed ?", "words": ["your", "bed", "by"]}, {"sent": "but you were in the bed at the edge .", "words": ["the", "bed", "you", "in", "at", "were", "but"]}, {"sent": "I should turn off all the lights so we can go to bed .", "words": ["off", "all", "the", "can", "we", "go", "so", "to", "bed"]}, {"sent": "thats a funny way to get down from your bed .", "words": ["your", "to", "bed", "get", "down", "a"]}, {"sent": "Im get you when I get on that bed .", "words": ["when", "bed", "you", "get", "on", "that"]}, {"sent": "thats a crib thats a bed see ?", "words": ["see", "crib", "bed", "a"]}, {"sent": "but actually in your bed youve always got Po .", "words": ["in", "your", "bed", "but"]}, {"sent": "she gave the bed a coat of red paint .", "words": ["the", "red", "paint", "of", "bed", "she", "coat", "a"]}, {"sent": "uh big boy bed a new bed just like yours .", "words": ["bed", "like", "new", "big", "a"]}, {"sent": "dogs bed .", "words": ["bed"]}, {"sent": "what kind of bed did he sleep on ?", "words": ["what", "of", "sleep", "bed", "he", "did", "on"]}, {"sent": "how about this will you sell me this for the bed ?", "words": ["the", "how", "for", "bed", "you", "this", "about", "will", "me"]}, {"sent": "um they had him ready for bed his pyjamas and all on him .", "words": ["all", "for", "bed", "and", "they", "him", "on", "his"]}, {"sent": "oh youre putting Bert to bed .", "words": ["to", "bed"]}, {"sent": "thats a good idea did you close that crayon box so he cant get it open how about you even um put it back up on mommys bed sothat Joseph cant get it and you better move your coloring book if you dont want him to rip it .", "words": ["put", "crayon", "bed", "about", "book", "it", "on", "that", "up", "how", "box", "if", "so", "to", "rip", "you", "and", "dont", "him", "good", "close", "a", "better", "your", "back", "open", "get", "he", "did"]}, {"sent": "is that where his bed goes ?", "words": ["where", "is", "bed", "his", "that"]}, {"sent": "youve got the new one on your bed havent you ?", "words": ["the", "your", "bed", "you", "on", "new"]}, {"sent": "you can hang it over the edge of your bed .", "words": ["can", "the", "your", "of", "bed", "you", "it", "over"]}, {"sent": "its time for all of them to go to bed ?", "words": ["all", "for", "go", "of", "to", "them", "bed"]}, {"sent": "whose bed is this ?", "words": ["this", "bed", "is"]}, {"sent": "oh theres his bed .", "words": ["his", "bed"]}, {"sent": "its over there by your bed .", "words": ["your", "bed", "by", "there", "over"]}, {"sent": "this not the right kind of a bed the cat said .", "words": ["the", "of", "bed", "this", "cat", "not", "a"]}, {"sent": "put you to bed ?", "words": ["to", "put", "bed", "you"]}, {"sent": "time to go to bed ?", "words": ["to", "bed", "go"]}, {"sent": "and then but shell also sometimes shell pat them and sometimes we put them to bed .", "words": ["put", "we", "to", "them", "bed", "and", "then", "but"]}, {"sent": "for some reason he think he hasta have dessert every night before he go to bed .", "words": ["think", "for", "go", "to", "bed", "have", "he", "every", "some"]}, {"sent": "or perhaps when shes tired and she wants a little sleep in a bed .", "words": ["sleep", "bed", "and", "tired", "little", "in", "when", "she", "a"]}, {"sent": "bed .", "words": ["bed"]}, {"sent": "she takes panda to bed with her .", "words": ["to", "with", "bed", "her", "she"]}, {"sent": "she had a bunk bed didnt she ?", "words": ["she", "bed", "a"]}, {"sent": "you fight in bed ?", "words": ["in", "bed", "you"]}, {"sent": "found and it says the date it was found and on which bus so when you think back to that picture there where the little boys in bed with a tear in his eye .", "words": ["the", "think", "bed", "when", "which", "bus", "where", "with", "eye", "there", "it", "on", "that", "was", "so", "to", "and", "you", "little", "a", "back", "picture", "tear", "in", "his"]}, {"sent": "was he in the bed ?", "words": ["the", "was", "bed", "in", "he"]}, {"sent": "you better get to bed then .", "words": ["better", "to", "bed", "you", "get", "then"]}, {"sent": "put the car to bed .", "words": ["the", "put", "car", "to", "bed"]}, {"sent": "how long will she hafta stay in bed for ?", "words": ["how", "for", "bed", "long", "in", "will", "stay", "she"]}, {"sent": "ah ThomasTheTankEngine the first out of bed .", "words": ["the", "of", "first", "bed", "out"]}, {"sent": "does Mommys lap make a good bed ?", "words": ["does", "bed", "good", "make", "a"]}, {"sent": "bed .", "words": ["bed"]}, {"sent": "is PoohBear in your bed ?", "words": ["in", "your", "bed", "is"]}, {"sent": "a train and a bed and a train .", "words": ["train", "bed", "and", "a"]}, {"sent": "ah look are you gonna just just let Bunny go to bed crying look Eleanor ?", "words": ["go", "to", "bed", "you", "are", "look"]}, {"sent": "are you going to bed ?", "words": ["to", "are", "bed", "you"]}, {"sent": "why dont you go up to bed Rachael if youre tired ?", "words": ["up", "if", "go", "to", "bed", "you", "tired", "dont", "why"]}, {"sent": "Ill take you to bed .", "words": ["to", "take", "bed", "you"]}, {"sent": "oh you gonna put them back in the bed ?", "words": ["the", "put", "back", "them", "bed", "you", "in"]}, {"sent": "bump your head on that bed .", "words": ["bump", "your", "head", "bed", "on", "that"]}, {"sent": "juice and bed .", "words": ["juice", "bed", "and"]}, {"sent": "hes in under the bed with xxx again .", "words": ["the", "with", "bed", "under", "in"]}, {"sent": "what is he doing to his bed ?", "words": ["what", "is", "to", "bed", "he", "his"]}, {"sent": "xxx in bed .", "words": ["in", "bed"]}, {"sent": "or do you wanna sleep in Mummy and Daddys big bed ?", "words": ["do", "sleep", "bed", "you", "and", "in", "wanna", "big"]}, {"sent": "I was sleeping like this in my bed .", "words": ["my", "was", "like", "bed", "this", "in"]}, {"sent": "are you gonna put that under the bed ?", "words": ["the", "put", "bed", "you", "under", "are", "that"]}, {"sent": "there right lets find teddy in bed .", "words": ["there", "in", "bed", "find"]}, {"sent": "theyre going to bed now arent they ?", "words": ["to", "bed", "they"]}, {"sent": "its fifteen minutes prior to Michaels bed .", "words": ["to", "bed"]}, {"sent": "so they sleep down on the floor and we get first choice .", "words": ["the", "we", "sleep", "so", "first", "and", "get", "they", "on", "down"]}, {"sent": "Im gonna go to sleep .", "words": ["sleep", "to", "go"]}, {"sent": "and she is in a good mood and she had a two hour nap which started out as a forty five minute nap then nursed back to sleep .", "words": ["back", "is", "sleep", "to", "and", "in", "good", "out", "then", "which", "she", "a"]}, {"sent": "is she going to sleep now ?", "words": ["sleep", "to", "she", "is"]}, {"sent": "are you gonna go and have a sleep ?", "words": ["go", "sleep", "and", "have", "you", "are", "a"]}, {"sent": "oh youre going to sleep right there .", "words": ["sleep", "to", "there"]}, {"sent": "you know when you go to sleep .", "words": ["go", "sleep", "to", "you", "when"]}, {"sent": "you gonna go and sleep with Grandma and Granddad then ?", "words": ["go", "sleep", "with", "and", "you", "then"]}, {"sent": "oh I think my foots gone to sleep .", "words": ["sleep", "to", "think", "my"]}, {"sent": "she doesnt does she wanna sleep up here ?", "words": ["up", "does", "sleep", "here", "wanna", "she"]}, {"sent": "that was a quick sleep wasnt it ?", "words": ["sleep", "was", "it", "that", "a"]}, {"sent": "and they could all sleep on the beanbag chair couldnt they ?", "words": ["the", "all", "sleep", "chair", "and", "they", "on", "could"]}, {"sent": "you cant start to sleep on the job now youre too little .", "words": ["the", "sleep", "to", "you", "little", "too", "on"]}, {"sent": "do you sleep in the bed ?", "words": ["the", "do", "sleep", "bed", "you", "in"]}, {"sent": "did you wanna go to sleep last night ?", "words": ["go", "sleep", "to", "last", "you", "did", "wanna"]}, {"sent": "youre going to sleep with me tonight .", "words": ["sleep", "to", "me", "with"]}, {"sent": "do you think when buss go to sleep they lie on their side ?", "words": ["do", "think", "their", "go", "sleep", "to", "you", "they", "on", "when"]}, {"sent": "are you going to sleep aswell ?", "words": ["sleep", "to", "are", "you"]}, {"sent": "he hadta sleep in one of Grandpas shirts didnt he ?", "words": ["sleep", "of", "in", "he"]}, {"sent": "teepee is like a triangle indians sleep in .", "words": ["is", "sleep", "like", "in", "a"]}, {"sent": "yeah Mommy and Daddy are gonna sleep on it once we get the bed .", "words": ["the", "we", "sleep", "bed", "and", "get", "it", "are", "on"]}, {"sent": "arent you gonna go to sleep ?", "words": ["go", "to", "sleep", "you"]}, {"sent": "did you have a nice sleep ?", "words": ["sleep", "nice", "you", "have", "did", "a"]}, {"sent": "do you wanna go to sleep ?", "words": ["do", "go", "sleep", "to", "you", "wanna"]}, {"sent": "are you going to read dolly a book before she goes to sleep ?", "words": ["sleep", "to", "you", "are", "read", "book", "she", "a"]}, {"sent": "sleep ?", "words": ["sleep"]}, {"sent": "doesnt she sleep in there ?", "words": ["sleep", "there", "in", "she"]}, {"sent": "you wanna go to sleep ?", "words": ["go", "sleep", "to", "you", "wanna"]}, {"sent": "okay you sleep in there .", "words": ["sleep", "there", "in", "you"]}, {"sent": "the fire engine didnt have any sleep last night ?", "words": ["the", "sleep", "last", "any", "have"]}, {"sent": "she can sleep while shes in the carriage .", "words": ["can", "the", "sleep", "in", "she"]}, {"sent": "Ill pat me to sleep .", "words": ["to", "me", "sleep"]}, {"sent": "its gonna sleep ?", "words": ["sleep"]}, {"sent": "and if you went to try and go to sleep and you think oh Im on my own just put your hand out and hold onto Po or Freddy and youll .", "words": ["think", "my", "put", "if", "your", "go", "sleep", "to", "hand", "hold", "you", "and", "out", "on", "try"]}, {"sent": "you wanna sleep on your own like a big boy .", "words": ["your", "sleep", "like", "you", "wanna", "on", "big", "a"]}, {"sent": "you dont you dont go to sleep in the chair do you ?", "words": ["the", "do", "go", "sleep", "to", "chair", "you", "dont", "in"]}, {"sent": "are you going to go to sleep ?", "words": ["go", "sleep", "to", "you", "are"]}, {"sent": "Im gonna lie down a little bit and listen to you talk and go back to sleep .", "words": ["listen", "go", "back", "to", "sleep", "and", "little", "you", "talk", "down", "a"]}, {"sent": "no you sleep on the floor .", "words": ["sleep", "the", "on", "you"]}, {"sent": "time to go to sleep .", "words": ["sleep", "to", "go"]}, {"sent": "do you sleep like that with your feet up in the air ?", "words": ["up", "do", "the", "your", "sleep", "with", "like", "you", "in", "that"]}, {"sent": "because you look like youre going to sleep and you dont wanna talk do you ?", "words": ["do", "sleep", "to", "like", "you", "and", "dont", "wanna", "look", "talk", "because"]}, {"sent": "okay where do the dollies sleep ?", "words": ["where", "do", "sleep", "the"]}, {"sent": "theres that baby go to sleep .", "words": ["sleep", "to", "that", "go"]}, {"sent": "it curls up when it wants to got to sleep .", "words": ["up", "sleep", "to", "it", "when"]}, {"sent": "its just that John likes to sleep with you sometimes doesnt he ?", "words": ["sleep", "to", "with", "you", "he", "that"]}, {"sent": "what about the ducks are they going to sleep ?", "words": ["the", "what", "sleep", "to", "they", "are", "about"]}, {"sent": "would you like a little sleep after lunch ?", "words": ["sleep", "like", "little", "you", "would", "a"]}, {"sent": "so Mickey gonna go to sleep ?", "words": ["to", "so", "sleep", "go"]}, {"sent": "children needta sleep in it .", "words": ["sleep", "in", "it"]}, {"sent": "well then why dont you go to sleep at the point when youre given the opportunity ?", "words": ["the", "go", "sleep", "to", "you", "dont", "at", "then", "when", "why"]}, {"sent": "and you let me sleep until nine oclock and it .", "words": ["sleep", "you", "and", "it", "me"]}, {"sent": "Peter want ta go to sleep ?", "words": ["sleep", "to", "go"]}, {"sent": "maybe he can sleep on the settee uhhuh .", "words": ["can", "the", "sleep", "he", "on"]}, {"sent": "sleep tight .", "words": ["sleep"]}, {"sent": "sleep .", "words": ["sleep"]}, {"sent": "theyve gone to sleep .", "words": ["sleep", "to"]}, {"sent": "Im tired and I cant sleep ?", "words": ["sleep", "and", "tired"]}, {"sent": "why couldnt you sleep ?", "words": ["sleep", "why", "you"]}, {"sent": "but I think Ethan is going to sleep .", "words": ["think", "is", "sleep", "to", "but"]}, {"sent": "and Im the sleep .", "words": ["the", "sleep", "and"]}, {"sent": "it says Daddy reads us a story before we go to sleep .", "words": ["we", "go", "to", "sleep", "us", "it", "story", "a"]}, {"sent": "oh did you sleep in the same room as daddy when you were staying in Miami so that he could look after you ?", "words": ["the", "look", "sleep", "so", "you", "in", "room", "same", "did", "that", "he", "could", "when", "were"]}, {"sent": "does he sleep in the afternoon then ?", "words": ["the", "does", "sleep", "in", "he", "then"]}, {"sent": "ooh is Pilchard going to sleep ?", "words": ["sleep", "to", "is"]}, {"sent": "are you going to sleep ?", "words": ["sleep", "to", "are", "you"]}, {"sent": "then the little kids say goodnight and go to sleep in their room goodnight .", "words": ["the", "their", "go", "sleep", "to", "say", "and", "little", "in", "room", "then"]}, {"sent": "what does she do before she goes to sleep ?", "words": ["what", "do", "does", "sleep", "to", "she"]}, {"sent": "oh does the baby sleep in the living room ?", "words": ["the", "does", "sleep", "living room", "in", "room"]}, {"sent": "oh my so you can sleep in the kitchen ?", "words": ["can", "the", "my", "sleep", "so", "you", "in", "kitchen"]}, {"sent": "ts my I needta go to sleep ?", "words": ["sleep", "to", "my", "go"]}, {"sent": "to sleep .", "words": ["sleep", "to"]}, {"sent": "you tell Helen where you gonna sleep ?", "words": ["where", "sleep", "you"]}, {"sent": "baby can go to sleep much better now .", "words": ["can", "better", "go", "sleep", "to", "much"]}, {"sent": "you sleep in what kind of bed ?", "words": ["what", "sleep", "of", "bed", "you", "in"]}, {"sent": "she was pretending to sleep on the floor .", "words": ["the", "sleep", "to", "was", "on", "she"]}, {"sent": "cantchoo sleep Little Bear asked Big Bear putting down his bear book which was just getting to the interesting part and padding over to the bed .", "words": ["the", "over", "sleep", "was", "to", "bed", "bear", "and", "his", "book", "down", "which"]}, {"sent": "and so then I cant sleep .", "words": ["sleep", "so", "then", "and"]}, {"sent": "where do you sleep now ?", "words": ["where", "do", "sleep", "you"]}, {"sent": "and when its over you can go to sleep .", "words": ["can", "when", "go", "sleep", "to", "you", "and", "over"]}, {"sent": "now Im going to sleep and I dont know what else you do .", "words": ["what", "do", "sleep", "to", "and", "you", "dont"]}, {"sent": "Eleanor going to sleep .", "words": ["sleep", "to"]}, {"sent": "you wanna go and sleep in your bed ?", "words": ["your", "go", "sleep", "bed", "and", "you", "in", "wanna"]}, {"sent": "I didnt have a very good sleep .", "words": ["have", "sleep", "good", "a"]}, {"sent": "is that where hes going to sleep ?", "words": ["where", "is", "sleep", "to", "that"]}, {"sent": "colts going to sleep .", "words": ["sleep", "to"]}, {"sent": "he can go to sleep in his play pen .", "words": ["can", "go", "sleep", "to", "play", "play pen", "in", "he", "pen", "his"]}, {"sent": "wanna sleep on my lap ?", "words": ["sleep", "on", "wanna", "my"]}, {"sent": "did you sleep last night ?", "words": ["sleep", "last", "did", "you"]}, {"sent": "looks asif Daddys gone to sleep Anne .", "words": ["sleep", "to"]}, {"sent": "good night go to sleep .", "words": ["sleep", "to", "good", "go"]}, {"sent": "no Mummas not going to sleep .", "words": ["sleep", "to", "not"]}, {"sent": "gone to sleep ?", "words": ["sleep", "to"]}, {"sent": "they like to sleep .", "words": ["to", "they", "like", "sleep"]}, {"sent": "are you going to go and sleep with Grandma and Grandad then ?", "words": ["go", "sleep", "to", "with", "and", "you", "are", "then"]}, {"sent": "but all who heard it snuggled down to a cosy cheerful sleep .", "words": ["all", "sleep", "to", "it", "who", "down", "but", "a"]}, {"sent": "what do you sleep in ?", "words": ["what", "do", "sleep", "you", "in"]}, {"sent": "maybe shes tired let her sleep awhile .", "words": ["sleep", "her", "tired"]}, {"sent": "byebye bus .", "words": ["bus"]}, {"sent": "oh yeah okay yes these bus drivers have one of them .", "words": ["these", "of", "them", "have", "bus"]}, {"sent": "we put them in the bus .", "words": ["the", "put", "we", "them", "in", "bus"]}, {"sent": "youre a bus driver and I dont know the way back to the bus depot ?", "words": ["the", "back", "to", "and", "dont", "bus", "a"]}, {"sent": "how about the wipers on the bus ?", "words": ["the", "how", "about", "on", "bus"]}, {"sent": "the people on the bus go .", "words": ["the", "on", "bus", "go"]}, {"sent": "its the horn on the bus that goes beep beep beep .", "words": ["the", "that", "on", "bus"]}, {"sent": "he got on the bus .", "words": ["on", "the", "he", "bus"]}, {"sent": "the bus ?", "words": ["the", "bus"]}, {"sent": "shall we have this as the bus stop outside the house ?", "words": ["the", "we", "outside", "stop", "have", "this", "house", "bus"]}, {"sent": "bus bus bus bus .", "words": ["bus"]}, {"sent": "and then from YorkStation we got a bus to where Grandma and Grandad at xxx .", "words": ["where", "we", "to", "and", "at", "then", "bus", "a"]}, {"sent": "do you know a song about wheels on a bus ?", "words": ["do", "you", "about", "on", "bus", "a"]}, {"sent": "is that your school bus ?", "words": ["your", "is", "school", "that", "bus"]}, {"sent": "the wheels on the bus go round and round round and round round and round the wheels on the bus go round and round all through the town .", "words": ["the", "all", "go", "and", "on", "bus"]}, {"sent": "the wheels on the bus ?", "words": ["the", "on", "bus"]}, {"sent": "and you walked from the bus stop through Didsbury park to their house .", "words": ["the", "their", "to", "stop", "you", "and", "park", "house", "bus"]}, {"sent": "but why would the little boy and the little and the Mummy go back to the bus depot what were they hoping to find ?", "words": ["the", "what", "were", "back", "go", "to", "find", "little", "and", "they", "but", "would", "bus", "why"]}, {"sent": "a twobus because I think you useta think of one bus on top of another bus because its a double decker bus .", "words": ["think", "of", "you", "another", "on", "bus", "because", "a"]}, {"sent": "oh and whats happened to that big red bus ?", "words": ["red", "to", "and", "that", "big", "bus"]}, {"sent": "to the bus depot .", "words": ["the", "to", "bus"]}, {"sent": "this is a mini bus ?", "words": ["this", "bus", "a", "is"]}, {"sent": "Im not sure if its a train driver or a bus driver .", "words": ["if", "train", "bus", "not", "a"]}, {"sent": "because on the bus I said .", "words": ["on", "the", "bus", "because"]}, {"sent": "well I had my handbag trapped in the bus all the way here and now Ive left it on the bus .", "words": ["the", "all", "my", "and", "here", "in", "it", "on", "bus"]}, {"sent": "well we could go on the bus Thomas into Didsbury .", "words": ["the", "into", "go", "we", "on", "could", "bus"]}, {"sent": "thats the busy busy world bus .", "words": ["the", "bus"]}, {"sent": "get your bus .", "words": ["get", "your", "bus"]}, {"sent": "and then he gets on the bus and he sits down with the box .", "words": ["the", "box", "with", "and", "he", "on", "then", "down", "bus"]}, {"sent": "what sort of bus is it ?", "words": ["what", "is", "of", "it", "bus"]}, {"sent": "the bus station .", "words": ["the", "bus"]}, {"sent": "I wonder why he was banging his bus .", "words": ["was", "he", "his", "bus", "why"]}, {"sent": "and heres a bus .", "words": ["bus", "a", "and"]}, {"sent": "and your bus .", "words": ["your", "bus", "and"]}, {"sent": "bus .", "words": ["bus"]}, {"sent": "and theyre all getting inside the school bus .", "words": ["the", "all", "school", "inside", "and", "bus"]}, {"sent": "the ice cream bus doesnt come till much later on does he ?", "words": ["the", "does", "ice cream", "much", "on", "he", "ice", "bus"]}, {"sent": "but if we go on the bus it gives you a nice ride out doesnt it ?", "words": ["the", "if", "go", "we", "nice", "ride", "you", "it", "out", "on", "bus", "but", "a"]}, {"sent": "do you wanna do the the teddybear and the the bus one ?", "words": ["the", "do", "teddybear", "you", "and", "wanna", "bus"]}, {"sent": "did we come on the bus ?", "words": ["the", "we", "did", "on", "bus"]}, {"sent": "people at the bus stop .", "words": ["the", "bus", "stop", "at"]}, {"sent": "do you see her bus ?", "words": ["do", "you", "see", "her", "bus"]}, {"sent": "yeah thats a school bus .", "words": ["school", "bus", "a"]}, {"sent": "bus huh ?", "words": ["bus"]}, {"sent": "weve got the entire Tweenies on the bus .", "words": ["the", "on", "bus"]}, {"sent": "its time we had a day off said driver Pete to the little yellow bus .", "words": ["off", "the", "we", "to", "little", "yellow", "bus", "a"]}, {"sent": "whats this thing whats red on the school bus ?", "words": ["red", "the", "school", "this", "on", "bus"]}, {"sent": "does the big bus fit ?", "words": ["the", "does", "fit", "big", "bus"]}, {"sent": "why do you think the bus did eeh ?", "words": ["the", "do", "think", "you", "did", "bus", "why"]}, {"sent": "a bus .", "words": ["bus", "a"]}, {"sent": "wait at the bus stop for the bus ?", "words": ["the", "wait", "for", "stop", "at", "bus"]}, {"sent": "or the wheels on the bus or any other of your favorite songs .", "words": ["the", "your", "of", "any", "other", "on", "bus"]}, {"sent": "did you enjoy looking at the bus book ?", "words": ["the", "you", "did", "at", "book", "bus"]}, {"sent": "the bus station .", "words": ["the", "bus"]}, {"sent": "they hafta wait for the school bus to come the school bus isnt there .", "words": ["the", "wait", "for", "school", "to", "they", "there", "bus"]}, {"sent": "and we were going over by the bus .", "words": ["the", "we", "by", "and", "over", "bus", "were"]}, {"sent": "make a bus .", "words": ["make", "bus", "a"]}, {"sent": "that bus is noisy .", "words": ["noisy", "that", "bus", "is"]}, {"sent": "are they coming off the school bus ?", "words": ["off", "the", "school", "they", "are", "bus"]}, {"sent": "big green bus .", "words": ["big", "bus", "green"]}, {"sent": "a bus .", "words": ["bus", "a"]}, {"sent": "where are you going on the bus ?", "words": ["where", "the", "you", "are", "on", "bus"]}, {"sent": "have we been on a bus today ?", "words": ["we", "have", "on", "bus", "a"]}, {"sent": "do you wanna go with them sweetie when we take them down to the bus station ?", "words": ["the", "do", "go", "we", "to", "with", "them", "you", "wanna", "take", "down", "when", "bus"]}, {"sent": "that bus went away ?", "words": ["away", "that", "bus"]}, {"sent": "hey tell me about the bus .", "words": ["the", "me", "bus", "about"]}, {"sent": "what else happens on the bus ?", "words": ["on", "what", "bus", "the"]}, {"sent": "blue bus ?", "words": ["bus", "blue"]}, {"sent": "what about doing a bus to stop at the bus stop ?", "words": ["the", "what", "to", "stop", "about", "at", "bus", "a"]}, {"sent": "lets go on a bus huh ?", "words": ["on", "bus", "a", "go"]}, {"sent": "whats that on the bus ?", "words": ["on", "the", "that", "bus"]}, {"sent": "is this your bus ?", "words": ["this", "your", "bus", "is"]}, {"sent": "bus .", "words": ["bus"]}, {"sent": "wheres the bus gonna take you today ?", "words": ["the", "take", "bus", "you"]}, {"sent": "before it was a bus .", "words": ["was", "it", "bus", "a"]}, {"sent": "a bit smaller than a mini bus but larger than a car .", "words": ["car", "bus", "but", "a"]}, {"sent": "thats a bus .", "words": ["bus", "a"]}, {"sent": "your bus and the yellow car the sports car that Granddad has bought you .", "words": ["the", "your", "car", "and", "you", "that", "yellow", "bus"]}, {"sent": "bus and trucks .", "words": ["bus", "and"]}, {"sent": "we went on the bus didnt we ?", "words": ["on", "the", "bus", "we"]}, {"sent": "the wheels on the bus .", "words": ["the", "on", "bus"]}, {"sent": "a magic school bus ?", "words": ["school", "bus", "a"]}, {"sent": "wheres your school bus ?", "words": ["school", "your", "bus"]}, {"sent": "no its next to the bus .", "words": ["the", "to", "bus"]}, {"sent": "the wipers on the bus go swish swish swish .", "words": ["the", "on", "bus", "go"]}, {"sent": "the riders on the bus go bumpity bump bump .", "words": ["the", "bump", "go", "on", "bus"]}, {"sent": "one day on his way to the bus stop he found a big beautiful red balloon .", "words": ["the", "red", "to", "stop", "his", "big", "he", "balloon", "on", "bus", "a"]}, {"sent": "climbing up on the bus .", "words": ["up", "the", "on", "bus"]}, {"sent": "oh theyre all getting off the school bus .", "words": ["off", "all", "the", "school", "bus"]}, {"sent": "a little boy found a red balloon on his way to the bus stop .", "words": ["red", "the", "to", "stop", "little", "his", "balloon", "on", "bus", "a"]}, {"sent": "would you like to play with those and the school bus ?", "words": ["the", "those", "to", "school", "play", "with", "like", "you", "and", "would", "bus"]}, {"sent": "but the driver of the bus is called a bus driver .", "words": ["the", "is", "of", "bus", "but", "a"]}, {"sent": "the bus stopped at the bus stop .", "words": ["the", "bus", "stop", "at"]}, {"sent": "weve put some of those in the bus tin .", "words": ["the", "put", "those", "of", "in", "some", "bus"]}, {"sent": "the money on the bus goes clink clink clink .", "words": ["the", "on", "bus", "money"]}, {"sent": "what happens to the babies on the bus ?", "words": ["the", "what", "to", "on", "bus"]}, {"sent": "youre to go out with your mommy to get the bus to go see Jenny to Nanas .", "words": ["the", "your", "go", "to", "with", "get", "out", "see", "bus"]}, {"sent": "is that my bus ?", "words": ["my", "that", "bus", "is"]}, {"sent": "under your boat about to get squashed by it I think .", "words": ["think", "your", "to", "by", "get", "under", "it", "about", "boat"]}, {"sent": "under the table .", "words": ["the", "under", "table"]}, {"sent": "let me see under there .", "words": ["see", "me", "under", "there"]}, {"sent": "get stuck under the bed ?", "words": ["the", "bed", "get", "under", "stuck"]}, {"sent": "are you putting it under the doggies arm ?", "words": ["the", "you", "under", "it", "are", "arm"]}, {"sent": "youve put them under the arches of the bridges so theyve gone to bed ?", "words": ["the", "put", "of", "so", "to", "them", "bed", "under"]}, {"sent": "whos under the hat ?", "words": ["the", "under", "hat"]}, {"sent": "shouldve made an Easter basket and stuck it under the tree .", "words": ["the", "tree", "and", "under", "an", "it", "stuck", "basket"]}, {"sent": "and theyre looking under .", "words": ["under", "and"]}, {"sent": "under the chair .", "words": ["the", "under", "chair"]}, {"sent": "get the napkin from under your desk .", "words": ["the", "your", "get", "under", "napkin"]}, {"sent": "its under the couch .", "words": ["the", "under", "couch"]}, {"sent": "no but you can come under .", "words": ["can", "under", "but", "you"]}, {"sent": "under .", "words": ["under"]}, {"sent": "Im going to put it under this cup okay ?", "words": ["cup", "put", "to", "under", "this", "it"]}, {"sent": "under the bed .", "words": ["the", "under", "bed"]}, {"sent": "oh were gonna hide under the table .", "words": ["the", "table", "hide", "under", "were"]}, {"sent": "xxx I dont see oh okay wheres our m spell mommy under here .", "words": ["our", "here", "dont", "under", "see"]}, {"sent": "one under the one .", "words": ["the", "under"]}, {"sent": "all you wanna do is look under that chair dont you today ?", "words": ["all", "do", "is", "chair", "you", "under", "dont", "wanna", "look", "that"]}, {"sent": "unless its its been kicked under the sofa .", "words": ["the", "under", "sofa"]}, {"sent": "Mummys just going to rinse the apple under the tap .", "words": ["the", "to", "apple", "under"]}, {"sent": "you dont have stockings on under there .", "words": ["you", "have", "dont", "under", "there", "on"]}, {"sent": "whos under the hat ?", "words": ["the", "under", "hat"]}, {"sent": "oh under the bridge ?", "words": ["the", "under"]}, {"sent": "under your sweater .", "words": ["under", "your", "sweater"]}, {"sent": "wash wash under your arm .", "words": ["your", "under", "wash", "arm"]}, {"sent": "and do you have a belly under that dress too ?", "words": ["do", "dress", "you", "have", "and", "under", "too", "that", "a"]}, {"sent": "Nathan is she going to go under the chair or on the chair ?", "words": ["the", "go", "is", "to", "chair", "under", "on", "she"]}, {"sent": "youre pushing it under the television ?", "words": ["the", "under", "it"]}, {"sent": "whats under here ?", "words": ["here", "under"]}, {"sent": "under all that stuff huh ?", "words": ["all", "under", "that"]}, {"sent": "oh no I think I see it hiding under there .", "words": ["think", "under", "there", "it", "see"]}, {"sent": "Henry no you cant go under there .", "words": ["go", "under", "there", "you"]}, {"sent": "its under the table by the harmonica .", "words": ["the", "under", "table", "by"]}, {"sent": "he wants to go under the you wanna go under the blanket .", "words": ["the", "go", "to", "you", "under", "he", "wanna", "blanket"]}, {"sent": "xxx sitting under the table .", "words": ["the", "under", "table"]}, {"sent": "whats under there whats under it ?", "words": ["under", "it", "there"]}, {"sent": "look under the rug there .", "words": ["the", "under", "there", "look"]}, {"sent": "under the chair JJ .", "words": ["the", "under", "chair"]}, {"sent": "under her .", "words": ["under", "her"]}, {"sent": "look can you see me under the car ?", "words": ["can", "the", "car", "me", "you", "under", "look", "see"]}, {"sent": "oh there goes the car under the bridge .", "words": ["the", "there", "under", "car"]}, {"sent": "his tractors wanna dig under the street ?", "words": ["the", "under", "wanna", "his", "street"]}, {"sent": "into bed under the covers .", "words": ["the", "under", "into", "bed"]}, {"sent": "so this goes under here and that goes under there .", "words": ["so", "and", "here", "under", "this", "there", "that"]}, {"sent": "who else is gonna go under the bridge Carl ?", "words": ["the", "go", "is", "under", "who"]}, {"sent": "is that a baby under there ?", "words": ["is", "under", "there", "that", "a"]}, {"sent": "under the table ?", "words": ["the", "under", "table"]}, {"sent": "whos under the umbrella ?", "words": ["the", "under"]}, {"sent": "your socksocks are under your slippers are they ?", "words": ["under", "are", "your", "they"]}, {"sent": "slipped under the gate and was safely in the wood .", "words": ["the", "was", "and", "under", "in"]}, {"sent": "underneath the ground and what sort of people would live under the ground ?", "words": ["the", "what", "of", "and", "under", "would"]}, {"sent": "is he under here ?", "words": ["here", "under", "he", "is"]}, {"sent": "the bus is going under the tunnel .", "words": ["the", "under", "bus", "is"]}, {"sent": "think theyre just for under the house spare ones .", "words": ["the", "think", "for", "under", "house"]}, {"sent": "you can go under .", "words": ["can", "under", "go", "you"]}, {"sent": "now you have playdough under your fingernails .", "words": ["have", "under", "your", "you"]}, {"sent": "xxx whats under the turtle .", "words": ["the", "under", "turtle"]}, {"sent": "is that spot under there ?", "words": ["under", "that", "there", "is"]}, {"sent": "under here ?", "words": ["here", "under"]}, {"sent": "no he plays with our pots and pans like hell go under the cabinet and pull pots out and well give him a spoon and hell walk around and like play with the pots and pans and stuff .", "words": ["the", "go", "play", "with", "like", "and", "our", "under", "pull", "him", "he", "spoon", "out", "around", "give", "walk", "a"]}, {"sent": "you put it upstairs under your plants .", "words": ["put", "your", "you", "under", "it"]}, {"sent": "no it its not under my foot .", "words": ["my", "under", "foot", "it", "not"]}, {"sent": "hes still under there .", "words": ["under", "there"]}, {"sent": "xxx that baby under a hat ?", "words": ["hat", "under", "that", "a"]}, {"sent": "daddys balls under the coffee table look .", "words": ["the", "table", "under", "coffee", "look"]}, {"sent": "not under the chair .", "words": ["the", "under", "not", "chair"]}, {"sent": "a toadstool and Toad is sitting under a mushroom called a toadstool .", "words": ["under", "is", "and", "a"]}, {"sent": "oh it got stuck under the sit and spin .", "words": ["the", "and", "under", "it", "stuck", "sit"]}, {"sent": "theres a train under there ?", "words": ["under", "train", "there", "a"]}, {"sent": "whos under that whos under the hat ?", "words": ["the", "under", "that", "hat"]}, {"sent": "stitches under his .", "words": ["his", "under"]}, {"sent": "whos that under the hat ?", "words": ["the", "under", "that", "hat"]}, {"sent": "oh you found a card under there .", "words": ["under", "there", "a", "you"]}, {"sent": "whos under the umbrella is that you ?", "words": ["the", "is", "you", "under", "that"]}, {"sent": "he willnt fit under the bridge .", "words": ["the", "under", "fit", "he"]}, {"sent": "one more missing I see it over there under the pole almost under the .", "words": ["the", "there", "more", "it", "under", "see", "over"]}, {"sent": "we hafta get that under the axle .", "words": ["the", "we", "get", "under", "that"]}, {"sent": "we get one under the train every half an hour that goes to Didsbury .", "words": ["the", "train", "we", "to", "get", "under", "an", "every", "that"]}, {"sent": "whats it doing under the bed ?", "words": ["the", "under", "it", "bed"]}, {"sent": "look under Daisy does she have a cinnamon stick under her ?", "words": ["does", "stick", "have", "under", "look", "her", "she", "a"]}, {"sent": "Daddy says did you look under your bed ?", "words": ["your", "bed", "you", "under", "did", "look"]}, {"sent": "whos under there ?", "words": ["under", "there"]}, {"sent": "thats the jack that you put put under the car .", "words": ["the", "put", "car", "you", "under", "that"]}, {"sent": "under your head .", "words": ["under", "head", "your"]}, {"sent": "maybe it can go under the bridge .", "words": ["can", "the", "go", "under", "it"]}, {"sent": "that seat is so I dont know if he can get his legs under there .", "words": ["can", "if", "is", "so", "get", "dont", "under", "there", "he", "his", "that"]}, {"sent": "we found the man under the clothes didnt we ?", "words": ["the", "under", "we"]}, {"sent": "oh hes going hide under there as well is he now ?", "words": ["hide", "is", "under", "there", "he"]}, {"sent": "is someone giggling under there ?", "words": ["under", "there", "is"]}, {"sent": "the only thing thats different about the room is that Jaspers out from under the bed .", "words": ["the", "is", "bed", "under", "room", "about", "out", "that"]}, {"sent": "Michael doesnt have a shirt under that ?", "words": ["shirt", "have", "under", "that", "a"]}, {"sent": "uhoh Bro it went way under the table .", "words": ["the", "under", "it", "table"]}, {"sent": "under the bridge .", "words": ["the", "under"]}, {"sent": "theyre under your big bed ?", "words": ["under", "big", "your", "bed"]}, {"sent": "its a good job you took the man off because he wouldnt have gone under that bridge with the man on would he ?", "words": ["the", "off", "with", "you", "have", "under", "he", "good", "would", "on", "that", "because", "a"]}, {"sent": "under your chin .", "words": ["under", "your", "chin"]}, {"sent": "this is the one that was under the bed and I went to get it .", "words": ["the", "is", "was", "to", "bed", "and", "get", "this", "under", "it", "that"]}, {"sent": "is this a red shirt ?", "words": ["red", "is", "shirt", "this", "a"]}, {"sent": "first Im going to take your shirt first Im going to take your shirt .", "words": ["your", "to", "shirt", "first", "take"]}, {"sent": "lemme help you you got a b down your shirt down your shirt you gotta b down your shirt down your shirt .", "words": ["help", "your", "shirt", "you", "down", "a"]}, {"sent": "um excuse me can you take your hand out of my shirt please thank you .", "words": ["can", "my", "your", "hand", "of", "shirt", "you", "out", "me", "take"]}, {"sent": "how the hell xxx cooperate xxx and the shirt that goes xxx okay ?", "words": ["the", "how", "shirt", "and", "that"]}, {"sent": "well it doesnt it my microphone is already on my shirt so you gotta say hello .", "words": ["my", "is", "shirt", "so", "say", "you", "it", "on"]}, {"sent": "so we hadta put a vest on top of the shirt which is on top of a shortsleeved shirt .", "words": ["the", "put", "we", "of", "so", "shirt", "is", "on", "which", "a"]}, {"sent": "anyway I got shame shirt .", "words": ["shirt"]}, {"sent": "Mommy have flowers on her shirt ?", "words": ["have", "shirt", "on", "her"]}, {"sent": "does it go on your shirt ?", "words": ["does", "your", "go", "shirt", "it", "on"]}, {"sent": "put his shirt on then .", "words": ["put", "shirt", "on", "his", "then"]}, {"sent": "Daddys shirt .", "words": ["shirt"]}, {"sent": "youre gonna put an O down your shirt down your shirt ?", "words": ["put", "your", "shirt", "an", "down"]}, {"sent": "looks like your shirt actually .", "words": ["shirt", "your", "like"]}, {"sent": "for his shirt .", "words": ["his", "shirt", "for"]}, {"sent": "lets put a shirt on and go and have breakfast .", "words": ["put", "go", "shirt", "and", "have", "on", "a"]}, {"sent": "so how do feel with that shirt on ?", "words": ["do", "how", "so", "shirt", "with", "on", "that"]}, {"sent": "your new shirt all stained up xxx .", "words": ["up", "all", "your", "shirt", "new"]}, {"sent": "wanna put a shirt on ?", "words": ["put", "shirt", "wanna", "on", "a"]}, {"sent": "or this shirt ?", "words": ["this", "shirt"]}, {"sent": "no this is his shirt .", "words": ["his", "this", "shirt", "is"]}, {"sent": "I think its Daddys shirt .", "words": ["shirt", "think"]}, {"sent": "your guy gonna wear a shirt ?", "words": ["shirt", "your", "a"]}, {"sent": "dont get pen on your shirt .", "words": ["your", "shirt", "get", "dont", "pen", "on"]}, {"sent": "well you cant put the shirt on his head .", "words": ["the", "put", "shirt", "head", "you", "his", "on"]}, {"sent": "dont suck your thumb along with my shirt .", "words": ["my", "your", "shirt", "with", "dont"]}, {"sent": "putting a lego in my shirt ?", "words": ["shirt", "in", "my", "a"]}, {"sent": "what shirt should she wear ?", "words": ["what", "she", "shirt"]}, {"sent": "are you pulling your shirt like that ?", "words": ["your", "shirt", "like", "you", "are", "that"]}, {"sent": "yeah let him take the shirt off .", "words": ["the", "off", "shirt", "him", "take"]}, {"sent": "red shirt blue shirt yellow shirt oops .", "words": ["red", "shirt", "yellow", "blue"]}, {"sent": "do you wanna put on the blue shirt or the pink shirt ?", "words": ["the", "do", "put", "shirt", "you", "blue", "wanna", "on"]}, {"sent": "is he through with the shirt ?", "words": ["the", "is", "shirt", "with", "he"]}, {"sent": "shirt .", "words": ["shirt"]}, {"sent": "there is his shirt .", "words": ["his", "there", "shirt", "is"]}, {"sent": "where does his shirt go ?", "words": ["where", "does", "go", "shirt", "his"]}, {"sent": "your shirt and pants dont meet .", "words": ["your", "shirt", "and", "dont", "pants"]}, {"sent": "okay lets put your Barney shirt on .", "words": ["on", "shirt", "put", "your"]}, {"sent": "thats attached to your shirt ?", "words": ["to", "your", "shirt"]}, {"sent": "Lastname go get yourself a shirt before you catch pneumonia .", "words": ["yourself", "go", "shirt", "catch", "you", "get", "a"]}, {"sent": "what shirt do you wanna wear ?", "words": ["what", "do", "shirt", "you", "wanna"]}, {"sent": "all down your blue trousers and your shirt .", "words": ["all", "your", "shirt", "and", "blue", "down"]}, {"sent": "lets see if we can find some more of this checkered shirt .", "words": ["can", "if", "we", "of", "shirt", "this", "more", "find", "see", "some"]}, {"sent": "Ronald did your wear that shirt to school or is that one uh the .", "words": ["the", "your", "is", "to", "shirt", "school", "did", "that"]}, {"sent": "his shirt ?", "words": ["his", "shirt"]}, {"sent": "which has melted all over your nice shirt .", "words": ["all", "your", "shirt", "nice", "which", "over"]}, {"sent": "well I dont really want this shirt to get messy cause this is a shirt that I like .", "words": ["is", "to", "shirt", "like", "get", "dont", "this", "that", "a"]}, {"sent": "shirt .", "words": ["shirt"]}, {"sent": "I put this shirt on ?", "words": ["on", "this", "put", "shirt"]}, {"sent": "whats he got on his shirt ?", "words": ["on", "his", "he", "shirt"]}, {"sent": "snap my shirt .", "words": ["shirt", "my"]}, {"sent": "can you put her shirt on ?", "words": ["can", "put", "shirt", "you", "on", "her"]}, {"sent": "is this his shirt ?", "words": ["his", "this", "shirt", "is"]}, {"sent": "just you you have the special shirt ?", "words": ["have", "the", "shirt", "you"]}, {"sent": "no put the shirt right there .", "words": ["the", "shirt", "put", "there"]}, {"sent": "go in there and let him take the the shirt off first .", "words": ["the", "off", "go", "shirt", "first", "and", "there", "in", "him", "take"]}, {"sent": "this one this shirt I need to .", "words": ["need", "need to", "to", "shirt", "this"]}, {"sent": "who is on your new shirt ?", "words": ["your", "is", "shirt", "who", "on", "new"]}, {"sent": "bring the shirt over so I can put it on her .", "words": ["the", "can", "put", "shirt", "so", "her", "bring", "it", "on", "over"]}, {"sent": "lets put this red shirt on today shall we ?", "words": ["red", "put", "we", "shirt", "this", "on"]}, {"sent": "do you think that shirt might be a little big for Barney ?", "words": ["do", "think", "for", "shirt", "be", "you", "little", "big", "that", "a"]}, {"sent": "a sweater another shirt .", "words": ["another", "shirt", "sweater", "a"]}, {"sent": "looks like a shirt I guess .", "words": ["shirt", "like", "a"]}, {"sent": "green shirt .", "words": ["shirt", "green"]}, {"sent": "would you wear the shirt then ?", "words": ["the", "shirt", "you", "would", "then"]}, {"sent": "the shirt ?", "words": ["the", "shirt"]}, {"sent": "look at that pretty yellow shirt .", "words": ["look", "shirt", "pretty", "at", "yellow", "that"]}, {"sent": "shall we put on a red shirt or a white shirt ?", "words": ["red", "put", "we", "shirt", "on", "white", "a"]}, {"sent": "put can you put that shirt back in ?", "words": ["can", "put", "back", "shirt", "you", "in", "that"]}, {"sent": "is that shirt for girls ?", "words": ["shirt", "for", "that", "is"]}, {"sent": "xxx a cute sweat shirt on .", "words": ["cute", "shirt", "on", "a"]}, {"sent": "fingers got Daddys shirt .", "words": ["shirt"]}, {"sent": "whats on your shirt ?", "words": ["on", "shirt", "your"]}, {"sent": "you do want a shirt on ?", "words": ["do", "shirt", "you", "on", "a"]}, {"sent": "look he has a red shirt on .", "words": ["red", "shirt", "he", "look", "on", "a"]}, {"sent": "see Tom has a red shirt on .", "words": ["red", "shirt", "on", "see", "a"]}, {"sent": "shall we put this shirt on you ?", "words": ["put", "we", "shirt", "you", "this", "on"]}, {"sent": "well hafta spray that with the shirt with the spray .", "words": ["the", "shirt", "that", "with"]}, {"sent": "you do you have a lil pink teddy bear on your shirt .", "words": ["do", "your", "shirt", "bear", "have", "you", "on", "a"]}, {"sent": "in the back of your shirt .", "words": ["the", "your", "back", "of", "shirt", "in"]}, {"sent": "you wanna try and put the shirt on ?", "words": ["the", "put", "shirt", "and", "you", "wanna", "on", "try"]}, {"sent": "youve got a lumberjack shirt on today .", "words": ["on", "shirt", "a"]}, {"sent": "this same as mommys shirt .", "words": ["this", "same", "shirt"]}, {"sent": "why do you think its mommys shirt ?", "words": ["do", "think", "shirt", "you", "why"]}, {"sent": "is that his shirt ?", "words": ["his", "shirt", "that", "is"]}, {"sent": "so I think well take your shirt off .", "words": ["off", "think", "your", "so", "shirt", "take"]}, {"sent": "and youve got crumbs all over your shirt .", "words": ["all", "your", "shirt", "and", "over"]}, {"sent": "now gonna put your shirt on ?", "words": ["on", "shirt", "put", "your"]}, {"sent": "put the little shirt on the line like that .", "words": ["the", "put", "shirt", "like", "little", "on", "that"]}, {"sent": "look at your pretty striped shirt .", "words": ["look", "your", "shirt", "pretty", "at"]}, {"sent": "does that zookeeper have the word zoo on his shirt ?", "words": ["zoo", "does", "the", "shirt", "have", "his", "on", "that"]}, {"sent": "oh am I gonna pick out a shirt ?", "words": ["pick", "shirt", "am", "out", "a"]}, {"sent": "LaaLaa has a yellow shirt right .", "words": ["yellow", "shirt", "a"]}, {"sent": "just dont get it on your shirt .", "words": ["your", "shirt", "get", "dont", "it", "on"]}, {"sent": "what colors her shirt ?", "words": ["what", "her", "shirt"]}, {"sent": "would you um go upstairs and get me a shirt to put on him .", "words": ["put", "go", "shirt", "to", "and", "get", "you", "him", "would", "on", "me", "a"]}, {"sent": "oh little shirt ?", "words": ["shirt", "little"]}, {"sent": "no you dont paint on the table .", "words": ["the", "paint", "table", "you", "dont", "on"]}, {"sent": "oh I dont think that you have any paint here .", "words": ["think", "paint", "any", "you", "have", "dont", "here", "that"]}, {"sent": "with seeds in and paint in and things like that .", "words": ["paint", "with", "like", "and", "in", "that"]}, {"sent": "do you want some black paint ?", "words": ["do", "paint", "you", "black", "some"]}, {"sent": "well did you did you paint the face on it ?", "words": ["the", "paint", "face", "you", "it", "did", "on"]}, {"sent": "lets pour a little bit of paint there .", "words": ["paint", "of", "pour", "little", "there", "a"]}, {"sent": "you wanna paint ?", "words": ["paint", "wanna", "you"]}, {"sent": "they paint ?", "words": ["they", "paint"]}, {"sent": "no green paint said hush and brush .", "words": ["brush", "paint", "green", "and"]}, {"sent": "why dont you save it and you you and Stephanie and Jenny can paint together ?", "words": ["can", "paint", "and", "you", "dont", "it", "why"]}, {"sent": "you can use the finger you can use the paint on the paper already with your fingers .", "words": ["can", "the", "paint", "your", "paper", "with", "you", "finger", "on"]}, {"sent": "with a little bit of paint .", "words": ["paint", "of", "with", "little", "a"]}, {"sent": "are you gonna paint Mummy a big picture ?", "words": ["paint", "picture", "you", "are", "big", "a"]}, {"sent": "because I may never get a chance to paint such a nice picture again .", "words": ["paint", "to", "nice", "picture", "get", "because", "a"]}, {"sent": "paint ?", "words": ["paint"]}, {"sent": "the paint dropped off our wall ?", "words": ["the", "paint", "off", "our"]}, {"sent": "but you have to wear this if were gonna paint okay weregonna put it on your back like this .", "words": ["put", "paint", "if", "your", "back", "to", "like", "you", "have", "this", "it", "on", "were", "but"]}, {"sent": "and just get the paint on the dolphin direct .", "words": ["the", "paint", "and", "get", "on"]}, {"sent": "dried paint .", "words": ["paint"]}, {"sent": "bits of paint ?", "words": ["of", "paint"]}, {"sent": "did you paint this ?", "words": ["this", "paint", "did", "you"]}, {"sent": "do you hafta suck a little bit of paint in the straw ?", "words": ["the", "do", "paint", "of", "little", "you", "in", "a"]}, {"sent": "so how do you get the paint from in the paint holder to on the paper ?", "words": ["the", "do", "how", "paint", "so", "to", "paper", "you", "get", "in", "on"]}, {"sent": "or they just say wet paint do not touch .", "words": ["do", "wet", "paint", "say", "touch", "they", "not"]}, {"sent": "oh dear youve dropped the red paint .", "words": ["the", "red", "paint"]}, {"sent": "mums going upstairs to paint the new bedroom .", "words": ["the", "paint", "to", "new", "bedroom"]}, {"sent": "why did you put water with paint in there ?", "words": ["put", "paint", "with", "you", "there", "in", "did", "water", "why"]}, {"sent": "brush mixed some red paint and white paint together and what did that make ?", "words": ["red", "what", "paint", "brush", "and", "make", "did", "some", "white", "that"]}, {"sent": "do you think theyll let you paint ?", "words": ["do", "think", "paint", "you"]}, {"sent": "oh okay well we can make some more this afternoon and then you can paint them .", "words": ["can", "paint", "we", "them", "and", "you", "this", "more", "some", "then", "make"]}, {"sent": "what do you use to paint pictures with ?", "words": ["what", "do", "paint", "to", "with", "you"]}, {"sent": "now paint him on the guy .", "words": ["on", "the", "him", "paint"]}, {"sent": "is that paint ?", "words": ["paint", "that", "is"]}, {"sent": "and you paint such beautiful pictures .", "words": ["paint", "you", "and"]}, {"sent": "it totally looks like paint .", "words": ["it", "paint", "like"]}, {"sent": "youre covered in paint .", "words": ["in", "paint"]}, {"sent": "and you just rub the paint in like that .", "words": ["the", "paint", "like", "you", "and", "in", "that"]}, {"sent": "thats dried paint .", "words": ["paint"]}, {"sent": "what you going to paint now ?", "words": ["to", "what", "paint", "you"]}, {"sent": "you wanna paint .", "words": ["paint", "wanna", "you"]}, {"sent": "because the new house smells like paint .", "words": ["the", "paint", "like", "house", "new", "because"]}, {"sent": "paint .", "words": ["paint"]}, {"sent": "I was gonna paint them green xxx .", "words": ["was", "paint", "green", "them"]}, {"sent": "dont put any face paint in your eye .", "words": ["put", "paint", "face", "your", "any", "eye", "dont", "in"]}, {"sent": "you got some paint left yet .", "words": ["some", "paint", "you"]}, {"sent": "rub both hands together with yellow paint .", "words": ["yellow", "paint", "with"]}, {"sent": "hope youre gonna paint after this .", "words": ["this", "paint"]}, {"sent": "Abe are you going to paint a picture ?", "words": ["paint", "to", "picture", "you", "are", "a"]}, {"sent": "did she paint too ?", "words": ["too", "paint", "did", "she"]}, {"sent": "hes trying to paint his face .", "words": ["his", "to", "paint", "face"]}, {"sent": "you could paint every side different .", "words": ["could", "paint", "every", "you"]}, {"sent": "lets put this jumper on and its an old jumper so it doesnt matter if you get paint on it .", "words": ["old", "put", "paint", "if", "so", "and", "you", "this", "an", "it", "get", "on"]}, {"sent": "did you put your paint in the ha did you put your hand in the paint ?", "words": ["the", "put", "paint", "your", "hand", "you", "in", "did"]}, {"sent": "you used up a lot of the blue paint and .", "words": ["up", "the", "paint", "of", "a lot", "you", "blue", "and", "a"]}, {"sent": "maybe the paint spilt .", "words": ["the", "paint"]}, {"sent": "you could paint a painting of your garden .", "words": ["paint", "your", "of", "you", "garden", "could", "a"]}, {"sent": "do you want me to paint it ?", "words": ["do", "paint", "to", "you", "it", "me"]}, {"sent": "dont stack the saucers up love because theyve still got paint in them havent they ?", "words": ["the", "up", "paint", "them", "dont", "in", "they", "love", "because"]}, {"sent": "and then put it in the paint .", "words": ["the", "put", "paint", "and", "in", "it", "then"]}, {"sent": "if you can fix every paint in it .", "words": ["can", "paint", "if", "fix", "you", "in", "it", "every"]}, {"sent": "lets take this one off because all your paint will go on your nice nice top .", "words": ["off", "all", "your", "paint", "go", "nice", "this", "on", "will", "take", "because"]}, {"sent": "if you wanna paint you should play with the paint .", "words": ["the", "paint", "if", "play", "with", "you", "wanna"]}, {"sent": "so if you paint a nice picture .", "words": ["paint", "if", "so", "nice", "picture", "you", "a"]}, {"sent": "what color shall we paint you ?", "words": ["what", "paint", "you", "we"]}, {"sent": "paint ?", "words": ["paint"]}, {"sent": "that way the paint wont go all over the table .", "words": ["the", "all", "paint", "table", "go", "over", "that"]}, {"sent": "and get paint in her mouth ?", "words": ["paint", "mouth", "and", "get", "in", "her"]}, {"sent": "oil is used in paint some kinds of paint to paint walls and houses and furniture and things like that .", "words": ["paint", "is", "of", "to", "like", "and", "in", "some", "that"]}, {"sent": "you glue and paint ?", "words": ["glue", "paint", "and", "you"]}, {"sent": "you can paint that and put your name on it .", "words": ["can", "put", "paint", "your", "and", "you", "it", "on", "that"]}, {"sent": "why did you waste your paint ?", "words": ["paint", "your", "you", "did", "why"]}, {"sent": "paint .", "words": ["paint"]}, {"sent": "just paint alotof things in the background .", "words": ["the", "in", "paint"]}, {"sent": "and then youve got loads of paint on see ?", "words": ["paint", "of", "and", "on", "see", "then"]}, {"sent": "should we paint should we use the black paint then ?", "words": ["the", "paint", "we", "black", "then"]}, {"sent": "and then were gonna paint it .", "words": ["paint", "and", "it", "then", "were"]}, {"sent": "with paint you know how to paint with paint now dont you ?", "words": ["how", "paint", "to", "with", "you", "dont"]}, {"sent": "oh youre talking about mouse paint huh ?", "words": ["mouse", "paint", "about"]}, {"sent": "or paint on ?", "words": ["on", "paint"]}, {"sent": "some black paint .", "words": ["black", "some", "paint"]}, {"sent": "you need lotsof paint on your brush really .", "words": ["paint", "need", "your", "brush", "you", "on"]}, {"sent": "should we get you some kind of paint that you can use with brushes ?", "words": ["can", "paint", "we", "of", "with", "you", "get", "some", "that"]}, {"sent": "are or were xxx upset that you cant paint ?", "words": ["paint", "you", "are", "that", "were"]}, {"sent": "so what did you paint ?", "words": ["what", "paint", "so", "you", "did"]}, {"sent": "you gonna paint it black ?", "words": ["black", "it", "paint", "you"]}, {"sent": "xxx Tracy xxx we got here first cause the teacher said we can paint .", "words": ["the", "can", "paint", "we", "first", "here"]}, {"sent": "well have to buy more paint things .", "words": ["paint", "to", "have", "more", "buy"]}, {"sent": "dont paint on the table please .", "words": ["the", "paint", "table", "dont", "on"]}, {"sent": "I wanna paint xxx .", "words": ["paint", "wanna"]}, {"sent": "peent as in paint like painting a wall ?", "words": ["in", "paint", "like", "a"]}, {"sent": "and then you paint the house .", "words": ["the", "paint", "you", "and", "house", "then"]}, {"sent": "maybe well paint maybe well make things .", "words": ["make", "paint"]}, {"sent": "paint ?", "words": ["paint"]}, {"sent": "this was a book to paint in .", "words": ["paint", "was", "to", "this", "in", "book", "a"]}, {"sent": "okay student paint .", "words": ["paint"]}, {"sent": "and youve got blue ink on your hands and were now putting yellow paint on top .", "words": ["your", "paint", "blue", "and", "on", "yellow", "were"]}, {"sent": "hes using many different colors of paint .", "words": ["of", "paint"]}, {"sent": "so theyre not gonna watch mom paint anymore are they ?", "words": ["watch", "paint", "so", "they", "are", "not"]}, {"sent": "oh heres pizza .", "words": ["pizza"]}, {"sent": "pizza ?", "words": ["pizza"]}, {"sent": "mm pizza .", "words": ["pizza"]}, {"sent": "oh the pizza huh ?", "words": ["the", "pizza"]}, {"sent": "yeah and she doesnt eat pizza that looks like th with all that stuff on it either so I dunno .", "words": ["all", "so", "with", "like", "pizza", "and", "eat", "it", "on", "that", "she"]}, {"sent": "youre having pizza .", "words": ["pizza"]}, {"sent": "how many slices of pizza are there ?", "words": ["how", "of", "pizza", "there", "are"]}, {"sent": "put the pizza on a plate .", "words": ["the", "put", "plate", "pizza", "on", "a"]}, {"sent": "you dont know what that is but heres some pizza .", "words": ["what", "is", "pizza", "you", "dont", "some", "that", "but"]}, {"sent": "pizza ?", "words": ["pizza"]}, {"sent": "Shakys pizza parlor speaking .", "words": ["pizza"]}, {"sent": "thats for pizza .", "words": ["for", "pizza"]}, {"sent": "another piece of pizza ?", "words": ["another", "of", "pizza"]}, {"sent": "back to the pizza .", "words": ["the", "to", "back", "pizza"]}, {"sent": "does daddy eat pizza ?", "words": ["eat", "does", "pizza"]}, {"sent": "should we get some pizza for lunch ?", "words": ["for", "we", "pizza", "get", "some"]}, {"sent": "you wanna find where your pizza is ?", "words": ["where", "your", "is", "pizza", "you", "wanna", "find"]}, {"sent": "more pizza ?", "words": ["more", "pizza"]}, {"sent": "is that pizza ?", "words": ["that", "pizza", "is"]}, {"sent": "does Wes want a bite of pizza no ?", "words": ["does", "of", "pizza", "bite", "a"]}, {"sent": "pizza and ice cream ?", "words": ["ice", "ice cream", "pizza", "and"]}, {"sent": "oh my they have so much pizza that its hanging out the window now .", "words": ["the", "my", "so", "pizza", "have", "they", "that", "out", "window", "much"]}, {"sent": "well Trevors in the middle of eating a pizza .", "words": ["the", "of", "pizza", "in", "a"]}, {"sent": "oh the pizza is too tight in there .", "words": ["the", "is", "pizza", "there", "in", "too"]}, {"sent": "and pizza ?", "words": ["pizza", "and"]}, {"sent": "pizza and cookies .", "words": ["pizza", "and"]}, {"sent": "oh my pizza .", "words": ["my", "pizza"]}, {"sent": "he loves pizza .", "words": ["he", "pizza"]}, {"sent": "pizza ?", "words": ["pizza"]}, {"sent": "you got the pizza ?", "words": ["the", "pizza", "you"]}, {"sent": "the pizza .", "words": ["the", "pizza"]}, {"sent": "youre making a pizza parlor ?", "words": ["pizza", "a"]}, {"sent": "you like that colorful piece of pizza .", "words": ["of", "like", "pizza", "you", "that"]}, {"sent": "can you find some pizza ?", "words": ["can", "pizza", "you", "find", "some"]}, {"sent": "the pizza the spaghetti uh spoons she loves spoons .", "words": ["the", "she", "pizza", "spaghetti"]}, {"sent": "youknow shell have pizza but itll be a few pieces of youknow a few little bites of pizza not the full slice .", "words": ["the", "full", "of", "be", "pizza", "little", "have", "not", "but", "a"]}, {"sent": "pizza .", "words": ["pizza"]}, {"sent": "why is it pizza ?", "words": ["pizza", "it", "why", "is"]}, {"sent": "whats for shall we have our pizza for our lunch ?", "words": ["for", "we", "pizza", "our", "have"]}, {"sent": "my mom at her house and we go over to her house quite often has clay pizza and clay spaghetti .", "words": ["my", "we", "go", "to", "spaghetti", "pizza", "and", "house", "at", "over", "her"]}, {"sent": "an entire pizza pie instead of just slices .", "words": ["of", "an", "pizza"]}, {"sent": "pizza thats right .", "words": ["pizza"]}, {"sent": "oh back to the pizza ?", "words": ["the", "to", "back", "pizza"]}, {"sent": "he brought some pizza .", "words": ["some", "he", "pizza"]}, {"sent": "pretend pizza .", "words": ["pretend", "pizza"]}, {"sent": "pizza in there ?", "words": ["there", "in", "pizza"]}, {"sent": "look at the pizza .", "words": ["the", "at", "pizza", "look"]}, {"sent": "pizza .", "words": ["pizza"]}, {"sent": "did you tell Amanda what type of pizza you like to eat ?", "words": ["what", "of", "to", "like", "pizza", "you", "eat", "did"]}, {"sent": "and a piece of pizza .", "words": ["of", "pizza", "a", "and"]}, {"sent": "thats right youve got pizza .", "words": ["pizza"]}, {"sent": "youre gonna turn into a pizza if you eat any more pizzas .", "words": ["if", "into", "any", "pizza", "you", "eat", "more", "a"]}, {"sent": "should we put the pizza away ?", "words": ["the", "put", "we", "pizza", "away"]}, {"sent": "pizza ?", "words": ["pizza"]}, {"sent": "the um pizza .", "words": ["the", "pizza"]}, {"sent": "she says I heard there was some pizza in your car I love pizza can I have some pizza too ?", "words": ["can", "your", "was", "car", "pizza", "have", "there", "in", "love", "too", "some", "she"]}, {"sent": "the sauce pizza thats true but whada you like to eat mom whadoes mommy like to eat on her pizza ?", "words": ["the", "sauce", "to", "like", "pizza", "you", "eat", "on", "her", "but"]}, {"sent": "for pizza .", "words": ["for", "pizza"]}, {"sent": "bet you we can make a whole pizza pie .", "words": ["can", "we", "pizza", "you", "make", "a"]}, {"sent": "did you eat the other slice of pizza ?", "words": ["the", "of", "pizza", "other", "eat", "you", "did"]}, {"sent": "mm you want some pizza huh ?", "words": ["some", "pizza", "you"]}, {"sent": "oh did we forget we had pizza on the oven ?", "words": ["the", "we", "pizza", "did", "on", "oven"]}, {"sent": "and now she wants the pizza .", "words": ["the", "she", "pizza", "and"]}, {"sent": "more pizza ?", "words": ["more", "pizza"]}, {"sent": "you want pizza ?", "words": ["pizza", "you"]}, {"sent": "piece of pizza ?", "words": ["of", "pizza"]}, {"sent": "thats pizza .", "words": ["pizza"]}, {"sent": "there was only once piece of pizza but I know in the past hes picked up the pizza and said pizza .", "words": ["the", "up", "of", "was", "pizza", "and", "there", "in", "but"]}, {"sent": "so she knows that youknow that we make pizza and you cut the pizza and stuff like that .", "words": ["the", "cut", "we", "so", "like", "pizza", "and", "you", "that", "make", "she"]}, {"sent": "I got Ive eaten my pizza .", "words": ["my", "pizza"]}, {"sent": "and pizza .", "words": ["pizza", "and"]}, {"sent": "waffle and pizza ?", "words": ["pizza", "and"]}, {"sent": "pizza ?", "words": ["pizza"]}, {"sent": "more pizza .", "words": ["more", "pizza"]}, {"sent": "yah thats the pizza .", "words": ["the", "pizza"]}, {"sent": "and put your pizza on there .", "words": ["put", "your", "pizza", "and", "there", "on"]}, {"sent": "how many pieces of pizza do we have ?", "words": ["do", "how", "we", "of", "pizza", "have"]}, {"sent": "would I like pizza ?", "words": ["like", "pizza", "would"]}, {"sent": "that cuts pizza .", "words": ["that", "pizza"]}, {"sent": "do you want some pizza ?", "words": ["pizza", "do", "some", "you"]}, {"sent": "pizza pizza pizza .", "words": ["pizza"]}, {"sent": "the waffle is not as interesting as the pizza ?", "words": ["the", "not", "pizza", "is"]}, {"sent": "babies dont use pizza cutters .", "words": ["dont", "pizza"]}, {"sent": "we eat pizza a lot .", "words": ["we", "a lot", "pizza", "eat", "a"]}, {"sent": "no just the pizza ?", "words": ["the", "pizza"]}, {"sent": "lets order a pizza huh .", "words": ["pizza", "a"]}, {"sent": "pizza ?", "words": ["pizza"]}, {"sent": "oh yeah the p oh the pizza slices .", "words": ["the", "pizza"]}, {"sent": "he loved the pizza wedges the most .", "words": ["the", "he", "pizza"]}, {"sent": "is that my pizza ?", "words": ["my", "that", "pizza", "is"]}, {"sent": "cuts the pizza .", "words": ["the", "pizza"]}, {"sent": "Imean do we have pizza at home ?", "words": ["do", "we", "pizza", "have", "at", "home"]}, {"sent": "daddy have a pizza cutter like that ?", "words": ["like", "pizza", "have", "that", "a"]}, {"sent": "oh my goodness look its pizza .", "words": ["my", "pizza", "look"]}, {"sent": "thats another piece of pizza .", "words": ["another", "of", "pizza"]}, {"sent": "uhhuh are you sharing the pizza ?", "words": ["the", "are", "pizza", "you"]}, {"sent": "look look its pizza .", "words": ["pizza", "look"]}, {"sent": "you can take the pizza cutter with you .", "words": ["can", "the", "with", "pizza", "you", "take"]}, {"sent": "pizza .", "words": ["pizza"]}, {"sent": "one day in the market Aladin took some bread .", "words": ["the", "some", "in", "bread"]}, {"sent": "do you think we should put chocolate chips in our banana bread Honey ?", "words": ["do", "think", "put", "bread", "we", "chocolate", "you", "our", "banana", "in"]}, {"sent": "piece of bread .", "words": ["of", "bread"]}, {"sent": "because you say bread .", "words": ["bread", "say", "because", "you"]}, {"sent": "thats Daddys pepperoni bread hes been asking Mommy to make for three days .", "words": ["to", "bread", "for", "make"]}, {"sent": "bread .", "words": ["bread"]}, {"sent": "um and bread .", "words": ["bread", "and"]}, {"sent": "Papas bread .", "words": ["bread"]}, {"sent": "thats the bread .", "words": ["the", "bread"]}, {"sent": "its thats a bread roll .", "words": ["bread", "a"]}, {"sent": "byebye bread .", "words": ["bread"]}, {"sent": "how bout the bread can you put the bread over there too ?", "words": ["the", "can", "how", "bread", "put", "you", "there", "too", "over"]}, {"sent": "would you like some toast or some bread ?", "words": ["bread", "toast", "like", "you", "would", "some"]}, {"sent": "look at my hand I made with my bread .", "words": ["my", "look", "bread", "hand", "with", "at"]}, {"sent": "and did did the pigeons eat all your bread Anne ?", "words": ["the", "all", "bread", "your", "and", "eat", "did"]}, {"sent": "is that bread ?", "words": ["bread", "that", "is"]}, {"sent": "theres a piece of bread oh theres the apple juice .", "words": ["the", "bread", "of", "apple", "juice", "a"]}, {"sent": "another loaf of bread .", "words": ["another", "of", "bread"]}, {"sent": "bread .", "words": ["bread"]}, {"sent": "or would you prefer bread ?", "words": ["bread", "you", "would"]}, {"sent": "bread ?", "words": ["bread"]}, {"sent": "its time for me to put the garlic bread in a minute .", "words": ["the", "put", "for", "bread", "to", "in", "me", "a"]}, {"sent": "some bread to the .", "words": ["to", "some", "bread", "the"]}, {"sent": "would you like some more bread ?", "words": ["bread", "like", "you", "more", "would", "some"]}, {"sent": "bread .", "words": ["bread"]}, {"sent": "some bread .", "words": ["some", "bread"]}, {"sent": "oh bread .", "words": ["bread"]}, {"sent": "I make bread xxx .", "words": ["bread", "make"]}, {"sent": "and the bread .", "words": ["the", "bread", "and"]}, {"sent": "and bread .", "words": ["bread", "and"]}, {"sent": "are we gonna give old bread to them ?", "words": ["old", "bread", "we", "to", "them", "are", "give"]}, {"sent": "is there any bread in there or not ?", "words": ["bread", "is", "any", "there", "in", "not"]}, {"sent": "and bread .", "words": ["bread", "and"]}, {"sent": "its not made from bread crumbs but it looks like bread crumbs doesnt it ?", "words": ["bread", "like", "it", "not", "but"]}, {"sent": "did you want a piece of this bread ?", "words": ["bread", "of", "you", "this", "did", "a"]}, {"sent": "look at all that bread on the wagon .", "words": ["the", "all", "bread", "look", "at", "on", "that"]}, {"sent": "um she liked the bread .", "words": ["the", "bread", "she"]}, {"sent": "bread or toast .", "words": ["bread", "toast"]}, {"sent": "or the sides of the bread .", "words": ["the", "of", "bread"]}, {"sent": "xxx he is nibble the bread xxx .", "words": ["the", "bread", "he", "is"]}, {"sent": "theres bread .", "words": ["bread"]}, {"sent": "what does Mommy do with bread ?", "words": ["what", "bread", "do", "does", "with"]}, {"sent": "wonder bread ooh .", "words": ["bread"]}, {"sent": "do some with the bread too ?", "words": ["the", "do", "bread", "with", "too", "some"]}, {"sent": "I dont think xxx bread then .", "words": ["bread", "dont", "think", "then"]}, {"sent": "oh thats alotof bread isnt it ?", "words": ["bread", "it"]}, {"sent": "see hes giving them bread crumbs .", "words": ["see", "bread", "them"]}, {"sent": "if she sees you go for the bread .", "words": ["the", "bread", "for", "if", "go", "you", "she"]}, {"sent": "uhuh lets eat the bread lets eat the bread .", "words": ["eat", "the", "bread"]}, {"sent": "bread .", "words": ["bread"]}, {"sent": "and bread .", "words": ["bread", "and"]}, {"sent": "bread .", "words": ["bread"]}, {"sent": "I want some bread I wanna take it .", "words": ["bread", "it", "wanna", "some", "take"]}, {"sent": "Im not sure what has happened to the bread knife .", "words": ["the", "what", "bread", "to", "knife", "not"]}, {"sent": "a slice of bread ?", "words": ["of", "bread", "a"]}, {"sent": "he might like some pita bread .", "words": ["some", "bread", "he", "like"]}, {"sent": "should we put the other bread on top ?", "words": ["the", "put", "bread", "we", "other", "on"]}, {"sent": "you want a bite of bread ?", "words": ["bread", "of", "you", "bite", "a"]}, {"sent": "and I went to see the ginger bread man .", "words": ["the", "bread", "to", "and", "see"]}, {"sent": "gonna eat the bread roll is it ?", "words": ["the", "bread", "is", "eat", "it"]}, {"sent": "some bread ?", "words": ["some", "bread"]}, {"sent": "you make some bread and butter .", "words": ["bread", "and", "you", "butter", "some", "make"]}, {"sent": "first of all weve got to put this bread away .", "words": ["all", "put", "bread", "of", "to", "first", "this", "away"]}, {"sent": "the orange bread is nice isnt it ?", "words": ["the", "bread", "is", "nice", "orange", "it"]}, {"sent": "bread .", "words": ["bread"]}, {"sent": "mm the yummy bread .", "words": ["the", "bread"]}, {"sent": "well I hadta move the box so that I could open the bread bin to get the bread out to make toast .", "words": ["the", "bread", "box", "toast", "open", "so", "to", "get", "make", "out", "could", "that"]}, {"sent": "bread and stuff but .", "words": ["bread", "but", "and"]}, {"sent": "uh should we get you another piece of bread ?", "words": ["bread", "we", "of", "you", "get", "another"]}, {"sent": "that bread .", "words": ["bread", "that"]}, {"sent": "does that bread taste yummy ?", "words": ["does", "bread", "that", "taste"]}, {"sent": "bread looks nice and soft .", "words": ["soft", "bread", "nice", "and"]}, {"sent": "is that bread nice ?", "words": ["bread", "nice", "that", "is"]}, {"sent": "yeah you got that bread .", "words": ["bread", "that", "you"]}, {"sent": "if I could give you bread like that I would .", "words": ["bread", "if", "like", "you", "would", "give", "that", "could"]}, {"sent": "that my bread .", "words": ["bread", "my", "that"]}, {"sent": "mhm the bread .", "words": ["the", "bread"]}, {"sent": "this bread .", "words": ["this", "bread"]}, {"sent": "that I think thats like a piece of bread .", "words": ["think", "bread", "of", "like", "that", "a"]}, {"sent": "thats your left remember you just said that and thats holding bread .", "words": ["bread", "your", "and", "you", "that"]}, {"sent": "hm I want bread .", "words": ["bread"]}, {"sent": "and do you know how to make garlic bread ?", "words": ["do", "how", "bread", "to", "you", "and", "make"]}, {"sent": "eat jam and bread .", "words": ["eat", "bread", "and"]}, {"sent": "you mix the bread ?", "words": ["the", "bread", "you"]}, {"sent": "bread ?", "words": ["bread"]}, {"sent": "now please have some bread .", "words": ["have", "some", "bread"]}, {"sent": "youre sposta eat potatoes youre eating bread .", "words": ["eat", "bread"]}, {"sent": "is he giving the elephant bread ?", "words": ["the", "bread", "is", "elephant", "he"]}, {"sent": "brioche bread and some .", "words": ["some", "bread", "and"]}, {"sent": "and the bread even hes never had bread that looks like that .", "words": ["the", "bread", "like", "and", "that"]}, {"sent": "bread crumbs .", "words": ["bread"]}, {"sent": "rub it all in so it all looks like bread crumbs .", "words": ["all", "bread", "so", "like", "in", "it"]}, {"sent": "bread and this and that .", "words": ["this", "bread", "that", "and"]}, {"sent": "you have bread .", "words": ["have", "bread", "you"]}, {"sent": "he knows bread cheese the drinks the bottle waffle .", "words": ["the", "bread", "he", "bottle", "cheese"]}, {"sent": "okay this is all of our bread .", "words": ["all", "bread", "is", "of", "our", "this"]}, {"sent": "no more bread .", "words": ["bread", "more"]}, {"sent": "I think I actually found a piece of bread .", "words": ["of", "think", "bread", "a"]}, {"sent": "lets go sit .", "words": ["sit", "go"]}, {"sent": "Mommy sit you down .", "words": ["sit", "down", "you"]}, {"sent": "you want Mommy to sit there ?", "words": ["to", "sit", "there", "you"]}, {"sent": "sit him down .", "words": ["sit", "down", "him"]}, {"sent": "sit you down Cathy xxx standing .", "words": ["sit", "down", "you"]}, {"sent": "sit up .", "words": ["up", "sit"]}, {"sent": "do you wanna sit in the green chair ?", "words": ["the", "do", "chair", "you", "in", "wanna", "green", "sit"]}, {"sent": "sit down Mummy .", "words": ["sit", "down"]}, {"sent": "thats a silly thing to sit on Thomas .", "words": ["on", "to", "sit", "a"]}, {"sent": "do you wanna give me those magazines so you can sit on the rocking chair ?", "words": ["can", "do", "the", "those", "so", "sit", "rocking chair", "chair", "you", "wanna", "on", "me", "give"]}, {"sent": "yeah why dont you come sit here ?", "words": ["you", "here", "dont", "sit", "why"]}, {"sent": "would you sit on my lap ?", "words": ["my", "you", "would", "on", "sit"]}, {"sent": "can I come in your store and sit down for a little while ?", "words": ["can", "for", "your", "store", "and", "little", "in", "sit", "down", "a"]}, {"sent": "go sit in the living room and watch some t v and sit in those big comfortable chairs .", "words": ["the", "watch", "those", "go", "living room", "and", "in", "room", "some", "sit", "big"]}, {"sent": "and Ill sit here and Ill go .", "words": ["here", "sit", "go", "and"]}, {"sent": "I sit you on there a second dont I ?", "words": ["you", "there", "dont", "on", "sit", "a"]}, {"sent": "sit down please Thomas .", "words": ["sit", "down"]}, {"sent": "go sit down .", "words": ["sit", "down", "go"]}, {"sent": "dont dont sit dont sit on the road .", "words": ["on", "dont", "sit", "the"]}, {"sent": "you gonna sit in that chair ?", "words": ["chair", "you", "in", "sit", "that"]}, {"sent": "lets sit in your chair and have your apple juice .", "words": ["your", "chair", "and", "have", "in", "apple", "sit", "juice"]}, {"sent": "wanna come sit over here by mommy and do it ?", "words": ["do", "by", "here", "and", "it", "wanna", "sit", "over"]}, {"sent": "hey watch me sit on this guy and flatten him out like a pancake .", "words": ["watch", "pancake", "like", "and", "this", "him", "out", "on", "me", "sit", "a"]}, {"sent": "are you gonna sit down ?", "words": ["down", "sit", "are", "you"]}, {"sent": "sit an hour .", "words": ["sit", "an"]}, {"sent": "sit down and eat .", "words": ["eat", "sit", "down", "and"]}, {"sent": "xxx sit down .", "words": ["sit", "down"]}, {"sent": "come and sit over here then .", "words": ["then", "and", "here", "sit", "over"]}, {"sent": "just sit down and you can start eating about now .", "words": ["can", "you", "and", "about", "sit", "down"]}, {"sent": "go and sit over there .", "words": ["go", "and", "there", "sit", "over"]}, {"sent": "xxx sit down in your chair .", "words": ["your", "chair", "in", "sit", "down"]}, {"sent": "that goes on the sand sothat you can sit on it .", "words": ["the", "can", "you", "it", "on", "sit", "that"]}, {"sent": "or sit on the subway to Brooklyn .", "words": ["on", "the", "sit", "to"]}, {"sent": "sit down .", "words": ["sit", "down"]}, {"sent": "just tidying the cushions so that Nana and Granddad can sit down when they arrive .", "words": ["the", "can", "when", "so", "and", "they", "sit", "down", "that"]}, {"sent": "and I will sit with him on my lap and well look in and Ill let him do the hand signs and he enjoys that too like he thinks its very cool to watch the baby do it .", "words": ["the", "like", "hand", "with", "it", "on", "that", "to", "and", "him", "too", "will", "do", "my", "watch", "in", "he", "look", "sit"]}, {"sent": "we could sit and do nothing .", "words": ["do", "we", "and", "sit", "could"]}, {"sent": "no no sit down sit down .", "words": ["sit", "down"]}, {"sent": "hey sit down now .", "words": ["sit", "down"]}, {"sent": "where do you wanna sit ?", "words": ["where", "do", "you", "wanna", "sit"]}, {"sent": "you sit next to Mama .", "words": ["to", "sit", "you"]}, {"sent": "when people have been naughty they useta make them sit in there .", "words": ["when", "them", "have", "they", "in", "there", "sit", "make", "naughty"]}, {"sent": "okay you gonna sit with me ?", "words": ["me", "sit", "with", "you"]}, {"sent": "sit down Jack .", "words": ["sit", "down"]}, {"sent": "sit down please sweetheart .", "words": ["sit", "down"]}, {"sent": "sit right there hell come .", "words": ["there", "sit"]}, {"sent": "are you going to just going to sit still on the potty ?", "words": ["the", "to", "you", "are", "potty", "on", "sit"]}, {"sent": "sit down David .", "words": ["sit", "down"]}, {"sent": "and see right here they sit and have ice cream .", "words": ["ice cream", "and", "here", "they", "have", "ice", "see", "sit"]}, {"sent": "well maybe if you sit him down .", "words": ["if", "you", "him", "sit", "down"]}, {"sent": "you wanna sit down and roll the ball ?", "words": ["the", "and", "you", "ball", "wanna", "sit", "down"]}, {"sent": "sit still now .", "words": ["sit"]}, {"sent": "sit up straight .", "words": ["up", "sit"]}, {"sent": "Samantha sit down .", "words": ["sit", "down"]}, {"sent": "sit down and play with me .", "words": ["play", "with", "and", "me", "sit", "down"]}, {"sent": "are you gonna sit still ?", "words": ["sit", "are", "you"]}, {"sent": "she probably thought if I sit here longer Im going to get RickKrispies all over my tail .", "words": ["all", "my", "if", "to", "here", "get", "sit", "over", "she"]}, {"sent": "and then he can sit there xxx .", "words": ["can", "then", "and", "there", "he", "sit"]}, {"sent": "Im gonna move in there so I can sit because hes xxx .", "words": ["can", "so", "there", "in", "sit", "because"]}, {"sent": "and it was easier for you to sit in your highchair wasnt it ?", "words": ["for", "your", "was", "to", "you", "and", "in", "it", "sit"]}, {"sent": "come and sit here darling .", "words": ["here", "sit", "and"]}, {"sent": "theres just about enough room for them to sit next to eachother in the car isnt there ?", "words": ["the", "for", "to", "car", "them", "there", "in", "room", "about", "sit"]}, {"sent": "do you think we could sit on the couch ?", "words": ["the", "do", "think", "we", "sit", "couch", "you", "on", "could"]}, {"sent": "you come and sit down .", "words": ["sit", "down", "and", "you"]}, {"sent": "Megan sit down and try to build something .", "words": ["to", "and", "build", "sit", "down", "try", "try to"]}, {"sent": "sit down .", "words": ["sit", "down"]}, {"sent": "get up here and sit down .", "words": ["up", "and", "get", "here", "sit", "down"]}, {"sent": "Mommys gonna take your clothes off because youre a mess sit .", "words": ["off", "your", "sit", "take", "because", "a"]}, {"sent": "um do you wanna sit down and look at some pictures ?", "words": ["do", "you", "and", "at", "wanna", "look", "some", "sit", "down"]}, {"sent": "go and sit down .", "words": ["sit", "down", "and", "go"]}, {"sent": "because Id sort of like to sit down .", "words": ["of", "to", "like", "sit", "down", "because"]}, {"sent": "did you sit on Mrs Andersons knee ?", "words": ["knee", "you", "did", "on", "sit"]}, {"sent": "sit down dolly .", "words": ["sit", "down"]}, {"sent": "sit down ?", "words": ["sit", "down"]}, {"sent": "no you sit down stay and eat your dinner .", "words": ["your", "and", "you", "eat", "stay", "sit", "down"]}, {"sent": "if you sit up .", "words": ["up", "sit", "if", "you"]}, {"sent": "well you hafta sit over here .", "words": ["here", "sit", "over", "you"]}, {"sent": "sit right Joey .", "words": ["sit"]}, {"sent": "youre gonna sit there in your chair and eat your breakfast .", "words": ["your", "chair", "and", "eat", "there", "in", "sit"]}, {"sent": "oh lets sit up at the table and draw .", "words": ["up", "the", "table", "draw", "and", "at", "sit"]}, {"sent": "go and sit over there nicely and eat it .", "words": ["go", "and", "eat", "there", "it", "sit", "over"]}, {"sent": "well thats okay he can sit down .", "words": ["can", "sit", "down", "he"]}, {"sent": "Bobs gonna sit down .", "words": ["sit", "down"]}, {"sent": "do you wanna sit the monkey on the back ?", "words": ["the", "do", "back", "monkey", "you", "wanna", "on", "sit"]}, {"sent": "if they dont have if she didnt have all this distraction shell just sit there and let you read the book to her .", "words": ["the", "all", "if", "to", "and", "have", "dont", "they", "this", "there", "you", "read", "book", "sit", "her", "she"]}, {"sent": "can I sit down with you ?", "words": ["can", "with", "you", "sit", "down"]}, {"sent": "now sit back in that chair .", "words": ["back", "chair", "in", "sit", "that"]}, {"sent": "sit still .", "words": ["sit"]}, {"sent": "youre going to sit .", "words": ["to", "sit"]}, {"sent": "sit up .", "words": ["up", "sit"]}, {"sent": "you sit on that .", "words": ["on", "sit", "that", "you"]}, {"sent": "sit up .", "words": ["up", "sit"]}, {"sent": "did you sit down ?", "words": ["sit", "down", "did", "you"]}, {"sent": "sit down .", "words": ["sit", "down"]}, {"sent": "no sit down .", "words": ["sit", "down"]}, {"sent": "sit up sit up .", "words": ["up", "sit"]}, {"sent": "just look at it lets sit down .", "words": ["look", "it", "at", "sit", "down"]}, {"sent": "come over here and sit down .", "words": ["and", "here", "sit", "down", "over"]}, {"sent": "well wheres he gonna sit ?", "words": ["sit", "he"]}, {"sent": "Imean your face and all is like that .", "words": ["all", "your", "face", "is", "like", "and", "that"]}, {"sent": "but hes got hives on his face and its not severe but to you know Doctor xxx always told me that if he has a reaction that I should give him Benadryl because the reaction could be worse later .", "words": ["the", "me", "could", "not", "be", "on", "that", "face", "if", "to", "and", "you", "him", "because", "a", "he", "his", "give", "but"]}, {"sent": "look at that cats face now .", "words": ["at", "that", "face", "look"]}, {"sent": "or else we can lets just wash his hands and face .", "words": ["can", "face", "we", "and", "his", "wash"]}, {"sent": "oh shes making a face isnt she ?", "words": ["she", "face", "a"]}, {"sent": "theres a hippo with no face .", "words": ["with", "face", "a"]}, {"sent": "did it splash on your face ?", "words": ["your", "face", "it", "splash", "did", "on"]}, {"sent": "watch your face on the window .", "words": ["the", "watch", "your", "face", "on", "window"]}, {"sent": "oh show that face to Amanda .", "words": ["show", "to", "that", "face"]}, {"sent": "a face it looks like a face .", "words": ["it", "face", "like", "a"]}, {"sent": "youve got a grubby little face this morning as well .", "words": ["this", "face", "little", "a"]}, {"sent": "heres a funny face .", "words": ["face", "a"]}, {"sent": "if we do Dougal weve got to go onto the face .", "words": ["the", "do", "if", "face", "go", "we", "to"]}, {"sent": "right a clowns face .", "words": ["face", "a"]}, {"sent": "you eat the babys face .", "words": ["eat", "the", "face", "you"]}, {"sent": "yeah thats a face cake uhn isnt it ?", "words": ["it", "face", "cake", "a"]}, {"sent": "this one youve gotta happy face on the back .", "words": ["the", "face", "happy", "back", "this", "on"]}, {"sent": "you funny face .", "words": ["face", "you"]}, {"sent": "you want some on your face ?", "words": ["your", "face", "you", "on", "some"]}, {"sent": "shall we do a face ?", "words": ["do", "face", "a", "we"]}, {"sent": "is she washing her face ?", "words": ["face", "her", "she", "is"]}, {"sent": "xxx scary face .", "words": ["face"]}, {"sent": "youre gonna show Jeannine your cross face ?", "words": ["show", "your", "face"]}, {"sent": "washing his face ?", "words": ["his", "face"]}, {"sent": "apple face ?", "words": ["apple", "face"]}, {"sent": "and what about the sad face ?", "words": ["the", "what", "face", "and", "about", "sad"]}, {"sent": "wow youve painted mummys face .", "words": ["face"]}, {"sent": "it had a little black face with red eyes .", "words": ["red", "face", "with", "little", "black", "it", "a"]}, {"sent": "theres a face .", "words": ["face", "a"]}, {"sent": "now you feel Daddys scratchy face .", "words": ["face", "you"]}, {"sent": "thats a clock face .", "words": ["clock", "face", "a"]}, {"sent": "make a circle around that and that can be her face .", "words": ["can", "face", "be", "her", "and", "make", "around", "that", "a"]}, {"sent": "get rid of that sad face .", "words": ["face", "of", "get", "sad", "that"]}, {"sent": "first he washes his face and hands .", "words": ["face", "first", "and", "he", "his"]}, {"sent": "but dah face doesnt really look back .", "words": ["back", "face", "but", "look"]}, {"sent": "no dont put the blue straw in Purdies face .", "words": ["the", "put", "face", "blue", "dont", "in"]}, {"sent": "look at that happy face .", "words": ["look", "face", "happy", "at", "that"]}, {"sent": "dont put your hand in my face .", "words": ["put", "my", "your", "face", "hand", "dont", "in"]}, {"sent": "he is washing his face right .", "words": ["his", "he", "face", "is"]}, {"sent": "oh you washing your face Nathaniel .", "words": ["your", "face", "you"]}, {"sent": "can you see his little face appeared at the window .", "words": ["can", "the", "face", "little", "you", "at", "his", "see", "window"]}, {"sent": "a face ?", "words": ["face", "a"]}, {"sent": "lets see your face darling .", "words": ["see", "your", "face"]}, {"sent": "hello teddys face .", "words": ["face"]}, {"sent": "thats an angry face isnt it ?", "words": ["an", "it", "face"]}, {"sent": "you covered your face up ?", "words": ["up", "your", "face", "you"]}, {"sent": "think Fraser pull a funny face ?", "words": ["think", "face", "a", "pull"]}, {"sent": "look what this face is called .", "words": ["what", "face", "is", "this", "look"]}, {"sent": "we make a happy face ?", "words": ["face", "happy", "we", "make", "a"]}, {"sent": "you really dont have a line in your face .", "words": ["your", "face", "you", "have", "dont", "in", "a"]}, {"sent": "xxx you want me to splash this chocolate in your face ?", "words": ["your", "face", "to", "chocolate", "you", "this", "in", "splash", "me"]}, {"sent": "should I make a face ?", "words": ["make", "face", "a"]}, {"sent": "I covered his face up .", "words": ["his", "up", "face"]}, {"sent": "Daddys face .", "words": ["face"]}, {"sent": "your face right against that tv .", "words": ["tv", "that", "your", "face"]}, {"sent": "face .", "words": ["face"]}, {"sent": "let me see did you get it all over your face ?", "words": ["all", "your", "face", "me", "you", "get", "it", "did", "see", "over"]}, {"sent": "a face of daddy .", "words": ["of", "face", "a"]}, {"sent": "cmere lemme see your face .", "words": ["see", "your", "face"]}, {"sent": "and where is the towel that the bunny is using to clean to wipe his face ?", "words": ["where", "the", "clean", "bunny", "face", "is", "to", "and", "towel", "his", "wipe", "that"]}, {"sent": "face right in through here .", "words": ["here", "in", "face"]}, {"sent": "got smiley face on it ?", "words": ["on", "it", "face"]}, {"sent": "lets just do his face okay ?", "words": ["his", "do", "face"]}, {"sent": "Im just going to wash my face .", "words": ["to", "my", "wash", "face"]}, {"sent": "got some spaghetti sauce on your face .", "words": ["your", "face", "sauce", "spaghetti", "on", "some"]}, {"sent": "theres Pos face and well put some eyes on there .", "words": ["put", "face", "and", "there", "on", "some"]}, {"sent": "you have a glare on your face .", "words": ["your", "face", "you", "have", "on", "a"]}, {"sent": "the surprised face .", "words": ["the", "face"]}, {"sent": "I dont think youll be able to do it with your that crying face .", "words": ["do", "think", "your", "face", "to", "be", "with", "dont", "it", "that"]}, {"sent": "look at his little face .", "words": ["look", "face", "little", "at", "his"]}, {"sent": "with her big brown eyes and her smily face .", "words": ["face", "brown", "her", "with", "and", "big"]}, {"sent": "come here honey come and wash your face and hands .", "words": ["your", "face", "and", "here", "wash"]}, {"sent": "thats a funny face .", "words": ["face", "a"]}, {"sent": "you put your cheeky face on ?", "words": ["put", "your", "face", "you", "on"]}, {"sent": "youre going red in the face .", "words": ["red", "the", "in", "face"]}, {"sent": "his face .", "words": ["his", "face"]}, {"sent": "his face his face is annoying him a xxx .", "words": ["face", "is", "him", "his", "a"]}, {"sent": "oh my did you feel that on your face ?", "words": ["my", "your", "face", "you", "did", "on", "that"]}, {"sent": "show how how the face changes .", "words": ["show", "the", "how", "face"]}, {"sent": "wash your face .", "words": ["your", "wash", "face"]}, {"sent": "make a big smily face .", "words": ["big", "make", "face", "a"]}, {"sent": "thats Scoops face isnt it ?", "words": ["it", "face"]}, {"sent": "just wipe your face .", "words": ["wipe", "your", "face"]}, {"sent": "why do you wash your face before eating ?", "words": ["do", "your", "face", "you", "wash", "why"]}, {"sent": "lemme see da face .", "words": ["see", "face"]}, {"sent": "his face is up here .", "words": ["up", "face", "is", "here", "his"]}, {"sent": "so were still looking for Maisys face .", "words": ["so", "for", "were", "face"]}, {"sent": "you can do you make a face .", "words": ["can", "do", "face", "you", "make", "a"]}, {"sent": "I havent got a red face .", "words": ["red", "face", "a"]}, {"sent": "look at the face .", "words": ["the", "at", "face", "look"]}, {"sent": "Ill just give your face a little wipe .", "words": ["your", "face", "little", "give", "wipe", "a"]}, {"sent": "what a funny face .", "words": ["what", "face", "a"]}, {"sent": "theres something on your face ?", "words": ["on", "your", "face"]}, {"sent": "yeah some of them do put alotof makeup on their face .", "words": ["do", "put", "their", "face", "of", "them", "on", "some"]}, {"sent": "Ill go blow up his face ?", "words": ["up", "face", "go", "his", "blow"]}, {"sent": "is it a happy face or a sad face ?", "words": ["face", "happy", "is", "it", "sad", "a"]}, {"sent": "you wipe your face while daddy wipes your bum .", "words": ["wipe", "your", "face", "you"]}, {"sent": "she painted lovely smiley face on it .", "words": ["face", "on", "it", "she"]}, {"sent": "wheres the dogs face ?", "words": ["the", "face"]}, {"sent": "hah he went boom and fell on his face .", "words": ["face", "and", "his", "he", "on"]}, {"sent": "sing a cry Felipe .", "words": ["sing", "cry", "a"]}, {"sent": "know what was making me cry all the more the policeman was saying about the camera ?", "words": ["the", "what", "all", "camera", "was", "cry", "more", "about", "me"]}, {"sent": "dont cry .", "words": ["dont", "cry"]}, {"sent": "did Jimmy cry .", "words": ["cry", "did"]}, {"sent": "Amy will cry .", "words": ["will", "cry"]}, {"sent": "oh Kyles not allowed to cry or anything .", "words": ["to", "cry", "not"]}, {"sent": "make panda cry .", "words": ["cry", "make"]}, {"sent": "dont cry .", "words": ["dont", "cry"]}, {"sent": "dont cry and dont .", "words": ["dont", "cry", "and"]}, {"sent": "well it dont cry if it burns your mouth .", "words": ["your", "if", "cry", "mouth", "dont", "it"]}, {"sent": "and you dont wanna make Granddad cry do you ?", "words": ["do", "cry", "you", "and", "dont", "wanna", "make"]}, {"sent": "but three little kittens have found their mittens and they began to cry .", "words": ["mittens", "their", "to", "cry", "little", "have", "and", "they", "but"]}, {"sent": "mommy cry xxx hes ugly .", "words": ["cry"]}, {"sent": "Rachel was at the group on Tuesday but she didnt cry on Tuesday .", "words": ["the", "was", "cry", "at", "on", "she", "but"]}, {"sent": "one of your dollies will cry .", "words": ["of", "cry", "will", "your"]}, {"sent": "dont cry darling .", "words": ["dont", "cry"]}, {"sent": "shall we read a nice story do you wanna read the story about oh we can read the story about Sunnys birthday cry .", "words": ["the", "do", "can", "we", "cry", "nice", "you", "read", "wanna", "about", "story", "a"]}, {"sent": "Ben began to cry then he yelled .", "words": ["he", "to", "cry", "then"]}, {"sent": "did you cry ?", "words": ["cry", "did", "you"]}, {"sent": "is the baby cry ?", "words": ["the", "cry", "is"]}, {"sent": "shes not a cry baby ?", "words": ["cry", "not", "a"]}, {"sent": "then dont cry .", "words": ["dont", "then", "cry"]}, {"sent": "so he doesnt cry does he ?", "words": ["so", "cry", "does", "he"]}, {"sent": "he stopped to cry .", "words": ["to", "cry", "he"]}, {"sent": "did he cry ?", "words": ["cry", "he", "did"]}, {"sent": "oh my god Im gonna cry .", "words": ["cry", "my"]}, {"sent": "xxx cry too .", "words": ["cry", "too"]}, {"sent": "do you want her to cry ?", "words": ["do", "to", "cry", "you", "her"]}, {"sent": "the babies on the bus cry wahwahwah .", "words": ["the", "cry", "on", "bus"]}, {"sent": "why will you cry ?", "words": ["will", "cry", "why", "you"]}, {"sent": "dont bash the cars because youll break them and then youll cry .", "words": ["the", "cry", "them", "and", "dont", "then", "break", "because"]}, {"sent": "oh come on you dont need to cry .", "words": ["need", "need to", "to", "cry", "you", "dont", "on"]}, {"sent": "did he cry ?", "words": ["cry", "he", "did"]}, {"sent": "how does the baby cry Aislinn .", "words": ["the", "does", "how", "cry"]}, {"sent": "did you cry on Mummys birthday with a candle ?", "words": ["cry", "with", "you", "did", "on", "a"]}, {"sent": "who pooed and did cry ?", "words": ["cry", "did", "and", "who"]}, {"sent": "cry boys .", "words": ["cry"]}, {"sent": "shes just a cry baby isnt she ?", "words": ["cry", "she", "a"]}, {"sent": "oh we dont want Sleepy to cry do we ?", "words": ["do", "we", "to", "cry", "dont"]}, {"sent": "you gonna cry ?", "words": ["cry", "you"]}, {"sent": "whos going to cry ?", "words": ["to", "cry"]}, {"sent": "Annall cry and I dont wanna make Anna cry .", "words": ["cry", "and", "dont", "wanna", "make"]}, {"sent": "no Lola doesnt cry .", "words": ["cry"]}, {"sent": "you need me to cry .", "words": ["need", "to", "cry", "you", "me"]}, {"sent": "and whenever he starts to cry all he really needs is someone to rub him and help him .", "words": ["all", "help", "is", "to", "cry", "and", "him", "he"]}, {"sent": "dont cry .", "words": ["dont", "cry"]}, {"sent": "the babys gonna cry .", "words": ["the", "cry"]}, {"sent": "its like a guessing game trying to guess what sort of cry it is .", "words": ["what", "game", "is", "of", "to", "cry", "like", "it", "a"]}, {"sent": "if she grabs your toe it will make you cry and yell .", "words": ["your", "if", "toe", "cry", "you", "and", "it", "will", "make", "she"]}, {"sent": "well youd cry wouldnt you ?", "words": ["cry", "you"]}, {"sent": "let me see you cry .", "words": ["see", "me", "cry", "you"]}, {"sent": "will you cry ?", "words": ["will", "cry", "you"]}, {"sent": "you tell Zara not to cry .", "words": ["to", "cry", "not", "you"]}, {"sent": "you cry ?", "words": ["cry", "you"]}, {"sent": "Eleanor doesnt cry .", "words": ["cry"]}, {"sent": "oh dont cry .", "words": ["dont", "cry"]}, {"sent": "theres no needta cry though is there ?", "words": ["there", "cry", "is"]}, {"sent": "xxx cry .", "words": ["cry"]}, {"sent": "cry .", "words": ["cry"]}, {"sent": "am I gonna cry ?", "words": ["cry", "am"]}, {"sent": "dont cry the minute you dont succeed at something .", "words": ["the", "cry", "you", "dont", "at"]}, {"sent": "sometimes onions make your eyes cry .", "words": ["your", "cry", "make"]}, {"sent": "is that something to cry about ?", "words": ["is", "to", "cry", "about", "that"]}, {"sent": "oh no the dollys going to cry .", "words": ["the", "to", "cry"]}, {"sent": "and then the children would cry because theyd lost their Mother hen and her chicks .", "words": ["the", "their", "cry", "hen", "and", "would", "then", "her", "because"]}, {"sent": "dang you dont hafta cry about it .", "words": ["cry", "you", "dont", "it", "about"]}, {"sent": "why do you want him to cry ?", "words": ["do", "to", "cry", "you", "him", "why"]}, {"sent": "hell cry .", "words": ["cry"]}, {"sent": "I will cry because Ive missed you so much .", "words": ["so", "cry", "you", "will", "much", "because"]}, {"sent": "you cry when you have dirt on your shoes dont you ?", "words": ["your", "cry", "you", "have", "dont", "on", "when"]}, {"sent": "didnt she cry ?", "words": ["cry", "she"]}, {"sent": "you cry .", "words": ["cry", "you"]}, {"sent": "did you cry ?", "words": ["cry", "did", "you"]}, {"sent": "do you cry if somethings funny ?", "words": ["do", "cry", "if", "you"]}, {"sent": "dont cry .", "words": ["dont", "cry"]}, {"sent": "now you wanna cry .", "words": ["cry", "wanna", "you"]}, {"sent": "and did you cry ?", "words": ["cry", "did", "you", "and"]}, {"sent": "if people are sad they cry .", "words": ["if", "cry", "they", "are", "sad"]}, {"sent": "did you cry when you went to have your hair cut at the barbers ?", "words": ["the", "your", "cut", "to", "cry", "you", "have", "did", "at", "hair", "when"]}, {"sent": "dont cry ?", "words": ["dont", "cry"]}, {"sent": "shell cry .", "words": ["cry"]}, {"sent": "you didnt cry .", "words": ["cry", "you"]}, {"sent": "Jwww gets upset as well when you cry doesnt he ?", "words": ["he", "cry", "when", "you"]}, {"sent": "dont cry .", "words": ["dont", "cry"]}, {"sent": "Annall cry if we go out .", "words": ["if", "go", "we", "cry", "out"]}, {"sent": "well I didnt cry but I was little bit sad I couldnt go .", "words": ["go", "was", "cry", "little", "sad", "but"]}, {"sent": "dont cry sweetheart .", "words": ["dont", "cry"]}, {"sent": "did you cry ?", "words": ["cry", "did", "you"]}, {"sent": "oh dont cry .", "words": ["dont", "cry"]}, {"sent": "youve nothing to cry about .", "words": ["to", "cry", "about"]}, {"sent": "did the onions make Thomas cry ?", "words": ["the", "cry", "make", "did"]}, {"sent": "can I cry on your shoulder ?", "words": ["can", "your", "cry", "shoulder", "on"]}, {"sent": "cry .", "words": ["cry"]}, {"sent": "I hope not because Im a little bit too big to cry on a train .", "words": ["train", "to", "cry", "little", "too", "on", "big", "not", "because", "a"]}, {"sent": "I dont want it to go in her eye because shell cry .", "words": ["go", "to", "cry", "eye", "dont", "in", "it", "her", "because"]}, {"sent": "Mama said dont cry didnt she ?", "words": ["dont", "cry", "she"]}, {"sent": "maybe he wont even cry see ?", "words": ["see", "cry", "he"]}, {"sent": "did I cry ?", "words": ["cry", "did"]}, {"sent": "alright dont cry .", "words": ["dont", "cry"]}, {"sent": "show me .", "words": ["show", "me"]}, {"sent": "he should notice these because we watch this show all the time .", "words": ["the", "these", "all", "watch", "we", "show", "this", "he", "because"]}, {"sent": "you show him the pictures and tell him whats in them .", "words": ["the", "show", "them", "and", "you", "him", "in"]}, {"sent": "you wanna show me the cake ?", "words": ["the", "cake", "show", "you", "wanna", "me"]}, {"sent": "Im gonna show you something new .", "words": ["show", "new", "you"]}, {"sent": "you you you show her .", "words": ["show", "her", "you"]}, {"sent": "show if off for a while before you pop it in ?", "words": ["off", "for", "pop", "if", "show", "you", "in", "it", "a"]}, {"sent": "show me .", "words": ["show", "me"]}, {"sent": "show her how you honk his nose .", "words": ["how", "show", "you", "his", "her", "nose"]}, {"sent": "you bring it over here and show me .", "words": ["show", "bring", "you", "here", "and", "it", "me", "over"]}, {"sent": "come and show me .", "words": ["show", "me", "and"]}, {"sent": "Ill show you .", "words": ["show", "you"]}, {"sent": "show me .", "words": ["show", "me"]}, {"sent": "can you show her how you can build a tower ?", "words": ["can", "how", "show", "you", "build", "her", "a"]}, {"sent": "show Mommy the teddybear .", "words": ["show", "the", "teddybear"]}, {"sent": "can I show you something ?", "words": ["can", "you", "show"]}, {"sent": "gonna to stand up and show us .", "words": ["up", "show", "to", "stand", "and", "us"]}, {"sent": "the show will be on in just a little while Abe .", "words": ["the", "show", "be", "will", "little", "in", "on", "a"]}, {"sent": "you show me how they all go together .", "words": ["all", "how", "go", "show", "you", "they", "me"]}, {"sent": "so you wanna watch one more tv show ?", "words": ["watch", "show", "so", "you", "more", "tv", "wanna"]}, {"sent": "can you show pictures with that ?", "words": ["can", "show", "with", "you", "that"]}, {"sent": "and what did they show you Beasey ?", "words": ["what", "show", "and", "you", "they", "did"]}, {"sent": "you show me you show me where .", "words": ["show", "me", "where", "you"]}, {"sent": "what do you think about that show ?", "words": ["what", "do", "think", "show", "you", "about", "that"]}, {"sent": "come show Mommy where the pencil is .", "words": ["the", "where", "is", "show", "pencil"]}, {"sent": "want me to show you ?", "words": ["to", "me", "you", "show"]}, {"sent": "show me red .", "words": ["show", "me", "red"]}, {"sent": "Ill show you .", "words": ["show", "you"]}, {"sent": "okay the show is over .", "words": ["the", "over", "is", "show"]}, {"sent": "xxx show me your teeth ?", "words": ["show", "me", "your"]}, {"sent": "shes got a couple of toys at home that you press the buttons and it will play music for her and shh same thing with com like if shes walking by and theres a commercial or a tv show at the end of a show youknow sometimes theyll have music at the end of a show shell stop to dance .", "words": ["the", "of", "stop", "like", "same", "with", "dance", "by", "have", "it", "that", "home", "for", "if", "to", "play", "you", "and", "tv", "will", "a", "show", "at", "her"]}, {"sent": "can you show Mommy the pickle ?", "words": ["can", "the", "show", "pickle", "you"]}, {"sent": "you show me what you want .", "words": ["show", "me", "what", "you"]}, {"sent": "want me to show you .", "words": ["to", "me", "you", "show"]}, {"sent": "can you show me the girl ?", "words": ["can", "the", "show", "you", "me"]}, {"sent": "go and show your Nanny .", "words": ["show", "your", "and", "go"]}, {"sent": "but like I tried to show him the peas which are about the same size I thought theyd be equally interesting and theyre not .", "words": ["the", "show", "to", "be", "peas", "like", "and", "him", "are", "same", "about", "which", "not", "but"]}, {"sent": "let me show you .", "words": ["show", "me", "you"]}, {"sent": "show John where your ears are .", "words": ["show", "are", "your", "where"]}, {"sent": "you wont show me yet ?", "words": ["show", "me", "you"]}, {"sent": "oh I was gonna show you where China was .", "words": ["show", "was", "where", "you"]}, {"sent": "can you show the baby owls to Manuela ?", "words": ["can", "the", "show", "to", "you"]}, {"sent": "so youve got to show them which path .", "words": ["show", "to", "so", "them", "which"]}, {"sent": "show me show me .", "words": ["show", "me"]}, {"sent": "I want you to show me how to play tictactoe because I I dont know how to .", "words": ["how", "show", "to", "play", "you", "dont", "me", "because"]}, {"sent": "show me where the words are .", "words": ["where", "the", "show", "are", "me"]}, {"sent": "you show me .", "words": ["show", "me", "you"]}, {"sent": "go and show Dimitra .", "words": ["show", "and", "go"]}, {"sent": "lets show him your magic .", "words": ["show", "him", "your"]}, {"sent": "you show me how youre gonna draw a house .", "words": ["how", "show", "draw", "you", "house", "me", "a"]}, {"sent": "show me where is your eye .", "words": ["where", "your", "is", "show", "eye", "me"]}, {"sent": "show me how it crashes .", "words": ["show", "me", "how", "it"]}, {"sent": "so youll hafta show me .", "words": ["show", "so", "me"]}, {"sent": "show it Carey will show ya go see ?", "words": ["go", "show", "it", "will", "see"]}, {"sent": "come here come here let me show you what to do .", "words": ["what", "do", "show", "to", "you", "here", "me"]}, {"sent": "then Ill show you .", "words": ["show", "then", "you"]}, {"sent": "show me .", "words": ["show", "me"]}, {"sent": "can you show me ?", "words": ["can", "me", "show", "you"]}, {"sent": "Ill show you how .", "words": ["show", "how", "you"]}, {"sent": "lemme show you how to do it .", "words": ["do", "how", "show", "to", "you", "it"]}, {"sent": "show baby shapes .", "words": ["show"]}, {"sent": "right right let me show you what .", "words": ["show", "me", "what", "you"]}, {"sent": "theyre watching a puppet show .", "words": ["show", "a"]}, {"sent": "show her how you zipper .", "words": ["how", "show", "you", "zipper", "her"]}, {"sent": "d Ali Cat was painting the direction sign to show drivers which way to go and Dingo .", "words": ["the", "go", "show", "was", "to", "and", "which"]}, {"sent": "and you really want to show it .", "words": ["show", "to", "you", "and", "it", "want to"]}, {"sent": "show me a duck .", "words": ["show", "me", "duck", "a"]}, {"sent": "can you show me Daisys tail honey ?", "words": ["can", "me", "show", "you"]}, {"sent": "show me show me show me show me .", "words": ["show", "me"]}, {"sent": "now let me show you another time hundredth time .", "words": ["show", "me", "another", "you"]}, {"sent": "no Ill show you .", "words": ["show", "you"]}, {"sent": "are you going to show me how how you have your milk on Mummys knee ?", "words": ["how", "knee", "your", "show", "to", "you", "have", "on", "are", "milk", "me"]}, {"sent": "Ill show you ?", "words": ["show", "you"]}, {"sent": "show her where it is .", "words": ["where", "is", "show", "it", "her"]}, {"sent": "let me show you these stripes .", "words": ["show", "me", "these", "you"]}, {"sent": "you show me where a finger is .", "words": ["where", "is", "show", "you", "finger", "me", "a"]}, {"sent": "yeah it has a little map and it has little sticky things and little pins right to show where we are in New York and in California right .", "words": ["where", "we", "sticky", "to", "show", "and", "little", "in", "it", "are", "a"]}, {"sent": "you show me what you think a lighthouse is like .", "words": ["what", "think", "is", "show", "like", "you", "me", "a"]}, {"sent": "I think you might hafta show me .", "words": ["show", "me", "think", "you"]}, {"sent": "let me show you .", "words": ["show", "me", "you"]}, {"sent": "can you show Mommy ?", "words": ["can", "show", "you"]}, {"sent": "show me show Anna what a froggie des .", "words": ["show", "me", "what", "a"]}, {"sent": "Ill show you how much is half .", "words": ["how", "is", "show", "you", "much"]}, {"sent": "here let me show you how .", "words": ["how", "show", "you", "here", "me"]}, {"sent": "let Mummy show you what you do .", "words": ["show", "what", "do", "you"]}, {"sent": "show me your nose .", "words": ["show", "me", "your", "nose"]}, {"sent": "show me .", "words": ["show", "me"]}, {"sent": "Ill show you .", "words": ["show", "you"]}, {"sent": "you wanna show me the fish ?", "words": ["the", "show", "you", "wanna", "fish", "me"]}, {"sent": "maybe but I I dont know how he would show that he knows it .", "words": ["how", "show", "dont", "it", "he", "would", "that", "but"]}, {"sent": "show xxx .", "words": ["show"]}, {"sent": "Ill show you the drying place its real nice and in fact s if whenever Shems ready to wash up you can wash up here theres a nice xxx .", "words": ["the", "up", "can", "if", "show", "to", "nice", "and", "you", "here", "in", "wash", "a"]}, {"sent": "Ill show you .", "words": ["show", "you"]}, {"sent": "Ill show you what they look like .", "words": ["what", "show", "like", "you", "they", "look"]}, {"sent": "can you show me ?", "words": ["can", "me", "show", "you"]}, {"sent": "Ill show you how .", "words": ["show", "how", "you"]}, {"sent": "I thought you were going to show Jeannine these .", "words": ["these", "show", "to", "you", "were"]}, {"sent": "oh show her .", "words": ["show", "her"]}, {"sent": "show the baby your keys ?", "words": ["show", "the", "keys", "your"]}, {"sent": "you wont be able to show Daddy your drawings when he comes home if you ruin them will you ?", "words": ["your", "if", "show", "to", "be", "them", "you", "he", "will", "when", "home"]}, {"sent": "lemme see blue .", "words": ["see", "blue"]}, {"sent": "blue wiggly worms .", "words": ["blue"]}, {"sent": "the little blue eyes across the table .", "words": ["the", "blue", "table", "little"]}, {"sent": "thats blue .", "words": ["blue"]}, {"sent": "youre giving the blue truck some more juice ?", "words": ["the", "truck", "blue", "more", "some", "juice"]}, {"sent": "you wanna wear your blue jeans today ?", "words": ["your", "blue", "you", "wanna", "jeans"]}, {"sent": "Im blue and green .", "words": ["green", "and", "blue"]}, {"sent": "this man has got a blue bag over his shoulder .", "words": ["blue", "shoulder", "this", "his", "over", "a"]}, {"sent": "its blue .", "words": ["blue"]}, {"sent": "wheres your blue ?", "words": ["your", "blue"]}, {"sent": "because a normal police car would have a red light uh blue light wouldnt it ?", "words": ["red", "light", "car", "blue", "have", "it", "would", "because", "a"]}, {"sent": "can you say blue ?", "words": ["can", "blue", "say", "you"]}, {"sent": "and the blue one .", "words": ["the", "blue", "and"]}, {"sent": "and blue .", "words": ["blue", "and"]}, {"sent": "okay Jonathan I want you Jonathan I want you to take all the blue cups and put them on the board there .", "words": ["the", "all", "put", "to", "them", "you", "blue", "and", "there", "on", "take"]}, {"sent": "a blue straw .", "words": ["blue", "a"]}, {"sent": "red fish blue fish .", "words": ["red", "blue", "fish"]}, {"sent": "blue chicken nuggets ?", "words": ["chicken", "blue"]}, {"sent": "its a bright blue .", "words": ["blue", "a"]}, {"sent": "an eecan blue .", "words": ["an", "blue"]}, {"sent": "its like blue paint isnt it ?", "words": ["it", "paint", "like", "blue"]}, {"sent": "blue juice ?", "words": ["juice", "blue"]}, {"sent": "shes blue .", "words": ["blue"]}, {"sent": "stays on the blue Mommy stays on the yellow now Mommy .", "words": ["on", "the", "yellow", "blue"]}, {"sent": "I dont want the blue aeroplane .", "words": ["the", "dont", "blue"]}, {"sent": "a blue ball .", "words": ["ball", "blue", "a"]}, {"sent": "where is blue ?", "words": ["where", "blue", "is"]}, {"sent": "blue .", "words": ["blue"]}, {"sent": "blue and what do you see there in the sky ?", "words": ["the", "what", "do", "sky", "you", "and", "blue", "there", "in", "see"]}, {"sent": "thats not blue .", "words": ["not", "blue"]}, {"sent": "blue .", "words": ["blue"]}, {"sent": "no this is blue what color is the car .", "words": ["the", "what", "is", "car", "blue", "this"]}, {"sent": "in this blue piece of paper .", "words": ["of", "paper", "blue", "this", "in"]}, {"sent": "no blue .", "words": ["blue"]}, {"sent": "theres only one blue .", "words": ["blue"]}, {"sent": "oh I have something in blue .", "words": ["have", "in", "blue"]}, {"sent": "I would be very startled if I saw a blue monkey .", "words": ["if", "be", "monkey", "blue", "would", "a"]}, {"sent": "then would you like your blue ?", "words": ["your", "like", "you", "blue", "would", "then"]}, {"sent": "blue .", "words": ["blue"]}, {"sent": "and thats a blue gingerbreadman .", "words": ["blue", "a", "and"]}, {"sent": "this is your blue .", "words": ["this", "blue", "your", "is"]}, {"sent": "are you all done with the blue ?", "words": ["the", "all", "with", "you", "blue", "are"]}, {"sent": "except its not a great blue heron .", "words": ["blue", "not", "a"]}, {"sent": "I like your blue line .", "words": ["your", "like", "blue"]}, {"sent": "Mummy hadta buy a new umbrella last www in Stockport because it was so wet and windy that the other umbrella blue inside out and broke .", "words": ["the", "wet", "was", "last", "so", "inside", "windy", "and", "other", "blue", "in", "it", "out", "new", "that", "buy", "because", "a"]}, {"sent": "gimme blue .", "words": ["blue"]}, {"sent": "got a blue dress on .", "words": ["on", "blue", "dress", "a"]}, {"sent": "whatever cutter you use it will be a blue train .", "words": ["train", "be", "you", "blue", "it", "will", "a"]}, {"sent": "the blue train .", "words": ["the", "train", "blue"]}, {"sent": "here we have a house with red bricks and a green door and green windows and blue curtains .", "words": ["red", "we", "with", "and", "here", "have", "blue", "house", "green", "door", "a"]}, {"sent": "its blue .", "words": ["blue"]}, {"sent": "written in blue .", "words": ["in", "blue"]}, {"sent": "are you coloring Lalas face blue ?", "words": ["blue", "are", "face", "you"]}, {"sent": "the blue ones coming now .", "words": ["the", "blue"]}, {"sent": "now find someone wearing blue .", "words": ["blue", "find"]}, {"sent": "and your tshirts a bright blue as well .", "words": ["blue", "your", "a", "and"]}, {"sent": "whos in your blue bowl anybody in your blue bowl can you reach your blue bowl ?", "words": ["bowl", "can", "your", "blue", "you", "in"]}, {"sent": "when the world leaves you feeling blue .", "words": ["the", "blue", "when", "you"]}, {"sent": "color on the back of the blue okay go ahead .", "words": ["the", "back", "go", "of", "blue", "on"]}, {"sent": "red yellow blue and black makes brown .", "words": ["red", "brown", "and", "black", "blue", "yellow"]}, {"sent": "weve lost the blue one havent we ?", "words": ["the", "we", "blue"]}, {"sent": "its blue thats right .", "words": ["blue"]}, {"sent": "bunny blue red .", "words": ["red", "bunny", "blue"]}, {"sent": "blue .", "words": ["blue"]}, {"sent": "its a blue man .", "words": ["blue", "a"]}, {"sent": "when you go cold you go blue not green .", "words": ["go", "not", "blue", "you", "green", "when", "cold"]}, {"sent": "blue ?", "words": ["blue"]}, {"sent": "and blue stripes .", "words": ["blue", "and"]}, {"sent": "blue okay .", "words": ["blue"]}, {"sent": "its a blue cat .", "words": ["cat", "blue", "a"]}, {"sent": "Laura do you want blue ?", "words": ["do", "blue", "you"]}, {"sent": "see a blue ?", "words": ["see", "blue", "a"]}, {"sent": "youve got a blue iron ?", "words": ["blue", "a"]}, {"sent": "djou remember who has all those who has that little hippo collection at their house all those little blue hippos ?", "words": ["all", "those", "their", "little", "blue", "at", "house", "who", "that"]}, {"sent": "it is blue .", "words": ["blue", "it", "is"]}, {"sent": "put the blue one away .", "words": ["the", "put", "away", "blue"]}, {"sent": "thats blue put that one here .", "words": ["here", "put", "that", "blue"]}, {"sent": "a blue snake or a purple snake .", "words": ["blue", "a"]}, {"sent": "its a blue and yellow digger isnt it ?", "words": ["blue", "and", "it", "yellow", "a"]}, {"sent": "Ive got a blue one now .", "words": ["blue", "a"]}, {"sent": "think the big blue bear might be on .", "words": ["the", "think", "be", "bear", "blue", "on", "big"]}, {"sent": "blue .", "words": ["blue"]}, {"sent": "well hafta take the blue one off then .", "words": ["the", "off", "blue", "then", "take"]}, {"sent": "no theyre not they are red and blue lego bricks .", "words": ["red", "and", "blue", "they", "are", "not"]}, {"sent": "blue pink .", "words": ["blue"]}, {"sent": "is it blue ?", "words": ["blue", "it", "is"]}, {"sent": "wheres the blue shirt ?", "words": ["the", "shirt", "blue"]}, {"sent": "get the blue .", "words": ["get", "the", "blue"]}, {"sent": "your blue tshirt .", "words": ["your", "blue"]}, {"sent": "this this ones blue .", "words": ["this", "blue"]}, {"sent": "Ive got a blue and a red lego brick there .", "words": ["red", "blue", "and", "there", "a"]}, {"sent": "is that blue ?", "words": ["blue", "that", "is"]}, {"sent": "the waters blue .", "words": ["the", "blue"]}, {"sent": "blue shoes .", "words": ["blue"]}, {"sent": "built a nice blue Gumdrop up havent we ?", "words": ["up", "we", "nice", "blue", "a"]}, {"sent": "so youve got three red and one blue .", "words": ["red", "so", "blue", "and"]}, {"sent": "Adam wouldnt you like to ride on the blue truck instead of the red one ?", "words": ["the", "red", "of", "to", "truck", "like", "you", "blue", "on", "ride"]}, {"sent": "blue trucks .", "words": ["blue"]}, {"sent": "night night yeah thats right because Im sleepy .", "words": ["sleepy", "because"]}, {"sent": "youre sleepy .", "words": ["sleepy"]}, {"sent": "she I think shes sleepy .", "words": ["think", "sleepy", "she"]}, {"sent": "I think Josephs getting a little sleepy .", "words": ["think", "sleepy", "little", "a"]}, {"sent": "is you sleepy ?", "words": ["sleepy", "you", "is"]}, {"sent": "sleepy ?", "words": ["sleepy"]}, {"sent": "youre sleepy arent you ?", "words": ["sleepy", "you"]}, {"sent": "if you are sleepy go to sleep .", "words": ["if", "go", "sleep", "to", "you", "are", "sleepy"]}, {"sent": "are you sleepy ?", "words": ["sleepy", "are", "you"]}, {"sent": "are you sleepy ?", "words": ["sleepy", "are", "you"]}, {"sent": "yeah you sleepy ?", "words": ["sleepy", "you"]}, {"sent": "Im sleepy should we go to bed after Erin leaves ?", "words": ["go", "we", "to", "bed", "sleepy"]}, {"sent": "but of course Olivias not at all sleepy only five books tonight mommy she says .", "words": ["all", "of", "she", "at", "sleepy", "not", "but"]}, {"sent": "the kitty is very sleepy .", "words": ["the", "sleepy", "kitty", "is"]}, {"sent": "youre a sleepy girl .", "words": ["sleepy", "a"]}, {"sent": "did she get sleepy ?", "words": ["get", "sleepy", "did", "she"]}, {"sent": "but Im not sleepy at all I said .", "words": ["all", "at", "sleepy", "not", "but"]}, {"sent": "lets go shopping sleepy .", "words": ["sleepy", "go"]}, {"sent": "little sleepy ?", "words": ["sleepy", "little"]}, {"sent": "you sleepy ?", "words": ["sleepy", "you"]}, {"sent": "yes shes sleepy .", "words": ["sleepy"]}, {"sent": "are you sleepy Thomas ?", "words": ["sleepy", "are", "you"]}, {"sent": "are you sleepy ?", "words": ["sleepy", "are", "you"]}, {"sent": "Im sleepy .", "words": ["sleepy"]}, {"sent": "are you sleepy ?", "words": ["sleepy", "are", "you"]}, {"sent": "he was looking very sleepy we thought .", "words": ["was", "sleepy", "he", "we"]}, {"sent": "are you sleepy ?", "words": ["sleepy", "are", "you"]}, {"sent": "every day when John gets sleepy .", "words": ["sleepy", "when", "every"]}, {"sent": "are you getting sleepy ?", "words": ["sleepy", "are", "you"]}, {"sent": "I may hafta go soon because shes a little on the sleepy side and .", "words": ["the", "go", "little", "and", "on", "sleepy", "because", "a"]}, {"sent": "but of course Olivias not at all sleepy .", "words": ["all", "of", "at", "sleepy", "not", "but"]}, {"sent": "Im sleepy too .", "words": ["sleepy", "too"]}, {"sent": "hes feeling very sleepy but I think hed like a word with you .", "words": ["think", "with", "like", "you", "sleepy", "but", "a"]}, {"sent": "hes starting to get sleepy .", "words": ["get", "to", "sleepy"]}, {"sent": "weve got a sleepy baby .", "words": ["sleepy", "a"]}, {"sent": "is she sleepy ?", "words": ["sleepy", "she", "is"]}, {"sent": "youre a sleepy guy .", "words": ["sleepy", "a"]}, {"sent": "I dont think youre sleepy still .", "words": ["dont", "think", "sleepy"]}, {"sent": "sleepy .", "words": ["sleepy"]}, {"sent": "Im feeling sleepy though Thomas .", "words": ["sleepy"]}, {"sent": "are you sleepy ?", "words": ["sleepy", "are", "you"]}, {"sent": "were very into squishy wushy sleepy things these days .", "words": ["these", "sleepy", "were", "into"]}, {"sent": "crabby and sleepy .", "words": ["sleepy", "and"]}, {"sent": "sleepy sleepy sleepy puss .", "words": ["sleepy"]}, {"sent": "sleepy Dadaji ?", "words": ["sleepy"]}, {"sent": "every day when John gets sleepy .", "words": ["sleepy", "when", "every"]}, {"sent": "yep Im sleepy .", "words": ["sleepy"]}, {"sent": "you sleepy puss .", "words": ["sleepy", "you"]}, {"sent": "no you was pretend you was too sleepy .", "words": ["was", "you", "too", "pretend", "sleepy"]}, {"sent": "sleepy baby .", "words": ["sleepy"]}, {"sent": "are you sleepy ?", "words": ["sleepy", "are", "you"]}, {"sent": "youre sleepy .", "words": ["sleepy"]}, {"sent": "that sort of sleepy .", "words": ["of", "sleepy", "that"]}, {"sent": "Im sleepy too .", "words": ["sleepy", "too"]}, {"sent": "hm did I click at you little sleepy boy ?", "words": ["little", "you", "did", "at", "sleepy"]}, {"sent": "youre sleepy huh ?", "words": ["sleepy"]}, {"sent": "ah heres your blanket sleepy baby .", "words": ["blanket", "sleepy", "your"]}, {"sent": "or is it sleepy ?", "words": ["sleepy", "it", "is"]}, {"sent": "are you sleepy you resting ?", "words": ["sleepy", "are", "you"]}, {"sent": "Maisy is sleepy .", "words": ["sleepy", "is"]}, {"sent": "and sleepy Jeff .", "words": ["sleepy", "and"]}, {"sent": "sleepy head wake up wake up get out of bed .", "words": ["up", "of", "head", "bed", "get", "out", "wake", "sleepy"]}, {"sent": "gone all sleepy Nicole ?", "words": ["all", "sleepy"]}, {"sent": "sleepy grandma is she sleepy ?", "words": ["sleepy", "she", "is"]}, {"sent": "is she sleepy ?", "words": ["sleepy", "she", "is"]}, {"sent": "every day when John gets sleepy .", "words": ["sleepy", "when", "every"]}, {"sent": "along came a crocodile sleepy as can be .", "words": ["can", "be", "sleepy", "a"]}, {"sent": "uhhuh are you sleepy ?", "words": ["sleepy", "are", "you"]}, {"sent": "make you sleepy ?", "words": ["sleepy", "make", "you"]}, {"sent": "sleepy sheep six is summoning the other sheep to bed .", "words": ["the", "is", "to", "bed", "other", "sheep", "sleepy"]}, {"sent": "is she sleepy ?", "words": ["sleepy", "she", "is"]}, {"sent": "hes sleepy ?", "words": ["sleepy"]}, {"sent": "are you sleepy ?", "words": ["sleepy", "are", "you"]}, {"sent": "do you wanna have a sleepy ?", "words": ["do", "you", "have", "wanna", "sleepy", "a"]}, {"sent": "are you still sleepy ?", "words": ["sleepy", "are", "you"]}, {"sent": "are you still sleepy ?", "words": ["sleepy", "are", "you"]}, {"sent": "the sleepy book .", "words": ["the", "book", "sleepy"]}, {"sent": "are you sleepy are you hot ?", "words": ["sleepy", "hot", "are", "you"]}, {"sent": "when youre ready for your beddy sleepy corners down the drive .", "words": ["the", "for", "your", "down", "drive", "sleepy", "when"]}, {"sent": "sleepy sheep five is putting on a nighttime cap soft and red .", "words": ["red", "soft", "is", "and", "sheep", "on", "sleepy", "a"]}, {"sent": "oh hes so sleepy .", "words": ["so", "sleepy"]}, {"sent": "are you sleepy Weener .", "words": ["sleepy", "are", "you"]}, {"sent": "are you sleepy ?", "words": ["sleepy", "are", "you"]}, {"sent": "I think thats a very sleepy baby .", "words": ["think", "sleepy", "a"]}, {"sent": "you getting sleepy already ?", "words": ["sleepy", "you"]}, {"sent": "she sleepy ?", "words": ["sleepy", "she"]}, {"sent": "are you still sleepy ?", "words": ["sleepy", "are", "you"]}, {"sent": "Maisie is sleepy .", "words": ["sleepy", "is"]}, {"sent": "are you still a bit sleepy ?", "words": ["sleepy", "are", "a", "you"]}, {"sent": "Id love to but I gotta tell you Im yawning and Im very sleepy .", "words": ["to", "and", "you", "love", "sleepy", "but"]}, {"sent": "you not hardly sleepy yet .", "words": ["sleepy", "not", "you"]}, {"sent": "did you have a nice sleepy ?", "words": ["nice", "you", "have", "did", "sleepy", "a"]}, {"sent": "Mummys sleepy today .", "words": ["sleepy"]}, {"sent": "okay here the sleepy book .", "words": ["here", "the", "book", "sleepy"]}, {"sent": "told you you were sleepy .", "words": ["sleepy", "were", "you"]}, {"sent": "daddys playing a sleepy game .", "words": ["game", "sleepy", "a"]}, {"sent": "you look a sleepy boy .", "words": ["sleepy", "look", "a", "you"]}, {"sent": "you look sleepy .", "words": ["sleepy", "look", "you"]}, {"sent": "hes sleepy .", "words": ["sleepy"]}, {"sent": "help help .", "words": ["help"]}, {"sent": "Anthony now you can help me take those down .", "words": ["can", "help", "those", "you", "me", "down", "take"]}, {"sent": "you can help him get dressed .", "words": ["can", "help", "you", "get", "him"]}, {"sent": "oh what else did you help did you help Daddy fix outside ?", "words": ["what", "help", "fix", "outside", "you", "did"]}, {"sent": "can mummy help ?", "words": ["can", "help"]}, {"sent": "would you like some help ?", "words": ["help", "like", "you", "would", "some"]}, {"sent": "why dont you help Mama ?", "words": ["dont", "help", "why", "you"]}, {"sent": "Ill help you .", "words": ["help", "you"]}, {"sent": "want me to help you build something ?", "words": ["help", "to", "you", "build", "me"]}, {"sent": "help me get in .", "words": ["get", "me", "help", "in"]}, {"sent": "will you help me do it ?", "words": ["do", "help", "you", "it", "will", "me"]}, {"sent": "did the teacher help you ?", "words": ["the", "help", "did", "you"]}, {"sent": "finally he asked Mr Owl to help him read his favorite book .", "words": ["help", "to", "him", "he", "read", "his", "book"]}, {"sent": "Mommyll help you .", "words": ["help", "you"]}, {"sent": "can you help me with this Joey ?", "words": ["can", "help", "with", "you", "this", "me"]}, {"sent": "do you need my help or are you gonna do it alone ?", "words": ["do", "my", "help", "need", "you", "are", "it"]}, {"sent": "help help .", "words": ["help"]}, {"sent": "help you do what ?", "words": ["do", "help", "what", "you"]}, {"sent": "Ill help you .", "words": ["help", "you"]}, {"sent": "help me push them out .", "words": ["help", "push", "them", "out", "me"]}, {"sent": "do you want some help ?", "words": ["do", "help", "some", "you"]}, {"sent": "because it might help you feel good .", "words": ["help", "you", "it", "good", "because"]}, {"sent": "Eleanor are you gonna help me tidy up ?", "words": ["up", "help", "you", "are", "me"]}, {"sent": "oh do you wanna help me ?", "words": ["do", "help", "you", "wanna", "me"]}, {"sent": "he might need help cutting .", "words": ["need", "help", "he"]}, {"sent": "can you help me find a circle .", "words": ["can", "help", "you", "find", "me", "a"]}, {"sent": "gonna help Mummy do this one ?", "words": ["do", "help", "this"]}, {"sent": "no would you like some help ?", "words": ["help", "like", "you", "would", "some"]}, {"sent": "do you want some help with that ?", "words": ["do", "help", "with", "you", "some", "that"]}, {"sent": "can you help us ?", "words": ["can", "us", "help", "you"]}, {"sent": "uh what did you help me make last night Sofia ?", "words": ["what", "help", "last", "you", "did", "me", "make"]}, {"sent": "it wont help if you take this out .", "words": ["help", "if", "you", "this", "it", "out", "take"]}, {"sent": "Im trying to help .", "words": ["to", "help"]}, {"sent": "well you get everything else out and Ill help you in a minute .", "words": ["help", "and", "get", "you", "in", "out", "a"]}, {"sent": "help me .", "words": ["me", "help"]}, {"sent": "you want Mommy to help ?", "words": ["to", "help", "you"]}, {"sent": "yeah wanna help ?", "words": ["help", "wanna"]}, {"sent": "wanna help me tidy up upstairs ?", "words": ["up", "me", "help", "wanna"]}, {"sent": "you need help finding the monkey ?", "words": ["the", "help", "need", "monkey", "you"]}, {"sent": "her Andrews gonna help .", "words": ["help", "her"]}, {"sent": "you want me to help you ?", "words": ["to", "me", "help", "you"]}, {"sent": "well I dont think that would help with separating the eggs .", "words": ["the", "think", "help", "with", "dont", "would", "that"]}, {"sent": "thats the kind of ear problem I have while Im here can you help me with another problem ?", "words": ["the", "can", "help", "of", "with", "you", "have", "here", "another", "ear", "me"]}, {"sent": "well you help me .", "words": ["me", "help", "you"]}, {"sent": "shall I help you MrsCwww ?", "words": ["help", "you"]}, {"sent": "lets get you dressed and then you can help Mummy with the washing up .", "words": ["can", "the", "help", "up", "with", "and", "get", "you", "then"]}, {"sent": "would you like to help me crush these ?", "words": ["these", "help", "to", "like", "you", "would", "me"]}, {"sent": "in fact when youve eaten that sandwich you can help me do those flowers .", "words": ["can", "do", "help", "those", "when", "you", "in", "me", "that", "sandwich"]}, {"sent": "help me make a puzzle .", "words": ["help", "puzzle", "me", "make", "a"]}, {"sent": "shall I help you ?", "words": ["help", "you"]}, {"sent": "are you gonna help daddy water ?", "words": ["water", "help", "are", "you"]}, {"sent": "do you think PostmanPatll help out ?", "words": ["do", "think", "help", "you", "out"]}, {"sent": "can you help me ?", "words": ["can", "me", "help", "you"]}, {"sent": "and Ill help you .", "words": ["help", "you", "and"]}, {"sent": "you hold the pencil and then you can if you help mummy draw it .", "words": ["the", "can", "help", "hold", "if", "draw", "pencil", "and", "you", "it", "then"]}, {"sent": "help ?", "words": ["help"]}, {"sent": "needs some help here .", "words": ["here", "some", "help"]}, {"sent": "why do you say help help ?", "words": ["do", "help", "say", "you", "why"]}, {"sent": "help help .", "words": ["help"]}, {"sent": "he needs help ?", "words": ["help", "he"]}, {"sent": "oh shes gonna help the baby bundle up .", "words": ["the", "help", "up"]}, {"sent": "sit up and help me .", "words": ["up", "help", "and", "me", "sit"]}, {"sent": "sure Ill help you .", "words": ["help", "you"]}, {"sent": "and then Ill help you open it .", "words": ["help", "open", "you", "and", "it", "then"]}, {"sent": "can you help me put everything back in the bag ?", "words": ["can", "the", "help", "put", "back", "you", "in", "me"]}, {"sent": "okay you want mommy to help you ?", "words": ["to", "help", "you"]}, {"sent": "Ill help you get it Babybabe .", "words": ["get", "help", "it", "you"]}, {"sent": "come on help me tidy up .", "words": ["on", "me", "help", "up"]}, {"sent": "can a firetruck help in this situation ?", "words": ["can", "help", "this", "in", "firetruck", "a"]}, {"sent": "he needs help ?", "words": ["help", "he"]}, {"sent": "should I help you get it ?", "words": ["get", "help", "it", "you"]}, {"sent": "its not very much help for me trying to tidy up is it Fraser ?", "words": ["up", "help", "for", "is", "to", "it", "me", "much", "not"]}, {"sent": "do you want me to help you ?", "words": ["do", "help", "to", "you", "me"]}, {"sent": "Ill help you .", "words": ["help", "you"]}, {"sent": "did you help your mom cut the flowers ?", "words": ["the", "help", "your", "cut", "you", "did"]}, {"sent": "help eebah xxx I had a dog .", "words": ["help", "dog", "a"]}, {"sent": "she said help help .", "words": ["help", "she"]}, {"sent": "but first of all I want you to help me with this wee thing .", "words": ["all", "help", "of", "to", "first", "with", "you", "this", "me", "but"]}, {"sent": "help she said .", "words": ["help", "she"]}, {"sent": "it had a bit of help didnt it ?", "words": ["of", "help", "it", "a"]}, {"sent": "come and help Mum .", "words": ["help", "and"]}, {"sent": "you want Mommy to help you open it ?", "words": ["help", "open", "to", "you", "it"]}, {"sent": "you ask Mommy for help I always help .", "words": ["help", "for", "you"]}, {"sent": "do you need me to help ?", "words": ["do", "help", "need", "to", "you", "me"]}, {"sent": "oh you want some help ?", "words": ["some", "help", "you"]}, {"sent": "do you need some help ?", "words": ["do", "help", "need", "you", "some"]}, {"sent": "I dont mind you helping sometimes but it doesnt help quite as much as you think .", "words": ["help", "think", "you", "dont", "it", "much", "but"]}, {"sent": "are you going to help ?", "words": ["to", "help", "are", "you"]}, {"sent": "help me .", "words": ["me", "help"]}, {"sent": "help help .", "words": ["help"]}, {"sent": "and here come all the animals to help her .", "words": ["the", "all", "help", "to", "and", "here", "her"]}, {"sent": "wouldjou like a little help with that ?", "words": ["help", "with", "like", "little", "that", "a"]}, {"sent": "who do you help ?", "words": ["do", "help", "you", "who"]}, {"sent": "help help .", "words": ["help"]}, {"sent": "tell me what you are doing so I can help you .", "words": ["can", "what", "help", "so", "you", "are", "me"]}, {"sent": "you need help ?", "words": ["help", "need", "you"]}, {"sent": "help Mommy .", "words": ["help"]}, {"sent": "I will help you .", "words": ["will", "help", "you"]}, {"sent": "do you need some help ?", "words": ["do", "help", "need", "you", "some"]}, {"sent": "enjoy yourself for the next so many years .", "words": ["the", "so", "yourself", "for"]}, {"sent": "see youre getting yourself all deep in chicken fat now .", "words": ["all", "yourself", "chicken", "in", "see"]}, {"sent": "then youll be able to hear yourself talking .", "words": ["yourself", "to", "be", "hear", "then"]}, {"sent": "if you wish you may go by a lions tail or stamp yourself and go by mail MarvinKMooney dont you know the time has come to go go go .", "words": ["the", "yourself", "if", "go", "to", "by", "wish", "you", "and", "dont", "a"]}, {"sent": "you can hear yourself .", "words": ["can", "hear", "yourself", "you"]}, {"sent": "make the tunnel by yourself ?", "words": ["the", "yourself", "make", "by"]}, {"sent": "why are you holding yourself like that ?", "words": ["yourself", "like", "you", "are", "that", "why"]}, {"sent": "did you do it all by yourself ?", "words": ["all", "do", "yourself", "you", "by", "it", "did"]}, {"sent": "yourself ?", "words": ["yourself"]}, {"sent": "did you carry that big stick all by yourself ?", "words": ["all", "stick", "yourself", "you", "carry", "by", "that", "did", "big"]}, {"sent": "are you controlling yourself ?", "words": ["yourself", "are", "you"]}, {"sent": "youve put the straps on yourself .", "words": ["the", "put", "yourself", "on"]}, {"sent": "youve got a lot to say for yourself havent you ?", "words": ["yourself", "for", "a lot", "to", "say", "you", "a"]}, {"sent": "you dont want ta break it and youll just hurt yourself .", "words": ["yourself", "hurt", "and", "you", "dont", "it", "break"]}, {"sent": "help yourself .", "words": ["help", "yourself"]}, {"sent": "I know well clean yourself off .", "words": ["clean", "yourself", "off"]}, {"sent": "can you stand up by yourself ?", "words": ["can", "up", "yourself", "stand", "by", "you"]}, {"sent": "youll fall and youll hurt yourself .", "words": ["hurt", "yourself", "fall", "and"]}, {"sent": "no you locked yourself in again turn it straight up .", "words": ["up", "yourself", "you", "in", "it"]}, {"sent": "do you put it on yourself do you ?", "words": ["do", "put", "yourself", "you", "it", "on"]}, {"sent": "is there anything you wanna see yourself ?", "words": ["yourself", "is", "you", "there", "wanna", "see"]}, {"sent": "get yourself dressed .", "words": ["get", "yourself"]}, {"sent": "you wanna feed yourself ?", "words": ["feed", "yourself", "wanna", "you"]}, {"sent": "you didnt hurt yourself that much .", "words": ["yourself", "hurt", "much", "you", "that"]}, {"sent": "are you gonna tidy up after yourself ?", "words": ["up", "are", "yourself", "you"]}, {"sent": "Eleanor youre gonna hurt yourself darling .", "words": ["hurt", "yourself"]}, {"sent": "you put your tshirt on all by yourself now havent you ?", "words": ["all", "put", "yourself", "your", "you", "by", "on"]}, {"sent": "dyou wanna read it yourself ?", "words": ["yourself", "it", "read", "wanna"]}, {"sent": "youre going to put it in your mouth all by yourself .", "words": ["all", "put", "yourself", "your", "to", "mouth", "by", "in", "it"]}, {"sent": "*SI1: lets see ya all by yourself .", "words": ["all", "see", "yourself", "by"]}, {"sent": "did you ever visit your sister like by yourself in Florida ?", "words": ["yourself", "your", "like", "by", "you", "in", "did"]}, {"sent": "and you made it all by yourself .", "words": ["all", "yourself", "by", "you", "and", "it"]}, {"sent": "did you make it yourself ?", "words": ["yourself", "you", "it", "did", "make"]}, {"sent": "you see yourself on the movie huh ?", "words": ["the", "yourself", "you", "on", "see"]}, {"sent": "can you carry that all home by yourself ?", "words": ["can", "all", "yourself", "you", "carry", "by", "that", "home"]}, {"sent": "youre getting ink all over yourself .", "words": ["all", "yourself", "over"]}, {"sent": "if you do it yourself youre gonna get do it carefully alright ?", "words": ["do", "yourself", "if", "you", "get", "it"]}, {"sent": "yeah you dont wanna hurt yourself do you ?", "words": ["do", "yourself", "hurt", "you", "dont", "wanna"]}, {"sent": "did you make it yourself ?", "words": ["yourself", "you", "it", "did", "make"]}, {"sent": "you can stay home all by yourself .", "words": ["can", "all", "yourself", "you", "by", "stay", "home"]}, {"sent": "did you hurt yourself my love .", "words": ["my", "yourself", "hurt", "you", "did", "love"]}, {"sent": "did you hurt yourself ?", "words": ["hurt", "yourself", "did", "you"]}, {"sent": "watch you dont burn yourself .", "words": ["yourself", "dont", "watch", "you"]}, {"sent": "well you would have hurt yourself .", "words": ["yourself", "hurt", "you", "have", "would"]}, {"sent": "*SI1: can ya do it by yourself ?", "words": ["can", "do", "yourself", "by", "it"]}, {"sent": "youre looking at yourself in the .", "words": ["the", "in", "yourself", "at"]}, {"sent": "youre going to hurt yourself .", "words": ["hurt", "to", "yourself"]}, {"sent": "you will scare yourself .", "words": ["will", "yourself", "you"]}, {"sent": "I think youve tired yourself out havent you ?", "words": ["think", "yourself", "you", "tired", "out"]}, {"sent": "and then you kept pulling yourself up .", "words": ["up", "yourself", "you", "and", "then"]}, {"sent": "uh oh did you get yourself ?", "words": ["get", "yourself", "did", "you"]}, {"sent": "you put that on all by yourself .", "words": ["all", "put", "yourself", "you", "by", "on", "that"]}, {"sent": "whereve you hurt yourself ?", "words": ["hurt", "yourself", "you"]}, {"sent": "youve done half of it by yourself already Fraser .", "words": ["of", "it", "yourself", "by"]}, {"sent": "careful youre gonna hit yourself yeah .", "words": ["yourself", "hit", "careful"]}, {"sent": "do you think you could put your boots on by yourself ?", "words": ["do", "think", "put", "your", "yourself", "you", "by", "boots", "on", "could"]}, {"sent": "you havent cut yourself yet .", "words": ["yourself", "cut", "you"]}, {"sent": "you would spend all day looking at yourself wouldnt you ?", "words": ["all", "yourself", "would", "you", "at"]}, {"sent": "are you gonna drink all those yourself ?", "words": ["all", "yourself", "those", "drink", "you", "are"]}, {"sent": "Thomas dont attach yourself to anything please .", "words": ["to", "dont", "yourself"]}, {"sent": "pump ow gentle dont hurt yourself .", "words": ["hurt", "dont", "gentle", "yourself"]}, {"sent": "watch you dont hurt yourself darling .", "words": ["watch", "yourself", "hurt", "you", "dont"]}, {"sent": "youre gonna hurt yourself doing that .", "words": ["hurt", "yourself", "that"]}, {"sent": "were gonna talk to your Mom a little bit if you wanna play with that by yourself .", "words": ["yourself", "your", "if", "to", "play", "with", "little", "you", "by", "that", "talk", "wanna", "were", "a"]}, {"sent": "did you hurt yourself ?", "words": ["hurt", "yourself", "did", "you"]}, {"sent": "youre getting really good at feeding yourself .", "words": ["yourself", "good", "at"]}, {"sent": "give yourself a spanking .", "words": ["give", "yourself", "a"]}, {"sent": "looking under my sweater Michael you oughta be ashamed of yourself .", "words": ["my", "yourself", "sweater", "of", "be", "you", "under"]}, {"sent": "you see yourself ?", "words": ["see", "yourself", "you"]}, {"sent": "youre gonna read it by yourself ?", "words": ["yourself", "it", "read", "by"]}, {"sent": "youre not yourself today .", "words": ["yourself", "not"]}, {"sent": "youre reading all by yourself .", "words": ["all", "yourself", "by"]}, {"sent": "where do you see yourself ?", "words": ["where", "do", "yourself", "you", "see"]}, {"sent": "do you go downtown and things like that by yourself ?", "words": ["do", "yourself", "go", "like", "and", "you", "by", "that"]}, {"sent": "you could go by yourself .", "words": ["yourself", "go", "by", "you", "could"]}, {"sent": "to hear yourself .", "words": ["to", "yourself", "hear"]}, {"sent": "wed never get you to put them on yourself .", "words": ["put", "yourself", "to", "them", "you", "get", "on"]}, {"sent": "you hurt yourself ?", "words": ["hurt", "yourself", "you"]}, {"sent": "xxx is the one who gave you give yourself to the rain that book .", "words": ["the", "yourself", "rain", "is", "to", "you", "who", "book", "give", "that"]}, {"sent": "can you get on yourself ?", "words": ["can", "yourself", "you", "get", "on"]}, {"sent": "I think youre going to do yourself more damage than anything else .", "words": ["do", "think", "yourself", "to", "more"]}, {"sent": "are there particular ones of these that you actually do have yourself ?", "words": ["these", "do", "yourself", "of", "you", "have", "there", "are", "that"]}, {"sent": "well you must be very proud of yourself .", "words": ["of", "be", "yourself", "you"]}, {"sent": "thats Playdoh you hafta make the stuff yourself .", "words": ["the", "yourself", "make", "you"]}, {"sent": "if you stood on a chair in front of the mirror youll probably see yourself with the hairband on .", "words": ["the", "yourself", "if", "of", "with", "chair", "you", "in", "on", "see", "a"]}, {"sent": "see if you can be a clever boy and do it by yourself .", "words": ["can", "do", "yourself", "if", "be", "you", "and", "by", "it", "see", "a"]}, {"sent": "can you stand up by yourself ?", "words": ["can", "up", "yourself", "stand", "by", "you"]}, {"sent": "might hurt yourself .", "words": ["hurt", "yourself"]}, {"sent": "because you drink the water and you make yourself poorly dont you ?", "words": ["the", "yourself", "drink", "you", "and", "dont", "water", "make", "because"]}, {"sent": "oh Violet dont give yourself spankings .", "words": ["dont", "give", "yourself"]}, {"sent": "you hurt yourself ?", "words": ["hurt", "yourself", "you"]}, {"sent": "you bit yourself ?", "words": ["yourself", "you"]}, {"sent": "you did it all by yourself .", "words": ["all", "yourself", "by", "you", "it", "did"]}, {"sent": "and and did you do anything about that yourself ?", "words": ["do", "yourself", "you", "and", "did", "about", "that"]}, {"sent": "you splash them on yourself ?", "words": ["yourself", "them", "you", "splash", "on"]}, {"sent": "for yourself xxx and left ?", "words": ["yourself", "for", "and"]}, {"sent": "youd rather go through it yourself .", "words": ["yourself", "it", "go"]}, {"sent": "you gonna walk by yourself ?", "words": ["yourself", "walk", "by", "you"]}, {"sent": "why do you just ram yourself in the tiniest places ?", "words": ["the", "do", "yourself", "you", "in", "why"]}, {"sent": "do you have a picture of Tressa ?", "words": ["do", "of", "picture", "you", "have", "a"]}, {"sent": "are you going to look at the picture .", "words": ["the", "to", "picture", "you", "are", "at", "look"]}, {"sent": "were at the end of our picture .", "words": ["the", "of", "picture", "our", "at", "were"]}, {"sent": "youre still drawing a picture of Sooty are you ?", "words": ["of", "picture", "you", "are", "a"]}, {"sent": "look now weve got a picture here .", "words": ["here", "picture", "a", "look"]}, {"sent": "wanna draw a picture of the tiger ?", "words": ["the", "of", "draw", "picture", "wanna", "tiger", "a"]}, {"sent": "how many other things can you spot in this picture that begin with the letter b ?", "words": ["can", "the", "how", "picture", "with", "you", "other", "this", "in", "that"]}, {"sent": "what a pretty picture .", "words": ["what", "picture", "pretty", "a"]}, {"sent": "its a picture .", "words": ["picture", "a"]}, {"sent": "well which picture up there ?", "words": ["up", "there", "picture", "which"]}, {"sent": "what picture looks like that ?", "words": ["what", "picture", "that", "like"]}, {"sent": "what do you see in that picture ?", "words": ["what", "do", "picture", "you", "in", "see", "that"]}, {"sent": "you see the lion hiding in that picture ?", "words": ["the", "picture", "you", "in", "that", "see", "lion"]}, {"sent": "how many kids are in that picture ?", "words": ["how", "picture", "in", "are", "that"]}, {"sent": "look at all the things in the picture .", "words": ["the", "all", "look", "picture", "in", "at"]}, {"sent": "just one more picture .", "words": ["more", "picture"]}, {"sent": "I should take a picture and send it to aunt Stephanie .", "words": ["to", "picture", "and", "it", "take", "a"]}, {"sent": "its a really lovely picture .", "words": ["picture", "a"]}, {"sent": "can you see all the mud on the picture ?", "words": ["can", "the", "all", "picture", "you", "on", "see"]}, {"sent": "oh the hat matches the picture on your shirt .", "words": ["the", "your", "shirt", "picture", "hat", "on"]}, {"sent": "what do you want do you wanna do those picture cards ?", "words": ["what", "do", "those", "picture", "you", "wanna"]}, {"sent": "theres the picture on the box of what the firemens doing .", "words": ["the", "what", "box", "of", "picture", "on"]}, {"sent": "this is a beautiful picture .", "words": ["this", "picture", "a", "is"]}, {"sent": "do you wanna play picture lotto ?", "words": ["do", "play", "picture", "you", "wanna"]}, {"sent": "let us make a picture .", "words": ["us", "picture", "make", "a"]}, {"sent": "whos in this picture ?", "words": ["this", "in", "picture"]}, {"sent": "thats a great picture .", "words": ["picture", "a"]}, {"sent": "mm yeah we need to draw an extra picture of the watermelon huh ?", "words": ["the", "need", "we", "of", "to", "need to", "draw", "picture", "an"]}, {"sent": "whats the picture of ?", "words": ["the", "of", "picture"]}, {"sent": "because its a nice picture .", "words": ["nice", "picture", "because", "a"]}, {"sent": "thats not a sticker thats just a picture .", "words": ["picture", "not", "a"]}, {"sent": "somebodys missing from that picture .", "words": ["picture", "that"]}, {"sent": "did you paint a nice picture ?", "words": ["paint", "nice", "picture", "you", "did", "a"]}, {"sent": "and lets see whats in the picture .", "words": ["the", "picture", "and", "in", "see"]}, {"sent": "Et the star press a picture .", "words": ["the", "star", "picture", "a"]}, {"sent": "can you see it on the picture ?", "words": ["can", "the", "picture", "you", "it", "on", "see"]}, {"sent": "I think its a picture of a football Thomas .", "words": ["of", "think", "picture", "a"]}, {"sent": "whatre you going to draw a picture of ?", "words": ["of", "to", "draw", "picture", "you", "a"]}, {"sent": "youve just got to watch the picture watch the picture and tell Mummy whats happening .", "words": ["the", "watch", "to", "picture", "and"]}, {"sent": "have I got a nose in your picture ?", "words": ["your", "picture", "have", "in", "a", "nose"]}, {"sent": "where are we gonna put your picture to dry where it will be ?", "words": ["where", "put", "your", "we", "to", "dry", "picture", "be", "are", "it", "will"]}, {"sent": "do you think shes drawing a picture ?", "words": ["do", "think", "picture", "you", "a"]}, {"sent": "shall we draw a picture of a horsie ?", "words": ["we", "of", "draw", "picture", "a"]}, {"sent": "you gonna take my picture ?", "words": ["my", "picture", "take", "you"]}, {"sent": "this is Lulus baby brother he can say Lulu hay you missed the picture of the lollipop .", "words": ["can", "the", "is", "of", "picture", "say", "you", "lollipop", "this", "he"]}, {"sent": "mummy make a picture look .", "words": ["look", "picture", "make", "a"]}, {"sent": "can you make him a picture ?", "words": ["can", "picture", "you", "him", "make", "a"]}, {"sent": "whats happening in this picture ?", "words": ["this", "in", "picture"]}, {"sent": "yeh theyre taking a picture of you eating raisins .", "words": ["of", "picture", "you", "a"]}, {"sent": "youre right not in that picture .", "words": ["in", "picture", "that", "not"]}, {"sent": "whats that a picture of Fraser ?", "words": ["of", "picture", "that", "a"]}, {"sent": "theres a picture of a car isnt there ?", "words": ["of", "car", "picture", "there", "a"]}, {"sent": "which picture ?", "words": ["picture", "which"]}, {"sent": "can you make a picture ?", "words": ["can", "picture", "you", "make", "a"]}, {"sent": "look at the picture .", "words": ["the", "at", "picture", "look"]}, {"sent": "when you see this picture I want you to tell me all about it .", "words": ["all", "to", "me", "picture", "you", "this", "it", "about", "see", "when"]}, {"sent": "um I dont see any seaweed in this picture but .", "words": ["picture", "any", "dont", "in", "this", "see", "but"]}, {"sent": "is that a picture of Nina ?", "words": ["is", "of", "picture", "that", "a"]}, {"sent": "take a crayon and make a picture .", "words": ["crayon", "picture", "and", "take", "make", "a"]}, {"sent": "here want a picture ?", "words": ["here", "picture", "a"]}, {"sent": "you wanna look at the picture of Mummy and Daddy ?", "words": ["the", "of", "picture", "you", "and", "at", "wanna", "look"]}, {"sent": "a story picture ?", "words": ["story", "picture", "a"]}, {"sent": "its a sidewalk in the safe picture yeah .", "words": ["the", "sidewalk", "picture", "in", "a"]}, {"sent": "I cant see this picture but you can .", "words": ["can", "picture", "you", "this", "see", "but"]}, {"sent": "once we start a picture we should finish it .", "words": ["we", "picture", "finish", "it", "a"]}, {"sent": "we havent got a dog on this picture .", "words": ["we", "picture", "this", "on", "dog", "a"]}, {"sent": "why dont you take a picture ?", "words": ["picture", "you", "dont", "take", "why", "a"]}, {"sent": "look at this big picture .", "words": ["look", "picture", "this", "at", "big"]}, {"sent": "they can draw a picture of the Eiffel tower .", "words": ["can", "the", "of", "draw", "picture", "they", "a"]}, {"sent": "I think it has fallen off the picture .", "words": ["off", "the", "think", "picture", "it"]}, {"sent": "shall we find the picture of Warren and Daddy ?", "words": ["the", "we", "of", "picture", "and", "find"]}, {"sent": "lets see if we can find a picture of a D if we can Thomas .", "words": ["can", "if", "we", "of", "picture", "find", "see", "a"]}, {"sent": "are you going to have a look at the picture ?", "words": ["the", "to", "picture", "you", "have", "are", "at", "look", "a"]}, {"sent": "have a look on the picture and see if you can find them .", "words": ["the", "can", "if", "picture", "them", "find", "and", "have", "you", "look", "on", "see", "a"]}, {"sent": "are we going to do this picture first ?", "words": ["do", "we", "to", "picture", "first", "this", "are"]}, {"sent": "the whole picture story come on .", "words": ["the", "story", "picture", "on"]}, {"sent": "what is this little girl in the picture doing ?", "words": ["the", "what", "is", "picture", "little", "this", "in"]}, {"sent": "where can you see the dog on that picture ?", "words": ["where", "can", "the", "picture", "you", "on", "see", "that", "dog"]}, {"sent": "an interesting picture .", "words": ["an", "picture"]}, {"sent": "wheres the picture of the butterfly John ?", "words": ["the", "of", "picture", "butterfly"]}, {"sent": "picture doesnt match .", "words": ["picture"]}, {"sent": "whats that a picture of ?", "words": ["of", "picture", "that", "a"]}, {"sent": "youre all doing a big picture of a big bonfire ?", "words": ["all", "of", "picture", "big", "a"]}, {"sent": "lets have a random picture .", "words": ["have", "picture", "a"]}, {"sent": "lets have a look on the picture .", "words": ["the", "picture", "have", "look", "on", "a"]}, {"sent": "its just a picture .", "words": ["picture", "a"]}, {"sent": "whats that a picture of ?", "words": ["of", "picture", "that", "a"]}, {"sent": "can you look at another picture ?", "words": ["can", "picture", "you", "another", "at", "look"]}, {"sent": "right hes got his picture in his mouth .", "words": ["his", "in", "picture", "mouth"]}, {"sent": "tell me one thing about that picture one thing about this .", "words": ["picture", "this", "about", "me", "that"]}, {"sent": "all finished will you tell us what whats in the picture ?", "words": ["the", "all", "what", "picture", "you", "us", "in", "will"]}, {"sent": "you gonna make a pretty picture ?", "words": ["picture", "you", "pretty", "make", "a"]}, {"sent": "youre a picture girl .", "words": ["picture", "a"]}, {"sent": "but I know JemimaPuddleducks in here because Ive just seen a picture .", "words": ["picture", "here", "in", "but", "because", "a"]}, {"sent": "heres a picture of your house Aisha come take a look .", "words": ["your", "of", "picture", "house", "look", "take", "a"]}, {"sent": "make a pretty picture for me .", "words": ["for", "picture", "pretty", "me", "make", "a"]}, {"sent": "ah and whos in that picture ?", "words": ["in", "picture", "that", "and"]}, {"sent": "a truck .", "words": ["truck", "a"]}, {"sent": "you want your dump truck ?", "words": ["truck", "dump", "your", "you"]}, {"sent": "oh it says its a bread truck .", "words": ["bread", "it", "truck", "a"]}, {"sent": "Bennys truck was wet .", "words": ["was", "truck", "wet"]}, {"sent": "put the freight from this truck onto this truck .", "words": ["the", "this", "put", "truck"]}, {"sent": "its a big truck isnt it ?", "words": ["truck", "it", "big", "a"]}, {"sent": "in the big truck .", "words": ["the", "truck", "in", "big"]}, {"sent": "thats right a trailer lorry truck yes .", "words": ["truck", "a"]}, {"sent": "theres a truck honey .", "words": ["truck", "a"]}, {"sent": "what is the truck doing ?", "words": ["the", "what", "truck", "is"]}, {"sent": "little truck .", "words": ["truck", "little"]}, {"sent": "I think the mail truck just got some gas .", "words": ["the", "some", "think", "truck"]}, {"sent": "what does he do with the truck ?", "words": ["the", "what", "do", "does", "with", "truck", "he"]}, {"sent": "hes in the truck .", "words": ["the", "in", "truck"]}, {"sent": "there goes the truck .", "words": ["the", "there", "truck"]}, {"sent": "weve not seen that big green truck that usually parks outside in the mornings have we ?", "words": ["the", "we", "outside", "truck", "have", "that", "in", "green", "big", "not"]}, {"sent": "that truck has an arm .", "words": ["truck", "that", "arm", "an"]}, {"sent": "maybe your dump truck can come and visit the village and bring things to the village .", "words": ["can", "the", "your", "to", "truck", "bring", "and", "dump"]}, {"sent": "that is a big truck .", "words": ["is", "truck", "that", "big", "a"]}, {"sent": "dump truck thats right .", "words": ["dump", "truck"]}, {"sent": "you gonna draw a picture of Roms own truck at home ?", "words": ["of", "draw", "picture", "truck", "you", "at", "home", "a"]}, {"sent": "maybe we can put the vegetables and food on the truck Patrick .", "words": ["can", "the", "put", "we", "truck", "and", "food", "on"]}, {"sent": "that looks like its out of a dumper truck .", "words": ["of", "truck", "like", "out", "that", "a"]}, {"sent": "thats kind of a tow truck .", "words": ["of", "truck", "a"]}, {"sent": "but they must have been jolly strong to pick up a police car and throw it into the dustbin truck .", "words": ["up", "the", "throw", "into", "pick", "to", "car", "truck", "and", "have", "they", "it", "but", "a"]}, {"sent": "put two people in the truck .", "words": ["the", "in", "put", "truck"]}, {"sent": "well whatll happen if Henry brings his truck trucks round ?", "words": ["his", "truck", "if"]}, {"sent": "a blue truck .", "words": ["blue", "truck", "a"]}, {"sent": "thats a truck .", "words": ["truck", "a"]}, {"sent": "is it stuck in the truck ?", "words": ["the", "is", "truck", "in", "it", "stuck"]}, {"sent": "for the for the mail truck .", "words": ["the", "for", "truck"]}, {"sent": "yeah wheres that truck going ?", "words": ["truck", "that"]}, {"sent": "crashed into the cattle truck ?", "words": ["the", "truck", "into"]}, {"sent": "put them in a truck .", "words": ["put", "truck", "them", "in", "a"]}, {"sent": "okay now she can sit in the truck .", "words": ["can", "the", "truck", "in", "sit", "she"]}, {"sent": "you know I dont know if this is a truck .", "words": ["if", "is", "truck", "you", "dont", "this", "a"]}, {"sent": "put the truck away .", "words": ["the", "put", "away", "truck"]}, {"sent": "because when we came home from music lesson yesterday the blue truck was there wasnt it ?", "words": ["the", "we", "was", "truck", "blue", "there", "it", "when", "because", "home"]}, {"sent": "and then it goes in the giant dumper truck .", "words": ["the", "truck", "and", "in", "it", "then"]}, {"sent": "can your truck make a sound ?", "words": ["can", "your", "truck", "make", "a"]}, {"sent": "play with the truck ?", "words": ["the", "truck", "play", "with"]}, {"sent": "Ill put those back in the truck .", "words": ["the", "put", "those", "back", "truck", "in"]}, {"sent": "the truck and the little car .", "words": ["the", "car", "truck", "little", "and"]}, {"sent": "see truck ?", "words": ["see", "truck"]}, {"sent": "is that a truck parking ?", "words": ["truck", "that", "a", "is"]}, {"sent": "this truck is great .", "words": ["this", "truck", "is"]}, {"sent": "a truck .", "words": ["truck", "a"]}, {"sent": "a truck ?", "words": ["truck", "a"]}, {"sent": "and then the wheeliebins go up on his big truck .", "words": ["the", "up", "go", "truck", "and", "his", "on", "then", "big"]}, {"sent": "get off my truck .", "words": ["get", "my", "truck", "off"]}, {"sent": "yep thats a truck all right .", "words": ["all", "truck", "a"]}, {"sent": "you would read them the truck book .", "words": ["the", "them", "truck", "you", "read", "would", "book"]}, {"sent": "look the blocks are in the truck .", "words": ["the", "truck", "in", "are", "look"]}, {"sent": "some guy go there right on the truck .", "words": ["the", "go", "truck", "there", "on", "some"]}, {"sent": "can we move the truck and the spoons ?", "words": ["can", "the", "we", "truck", "and"]}, {"sent": "oo have a little crash here we bumped the luggage truck .", "words": ["the", "we", "truck", "little", "have", "here", "a"]}, {"sent": "take it off and put it in his truck .", "words": ["off", "put", "truck", "and", "in", "it", "his", "take"]}, {"sent": "driving my truck .", "words": ["my", "truck"]}, {"sent": "those some truck ?", "words": ["some", "truck", "those"]}, {"sent": "theres a truck that hauls the suitcases around .", "words": ["the", "truck", "around", "that", "a"]}, {"sent": "Thumper is a truck .", "words": ["truck", "a", "is"]}, {"sent": "I think that was an oil truck .", "words": ["think", "was", "truck", "an", "that"]}, {"sent": "sounds like a big truck .", "words": ["truck", "big", "like", "a"]}, {"sent": "it got stuck in the corner of the truck underneath the plastic .", "words": ["the", "of", "truck", "in", "it", "stuck"]}, {"sent": "one truck and another truck .", "words": ["another", "truck", "and"]}, {"sent": "and theres a puppy in that brown truck .", "words": ["brown", "truck", "and", "in", "puppy", "that", "a"]}, {"sent": "should we put it in the truck ?", "words": ["the", "put", "we", "truck", "in", "it"]}, {"sent": "a truck ?", "words": ["truck", "a"]}, {"sent": "shes having a look at the truck now .", "words": ["the", "truck", "at", "look", "a"]}, {"sent": "yeah what kind of a truck ?", "words": ["of", "what", "truck", "a"]}, {"sent": "oh the truck was almost falling over ?", "words": ["the", "was", "truck", "over"]}, {"sent": "the only time that theyre not down is when the truck this truck is hooked up to the trailer .", "words": ["the", "up", "when", "is", "to", "truck", "this", "down", "that", "not"]}, {"sent": "wheres the breakdown truck ?", "words": ["the", "truck"]}, {"sent": "do you see your truck too ?", "words": ["do", "your", "truck", "you", "too", "see"]}, {"sent": "lets dont run the truck up on the recorder .", "words": ["the", "up", "truck", "dont", "run", "on"]}, {"sent": "a truck ?", "words": ["truck", "a"]}, {"sent": "the truck what darling ?", "words": ["the", "what", "truck"]}, {"sent": "now the dumper truck is coming along Thomas .", "words": ["the", "truck", "is"]}, {"sent": "thats a truck actually Dominic .", "words": ["truck", "a"]}, {"sent": "what are you going to do with your truck ?", "words": ["what", "do", "your", "to", "with", "truck", "you", "are"]}, {"sent": "are they gonna go for a ride in the truck ?", "words": ["the", "for", "go", "truck", "they", "in", "are", "ride", "a"]}, {"sent": "you calling it a truck ?", "words": ["it", "truck", "a", "you"]}, {"sent": "youre gonna put two people in the truck .", "words": ["the", "in", "put", "truck"]}, {"sent": "do you think that the dog will be able to keep up with the fire truck ?", "words": ["the", "do", "think", "up", "be", "to", "with", "truck", "you", "will", "that", "dog"]}, {"sent": "what kind of truck ?", "words": ["of", "what", "truck"]}, {"sent": "look at the truck look at the .", "words": ["the", "at", "truck", "look"]}, {"sent": "well its the truck with xxx it .", "words": ["the", "it", "truck", "with"]}, {"sent": "now the truck .", "words": ["the", "truck"]}, {"sent": "thats the handle on the truck xxx as well as the xxx .", "words": ["the", "on", "truck"]}, {"sent": "Hobos truck .", "words": ["truck"]}, {"sent": "lets take the truck away and the scissors away and the pen away .", "words": ["the", "scissors", "truck", "and", "away", "pen", "take"]}, {"sent": "we read a story today when you were in the waiting room about a truck didnt we ?", "words": ["the", "we", "truck", "you", "in", "room", "read", "about", "story", "when", "were", "a"]}, {"sent": "the plane lands and the truck takes the mail to the post office near Mollys house .", "words": ["the", "to", "truck", "and", "house"]}, {"sent": "you dont wanna make noise with the truck ?", "words": ["the", "with", "truck", "you", "dont", "wanna", "make"]}, {"sent": "do you think its a white truck ?", "words": ["do", "think", "truck", "you", "white", "a"]}, {"sent": "dump truck .", "words": ["dump", "truck"]}, {"sent": "oh why doesnt the dump truck come and bring some .", "words": ["the", "truck", "bring", "and", "some", "dump", "why"]}, {"sent": "the white truck has driven off hasnt it ?", "words": ["the", "off", "truck", "it", "white"]}, {"sent": "I said Ive put more of it all over my hand .", "words": ["all", "put", "my", "hand", "of", "more", "it", "over"]}, {"sent": "put your hand in there .", "words": ["put", "your", "hand", "there", "in"]}, {"sent": "he can sound like a hand on a door .", "words": ["can", "hand", "like", "he", "on", "door", "a"]}, {"sent": "which hand ?", "words": ["which", "hand"]}, {"sent": "right hand .", "words": ["hand"]}, {"sent": "hes holding his hand out .", "words": ["his", "out", "hand"]}, {"sent": "what do you have in you hand ?", "words": ["what", "do", "hand", "you", "have", "in"]}, {"sent": "thats a pentagon yeah but thats this other xxx got in your hand .", "words": ["your", "hand", "other", "this", "in", "but", "a"]}, {"sent": "Mummy has two rings and a watch on one hand .", "words": ["watch", "hand", "and", "on", "a"]}, {"sent": "hold my hand .", "words": ["my", "hold", "hand"]}, {"sent": "here the hour hand is the short hand .", "words": ["here", "the", "is", "hand"]}, {"sent": "Kimberly hand cant come out .", "words": ["out", "hand"]}, {"sent": "youre going to do what and hard on my hand ?", "words": ["do", "what", "my", "hand", "to", "and", "hard", "on"]}, {"sent": "Mummys hand ?", "words": ["hand"]}, {"sent": "lift up the lift up the cutter with that hand .", "words": ["up", "the", "hand", "with", "that"]}, {"sent": "the hats in your hand .", "words": ["the", "in", "your", "hand"]}, {"sent": "so this is his hand and this is his hand and these are the sticks see ?", "words": ["the", "these", "hand", "is", "so", "and", "this", "are", "his", "see"]}, {"sent": "because I dont want you to hurt your hand .", "words": ["your", "hand", "hurt", "to", "you", "dont", "because"]}, {"sent": "are you pushing Daddys hand away .", "words": ["away", "are", "hand", "you"]}, {"sent": "rub your hand .", "words": ["your", "hand"]}, {"sent": "we we can just see the plate holding sorry the hand holding the plate cant we ?", "words": ["can", "the", "plate", "we", "hand", "see"]}, {"sent": "give us your hand .", "words": ["us", "give", "your", "hand"]}, {"sent": "in your hand .", "words": ["in", "your", "hand"]}, {"sent": "no in your hand .", "words": ["in", "your", "hand"]}, {"sent": "hi Lauras hand .", "words": ["hand"]}, {"sent": "okay put your hand there where here at the end ?", "words": ["where", "the", "put", "your", "hand", "here", "there", "at"]}, {"sent": "here you put hand in mitten .", "words": ["put", "hand", "you", "here", "in"]}, {"sent": "hand picks an apple .", "words": ["apple", "an", "hand"]}, {"sent": "my hand .", "words": ["my", "hand"]}, {"sent": "hes eating your hand ?", "words": ["your", "hand"]}, {"sent": "shall I brush Thomass hand and arm ?", "words": ["brush", "arm", "and", "hand"]}, {"sent": "good girl and this is your right hand and wheres your wrist ?", "words": ["your", "is", "hand", "and", "this", "good"]}, {"sent": "just do it on Mummys hand .", "words": ["on", "do", "it", "hand"]}, {"sent": "let me give you a hand because its quite difficult .", "words": ["hand", "you", "me", "give", "because", "a"]}, {"sent": "theres mommys hand .", "words": ["hand"]}, {"sent": "when the little hands on the eight and the big hand is on the twelve thats eight oclock .", "words": ["the", "hand", "is", "and", "little", "big", "on", "when"]}, {"sent": "hold it with that hand .", "words": ["hold", "hand", "with", "it", "that"]}, {"sent": "I dont think its sposta be a hand .", "words": ["think", "hand", "be", "dont", "a"]}, {"sent": "hold your hand under so they dont go everywhere .", "words": ["your", "hold", "hand", "go", "so", "under", "they", "dont"]}, {"sent": "is it because they they got outof hand and kept misbehaving ?", "words": ["hand", "is", "and", "they", "it", "because"]}, {"sent": "youre making my hand better are you ?", "words": ["better", "my", "hand", "you", "are"]}, {"sent": "xxx theres your hand .", "words": ["your", "hand"]}, {"sent": "well put put your hand here and well a little closer .", "words": ["put", "your", "hand", "and", "here", "little", "a"]}, {"sent": "and whats else is in her hand ?", "words": ["hand", "is", "and", "in", "her"]}, {"sent": "no this one in my hand is macaroni and cheese .", "words": ["my", "is", "hand", "and", "this", "in", "cheese"]}, {"sent": "wait wait wait wait dont take it outof my hand .", "words": ["wait", "my", "hand", "dont", "it", "take"]}, {"sent": "you have that hand .", "words": ["have", "that", "hand", "you"]}, {"sent": "should we trace your hand ?", "words": ["your", "hand", "we"]}, {"sent": "there youve got it in your hand .", "words": ["your", "hand", "there", "in", "it"]}, {"sent": "thats the palm of your hand .", "words": ["the", "of", "your", "hand"]}, {"sent": "thats a big hand isnt it ?", "words": ["it", "big", "hand", "a"]}, {"sent": "your hand is inside there .", "words": ["your", "is", "hand", "inside", "there"]}, {"sent": "can ya draw your own hand ?", "words": ["can", "draw", "your", "hand"]}, {"sent": "Im sure youve got friends at school whats that little girls name who was holding your hand today ?", "words": ["your", "hand", "school", "was", "little", "at", "who", "that"]}, {"sent": "one hand .", "words": ["hand"]}, {"sent": "you draw Mamas hand .", "words": ["draw", "hand", "you"]}, {"sent": "one thing in each hand for shaking ?", "words": ["in", "for", "each", "hand"]}, {"sent": "you gotta do one hand walking Boo .", "words": ["do", "hand", "you"]}, {"sent": "hold the ruler with one hand .", "words": ["the", "with", "hold", "hand"]}, {"sent": "put your hand on top like that .", "words": ["put", "your", "hand", "like", "on", "that"]}, {"sent": "that wont come off in your hand huh ?", "words": ["off", "your", "hand", "in", "that"]}, {"sent": "who do you think the hand belongs to ?", "words": ["the", "do", "think", "hand", "to", "you", "who"]}, {"sent": "gimme other hand .", "words": ["hand", "other"]}, {"sent": "and you also have one on your left hand .", "words": ["your", "hand", "you", "have", "and", "on"]}, {"sent": "put your hand in there .", "words": ["put", "your", "hand", "there", "in"]}, {"sent": "on my hand ?", "words": ["on", "my", "hand"]}, {"sent": "you mean hes got a hook instead of a hand ?", "words": ["of", "hand", "a", "you"]}, {"sent": "other hand .", "words": ["hand", "other"]}, {"sent": "couple times he needed a hand but usually he was getting it most of the time .", "words": ["the", "hand", "of", "was", "it", "he", "but", "a"]}, {"sent": "pour it with your other hand .", "words": ["your", "hand", "pour", "with", "other", "it"]}, {"sent": "now youre getting it all over mommys hand .", "words": ["all", "it", "over", "hand"]}, {"sent": "the left hand .", "words": ["the", "hand"]}, {"sent": "put your hand in there .", "words": ["put", "your", "hand", "there", "in"]}, {"sent": "oh whats he doing with his hand ?", "words": ["his", "he", "with", "hand"]}, {"sent": "where his hand lands .", "words": ["where", "his", "hand"]}, {"sent": "look theres a wee cooker and a wash hand basin .", "words": ["hand", "and", "look", "wash", "a"]}, {"sent": "put your hand in there .", "words": ["put", "your", "hand", "there", "in"]}, {"sent": "lick your hand it says lick lick your hand .", "words": ["your", "it", "lick", "hand"]}, {"sent": "you can use your other hand to put him on there .", "words": ["can", "put", "your", "hand", "to", "other", "you", "there", "him", "on"]}, {"sent": "shall mummie give you a little hand to make sure theyre the right way round .", "words": ["the", "hand", "to", "little", "you", "give", "make", "a"]}, {"sent": "hold it with your other other hand and then .", "words": ["your", "hold", "hand", "with", "other", "and", "it", "then"]}, {"sent": "and show Daddy what has happened to your hand .", "words": ["what", "your", "hand", "show", "to", "and"]}, {"sent": "I wanna stick this hand in so Ill get it right .", "words": ["stick", "hand", "so", "get", "this", "in", "it", "wanna"]}, {"sent": "Lily can you hand mommy the bag that you took the food out of ?", "words": ["can", "the", "hand", "of", "you", "food", "out", "that"]}, {"sent": "okay move your hand .", "words": ["your", "hand"]}, {"sent": "what do you have in your hand ?", "words": ["what", "do", "your", "hand", "you", "have", "in"]}, {"sent": "no no this hand this hand .", "words": ["this", "hand"]}, {"sent": "I dont know wait wait move your hand I cant see the name .", "words": ["the", "wait", "your", "hand", "dont", "see"]}, {"sent": "and the little hand .", "words": ["the", "hand", "little", "and"]}, {"sent": "you wouldnt want to put your hand in the tire .", "words": ["the", "put", "your", "hand", "to", "you", "in", "want to"]}, {"sent": "unless youve got it in your hand .", "words": ["in", "it", "your", "hand"]}, {"sent": "you hold it in your hand .", "words": ["your", "hold", "hand", "you", "in", "it"]}, {"sent": "um other hand plays no .", "words": ["hand", "other"]}, {"sent": "and a bright yellow maraca in one hand .", "words": ["hand", "and", "in", "yellow", "a"]}, {"sent": "no use your other hand .", "words": ["your", "hand", "other"]}, {"sent": "dont you put your hand through there .", "words": ["put", "your", "hand", "you", "dont", "there"]}, {"sent": "or or roll it with the other hand .", "words": ["the", "hand", "with", "other", "it"]}, {"sent": "try the other ones in your hand .", "words": ["the", "your", "hand", "other", "in", "try"]}, {"sent": "this is the big hand .", "words": ["the", "hand", "is", "this", "big"]}, {"sent": "does the sweet in this hand feel heavier than the sweet in that hand ?", "words": ["the", "does", "hand", "this", "in", "that"]}, {"sent": "okay let us see if we can get them on so they stay on there though .", "words": ["can", "stay", "if", "we", "so", "them", "get", "us", "they", "there", "on", "see"]}, {"sent": "it willnt stay on Henry either .", "words": ["on", "stay", "it"]}, {"sent": "Thomas stay and talk to me please .", "words": ["to", "me", "and", "talk", "stay"]}, {"sent": "trying to stay in one place .", "words": ["to", "in", "stay"]}, {"sent": "if I stay ?", "words": ["stay", "if"]}, {"sent": "stay away Mummy .", "words": ["stay", "away"]}, {"sent": "I dont think hell stay up there .", "words": ["up", "think", "dont", "there", "stay"]}, {"sent": "to stay strong and healthy the brown horse eats all day in the stable or the field he munches on some hay .", "words": ["the", "all", "brown", "to", "and", "in", "on", "he", "horse", "stay", "some"]}, {"sent": "they stay on the camera .", "words": ["the", "camera", "they", "on", "stay"]}, {"sent": "I wanna stay way from ?", "words": ["stay", "wanna"]}, {"sent": "it doesnt stay .", "words": ["stay", "it"]}, {"sent": "but itll then hafta stay in the fridge .", "words": ["the", "in", "stay", "then", "but"]}, {"sent": "stay there .", "words": ["stay", "there"]}, {"sent": "shall we stay in here ?", "words": ["here", "stay", "in", "we"]}, {"sent": "we hafta stay with the wires .", "words": ["the", "stay", "with", "we"]}, {"sent": "youre going to tumble if you stay there .", "words": ["if", "to", "you", "there", "stay"]}, {"sent": "stay here a minute .", "words": ["here", "stay", "a"]}, {"sent": "alright so hell probably stay .", "words": ["so", "stay"]}, {"sent": "but when she went to stay in the country she decided she liked the country as well as the town .", "words": ["the", "to", "in", "stay", "when", "she", "but"]}, {"sent": "I could stay here for hours night night Poppy .", "words": ["here", "stay", "could", "for"]}, {"sent": "stay there then .", "words": ["stay", "then", "there"]}, {"sent": "no I want you to stay down here .", "words": ["to", "you", "here", "stay", "down"]}, {"sent": "stay outof the water .", "words": ["the", "stay", "water"]}, {"sent": "well this little cow just doesnt wanna stay on its feet .", "words": ["little", "this", "wanna", "on", "stay", "cow"]}, {"sent": "Gaspard and Lisa please try to stay out of trouble our teacher said as we got on the bus .", "words": ["the", "we", "of", "to", "and", "our", "out", "on", "stay", "try to", "try", "bus"]}, {"sent": "but you must stay safe in the bathtub .", "words": ["the", "you", "in", "bathtub", "stay", "but"]}, {"sent": "right you just stay here with Dada and Ill just go and get the doctors and get your prescription okay ?", "words": ["the", "your", "go", "with", "and", "here", "you", "get", "stay"]}, {"sent": "does this stay at the shop ?", "words": ["the", "does", "this", "at", "stay"]}, {"sent": "but she doesnt stay in here very long .", "words": ["long", "here", "in", "stay", "she", "but"]}, {"sent": "I wanna stay here .", "words": ["here", "stay", "wanna"]}, {"sent": "stay there .", "words": ["stay", "there"]}, {"sent": "och no shell stay here .", "words": ["here", "stay"]}, {"sent": "stay there .", "words": ["stay", "there"]}, {"sent": "stay there .", "words": ["stay", "there"]}, {"sent": "Im two and a half and I can stay up late .", "words": ["can", "up", "and", "stay", "a"]}, {"sent": "now itll stay on .", "words": ["on", "stay"]}, {"sent": "if you wanted to stay you could stay .", "words": ["if", "to", "you", "stay", "could"]}, {"sent": "did you hafta stay at home instead of going to playschool ?", "words": ["of", "to", "you", "did", "at", "stay", "home"]}, {"sent": "okay Ill stay .", "words": ["stay"]}, {"sent": "why it willnt stay down .", "words": ["stay", "down", "it", "why"]}, {"sent": "you stay here .", "words": ["here", "stay", "you"]}, {"sent": "Ill seal it up with a clip and then theyll stay fresh till tomorrow wont they ?", "words": ["up", "with", "and", "they", "it", "stay", "then", "a"]}, {"sent": "stay with Mummy a minute .", "words": ["stay", "with", "a"]}, {"sent": "Ill stay in there and maybe shell stay in there to .", "words": ["to", "and", "there", "in", "stay"]}, {"sent": "stay here please Lara .", "words": ["here", "stay"]}, {"sent": "to stay strong and healthy the brown horse eats all day the stabler the field he munches on some hay .", "words": ["the", "all", "brown", "to", "and", "on", "he", "horse", "stay", "some"]}, {"sent": "uhhuh Ill stay by you always and live by the swamp .", "words": ["the", "and", "you", "by", "stay"]}, {"sent": "do you want them to stay longer ?", "words": ["do", "to", "them", "you", "stay"]}, {"sent": "stay over here .", "words": ["here", "stay", "over"]}, {"sent": "let her stay here please .", "words": ["here", "stay", "her"]}, {"sent": "youre gonna stay here ?", "words": ["here", "stay"]}, {"sent": "mm foxes stay in a den during the day and they come out at night .", "words": ["the", "and", "they", "in", "out", "at", "stay", "a"]}, {"sent": "the dryer stay on the longest .", "words": ["the", "stay", "on", "dryer"]}, {"sent": "if I could find someone to stay on my nest .", "words": ["my", "if", "to", "find", "on", "stay", "could"]}, {"sent": "stay where you are .", "words": ["where", "stay", "are", "you"]}, {"sent": "do you wanna go to bed now or stay up for a while ?", "words": ["up", "do", "for", "go", "to", "bed", "you", "wanna", "stay", "a"]}, {"sent": "no are you just going to stay dirty ?", "words": ["to", "you", "are", "dirty", "stay"]}, {"sent": "so stay there for a little while .", "words": ["for", "so", "little", "there", "stay", "a"]}, {"sent": "well you stay here .", "words": ["here", "stay", "you"]}, {"sent": "were gonna stay in this town for a couple of days and then we can visit the dogs ?", "words": ["can", "the", "for", "we", "of", "and", "this", "in", "stay", "then", "were", "a"]}, {"sent": "then xxx hafta stay inside because its raining out .", "words": ["inside", "out", "stay", "then", "because"]}, {"sent": "youll hafta stay here then .", "words": ["here", "stay", "then"]}, {"sent": "stay still .", "words": ["stay"]}, {"sent": "they stay up there .", "words": ["up", "they", "stay", "there"]}, {"sent": "stay sitting .", "words": ["stay"]}, {"sent": "you stay here .", "words": ["here", "stay", "you"]}, {"sent": "Adam maybe your fire engine will stay on better .", "words": ["better", "your", "on", "will", "stay"]}, {"sent": "going to stay and Grandma and Granddads house .", "words": ["to", "stay", "house", "and"]}, {"sent": "stay right here .", "words": ["here", "stay"]}, {"sent": "so then if they both stay at home how do they have money for their house and for food ?", "words": ["money", "do", "how", "for", "if", "their", "so", "and", "have", "they", "food", "house", "at", "stay", "then", "home"]}, {"sent": "we went to stay with Mark first of all didnt we ?", "words": ["all", "we", "of", "to", "first", "with", "stay"]}, {"sent": "will you not stay ?", "words": ["will", "stay", "not", "you"]}, {"sent": "youve got to stay with me .", "words": ["to", "me", "stay", "with"]}, {"sent": "and you stay dry .", "words": ["stay", "dry", "you", "and"]}, {"sent": "right you stay in bed because you havent been a very good boy .", "words": ["bed", "you", "in", "good", "stay", "because", "a"]}, {"sent": "stay here .", "words": ["here", "stay"]}, {"sent": "tadah you got it to stay .", "words": ["to", "it", "stay", "you"]}, {"sent": "you stay with Thomas on the tracks and Ill get you the clock .", "words": ["the", "clock", "with", "you", "and", "get", "on", "stay"]}, {"sent": "now shes mainly breastfed um and its I stay at home with her now Im not um Im you know I took a year off from teaching to be with her .", "words": ["off", "to", "be", "with", "and", "you", "at", "stay", "her", "not", "a", "home"]}, {"sent": "oh did you go and stay in a hotel did you down there ?", "words": ["go", "and", "you", "there", "in", "did", "stay", "down", "a"]}, {"sent": "stay in bed .", "words": ["stay", "in", "bed"]}, {"sent": "she goes from one thing to the other as you can see shes pretty mobile so she doesnt stay on one toy or one thing for very long she switches .", "words": ["the", "can", "stay", "for", "to", "so", "long", "you", "other", "toy", "pretty", "on", "see", "she"]}, {"sent": "how long have I got to stay here for ?", "words": ["how", "for", "to", "long", "have", "here", "stay"]}, {"sent": "because they like to stay wet .", "words": ["wet", "to", "like", "they", "stay", "because"]}, {"sent": "stay up .", "words": ["up", "stay"]}, {"sent": "do you want Mummy to stay at home ?", "words": ["do", "to", "you", "at", "stay", "home"]}, {"sent": "just let her just let her stay there .", "words": ["stay", "her", "there"]}, {"sent": "because Dada wants to stay with you inside .", "words": ["to", "inside", "with", "you", "stay", "because"]}, {"sent": "howd you get it to stay like that ?", "words": ["to", "like", "you", "get", "it", "stay", "that"]}, {"sent": "stay right there .", "words": ["stay", "there"]}, {"sent": "well when we go to the car we definitely hafta check and I dont know if any of them are in your room do you think they might be in your room sometimes they theyll stay in Mommys purse when I bring them somewhere to be with you and they end up stay in my purse .", "words": ["the", "think", "go", "of", "car", "any", "when", "be", "with", "are", "stay", "up", "if", "to", "bring", "and", "you", "dont", "do", "my", "your", "purse", "we", "them", "they", "in", "room"]}, {"sent": "you stay down here please .", "words": ["here", "stay", "down", "you"]}, {"sent": "yeah Ariel gives up her beautiful voice to stay human Ariel must make Eric give her a kiss of true love within three days .", "words": ["up", "of", "to", "make", "love", "kiss", "stay", "give", "her", "a"]}, {"sent": "but it can stay up if you dont .", "words": ["can", "up", "if", "you", "dont", "it", "stay", "but"]}, {"sent": "you stay here .", "words": ["here", "stay", "you"]}, {"sent": "yeah the glue didnt stay very well .", "words": ["the", "glue", "stay"]}, {"sent": "Mummy said I could come to tea and stay the night .", "words": ["the", "to", "and", "stay", "could"]}, {"sent": "were gonna stay .", "words": ["stay", "were"]}, {"sent": "can we take the shoes off too or do they stay on ?", "words": ["can", "the", "off", "do", "we", "they", "too", "on", "stay", "take"]}, {"sent": "thats the light .", "words": ["the", "light"]}, {"sent": "tell me what the light is .", "words": ["the", "what", "is", "light", "me"]}, {"sent": "uhhuh because its very light stuff .", "words": ["light", "because"]}, {"sent": "holding up your fishing rod towards the light .", "words": ["up", "the", "your", "light"]}, {"sent": "oh no I dont know where the light is .", "words": ["where", "the", "is", "light", "dont"]}, {"sent": "that is a light blue car .", "words": ["is", "light", "car", "blue", "that", "a"]}, {"sent": "a light orange .", "words": ["light", "orange", "a"]}, {"sent": "uh Joseph wheres the light here we go .", "words": ["the", "go", "we", "light", "here"]}, {"sent": "every morning as the sun peeks over the horizon a giant rock formation catches the first rays of light .", "words": ["the", "of", "light", "sun", "first", "every", "rock", "over", "a"]}, {"sent": "light .", "words": ["light"]}, {"sent": "oh that has a the light is shining in your face .", "words": ["the", "your", "face", "is", "light", "in", "that", "a"]}, {"sent": "and this is your light on the top here that flashes like daddys ?", "words": ["the", "your", "is", "light", "like", "and", "here", "this", "on", "that"]}, {"sent": "light .", "words": ["light"]}, {"sent": "how did the dragon light the fire ?", "words": ["the", "how", "did", "light"]}, {"sent": "like my light .", "words": ["light", "my", "like"]}, {"sent": "light okay .", "words": ["light"]}, {"sent": "you turned the light on ?", "words": ["the", "light", "on", "you"]}, {"sent": "is the light on in the freezer ?", "words": ["the", "is", "light", "in", "on"]}, {"sent": "babye light .", "words": ["light"]}, {"sent": "theres a little red light .", "words": ["red", "light", "little", "a"]}, {"sent": "you turned the light off to eat ?", "words": ["the", "off", "light", "to", "you", "eat"]}, {"sent": "the moon is the brightest light in the night sky .", "words": ["the", "moon", "is", "light", "sky", "in"]}, {"sent": "when the light went when you pushed the button the lights came on right ?", "words": ["the", "when", "light", "you", "on", "button"]}, {"sent": "a lantern or a light .", "words": ["light", "a"]}, {"sent": "and this is their light sothat they can see when it gets dark .", "words": ["can", "their", "is", "light", "and", "this", "they", "it", "dark", "see", "when"]}, {"sent": "now pick up your blankie Ill turn out the light kiss Mama kiss daddy goodnight dear goodnight .", "words": ["the", "up", "your", "pick", "light", "out", "kiss"]}, {"sent": "are you pointing to the light ?", "words": ["the", "light", "to", "you", "are"]}, {"sent": "there is the light .", "words": ["the", "there", "light", "is"]}, {"sent": "but when the light hits that it sends out these little things from the center .", "words": ["the", "these", "when", "light", "little", "it", "out", "that", "but"]}, {"sent": "its alright if the light is still on .", "words": ["the", "if", "is", "light", "on"]}, {"sent": "you know you got burnt on the light yesterday ?", "words": ["on", "the", "light", "you"]}, {"sent": "no white light .", "words": ["light", "white"]}, {"sent": "Im gonna get a light .", "words": ["get", "light", "a"]}, {"sent": "oh you put off we turn off the light ?", "words": ["off", "the", "put", "we", "light", "you"]}, {"sent": "light on yeah .", "words": ["light", "on"]}, {"sent": "look through the light .", "words": ["the", "light", "look"]}, {"sent": "well put the light on because its a little bit dark this morning .", "words": ["the", "put", "light", "little", "this", "dark", "on", "because", "a"]}, {"sent": "red rhino sees a red traffic light but he cant see the red balloon .", "words": ["red", "the", "light", "he", "balloon", "see", "but", "a"]}, {"sent": "I can just about see it because its a very light color .", "words": ["can", "light", "it", "about", "see", "because", "a"]}, {"sent": "can you turn the light on ?", "words": ["can", "the", "light", "you", "on"]}, {"sent": "they do have a light its a lamp .", "words": ["do", "light", "have", "they", "lamp", "a"]}, {"sent": "yes its light .", "words": ["light"]}, {"sent": "in the light of the moon a little egg lay on a leaf .", "words": ["the", "moon", "egg", "of", "light", "little", "in", "on", "a"]}, {"sent": "a light .", "words": ["light", "a"]}, {"sent": "light on .", "words": ["light", "on"]}, {"sent": "can you see the light come on ?", "words": ["can", "the", "light", "you", "on", "see"]}, {"sent": "one looks like a red traffic light and one looks like a bridge .", "words": ["red", "light", "like", "and", "a"]}, {"sent": "and they found out why its dark on one side of earth and its light on the other didnt they ?", "words": ["the", "of", "light", "and", "other", "they", "out", "dark", "on", "why"]}, {"sent": "night light .", "words": ["light"]}, {"sent": "because I wanna turn the light on because its getting quite dark now isnt it ?", "words": ["the", "light", "it", "wanna", "dark", "on", "because"]}, {"sent": "put the light on darling .", "words": ["the", "put", "on", "light"]}, {"sent": "if we get a neon light I think neon light will kill .", "words": ["think", "if", "we", "light", "get", "will", "a"]}, {"sent": "you see thats too light .", "words": ["light", "see", "too", "you"]}, {"sent": "you want to see the bathroom light .", "words": ["the", "light", "to", "you", "want to", "bathroom", "see"]}, {"sent": "I turnt the light on .", "words": ["the", "on", "light"]}, {"sent": "what light ?", "words": ["light", "what"]}, {"sent": "do you know how to turn the light on ?", "words": ["the", "do", "how", "light", "to", "you", "on"]}, {"sent": "is the light broken ?", "words": ["the", "light", "broken", "is"]}, {"sent": "three little bears one with a light one with a stick and one with a rope .", "words": ["stick", "light", "with", "little", "and", "a"]}, {"sent": "the lady that lives in that house where the light keeps coming on .", "words": ["the", "where", "light", "in", "house", "on", "that"]}, {"sent": "he sees the light coming through and Im like uh .", "words": ["the", "light", "like", "and", "he"]}, {"sent": "the worker ant is light and small .", "words": ["the", "is", "light", "and", "ant"]}, {"sent": "so on NewYearsEve well be able to light that candle wont we ?", "words": ["we", "light", "so", "be", "to", "on", "that"]}, {"sent": "the light has lit up .", "words": ["the", "up", "light"]}, {"sent": "invented a light that plugs into the sun .", "words": ["the", "into", "light", "sun", "that", "a"]}, {"sent": "theres a green light .", "words": ["light", "green", "a"]}, {"sent": "Im light .", "words": ["light"]}, {"sent": "oh the danger light .", "words": ["the", "light"]}, {"sent": "you see its like looking in a mirror you see the light from the ceiling reflected in the ring .", "words": ["the", "light", "like", "you", "in", "see", "a"]}, {"sent": "well I put the red light here though right ?", "words": ["the", "red", "put", "light", "here"]}, {"sent": "about the switch for that light .", "words": ["the", "for", "light", "about", "that"]}, {"sent": "tell Jennifer to leave the light on .", "words": ["the", "to", "on", "light"]}, {"sent": "and when you went to sleep it was light and when you woke up it was .", "words": ["up", "sleep", "to", "was", "light", "you", "and", "it", "when"]}, {"sent": "oh thats what Daddy puts in the light bulb when we go away .", "words": ["the", "what", "we", "light", "go", "in", "away", "when"]}, {"sent": "shall I put the light on in here actually ?", "words": ["the", "put", "light", "here", "in", "on"]}, {"sent": "hes putting a light on .", "words": ["light", "on", "a"]}, {"sent": "I put a new light .", "words": ["light", "put", "new", "a"]}, {"sent": "you need the light at the front of the train .", "words": ["the", "need", "train", "light", "of", "you", "at"]}, {"sent": "you want me to put the light on ?", "words": ["the", "put", "light", "to", "you", "on", "me"]}, {"sent": "Maisie turns off the light .", "words": ["off", "the", "light"]}, {"sent": "yeah the light bulb is there .", "words": ["the", "there", "is", "light"]}, {"sent": "there theres a red light on the tape recorder isnt there ?", "words": ["red", "the", "light", "tape", "there", "on", "a"]}, {"sent": "and one of them didnt have the orange light on .", "words": ["the", "of", "light", "them", "and", "have", "orange", "on"]}, {"sent": "uhhuh theres another light .", "words": ["another", "light"]}, {"sent": "how did the dragon light the fire ?", "words": ["the", "how", "did", "light"]}, {"sent": "and light it .", "words": ["light", "it", "and"]}, {"sent": "say byebye light .", "words": ["light", "say"]}, {"sent": "no I want the deal where we turn off the light and you put your clothes back on .", "words": ["the", "off", "where", "put", "your", "back", "we", "light", "and", "you", "on"]}, {"sent": "do you want me to put the light on in your room ?", "words": ["the", "do", "put", "your", "light", "to", "you", "in", "room", "on", "me"]}, {"sent": "and light blue like Daddys car .", "words": ["light", "car", "like", "blue", "and"]}, {"sent": "because its light in there .", "words": ["light", "there", "in", "because"]}, {"sent": "perhaps its just the light .", "words": ["the", "light"]}, {"sent": "do you see the light ?", "words": ["the", "do", "light", "you", "see"]}, {"sent": "thats light green and dark green right .", "words": ["light", "dark", "green", "and"]}, {"sent": "okay and heres a light .", "words": ["light", "a", "and"]}, {"sent": "well I think before we do that let me just put some light on in the kitchen otherwise were going to be too dark wont we ?", "words": ["the", "do", "think", "put", "we", "light", "to", "be", "dark", "in", "kitchen", "too", "on", "me", "that", "were", "some"]}, {"sent": "thats a light .", "words": ["light", "a"]}, {"sent": "have you seen the lights light up on that till when you do that ?", "words": ["the", "up", "do", "when", "light", "you", "have", "on", "that"]}, {"sent": "mhm not at the light .", "words": ["the", "light", "not", "at"]}, {"sent": "and theyre painting a picture of the yellow sun arent they ?", "words": ["the", "of", "sun", "picture", "and", "they", "yellow", "a"]}, {"sent": "somebody painted her yellow .", "words": ["yellow", "her"]}, {"sent": "yellow .", "words": ["yellow"]}, {"sent": "that yellow one is the v .", "words": ["the", "yellow", "that", "is"]}, {"sent": "yellow thats a yellow block .", "words": ["yellow", "block", "a"]}, {"sent": "its a yellow tshirt isnt it ?", "words": ["yellow", "it", "a"]}, {"sent": "its yellow like yours .", "words": ["yellow", "like"]}, {"sent": "yellow .", "words": ["yellow"]}, {"sent": "that big yellow blanket .", "words": ["blanket", "yellow", "big", "that"]}, {"sent": "we saw a big yellow lorry .", "words": ["yellow", "big", "a", "we"]}, {"sent": "its yellow do you know ?", "words": ["yellow", "do", "you"]}, {"sent": "yellow hammer is their bird .", "words": ["hammer", "their", "is", "bird", "yellow"]}, {"sent": "so you say yellow boxes partly because of the color I think .", "words": ["the", "think", "of", "so", "say", "you", "yellow", "because"]}, {"sent": "little yellow the yellow flower .", "words": ["the", "yellow", "flower", "little"]}, {"sent": "xxx yellow hippo has a yellow cart she takes it for a walk .", "words": ["for", "it", "yellow", "walk", "she", "a"]}, {"sent": "and the blue one and the yellow one ?", "words": ["the", "yellow", "blue", "and"]}, {"sent": "and yellow for Chris .", "words": ["yellow", "for", "and"]}, {"sent": "a yellow wow wow ?", "words": ["yellow", "a"]}, {"sent": "is that yellow ?", "words": ["yellow", "that", "is"]}, {"sent": "shes wearing a yellow sweater .", "words": ["yellow", "sweater", "a"]}, {"sent": "yellow yellow yellow yellow .", "words": ["yellow"]}, {"sent": "yeah and then lets put the yellow one on .", "words": ["the", "put", "and", "on", "yellow", "then"]}, {"sent": "yellow .", "words": ["yellow"]}, {"sent": "which ones the yellow one ?", "words": ["the", "yellow", "which"]}, {"sent": "if this if this is yellow how can this be yellow ?", "words": ["can", "how", "if", "is", "be", "this", "yellow"]}, {"sent": "uh yellow ducks are yellow and what is the frog ?", "words": ["the", "what", "is", "and", "frog", "are", "yellow"]}, {"sent": "yellow .", "words": ["yellow"]}, {"sent": "I think I particularly like the yellow horse and the blue fence .", "words": ["the", "think", "like", "and", "blue", "horse", "yellow"]}, {"sent": "and thats a yellow one .", "words": ["yellow", "a", "and"]}, {"sent": "in there the yellow one .", "words": ["the", "there", "in", "yellow"]}, {"sent": "yellow plate ?", "words": ["yellow", "plate"]}, {"sent": "oh youve just seen your yellow helmet havent you ?", "words": ["yellow", "your", "you"]}, {"sent": "yellow .", "words": ["yellow"]}, {"sent": "Im colored in stripes with yellow and blue .", "words": ["with", "blue", "and", "in", "yellow"]}, {"sent": "where is yellow ?", "words": ["where", "yellow", "is"]}, {"sent": "is yellow your favorite color ?", "words": ["yellow", "your", "is"]}, {"sent": "yellow .", "words": ["yellow"]}, {"sent": "can I do the teddybear yellow ?", "words": ["can", "do", "the", "teddybear", "yellow"]}, {"sent": "yellow and .", "words": ["yellow", "and"]}, {"sent": "how can I set the yellow one up ?", "words": ["can", "the", "how", "up", "yellow"]}, {"sent": "its yellow .", "words": ["yellow"]}, {"sent": "where is yellow ?", "words": ["where", "yellow", "is"]}, {"sent": "a little yellow duckling swim across the pond and back .", "words": ["the", "back", "swim", "little", "and", "yellow", "a"]}, {"sent": "it is sort of a bright yellow .", "words": ["is", "of", "it", "yellow", "a"]}, {"sent": "red nose red nose no I like the yellow .", "words": ["red", "the", "like", "yellow", "nose"]}, {"sent": "well start with yellow .", "words": ["yellow", "with"]}, {"sent": "the nose is orange and this is orange and this is yellow .", "words": ["the", "is", "and", "this", "orange", "yellow", "nose"]}, {"sent": "this is a yellow one .", "words": ["this", "yellow", "a", "is"]}, {"sent": "here Fri do the yellow crayon .", "words": ["the", "do", "crayon", "here", "yellow"]}, {"sent": "do christmastrees have yellow spots ?", "words": ["have", "do", "yellow"]}, {"sent": "that yellow is ?", "words": ["yellow", "that", "is"]}, {"sent": "yellow Po ?", "words": ["yellow"]}, {"sent": "a nice piece of yellow paper .", "words": ["of", "paper", "nice", "yellow", "a"]}, {"sent": "youre picking yellow .", "words": ["yellow"]}, {"sent": "so I think you might hafta take the yellow one off .", "words": ["the", "off", "think", "so", "you", "yellow", "take"]}, {"sent": "yellow mommy .", "words": ["yellow"]}, {"sent": "yellow .", "words": ["yellow"]}, {"sent": "its a yellow one .", "words": ["yellow", "a"]}, {"sent": "I want yellow .", "words": ["yellow"]}, {"sent": "thats yellow and thats .", "words": ["yellow", "and"]}, {"sent": "so this is yellow then is it ?", "words": ["is", "so", "this", "it", "yellow", "then"]}, {"sent": "hes uh wheres the yellow ?", "words": ["the", "yellow"]}, {"sent": "Jake has got a yellow face hasnt he ?", "words": ["yellow", "he", "face", "a"]}, {"sent": "with our yellow horse and our pink pig and our blue yellow and black bird .", "words": ["bird", "with", "and", "our", "pig", "blue", "black", "horse", "yellow"]}, {"sent": "two ye yellow pockets on that little rhyme about the bee landing on somebodys nose .", "words": ["the", "bee", "little", "about", "on", "yellow", "that", "nose"]}, {"sent": "a yellow one .", "words": ["yellow", "a"]}, {"sent": "and youve got a purple and yellow and silver one .", "words": ["yellow", "a", "and"]}, {"sent": "two yellow ones huh ?", "words": ["yellow"]}, {"sent": "which yellow one ?", "words": ["yellow", "which"]}, {"sent": "well the driver is called Pete and he has a little yellow bus .", "words": ["the", "is", "and", "little", "he", "yellow", "bus", "a"]}, {"sent": "yellow .", "words": ["yellow"]}, {"sent": "look at the yellow tummy .", "words": ["the", "look", "tummy", "at", "yellow"]}, {"sent": "yellow .", "words": ["yellow"]}, {"sent": "yellow one .", "words": ["yellow"]}, {"sent": "is the little yellow train hitting the big blue train .", "words": ["the", "train", "is", "little", "blue", "yellow", "big"]}, {"sent": "yellow .", "words": ["yellow"]}, {"sent": "y and thats a yellow yolk .", "words": ["yellow", "a", "and"]}, {"sent": "yellow what ?", "words": ["yellow", "what"]}, {"sent": "yellow .", "words": ["yellow"]}, {"sent": "youve got these youve got these very bright yellow trousers on again havent you ?", "words": ["on", "these", "yellow", "you"]}, {"sent": "yellow .", "words": ["yellow"]}, {"sent": "yellow .", "words": ["yellow"]}, {"sent": "now pick up a yellow one .", "words": ["pick", "yellow", "up", "a"]}, {"sent": "whos yellow group two Thomas ?", "words": ["yellow"]}, {"sent": "you have a big yellow chimney .", "words": ["you", "have", "yellow", "big", "a"]}, {"sent": "and a yellow one .", "words": ["yellow", "a", "and"]}, {"sent": "thats where that other yellow thing goes .", "words": ["where", "yellow", "that", "other"]}, {"sent": "red and yellow .", "words": ["red", "yellow", "and"]}, {"sent": "so the yellow ones go on your picture do they and the blue ones go on Jwwws ?", "words": ["the", "do", "your", "go", "so", "picture", "and", "blue", "they", "on", "yellow"]}, {"sent": "well theyre both a yellow color arent they ?", "words": ["yellow", "they", "a"]}, {"sent": "yellow red brown and white .", "words": ["red", "brown", "and", "yellow", "white"]}, {"sent": "yellow cars broken down too ?", "words": ["yellow", "too", "down", "broken"]}, {"sent": "its yellow .", "words": ["yellow"]}, {"sent": "its not yellow .", "words": ["yellow", "not"]}, {"sent": "thats your middle finger you were just putting that yellow hexagon on your middle finger .", "words": ["your", "you", "finger", "on", "yellow", "that", "were"]}, {"sent": "we can make a nice yellow Bob .", "words": ["can", "we", "nice", "yellow", "make", "a"]}, {"sent": "I think you might needta take the yellow one off sweetheart .", "words": ["the", "off", "think", "you", "yellow", "take"]}, {"sent": "theyre yellow .", "words": ["yellow"]}, {"sent": "yellow one .", "words": ["yellow"]}, {"sent": "where did you brush your teeth ?", "words": ["where", "your", "brush", "you", "did"]}, {"sent": "now one more brush sweetheart .", "words": ["brush", "more"]}, {"sent": "is he saying brush my hair Dillon ?", "words": ["my", "is", "brush", "he", "hair"]}, {"sent": "I brush my teeth xxx .", "words": ["brush", "my"]}, {"sent": "Im gonna rinse off that brush or do you wanna rinse off that brush for me in the sink please ?", "words": ["off", "do", "the", "for", "brush", "you", "in", "sink", "wanna", "me", "that"]}, {"sent": "when I brush when Ive brushed the kitchen floor I shall need my tea break .", "words": ["the", "my", "break", "need", "brush", "kitchen", "when"]}, {"sent": "another brush ?", "words": ["another", "brush"]}, {"sent": "that is for cleaning the brush when you want to change colors remember to clean your brush .", "words": ["the", "clean", "for", "your", "when", "is", "brush", "to", "you", "want to", "that"]}, {"sent": "whats right underneath the brush ?", "words": ["the", "brush"]}, {"sent": "its only a quick brush .", "words": ["brush", "a"]}, {"sent": "and we cant do it without a brush .", "words": ["do", "we", "brush", "and", "it", "a"]}, {"sent": "can you brush her hair ?", "words": ["can", "brush", "you", "hair", "her"]}, {"sent": "brush brush brush brush brush brush .", "words": ["brush"]}, {"sent": "can I use your brush first ?", "words": ["can", "brush", "your", "first"]}, {"sent": "can I brush the babys hair ?", "words": ["can", "the", "hair", "brush"]}, {"sent": "brush them .", "words": ["brush", "them"]}, {"sent": "okay Im going to brush my hair .", "words": ["brush", "to", "my", "hair"]}, {"sent": "you brush them in the morning .", "words": ["the", "brush", "them", "you", "in"]}, {"sent": "we didnt brush the cat .", "words": ["brush", "the", "cat", "we"]}, {"sent": "did you have the brush ?", "words": ["the", "brush", "you", "have", "did"]}, {"sent": "thats a brush .", "words": ["brush", "a"]}, {"sent": "the brush is lost isnt it ?", "words": ["the", "it", "brush", "is"]}, {"sent": "brush very good .", "words": ["brush", "good"]}, {"sent": "brush this one .", "words": ["brush", "this"]}, {"sent": "found the brush ?", "words": ["the", "brush"]}, {"sent": "put Daddys brush back .", "words": ["brush", "put", "back"]}, {"sent": "can I brush your hair ?", "words": ["can", "hair", "brush", "your"]}, {"sent": "you can have the big brush .", "words": ["can", "the", "brush", "you", "have", "big"]}, {"sent": "if you use one brush for the orange and then you use the same brush for the green .", "words": ["the", "for", "if", "brush", "you", "and", "orange", "same", "green", "then"]}, {"sent": "whoops brush .", "words": ["brush"]}, {"sent": "a big brush for the garden for sweeping up leaves .", "words": ["the", "up", "for", "brush", "garden", "big", "a"]}, {"sent": "well she her favorite part is a comb and a brush and a bowl full of mush and the quiet old lady whispering and she goes hush .", "words": ["bowl", "the", "old", "full", "is", "brush", "of", "comb", "and", "quiet", "her", "she", "a"]}, {"sent": "no no brush the trees when youre outside .", "words": ["brush", "the", "when", "outside"]}, {"sent": "brush his hair .", "words": ["brush", "his", "hair"]}, {"sent": "is to ask you to brush your teeth again .", "words": ["your", "is", "brush", "to", "you"]}, {"sent": "now just brush the floor .", "words": ["brush", "the"]}, {"sent": "brush .", "words": ["brush"]}, {"sent": "wash your brush .", "words": ["your", "brush", "wash"]}, {"sent": "and with a brush she hadta get your hat down from the roof .", "words": ["the", "your", "brush", "with", "and", "get", "roof", "hat", "down", "she", "a"]}, {"sent": "brush brush brush brush brushu .", "words": ["brush"]}, {"sent": "well brush brush brush for me and you .", "words": ["for", "brush", "you", "and", "me"]}, {"sent": "brush your zoobs brush your zoobs brush your zoobs brush your zoobs .", "words": ["brush", "your"]}, {"sent": "brush your teeth .", "words": ["brush", "your"]}, {"sent": "Purdie doesnt brush trees .", "words": ["brush"]}, {"sent": "shall mummy wash up while you brush the floor ?", "words": ["up", "the", "brush", "you", "wash"]}, {"sent": "you really need to brush your teeth after corn ya know ?", "words": ["need", "your", "brush", "to", "need to", "you", "corn"]}, {"sent": "shall we brush it ?", "words": ["brush", "it", "we"]}, {"sent": "rinse your brush .", "words": ["brush", "your"]}, {"sent": "okay should we try and brush ?", "words": ["brush", "try", "and", "we"]}, {"sent": "they brush and brush and brush their teeth .", "words": ["brush", "they", "their", "and"]}, {"sent": "you want your brush okay .", "words": ["brush", "your", "you"]}, {"sent": "you know how we brush Purdie with that special brush ?", "words": ["how", "we", "brush", "with", "you", "that"]}, {"sent": "youre trying to get on Mummys knee so that you can brush Mummys hair .", "words": ["can", "knee", "brush", "to", "so", "you", "get", "on", "hair", "that"]}, {"sent": "you can use your paint brush with these too .", "words": ["can", "these", "paint", "your", "brush", "with", "you", "too"]}, {"sent": "broom thats with brush uhn isnt it ?", "words": ["brush", "broom", "it", "with"]}, {"sent": "brush .", "words": ["brush"]}, {"sent": "well clean your brush honey .", "words": ["clean", "brush", "your"]}, {"sent": "does wanna brush your hair ?", "words": ["does", "your", "brush", "wanna", "hair"]}, {"sent": "brush brush brush your teeth each day .", "words": ["brush", "your", "each"]}, {"sent": "brush .", "words": ["brush"]}, {"sent": "oh youre going to brush mommys hair .", "words": ["brush", "to", "hair"]}, {"sent": "can I brush your hair ?", "words": ["can", "hair", "brush", "your"]}, {"sent": "but and whats she using to brush her teeth ?", "words": ["brush", "to", "and", "her", "she", "but"]}, {"sent": "Im not sure whether youre saying wash or brush .", "words": ["brush", "wash", "not"]}, {"sent": "because we no brush a trees do we ?", "words": ["do", "we", "brush", "because", "a"]}, {"sent": "wheres the brush ?", "words": ["the", "brush"]}, {"sent": "heres a brush .", "words": ["brush", "a"]}, {"sent": "what do we brush ?", "words": ["brush", "what", "do", "we"]}, {"sent": "some of them have their hair messy why dont you brush their hair ?", "words": ["their", "of", "brush", "them", "you", "have", "dont", "some", "hair", "why"]}, {"sent": "you brush your hair .", "words": ["brush", "hair", "your", "you"]}, {"sent": "no brush the trees .", "words": ["brush", "the"]}, {"sent": "you brush the ponys hair .", "words": ["brush", "the", "hair", "you"]}, {"sent": "yeah I gave you a different brush because the other one it looked like the pieces of the brush were coming out on your paper because you were banging the brush on the paper see how theres little pieces of brush on your paper ?", "words": ["the", "how", "your", "brush", "of", "paper", "like", "you", "other", "little", "it", "out", "on", "see", "were", "because", "a"]}, {"sent": "what are ya gonna do with the the brush ?", "words": ["the", "what", "do", "brush", "with", "are"]}, {"sent": "we brush brush brush the whole day through .", "words": ["brush", "the", "we"]}, {"sent": "I brush brush brush my teeth I brush them left to right I brush them up and down to keep them clean and blue ?", "words": ["up", "clean", "my", "brush", "to", "them", "and", "blue", "down"]}, {"sent": "you hafta clean your brush first Fraser ?", "words": ["clean", "your", "brush", "first", "you"]}, {"sent": "no brush the trees .", "words": ["brush", "the"]}, {"sent": "okay lets brush our teeth and get ready to go .", "words": ["go", "brush", "to", "and", "get", "our"]}, {"sent": "you better go brush them .", "words": ["better", "go", "brush", "them", "you"]}, {"sent": "no brush the trees .", "words": ["brush", "the"]}, {"sent": "I found the brush ?", "words": ["the", "brush"]}, {"sent": "hold the brush .", "words": ["the", "brush", "hold"]}, {"sent": "you gonna brush your hair with the spoon ?", "words": ["the", "your", "brush", "with", "you", "spoon", "hair"]}, {"sent": "can I brush it ?", "words": ["can", "it", "brush"]}, {"sent": "wash my face brush my teeth and Im .", "words": ["my", "face", "brush", "and", "wash"]}, {"sent": "do you wanna brush your teeth now ?", "words": ["do", "your", "brush", "you", "wanna"]}, {"sent": "its the handle from the brush isnt it ?", "words": ["the", "it", "brush"]}, {"sent": "daddy you got my brush I mean my soos ?", "words": ["brush", "my", "you"]}, {"sent": "we brush brush brush the whole day through .", "words": ["brush", "the", "we"]}, {"sent": "shall we brush your hair ?", "words": ["brush", "hair", "your", "we"]}, {"sent": "mommy helps me brush my hair and checks to make sure my face and hands are clean .", "words": ["clean", "my", "face", "brush", "to", "and", "are", "me", "hair", "make"]}, {"sent": "whyve you just put a paint full of a brush full of paint in the water ?", "words": ["the", "put", "paint", "full", "of", "brush", "you", "in", "water", "a"]}, {"sent": "can you brush your hair ?", "words": ["can", "your", "brush", "you", "hair"]}, {"sent": "you brush your teeth .", "words": ["brush", "your", "you"]}, {"sent": "alright lets go brush your teeth and wash your hands .", "words": ["your", "go", "brush", "and", "wash"]}, {"sent": "is the stick for the kitty ?", "words": ["the", "stick", "for", "is", "kitty"]}, {"sent": "do they stick together ?", "words": ["do", "stick", "they"]}, {"sent": "lets stick another one under there .", "words": ["another", "under", "stick", "there"]}, {"sent": "stick your finger in the hole .", "words": ["the", "stick", "your", "finger", "in"]}, {"sent": "they stick .", "words": ["they", "stick"]}, {"sent": "just stick them in there then .", "words": ["stick", "them", "there", "in", "then"]}, {"sent": "then very gently stick .", "words": ["then", "stick"]}, {"sent": "do you wanna stick them up there for me ?", "words": ["up", "do", "stick", "for", "them", "you", "there", "wanna", "me"]}, {"sent": "is the stick for the kitty ?", "words": ["the", "stick", "for", "is", "kitty"]}, {"sent": "I dont think you can do stick ones with this one darling .", "words": ["can", "do", "think", "stick", "with", "you", "dont", "this"]}, {"sent": "no you dont needta stick it in your pocket you might forget and leave it in there .", "words": ["stick", "your", "you", "and", "dont", "in", "it", "there"]}, {"sent": "thats a stick .", "words": ["stick", "a"]}, {"sent": "theres this one with a house where you can stick all these people on the house .", "words": ["where", "can", "stick", "all", "these", "the", "with", "you", "this", "house", "on", "a"]}, {"sent": "stick .", "words": ["stick"]}, {"sent": "stick it in here like this .", "words": ["stick", "like", "here", "this", "in", "it"]}, {"sent": "you dont wanna eat the stick though Booboo .", "words": ["the", "stick", "you", "eat", "dont", "wanna"]}, {"sent": "look well stick paper together .", "words": ["stick", "paper", "look"]}, {"sent": "yeah but sometimes his teeth stick in to the top of his mouth .", "words": ["the", "stick", "of", "to", "mouth", "in", "his", "but"]}, {"sent": "you can do sticking and stick some eyes and a smile on your gingerbread man .", "words": ["can", "do", "stick", "your", "and", "you", "on", "some", "smile", "a"]}, {"sent": "and then stick your bean right on it .", "words": ["stick", "your", "and", "it", "on", "then"]}, {"sent": "and stick this I think .", "words": ["this", "stick", "think", "and"]}, {"sent": "xxx stick xxx in the drivers seat xxx .", "words": ["the", "in", "stick"]}, {"sent": "shall we stick the monkey on aswell ?", "words": ["the", "stick", "we", "monkey", "on"]}, {"sent": "he was hitting I with a stick on the backside .", "words": ["the", "stick", "was", "with", "he", "on", "a"]}, {"sent": "willnt stick on there will they ?", "words": ["stick", "will", "there", "they", "on"]}, {"sent": "did you shoot her with a piece of stick ?", "words": ["stick", "of", "with", "you", "did", "her", "a"]}, {"sent": "and then were going to stick all the things on .", "words": ["the", "all", "stick", "to", "and", "on", "then", "were"]}, {"sent": "stick your tongue in Bro Mey .", "words": ["tongue", "in", "stick", "your"]}, {"sent": "can I take the bits off and then you can stick it ?", "words": ["can", "the", "off", "stick", "and", "you", "it", "then", "take"]}, {"sent": "what about opening your measuring stick some more ?", "words": ["what", "stick", "your", "more", "about", "some"]}, {"sent": "well stick with the pennies today .", "words": ["the", "stick", "with"]}, {"sent": "can you stick your hand in there ?", "words": ["can", "stick", "your", "hand", "you", "there", "in"]}, {"sent": "oh we could stick it in the middle like that .", "words": ["the", "stick", "we", "like", "in", "it", "could", "that"]}, {"sent": "now hold it straight like that and stick it up Teddy .", "words": ["up", "stick", "hold", "like", "and", "it", "that"]}, {"sent": "and you stick the green one on there .", "words": ["the", "stick", "you", "and", "there", "green", "on"]}, {"sent": "Im a stick this down in your pants just a little bit so it doesnt trail out and make ya look funny okay ?", "words": ["stick", "your", "so", "little", "and", "this", "in", "it", "pants", "out", "look", "down", "make", "a"]}, {"sent": "now you stick the straw in and you give her a drink .", "words": ["the", "stick", "drink", "and", "you", "in", "give", "her", "a"]}, {"sent": "and you color clothes in and you stick them on her .", "words": ["stick", "them", "you", "and", "in", "on", "her"]}, {"sent": "stick a yellow one on .", "words": ["on", "yellow", "stick", "a"]}, {"sent": "no dont stick your fingers in there sweetie .", "words": ["stick", "your", "dont", "in", "there"]}, {"sent": "where do you stick that little piece of velcro ?", "words": ["where", "do", "stick", "of", "you", "little", "that"]}, {"sent": "stick your hand in the mitten .", "words": ["the", "stick", "your", "hand", "in"]}, {"sent": "see you just stick them like that .", "words": ["stick", "them", "like", "you", "see", "that"]}, {"sent": "let Mama stick it on .", "words": ["on", "stick", "it"]}, {"sent": "would you like to stick some gold stars on it ?", "words": ["stick", "to", "like", "you", "it", "would", "on", "some"]}, {"sent": "stick it on stick stick stick oh there wanna put on there ?", "words": ["stick", "put", "there", "it", "wanna", "on"]}, {"sent": "stick the chair there .", "words": ["the", "there", "stick", "chair"]}, {"sent": "I found one popsicle stick Abe .", "words": ["stick", "popsicle"]}, {"sent": "you hafta put the bits that stick out into the holes .", "words": ["the", "put", "stick", "into", "you", "out", "that"]}, {"sent": "you just you just stuck it where it needed to be stick ?", "words": ["where", "stick", "to", "be", "you", "it", "stuck"]}, {"sent": "no stick .", "words": ["stick"]}, {"sent": "you just stick red stuff in there and they become red ?", "words": ["red", "stick", "you", "and", "there", "in", "they"]}, {"sent": "orange on a stick .", "words": ["on", "stick", "orange", "a"]}, {"sent": "the stick man ?", "words": ["the", "stick"]}, {"sent": "xxx stick ones .", "words": ["stick"]}, {"sent": "do you wanna stick that one down ?", "words": ["do", "stick", "you", "wanna", "down", "that"]}, {"sent": "you can stick that in there .", "words": ["can", "stick", "you", "there", "in", "that"]}, {"sent": "what happened to you other stick ?", "words": ["what", "stick", "to", "other", "you"]}, {"sent": "stick your tongue out .", "words": ["tongue", "stick", "your", "out"]}, {"sent": "on a stick ?", "words": ["on", "stick", "a"]}, {"sent": "itll only stick onto the bobbly bit .", "words": ["the", "stick"]}, {"sent": "stick it on here .", "words": ["on", "stick", "it", "here"]}, {"sent": "did Jwww stick that there ?", "words": ["there", "stick", "that", "did"]}, {"sent": "Im gonna stick those on .", "words": ["on", "stick", "those"]}, {"sent": "Mummyll hafta stick that on there for you .", "words": ["stick", "for", "you", "there", "on", "that"]}, {"sent": "shape stick .", "words": ["stick"]}, {"sent": "am I trying to stick it on you ?", "words": ["stick", "to", "you", "it", "am", "on"]}, {"sent": "they stick together .", "words": ["they", "stick"]}, {"sent": "now well stick Dadas head on there .", "words": ["on", "there", "stick", "head"]}, {"sent": "Eleanor you mustnt stick it on the table like this darling .", "words": ["the", "stick", "table", "like", "you", "this", "it", "on"]}, {"sent": "stick them up .", "words": ["up", "stick", "them"]}, {"sent": "what did he do to get a yellow stick ?", "words": ["what", "do", "stick", "to", "get", "he", "did", "yellow", "a"]}, {"sent": "come on you stick horsie on .", "words": ["on", "stick", "you"]}, {"sent": "a stick for clicking .", "words": ["stick", "for", "a"]}, {"sent": "you gonna stick him on his hat ?", "words": ["stick", "you", "his", "him", "hat", "on"]}, {"sent": "Ill stick it from there .", "words": ["there", "stick", "it"]}, {"sent": "do you wanna lick the stick ?", "words": ["the", "do", "stick", "you", "wanna", "lick"]}, {"sent": "do you like to pick things up and stick them in your mouth ?", "words": ["up", "do", "stick", "your", "pick", "to", "mouth", "them", "like", "you", "and", "in"]}, {"sent": "Ill stick it on and this time you can write your name all by your self .", "words": ["can", "all", "stick", "your", "write", "and", "you", "this", "by", "it", "on"]}, {"sent": "are you going to stick that one on ?", "words": ["stick", "to", "you", "are", "on", "that"]}, {"sent": "lets wipe the stick off so its not blue anymore .", "words": ["the", "off", "stick", "so", "blue", "wipe", "not"]}, {"sent": "that is the stick part .", "words": ["the", "stick", "that", "is"]}, {"sent": "your dip stick .", "words": ["stick", "your"]}, {"sent": "so if youre to stick it on there .", "words": ["stick", "if", "to", "so", "there", "it", "on"]}, {"sent": "do you remember you brought that that little wooden stick in the centre of the drum and when you all banged what happened to the stick ?", "words": ["the", "do", "stick", "all", "what", "when", "of", "to", "little", "you", "and", "in", "that"]}, {"sent": "xxx stick .", "words": ["stick"]}, {"sent": "a stick for jumping and a stick for dumping .", "words": ["stick", "for", "and", "a"]}, {"sent": "I stick my foot in the fire .", "words": ["the", "stick", "my", "in", "foot"]}, {"sent": "and weve got a house and we can stick it on .", "words": ["can", "stick", "we", "and", "it", "house", "on", "a"]}, {"sent": "can you stick those two together ?", "words": ["can", "stick", "those", "you"]}, {"sent": "whoops whoops stick that there darling .", "words": ["there", "stick", "that"]}, {"sent": "no stick him in the piggybank .", "words": ["the", "in", "stick", "him"]}, {"sent": "stick over here go like that go on .", "words": ["stick", "go", "like", "here", "that", "on", "over"]}, {"sent": "stick some on the envelopes because were going to post these .", "words": ["the", "these", "stick", "to", "on", "some", "were", "because"]}, {"sent": "its a walking stick isnt it ?", "words": ["stick", "it", "a"]}, {"sent": "stick him in .", "words": ["in", "stick", "him"]}, {"sent": "you stick him on there and you put that on the back of the train .", "words": ["the", "stick", "put", "back", "train", "of", "you", "and", "there", "him", "on", "that"]}, {"sent": "shall we stick her in the highchair ?", "words": ["the", "stick", "we", "in", "her"]}, {"sent": "stick your hands stick your hands like this .", "words": ["this", "stick", "your", "like"]}, {"sent": "you fascinated by that stick ?", "words": ["stick", "that", "by", "you"]}, {"sent": "oh did Daddy cut it ?", "words": ["cut", "it", "did"]}, {"sent": "I cant cut it .", "words": ["it", "cut"]}, {"sent": "youve cut his nose off and youve cut his tail off .", "words": ["off", "cut", "and", "his", "nose"]}, {"sent": "do you get your hair cut by a barber ?", "words": ["do", "your", "cut", "you", "get", "by", "hair", "a"]}, {"sent": "I might cut my finger again .", "words": ["finger", "my", "cut"]}, {"sent": "where did your cut up pears go huh ?", "words": ["where", "up", "your", "cut", "go", "did"]}, {"sent": "because the glass is really broken and it can cut .", "words": ["the", "can", "cut", "is", "broken", "and", "it", "glass", "because"]}, {"sent": "you cant cut .", "words": ["cut", "you"]}, {"sent": "I can see its a good job Mummy cut an apple up for herself isnt it ?", "words": ["can", "up", "for", "cut", "an", "it", "good", "apple", "see", "a"]}, {"sent": "do you want me to cut it ?", "words": ["do", "cut", "to", "you", "it", "me"]}, {"sent": "told ya not to cut that salt on there .", "words": ["cut", "to", "there", "salt", "on", "that", "not"]}, {"sent": "lets cut your nails .", "words": ["your", "cut"]}, {"sent": "Adam shall I get your scissors and you may cut some paper .", "words": ["your", "cut", "scissors", "paper", "you", "get", "and", "some"]}, {"sent": "theyll cut your fingers off .", "words": ["off", "your", "cut"]}, {"sent": "that one was so fat its not even cut out is it ?", "words": ["cut", "is", "was", "so", "it", "out", "that", "not"]}, {"sent": "because he did he rolled right in there and these guys were like hey hey you cut in front of me go to the back of the line .", "words": ["the", "these", "cut", "back", "go", "of", "to", "like", "and", "you", "there", "in", "he", "did", "me", "were", "because"]}, {"sent": "the barber Granddads barber cut your hair didnt he ?", "words": ["the", "your", "cut", "he", "hair"]}, {"sent": "what are you gonna cut ?", "words": ["what", "are", "cut", "you"]}, {"sent": "got a little cut ?", "words": ["cut", "little", "a"]}, {"sent": "yoyo cut dat ute on me .", "words": ["on", "me", "cut"]}, {"sent": "you youve cut down a piece of red card there havent you ?", "words": ["red", "cut", "of", "you", "there", "down", "a"]}, {"sent": "do you want me to cut it ?", "words": ["do", "cut", "to", "you", "it", "me"]}, {"sent": "yeah were gonna cut this some more paper up .", "words": ["up", "cut", "paper", "this", "more", "some", "were"]}, {"sent": "shall I get the nailcutter and you can cut Ponys nails ?", "words": ["the", "can", "cut", "you", "get", "and"]}, {"sent": "cut cut .", "words": ["cut"]}, {"sent": "how did we cut your hair ?", "words": ["how", "your", "cut", "we", "did", "hair"]}, {"sent": "we need eh some we need some scissors to cut it off .", "words": ["off", "need", "cut", "we", "to", "scissors", "it", "some"]}, {"sent": "Abe we dont want you to get cut .", "words": ["cut", "we", "to", "you", "get", "dont"]}, {"sent": "because if it was a real knife youd cut yourself doing that wouldnt you ?", "words": ["yourself", "if", "cut", "was", "knife", "you", "it", "that", "because", "a"]}, {"sent": "Im going to cut the stamp off .", "words": ["the", "to", "cut", "off"]}, {"sent": "you cut the cellophane with fingernails .", "words": ["the", "with", "cut", "you"]}, {"sent": "you do it you can cut it .", "words": ["can", "do", "cut", "you", "it"]}, {"sent": "cut around and around .", "words": ["cut", "and", "around"]}, {"sent": "you cant cut that off .", "words": ["off", "that", "cut", "you"]}, {"sent": "shall I shall I cut the string because because you cant really manage them on that can you ?", "words": ["the", "can", "cut", "them", "you", "on", "that", "because"]}, {"sent": "xxx cut right and he cut his hair .", "words": ["cut", "and", "he", "his", "hair"]}, {"sent": "I bet you could use the pizza cutter to cut this .", "words": ["the", "cut", "to", "pizza", "you", "this", "could"]}, {"sent": "the wood cutter cut your scratch did he ?", "words": ["the", "your", "cut", "he", "did"]}, {"sent": "which fingernail shall I cut first ?", "words": ["first", "which", "cut"]}, {"sent": "oh look see heres another pie but this one is whole its not cut up .", "words": ["up", "cut", "is", "another", "this", "look", "see", "not", "but"]}, {"sent": "cut it out .", "words": ["it", "cut", "out"]}, {"sent": "and were going to cut up some cheese into cubes .", "words": ["up", "cut", "into", "to", "and", "some", "cheese", "were"]}, {"sent": "uhoh the deer cut her leg on a glass bottle .", "words": ["the", "cut", "deer", "leg", "glass", "on", "bottle", "her", "a"]}, {"sent": "no how did we cut it up ?", "words": ["up", "how", "cut", "we", "it", "did"]}, {"sent": "and emergency push cut off .", "words": ["off", "push", "cut", "and"]}, {"sent": "would you like to cut them ?", "words": ["cut", "to", "them", "like", "you", "would"]}, {"sent": "I had my hair cut last week yes .", "words": ["my", "hair", "last", "cut"]}, {"sent": "this is a whole kiwi that kiwis cut in half .", "words": ["cut", "is", "this", "in", "that", "a"]}, {"sent": "hey you want me to cut this for you ?", "words": ["for", "cut", "to", "you", "this", "me"]}, {"sent": "you just need them cut so you dont scratch yourself .", "words": ["yourself", "need", "cut", "so", "them", "you", "dont"]}, {"sent": "cut up there then look .", "words": ["up", "cut", "there", "look", "then"]}, {"sent": "what wants to be cut ?", "words": ["to", "what", "be", "cut"]}, {"sent": "Im going to cut your finger nails as well .", "words": ["to", "finger", "your", "cut"]}, {"sent": "cut what in little pieces ?", "words": ["what", "in", "cut", "little"]}, {"sent": "heres one of my hardest xxx to cut through the net use a pair of .", "words": ["the", "my", "cut", "of", "to", "a"]}, {"sent": "wow youre going to cut all those ?", "words": ["all", "to", "those", "cut"]}, {"sent": "and the bread crunched when we cut into it .", "words": ["the", "bread", "cut", "into", "we", "and", "it", "when"]}, {"sent": "we could cut these out for decorations couldnt we huh ?", "words": ["these", "for", "cut", "we", "out", "could"]}, {"sent": "Im gonna cut ya .", "words": ["cut"]}, {"sent": "she was a good girl when she had her hair cut huh ?", "words": ["cut", "was", "her", "good", "hair", "when", "she", "a"]}, {"sent": "do you want me to cut you some pieces of cheese as well ?", "words": ["do", "cut", "of", "to", "you", "me", "cheese", "some"]}, {"sent": "we needta take you to the hairdresser and get your hair cut dont we ?", "words": ["the", "your", "cut", "we", "to", "you", "and", "get", "dont", "hair", "take"]}, {"sent": "could you cut this down a little bit for me ?", "words": ["for", "cut", "you", "little", "this", "me", "could", "down", "a"]}, {"sent": "Daddy is gonna cut it .", "words": ["it", "cut", "is"]}, {"sent": "Jwww doesnt wanna look like a girl so hes having his nails cut like a good boy .", "words": ["cut", "so", "like", "wanna", "look", "his", "good", "a"]}, {"sent": "do you wanna cut him up ?", "words": ["up", "do", "cut", "you", "him", "wanna"]}, {"sent": "Ill cut the bread if you want some .", "words": ["the", "bread", "cut", "if", "you", "some"]}, {"sent": "just gonna cut it all up .", "words": ["up", "all", "it", "cut"]}, {"sent": "they probably havent adjusted it look since the first cut of the season .", "words": ["the", "cut", "of", "first", "they", "it", "look"]}, {"sent": "cut it ?", "words": ["it", "cut"]}, {"sent": "laying out sheets of vinyl to cut out the Wellingtons ?", "words": ["the", "cut", "of", "to", "out"]}, {"sent": "Ronnie cut it out .", "words": ["it", "cut", "out"]}, {"sent": "where do you go for a hair cut ?", "words": ["where", "do", "for", "cut", "go", "you", "hair", "a"]}, {"sent": "um pizza she I dont know if she would necessarily recognize the shape per say because we cut it up but youknow Ive said youknow this is shes had pizza before Ive cut up .", "words": ["the", "up", "if", "cut", "we", "is", "say", "pizza", "dont", "this", "it", "but", "would", "she", "because"]}, {"sent": "shall I draw where you cut it ?", "words": ["where", "cut", "draw", "you", "it"]}, {"sent": "well you already cut the paper .", "words": ["the", "paper", "cut", "you"]}, {"sent": "wheres cut ?", "words": ["cut"]}, {"sent": "bandaid you know like you put on your hand when you have a cut ?", "words": ["put", "your", "cut", "hand", "like", "you", "have", "on", "when", "a"]}, {"sent": "do you like getting your hair cut ?", "words": ["do", "your", "cut", "like", "you", "hair"]}, {"sent": "just cut like that .", "words": ["that", "cut", "like"]}, {"sent": "cut your bananas help .", "words": ["help", "your", "cut"]}, {"sent": "no not sharp knives because if you trip you might end up getting a bad cut .", "words": ["up", "if", "cut", "bad", "you", "not", "because", "a"]}, {"sent": "Mummys having her hair cut .", "words": ["hair", "her", "cut"]}, {"sent": "are you going to cut Eleanor ?", "words": ["to", "are", "cut", "you"]}, {"sent": "were gonna cut your hair .", "words": ["your", "hair", "were", "cut"]}, {"sent": "are you going to come and cut this into pieces for me ?", "words": ["for", "cut", "into", "to", "and", "you", "this", "are", "me"]}, {"sent": "you didnt have your hair cut this time when you went to see Grandma and Granddad did you ?", "words": ["your", "cut", "to", "you", "have", "this", "and", "did", "see", "hair", "when"]}, {"sent": "is he allowed to cut the tinsel ?", "words": ["the", "cut", "is", "to", "he"]}, {"sent": "he cut a hole in the door .", "words": ["the", "cut", "in", "he", "door", "a"]}, {"sent": "Mummy cut it off didnt she ?", "words": ["off", "it", "cut", "she"]}, {"sent": "cut that ?", "words": ["that", "cut"]}, {"sent": "and then because Ive cut it in half that way it looks like a mouth doesnt it ?", "words": ["cut", "mouth", "like", "and", "in", "it", "then", "that", "because", "a"]}, {"sent": "thats why it willnt cut because you havent got it the right way round .", "words": ["the", "cut", "you", "it", "why", "because"]}, {"sent": "but for the most part all of the food he gets at this point is still cut up into little .", "words": ["the", "all", "up", "for", "cut", "into", "is", "of", "little", "this", "food", "he", "at", "but"]}, {"sent": "would you like me to cut round this rabbit ?", "words": ["cut", "to", "like", "you", "this", "would", "me"]}, {"sent": "you can cut .", "words": ["can", "cut", "you"]}, {"sent": "okay now cut .", "words": ["cut"]}, {"sent": "you only ever cut paper with scissors .", "words": ["cut", "scissors", "paper", "with", "you"]}, {"sent": "yeah itll cut ya .", "words": ["cut"]}, {"sent": "ready to jump ?", "words": ["to", "jump"]}, {"sent": "a jump .", "words": ["jump", "a"]}, {"sent": "you can both jump together .", "words": ["can", "jump", "you"]}, {"sent": "jump jump jump .", "words": ["jump"]}, {"sent": "you gonna jump over your tower ?", "words": ["your", "jump", "over", "you"]}, {"sent": "jump .", "words": ["jump"]}, {"sent": "jump in .", "words": ["jump", "in"]}, {"sent": "jump horsie .", "words": ["jump"]}, {"sent": "how did they get over there did they just jump or how did they get over to their new land ?", "words": ["how", "their", "to", "get", "they", "there", "jump", "did", "new", "over"]}, {"sent": "Betty jump .", "words": ["jump"]}, {"sent": "and theyre supposed to jump over the moon jump jump .", "words": ["the", "moon", "to", "and", "jump", "over"]}, {"sent": "jump .", "words": ["jump"]}, {"sent": "Joseph wants to try to jump too can you jump for him ?", "words": ["can", "for", "to", "you", "jump", "him", "too", "try", "try to"]}, {"sent": "its not gonna jump over there by itself .", "words": ["by", "jump", "there", "over", "not"]}, {"sent": "duckie jump ?", "words": ["jump"]}, {"sent": "jump on the bed and jump off ?", "words": ["the", "off", "bed", "and", "jump", "on"]}, {"sent": "jump over .", "words": ["jump", "over"]}, {"sent": "she cant jump .", "words": ["jump", "she"]}, {"sent": "jump ?", "words": ["jump"]}, {"sent": "can he jump on the floor ?", "words": ["can", "the", "jump", "he", "on"]}, {"sent": "yeah that was a jump wasnt it ?", "words": ["was", "jump", "it", "that", "a"]}, {"sent": "jump .", "words": ["jump"]}, {"sent": "can I jump over the fence too ?", "words": ["can", "the", "jump", "too", "over"]}, {"sent": "jump in the water .", "words": ["the", "jump", "in", "water"]}, {"sent": "ooh whee jump ?", "words": ["jump"]}, {"sent": "you stand on this chair then jump out .", "words": ["stand", "chair", "you", "this", "jump", "out", "on", "then"]}, {"sent": "you certainly did jump .", "words": ["jump", "did", "you"]}, {"sent": "instead of wishing it occurred to him at the last moment to jump aboard .", "words": ["the", "of", "to", "last", "jump", "him", "it", "at"]}, {"sent": "jump on the pillow ?", "words": ["on", "jump", "pillow", "the"]}, {"sent": "and jump jump jump .", "words": ["jump", "and"]}, {"sent": "say jump .", "words": ["jump", "say"]}, {"sent": "he jump all up and down them like he crazy .", "words": ["up", "all", "them", "like", "and", "jump", "he", "down"]}, {"sent": "Carls man jump .", "words": ["jump"]}, {"sent": "the common flea is a champion jumper it can jump more than seven inches high and thirteen inches long .", "words": ["the", "can", "is", "long", "and", "jump", "more", "it", "high", "a"]}, {"sent": "hoping that Jess wont jump and get her .", "words": ["and", "get", "jump", "that", "her"]}, {"sent": "did that make you jump ?", "words": ["you", "jump", "that", "did", "make"]}, {"sent": "what other animals jump ?", "words": ["what", "jump", "other"]}, {"sent": "would you like to do jump jump ?", "words": ["do", "to", "like", "you", "jump", "would"]}, {"sent": "I can jump .", "words": ["can", "jump"]}, {"sent": "one two three jump .", "words": ["jump"]}, {"sent": "well you can jump .", "words": ["can", "jump", "you"]}, {"sent": "can cows jump ?", "words": ["can", "jump"]}, {"sent": "show me how you jump then .", "words": ["how", "show", "you", "jump", "me", "then"]}, {"sent": "and he tries to jump on Mogli .", "words": ["to", "and", "jump", "he", "on"]}, {"sent": "jump .", "words": ["jump"]}, {"sent": "Carl hes going to jump .", "words": ["to", "jump"]}, {"sent": "you guys jump together say one two three jump .", "words": ["jump", "say", "you"]}, {"sent": "can you jump over the lorry ?", "words": ["can", "the", "you", "jump", "over"]}, {"sent": "jack be nimble jack be quick jack jump over the candlestick .", "words": ["the", "be", "jump", "over"]}, {"sent": "make em jump out .", "words": ["jump", "make", "out"]}, {"sent": "did you jump in the pool ?", "words": ["the", "pool", "you", "jump", "in", "did"]}, {"sent": "see you know how to jump up and down .", "words": ["up", "how", "to", "you", "and", "jump", "see", "down"]}, {"sent": "do you wanna jump some more ?", "words": ["do", "you", "jump", "more", "wanna", "some"]}, {"sent": "the squirrels can jump uhn cant they ?", "words": ["the", "jump", "can", "they"]}, {"sent": "good jump .", "words": ["jump", "good"]}, {"sent": "youre not sposta jump out on me .", "words": ["jump", "out", "on", "me", "not"]}, {"sent": "that was a big jump .", "words": ["was", "jump", "that", "big", "a"]}, {"sent": "gosh xxx jump .", "words": ["jump"]}, {"sent": "so she can have a sleep and then jump into the window and have a little play outside .", "words": ["can", "the", "into", "sleep", "so", "outside", "play", "and", "have", "jump", "window", "little", "then", "she", "a"]}, {"sent": "one jump then you come put your pants on .", "words": ["put", "your", "you", "jump", "pants", "on", "then"]}, {"sent": "jump down jump .", "words": ["jump", "down"]}, {"sent": "I dont think the crocodile could jump .", "words": ["the", "think", "dont", "jump", "could"]}, {"sent": "jump on Billys nose ?", "words": ["on", "jump", "nose"]}, {"sent": "a jump a jump .", "words": ["jump", "a"]}, {"sent": "jump .", "words": ["jump"]}, {"sent": "one two three jump wooaoh .", "words": ["jump"]}, {"sent": "mustnt jump in puddles .", "words": ["jump", "in"]}, {"sent": "jump jump jump .", "words": ["jump"]}, {"sent": "a jump a jump a jump a jump .", "words": ["jump", "a"]}, {"sent": "you did make me jump .", "words": ["you", "jump", "did", "me", "make"]}, {"sent": "jump jump jump jump jump jump .", "words": ["jump"]}, {"sent": "the tigers gonna jump them is he ?", "words": ["the", "is", "them", "jump", "he"]}, {"sent": "do you want do you want the horsie to jump those Ruth ?", "words": ["the", "do", "those", "to", "you", "jump"]}, {"sent": "hafta jump out the roof .", "words": ["the", "jump", "roof", "out"]}, {"sent": "wanna make the goat jump ?", "words": ["the", "jump", "make", "wanna"]}, {"sent": "you cant jump .", "words": ["jump", "you"]}, {"sent": "is she going to jump from the fence ?", "words": ["the", "is", "to", "jump", "she"]}, {"sent": "can you jump very very high ?", "words": ["can", "jump", "high", "you"]}, {"sent": "yes Daddy is gonna jump down .", "words": ["jump", "down", "is"]}, {"sent": "a jump a jump a jump .", "words": ["jump", "a"]}, {"sent": "j jump joyful jumble around .", "words": ["jump", "around"]}, {"sent": "have Georgie jump .", "words": ["have", "jump"]}, {"sent": "when they were almost across the room the fox said the water is deepest here you had better jump on my nose gingerbread man .", "words": ["the", "better", "my", "is", "you", "here", "they", "jump", "room", "water", "on", "when", "were", "nose"]}, {"sent": "big jump ?", "words": ["jump", "big"]}, {"sent": "jump jump jump .", "words": ["jump"]}, {"sent": "watch Daddy jump .", "words": ["jump", "watch"]}, {"sent": "the people are gonna jump .", "words": ["the", "jump", "are"]}, {"sent": "jump set .", "words": ["jump"]}, {"sent": "one two three jump .", "words": ["jump"]}, {"sent": "you jump on the settee and fight ?", "words": ["the", "and", "you", "jump", "on"]}, {"sent": "you jump in the swimming pool .", "words": ["the", "pool", "you", "jump", "in"]}, {"sent": "jump again .", "words": ["jump"]}, {"sent": "you like to jump dont you ?", "words": ["to", "like", "you", "dont", "jump"]}, {"sent": "lets make it jump .", "words": ["jump", "it", "make"]}, {"sent": "a jump a jump a jump a jump a jump .", "words": ["jump", "a"]}, {"sent": "you can jump ?", "words": ["can", "jump", "you"]}, {"sent": "jump over walls ?", "words": ["jump", "over"]}, {"sent": "do you wanna jump down ?", "words": ["do", "you", "jump", "wanna", "down"]}, {"sent": "just let me jump in .", "words": ["me", "in", "jump"]}, {"sent": "you still youre still scared of him ?", "words": ["of", "him", "scared", "you"]}, {"sent": "and you got scared becauseof the blender .", "words": ["scared", "the", "you", "and"]}, {"sent": "oh you think hes scared ?", "words": ["think", "scared", "you"]}, {"sent": "are you scared ?", "words": ["are", "scared", "you"]}, {"sent": "and was still scared .", "words": ["was", "scared", "and"]}, {"sent": "youre scared ?", "words": ["scared"]}, {"sent": "youre scared ?", "words": ["scared"]}, {"sent": "dont worry Im not scared xxx orchu take Richard away .", "words": ["scared", "dont", "away", "take", "not"]}, {"sent": "what are they scared of Roman ?", "words": ["what", "of", "scared", "they", "are"]}, {"sent": "because you were scared that Jwww got hurt ?", "words": ["hurt", "because", "scared", "you", "that", "were"]}, {"sent": "youre not scared .", "words": ["not", "scared"]}, {"sent": "are they scared of Fraser ?", "words": ["of", "they", "are", "scared"]}, {"sent": "scared her ?", "words": ["her", "scared"]}, {"sent": "are you scared of bears ?", "words": ["of", "are", "scared", "you"]}, {"sent": "was your sister scared ?", "words": ["was", "your", "scared"]}, {"sent": "my friend the doctor knows I sometimes get a little scared .", "words": ["the", "my", "scared", "little", "get", "a"]}, {"sent": "hes scared yeah it would be scary to almost fall wouldnt it .", "words": ["to", "be", "fall", "scared", "it", "would"]}, {"sent": "Cathy it would have scared you .", "words": ["scared", "you", "have", "it", "would"]}, {"sent": "why are you scared ?", "words": ["scared", "are", "why", "you"]}, {"sent": "youre just scared .", "words": ["scared"]}, {"sent": "Percys scared of it isnt he ?", "words": ["of", "it", "he", "scared"]}, {"sent": "did she get scared ?", "words": ["get", "did", "scared", "she"]}, {"sent": "well dont you be scared xxx .", "words": ["dont", "be", "scared", "you"]}, {"sent": "you scared ?", "words": ["scared", "you"]}, {"sent": "do you think Amys scared of something ?", "words": ["do", "think", "of", "scared", "you"]}, {"sent": "and then you wont be scared .", "words": ["be", "scared", "you", "and", "then"]}, {"sent": "do you think hes scared of him ?", "words": ["do", "think", "of", "scared", "you", "him"]}, {"sent": "why would she be scared ?", "words": ["be", "scared", "would", "she", "why"]}, {"sent": "what would you tell her if she was scared and she thought they were real ?", "words": ["what", "if", "was", "scared", "you", "and", "they", "would", "her", "she", "were"]}, {"sent": "Im not scared of you .", "words": ["of", "not", "scared", "you"]}, {"sent": "you scared of it ?", "words": ["of", "it", "scared", "you"]}, {"sent": "shes scared of someone ?", "words": ["of", "scared"]}, {"sent": "oh you scared me .", "words": ["me", "scared", "you"]}, {"sent": "oy hes scared .", "words": ["scared"]}, {"sent": "were you scared ?", "words": ["were", "scared", "you"]}, {"sent": "scared the poor bird half to death .", "words": ["the", "to", "bird", "scared", "poor"]}, {"sent": "they get very scared dont they because that big doggie is chasing them .", "words": ["is", "them", "scared", "get", "they", "dont", "big", "that", "because"]}, {"sent": "dinosaur you scared them away with all your roaring .", "words": ["all", "your", "with", "them", "scared", "you", "away"]}, {"sent": "you scared me .", "words": ["me", "scared", "you"]}, {"sent": "youre scared uhoh .", "words": ["scared"]}, {"sent": "you scared me .", "words": ["me", "scared", "you"]}, {"sent": "were not scared .", "words": ["were", "not", "scared"]}, {"sent": "how come you were scared ?", "words": ["how", "were", "scared", "you"]}, {"sent": "he was scared .", "words": ["was", "he", "scared"]}, {"sent": "this fox is scared of humans actually the fox is scared whenever a human comes a fox will run away .", "words": ["the", "is", "of", "scared", "this", "run", "away", "will", "a"]}, {"sent": "were you scared Thomas ?", "words": ["were", "scared", "you"]}, {"sent": "scared you on picture day ?", "words": ["on", "picture", "scared", "you"]}, {"sent": "Shrek wasnt scared was he ?", "words": ["was", "he", "scared"]}, {"sent": "what was Ron scared of in the movie ?", "words": ["the", "what", "of", "was", "scared", "in"]}, {"sent": "wasnt he scared ?", "words": ["he", "scared"]}, {"sent": "youre not scared of the moon .", "words": ["the", "moon", "of", "scared", "not"]}, {"sent": "what are you scared of ?", "words": ["what", "of", "scared", "you", "are"]}, {"sent": "are you a little bit scared ?", "words": ["scared", "little", "you", "are", "a"]}, {"sent": "why are you scared of the car ?", "words": ["the", "of", "car", "scared", "you", "are", "why"]}, {"sent": "youre scared of the hippo ?", "words": ["of", "the", "scared"]}, {"sent": "did it scared your finger ?", "words": ["your", "scared", "finger", "it", "did"]}, {"sent": "youre scared ?", "words": ["scared"]}, {"sent": "this scared the people .", "words": ["the", "this", "scared"]}, {"sent": "you scared Purdie with her big tail ?", "words": ["her", "with", "scared", "you", "big"]}, {"sent": "youre not scared .", "words": ["not", "scared"]}, {"sent": "I think it scared you a little bit becauseof the waves .", "words": ["the", "think", "scared", "you", "little", "it", "a"]}, {"sent": "and why she looking scared whats happened to make her look scared .", "words": ["to", "her", "scared", "and", "look", "make", "she", "why"]}, {"sent": "were scared of that lion so I sent him back .", "words": ["back", "of", "so", "scared", "that", "him", "lion", "were"]}, {"sent": "you were too scared .", "words": ["too", "were", "scared", "you"]}, {"sent": "ohdear is he scared ?", "words": ["he", "scared", "is"]}, {"sent": "ah Im scared ah .", "words": ["scared"]}, {"sent": "is Fraser scared of spiders ?", "words": ["of", "scared", "is"]}, {"sent": "you scared of Jobey ?", "words": ["of", "scared", "you"]}, {"sent": "Timmys scared of the lobster ?", "words": ["of", "the", "scared"]}, {"sent": "they said they werent scared didnt they ?", "words": ["they", "scared"]}, {"sent": "and what did Mummy what was Mummy scared of the other day ?", "words": ["the", "what", "of", "was", "scared", "and", "other", "did"]}, {"sent": "you were scared .", "words": ["were", "scared", "you"]}, {"sent": "you scared Linda .", "words": ["scared", "you"]}, {"sent": "is momma scared ?", "words": ["scared", "is"]}, {"sent": "why is the fish scared ?", "words": ["the", "is", "scared", "fish", "why"]}, {"sent": "shes a bit scared of the clown .", "words": ["of", "the", "scared", "a"]}, {"sent": "animals get scared of things like tractors and cars and trucks big noises like that .", "words": ["of", "like", "scared", "and", "get", "that", "big"]}, {"sent": "dont be scared because theres three of us .", "words": ["of", "be", "scared", "dont", "us", "because"]}, {"sent": "Im scared .", "words": ["scared"]}, {"sent": "was AuntieAktar scared ?", "words": ["was", "scared"]}, {"sent": "but he scared it away .", "words": ["scared", "it", "he", "away", "but"]}, {"sent": "so the flower got scared of Emma ?", "words": ["the", "of", "so", "flower", "scared"]}, {"sent": "is the horse scared ?", "words": ["the", "horse", "scared", "is"]}, {"sent": "hes um being scared .", "words": ["scared"]}, {"sent": "scared me .", "words": ["me", "scared"]}, {"sent": "uhoh you scared the kitty away .", "words": ["the", "scared", "you", "kitty", "away"]}, {"sent": "I think I might be a bit scared of lions .", "words": ["think", "of", "be", "scared", "a"]}, {"sent": "Im scared .", "words": ["scared"]}, {"sent": "a bit scared of xxx .", "words": ["of", "scared", "a"]}, {"sent": "your mom was so scared ?", "words": ["was", "so", "your", "scared"]}, {"sent": "he scared you away ?", "words": ["away", "he", "scared", "you"]}, {"sent": "youre not scared of flies ?", "words": ["of", "not", "scared"]}, {"sent": "were not scared .", "words": ["were", "not", "scared"]}, {"sent": "oh it scared me .", "words": ["me", "it", "scared"]}, {"sent": "weve scared your Mummy scared the policeman and scared the seal .", "words": ["the", "your", "scared", "and"]}, {"sent": "were you scared of the horsie ?", "words": ["the", "of", "scared", "you", "were"]}, {"sent": "were gonna shake shake shake our sillies out shake shake shake our sillies out shake shake shake our sillies out wiggle our wobbles away hey why was number ten scared I dont know why was number ten scared because seven eight nine xxx .", "words": ["was", "shake", "scared", "our", "because", "dont", "away", "out", "were", "why"]}, {"sent": "is Pilchard scared ?", "words": ["scared", "is"]}, {"sent": "scared you .", "words": ["scared", "you"]}, {"sent": "oh Cathy she has a wee like a porch .", "words": ["porch", "she", "like", "a"]}, {"sent": "maybe we could dry it on the porch .", "words": ["the", "we", "porch", "dry", "it", "on", "could"]}, {"sent": "y you can have it inside yeah Im gonna sit with Joseph for a minute on the porch .", "words": ["can", "the", "for", "porch", "inside", "with", "you", "have", "it", "on", "sit", "a"]}, {"sent": "I will go out on the front porch and walk up and down said Toad .", "words": ["the", "up", "go", "porch", "and", "on", "out", "will", "down", "walk"]}, {"sent": "Frog and Toad sat on the porch feeling sad together .", "words": ["the", "porch", "and", "on", "sad"]}, {"sent": "tell Jessica were gonna try and go on the porch for a little bit .", "words": ["the", "for", "go", "porch", "and", "little", "on", "try", "were", "a"]}, {"sent": "on your porch .", "words": ["on", "porch", "your"]}, {"sent": "whats on our porch ?", "words": ["on", "porch", "our"]}, {"sent": "it was a front porch swing .", "words": ["swing", "was", "porch", "it", "a"]}, {"sent": "maybe well sit on the porch .", "words": ["on", "the", "sit", "porch"]}, {"sent": "theyre on their porch building Legos .", "words": ["on", "porch", "their"]}, {"sent": "look finish cleaning the porch first .", "words": ["the", "porch", "first", "finish", "look"]}, {"sent": "and we were sitting on the porch .", "words": ["the", "we", "porch", "and", "on", "were"]}, {"sent": "a little porch theres a little door .", "words": ["porch", "door", "little", "a"]}, {"sent": "well the ovens in the front porch .", "words": ["the", "porch", "in"]}, {"sent": "wanna go in the porch ?", "words": ["the", "go", "porch", "in", "wanna"]}, {"sent": "you gotta go eat on the porch ?", "words": ["the", "go", "porch", "you", "eat", "on"]}, {"sent": "I am looking on the porch .", "words": ["on", "the", "am", "porch"]}, {"sent": "okay we can go on that little porch .", "words": ["can", "go", "we", "porch", "little", "on", "that"]}, {"sent": "were gonna go out on this little porch here and look at the backyard huh ?", "words": ["the", "backyard", "go", "porch", "little", "here", "this", "and", "at", "out", "look", "on", "were"]}, {"sent": "yeah were gonna dry it out on the porch okay ?", "words": ["the", "porch", "dry", "it", "out", "on", "were"]}, {"sent": "oh hes sitting on the porch of the house .", "words": ["the", "of", "porch", "house", "on"]}, {"sent": "Im gonna make a porch .", "words": ["porch", "make", "a"]}, {"sent": "make a red porch and a yellow porch .", "words": ["red", "porch", "and", "yellow", "make", "a"]}, {"sent": "sitting out there on the porch .", "words": ["the", "porch", "there", "out", "on"]}, {"sent": "that looks like a porch up high .", "words": ["up", "porch", "like", "high", "that", "a"]}, {"sent": "doesnt that look like a porch up real high ?", "words": ["up", "porch", "like", "high", "look", "that", "a"]}, {"sent": "they just saw smoke coming off of the porch and they thought it was a fire .", "words": ["off", "the", "of", "porch", "was", "and", "they", "it", "a"]}, {"sent": "like a porch .", "words": ["porch", "like", "a"]}, {"sent": "all the people are going to walk up those stairs and come through the house and come out onto the porch ?", "words": ["the", "all", "up", "those", "to", "porch", "and", "are", "house", "out", "stairs", "walk"]}, {"sent": "cause Im not were not bringing all those things outside dear the porch is too dirty .", "words": ["the", "all", "those", "is", "outside", "porch", "dirty", "too", "were", "not"]}, {"sent": "xxx on the porch .", "words": ["on", "the", "porch"]}, {"sent": "come in from the porch step into the hall .", "words": ["the", "porch", "in", "into"]}, {"sent": "pushed him out of the house and on to the front porch .", "words": ["the", "of", "to", "porch", "and", "him", "house", "out", "on"]}, {"sent": "think of it said Frog well skip through the meadows well run through the woods well swim in the river in the evenings we will sit right here on this front porch and count the stars .", "words": ["the", "think", "we", "of", "porch", "swim", "and", "here", "run", "in", "it", "on", "this", "will", "sit"]}, {"sent": "Toad walked up and down on the porch for a long time .", "words": ["up", "the", "for", "porch", "long", "and", "on", "down", "a"]}, {"sent": "Toad was sitting on his front porch when Frog came along and said whats the matter Toad you look sad .", "words": ["the", "was", "porch", "and", "his", "you", "look", "on", "sad", "when"]}, {"sent": "and Frog and Toad went out to the front porch to wait for the mail .", "words": ["the", "wait", "for", "to", "porch", "and", "out"]}, {"sent": "would you like to go on the porch .", "words": ["the", "go", "to", "porch", "like", "you", "would", "on"]}, {"sent": "take a porch break .", "words": ["porch", "break", "take", "a"]}, {"sent": "the porch .", "words": ["the", "porch"]}, {"sent": "go ahead go sit on your xxx on your porch .", "words": ["your", "go", "porch", "on", "sit"]}, {"sent": "you can sit outside if you want or on the front porch since its such a nice day .", "words": ["can", "the", "if", "outside", "porch", "nice", "you", "on", "sit", "a"]}, {"sent": "then late at night he left them on the chickens porch .", "words": ["the", "porch", "them", "he", "at", "on", "then"]}, {"sent": "like a porch chair .", "words": ["porch", "like", "chair", "a"]}, {"sent": "do you get cardinals on your porch ?", "words": ["do", "your", "porch", "you", "get", "on"]}, {"sent": "uh really not too much goes on our porch .", "words": ["porch", "our", "too", "on", "much", "not"]}, {"sent": "on the porch ?", "words": ["on", "the", "porch"]}, {"sent": "thats a porch ?", "words": ["porch", "a"]}, {"sent": "do we have a porch outside our house ?", "words": ["do", "we", "porch", "outside", "our", "have", "house", "a"]}, {"sent": "whats on our porch downstairs ?", "words": ["on", "porch", "our"]}, {"sent": "he took his first steps in the doctors appointment after he fell from the swing on our front porch .", "words": ["the", "swing", "porch", "first", "our", "in", "on", "he", "his"]}, {"sent": "ya probably left em on the porch .", "words": ["on", "the", "porch"]}, {"sent": "because hell be standing on the porch looking out so youll hafta ask Craig .", "words": ["the", "be", "porch", "so", "out", "on", "because"]}, {"sent": "shes gonna have a nap on the porch .", "words": ["the", "porch", "have", "on", "a"]}, {"sent": "on the porch .", "words": ["on", "the", "porch"]}, {"sent": "in her bed on the porch .", "words": ["the", "porch", "bed", "in", "on", "her"]}, {"sent": "but if it cant then Ill move it like into the kitchen or maybe out onto the back porch .", "words": ["the", "if", "into", "back", "porch", "like", "it", "kitchen", "out", "then", "but"]}, {"sent": "on the awning of the porch .", "words": ["on", "the", "of", "porch"]}, {"sent": "that ladys sweeping off her porch .", "words": ["off", "porch", "her", "that"]}, {"sent": "we need the porch to be on the side of the house .", "words": ["the", "need", "we", "of", "porch", "to", "be", "house", "on"]}, {"sent": "you wanna sit on the porch ?", "words": ["the", "porch", "you", "wanna", "on", "sit"]}, {"sent": "they are fixing the porch next door .", "words": ["the", "porch", "they", "are", "door"]}, {"sent": "everythings going in the porch .", "words": ["the", "porch", "in"]}, {"sent": "and there he is sitting on his porch .", "words": ["is", "porch", "and", "his", "there", "he", "on"]}, {"sent": "theyre not on the sun porch ?", "words": ["the", "porch", "sun", "on", "not"]}, {"sent": "in the in the sun porch ?", "words": ["the", "porch", "in", "sun"]}, {"sent": "I think its on the sun porch .", "words": ["the", "think", "porch", "sun", "on"]}, {"sent": "remember you threw your kangaroo puppet off the porch and we dont know where it is ?", "words": ["off", "the", "where", "your", "we", "is", "porch", "you", "and", "dont", "it"]}, {"sent": "when were finished when were finished cleaning the porch okay .", "words": ["the", "porch", "when", "were"]}, {"sent": "do you remember the racoon family we useta feed out on the porch ?", "words": ["the", "do", "we", "porch", "you", "out", "feed", "on"]}, {"sent": "the little baby racoon useta come up on the porch and we would feed him things .", "words": ["the", "up", "feed", "we", "porch", "little", "and", "him", "would", "on"]}, {"sent": "do you remember we were talking out on the porch and we were playing .", "words": ["the", "do", "we", "porch", "you", "and", "out", "on", "were"]}, {"sent": "a porch .", "words": ["porch", "a"]}, {"sent": "a porch ?", "words": ["porch", "a"]}, {"sent": "what do you do on the porch ?", "words": ["the", "what", "do", "porch", "you", "on"]}, {"sent": "theres some on the porch that havent been carved yet .", "words": ["the", "porch", "on", "some", "that"]}, {"sent": "go get your puzzles off the porch and bring them in and well do them .", "words": ["the", "off", "do", "your", "go", "porch", "them", "bring", "and", "get", "in"]}, {"sent": "you wanna go play on the porch ?", "words": ["the", "go", "porch", "play", "you", "wanna", "on"]}, {"sent": "its cold on the porch .", "words": ["on", "the", "cold", "porch"]}, {"sent": "porch at Grandmas house that is good .", "words": ["is", "porch", "house", "good", "at", "that"]}, {"sent": "some people leave it out on their porch .", "words": ["their", "porch", "it", "out", "on", "some"]}, {"sent": "youre going to hafta take some pennies outof your bank and give them to Ursula because youre using all the tape .", "words": ["the", "all", "your", "to", "them", "tape", "and", "some", "give", "take", "because"]}, {"sent": "a tape recording for him ?", "words": ["him", "for", "tape", "a"]}, {"sent": "I dont think the tape will cut through the sticky tape Thomas .", "words": ["the", "think", "cut", "sticky", "tape", "dont", "will"]}, {"sent": "tape recorder .", "words": ["tape"]}, {"sent": "if you wanna put that PostmanPat tape in youve got to take that one out .", "words": ["put", "if", "to", "tape", "you", "in", "wanna", "out", "take", "that"]}, {"sent": "theres just some tape up there .", "words": ["up", "some", "tape", "there"]}, {"sent": "my tape recorder is working back there .", "words": ["my", "back", "is", "tape", "there"]}, {"sent": "whats that tape doing on the floor ?", "words": ["on", "the", "that", "tape"]}, {"sent": "what the tape recorder better than a book ?", "words": ["the", "what", "better", "tape", "book", "a"]}, {"sent": "Baaees gonna turn the tape recorder off .", "words": ["the", "off", "tape"]}, {"sent": "have you got a tape recorder at home ?", "words": ["tape", "you", "have", "at", "home", "a"]}, {"sent": "theres no tape in there Dominic .", "words": ["there", "in", "tape"]}, {"sent": "or otherwise I shall get Rachel to stop the tape and I shall slap your legs if youre going to start being silly .", "words": ["the", "your", "if", "to", "stop", "tape", "and", "get"]}, {"sent": "adhesive tape .", "words": ["tape"]}, {"sent": "then we wont tape so much .", "words": ["we", "so", "tape", "then", "much"]}, {"sent": "here is Daddys tape recorder .", "words": ["here", "tape", "is"]}, {"sent": "have you got a tape recorder ?", "words": ["have", "tape", "a", "you"]}, {"sent": "tape underneath .", "words": ["tape"]}, {"sent": "you want the tape ?", "words": ["the", "tape", "you"]}, {"sent": "then the tape recorder goes off .", "words": ["the", "then", "off", "tape"]}, {"sent": "theyre running like Dimitra runs when she runs to change the tape over when were doing a video .", "words": ["the", "to", "like", "tape", "over", "when", "she", "were", "a"]}, {"sent": "turning the tape on .", "words": ["the", "on", "tape"]}, {"sent": "okay this is taping over because we had a lot of problems and a big buzzing sound so Im redoing the rest of this tape .", "words": ["the", "we", "is", "of", "so", "a lot", "tape", "and", "this", "big", "over", "because", "a"]}, {"sent": "okay shall we take the tape recorder down ?", "words": ["the", "we", "tape", "down", "take"]}, {"sent": "can you find me the tape ?", "words": ["can", "the", "tape", "you", "find", "me"]}, {"sent": "are you still tape recording this ?", "words": ["this", "are", "tape", "you"]}, {"sent": "Im just winding the tape round .", "words": ["the", "tape"]}, {"sent": "tape recorder ?", "words": ["tape"]}, {"sent": "now its back to the tape again .", "words": ["the", "to", "back", "tape"]}, {"sent": "and Ill finish the tape when you get home .", "words": ["the", "finish", "tape", "and", "you", "get", "when", "home"]}, {"sent": "usually when we say tape were going to put a little recording tape in the machine arent we ?", "words": ["the", "put", "when", "we", "to", "say", "tape", "little", "in", "were", "a"]}, {"sent": "you know how tape is ?", "words": ["how", "is", "tape", "you"]}, {"sent": "thatll be good for the tape .", "words": ["the", "for", "be", "tape", "good"]}, {"sent": "tape players dont like to next to each other .", "words": ["each", "to", "like", "tape", "other", "dont"]}, {"sent": "and we dont need a tape right now .", "words": ["need", "we", "tape", "and", "dont", "a"]}, {"sent": "this is tape .", "words": ["this", "tape", "is"]}, {"sent": "you know the tape like the Starmarket has Edward ?", "words": ["the", "like", "tape", "you"]}, {"sent": "you go and get the tape .", "words": ["the", "go", "tape", "and", "get", "you"]}, {"sent": "whatre you gonna tape ?", "words": ["tape", "you"]}, {"sent": "which is why weve come through here to do the tape and why I thought wed better do the door .", "words": ["the", "do", "better", "is", "to", "tape", "and", "here", "which", "why", "door"]}, {"sent": "yeah what did you want me to do to the tape ?", "words": ["the", "what", "do", "to", "tape", "you", "did", "me"]}, {"sent": "think theres we got a Ralphie tape and one of the songs on the tape is Joshua Giraffe .", "words": ["the", "think", "we", "of", "is", "tape", "and", "on", "a"]}, {"sent": "thats the tape recorder the lady has .", "words": ["the", "tape"]}, {"sent": "this session is being recorded a week about a week later but immediately after that is on the tape after the previous session unlike how I usually do it .", "words": ["the", "do", "how", "is", "tape", "this", "it", "about", "on", "that", "but", "a"]}, {"sent": "lunch pretty soom youre gonna have lunch pretty soom come on in here you gonna help us turn the tape recorder off ?", "words": ["the", "off", "help", "tape", "you", "have", "here", "in", "us", "pretty", "on"]}, {"sent": "come here and Ill get you some tape .", "words": ["tape", "you", "here", "get", "and", "some"]}, {"sent": "I told you its in the tape recorder .", "words": ["the", "in", "tape", "you"]}, {"sent": "shes coming to do a tape .", "words": ["to", "do", "tape", "a"]}, {"sent": "let me see there see you could fix it without tape .", "words": ["fix", "me", "tape", "you", "there", "it", "see", "could"]}, {"sent": "hes on the tape isnt he ?", "words": ["on", "the", "he", "tape"]}, {"sent": "a tape ?", "words": ["tape", "a"]}, {"sent": "that is the tape recorder .", "words": ["the", "that", "tape", "is"]}, {"sent": "whats the tape xxx recorder doing eh ?", "words": ["the", "tape"]}, {"sent": "Im going to need another tape for my tape recorder isnt that right ?", "words": ["my", "for", "need", "to", "tape", "another", "that"]}, {"sent": "tape recorder .", "words": ["tape"]}, {"sent": "who comes over and we make the tape ?", "words": ["the", "we", "tape", "and", "who", "over", "make"]}, {"sent": "music tape .", "words": ["tape"]}, {"sent": "tape recorder its turning .", "words": ["tape"]}, {"sent": "or if theyre in a chatty mood if you have a tape recorder and tape you could tape him when you just knew you know .", "words": ["if", "tape", "you", "have", "and", "in", "him", "could", "when", "a"]}, {"sent": "yeah dont turn the tape recorder off darling .", "words": ["the", "dont", "off", "tape"]}, {"sent": "that is Daddys tape recorder .", "words": ["that", "tape", "is"]}, {"sent": "well weve lost the tape Thomas .", "words": ["the", "tape"]}, {"sent": "you can leave the tape on if you want .", "words": ["can", "the", "if", "tape", "you", "on"]}, {"sent": "thats Nathaniels tape recorder yeah .", "words": ["tape"]}, {"sent": "we have one more tape .", "words": ["have", "more", "tape", "we"]}, {"sent": "my tape .", "words": ["my", "tape"]}, {"sent": "I think its the tape recorder .", "words": ["the", "think", "tape"]}, {"sent": "I can tape more on that if you want .", "words": ["can", "if", "tape", "you", "more", "on", "that"]}, {"sent": "I thought you were gonna tape ?", "words": ["were", "tape", "you"]}, {"sent": "that is what you say when you see the tape recorder .", "words": ["the", "what", "when", "is", "say", "tape", "you", "see", "that"]}, {"sent": "putting tape on your leg .", "words": ["on", "your", "leg", "tape"]}, {"sent": "there is the tape recorder .", "words": ["the", "there", "tape", "is"]}, {"sent": "dont put the tape in ?", "words": ["the", "put", "tape", "dont", "in"]}, {"sent": "will you go in my car and get the tape of Aladar .", "words": ["the", "my", "go", "of", "car", "tape", "you", "and", "get", "in", "will"]}, {"sent": "no I didnt tell them to bring strawberries tomorrow but were going to talk to them later because I said daddy was in the bath and that we were in the middle of doing a tape .", "words": ["the", "we", "of", "to", "was", "them", "bring", "because", "and", "tape", "in", "that", "talk", "were", "but", "a"]}, {"sent": "taping Jennifers cheek you gonna tape Jennifers ear ?", "words": ["cheek", "ear", "tape", "you"]}, {"sent": "Jeannines coming to do a tape .", "words": ["to", "do", "tape", "a"]}, {"sent": "mhm its a different kind of tape .", "words": ["of", "tape", "a"]}, {"sent": "ohgosh the tape .", "words": ["the", "tape"]}, {"sent": "here I can give you some tape on this box .", "words": ["can", "box", "tape", "you", "here", "this", "on", "some", "give"]}, {"sent": "there is tape recorder .", "words": ["there", "tape", "is"]}, {"sent": "xxx tape recorder .", "words": ["tape"]}, {"sent": "the tape recorder is behind you ?", "words": ["the", "is", "behind", "tape", "you"]}, {"sent": "I think youre not putting the tape quite far enough over the edge .", "words": ["the", "think", "tape", "over", "not"]}, {"sent": "oh on a video tape .", "words": ["on", "tape", "a"]}, {"sent": "Maisie mends the hole with some tape .", "words": ["the", "some", "with", "tape"]}, {"sent": "I was just thinking whoevers listening to this tape wont appreciate the noise of us mixing .", "words": ["the", "of", "to", "was", "tape", "this", "us"]}, {"sent": "because I cant read because that makes me talk a lot on the tape and we wanna hear you talk .", "words": ["the", "we", "a lot", "tape", "and", "hear", "you", "read", "talk", "wanna", "on", "me", "that", "because", "a"]}, {"sent": "now well tape it on both tape recorders .", "words": ["on", "it", "tape"]}, {"sent": "now you got tape on your hand .", "words": ["your", "hand", "tape", "you", "on"]}, {"sent": "ya know what lemme get the scissors the tape is easier to cut than it is to rip off .", "words": ["the", "what", "off", "cut", "is", "to", "scissors", "rip", "tape", "get", "it"]}, {"sent": "you know what theyre gonna get on this tape ?", "words": ["what", "tape", "you", "get", "this", "on"]}, {"sent": "can you tell the tape who Jessie is ?", "words": ["can", "the", "is", "tape", "you", "who"]}, {"sent": "I just hafta put the tape sothat we can see ya .", "words": ["the", "can", "put", "we", "tape", "see"]}, {"sent": "wherere the tape recorder wheels ?", "words": ["the", "tape"]}, {"sent": "I think we better just pause the tape a minute I should have thought and done this sooner shouldnt I .", "words": ["the", "better", "think", "we", "tape", "and", "have", "this", "a"]}, {"sent": "xxx the tape recorders xxx .", "words": ["the", "tape"]}, {"sent": "and the tape was alright then was it ?", "words": ["the", "was", "tape", "and", "it", "then"]}, {"sent": "well this ones not a Charlie Brown tape .", "words": ["this", "not", "tape", "a"]}, {"sent": "Missus Sutton will put a piece a tape to hold it .", "words": ["put", "hold", "to", "tape", "it", "will", "a"]}, {"sent": "whered my game go ?", "words": ["go", "my", "game"]}, {"sent": "are your friends stuck in that game ?", "words": ["game", "your", "in", "are", "stuck", "that"]}, {"sent": "yeah I wanna know how to play that game .", "words": ["how", "game", "to", "play", "wanna", "that"]}, {"sent": "well Joseph this is not a game if you wanna eat them you can eat them .", "words": ["can", "game", "if", "is", "them", "you", "eat", "this", "wanna", "not", "a"]}, {"sent": "yes were going to play another game .", "words": ["game", "to", "play", "another", "were"]}, {"sent": "and go and get some and play that fun game again ?", "words": ["game", "go", "play", "and", "get", "some", "that"]}, {"sent": "are we having this game of snap today ?", "words": ["game", "we", "of", "this", "are"]}, {"sent": "so we opened each of the buildings everyday but I think you can play the the game .", "words": ["the", "can", "think", "game", "each", "we", "of", "so", "play", "you", "but"]}, {"sent": "this is the game here now mummy needs a teddy .", "words": ["the", "game", "is", "here", "this", "a"]}, {"sent": "this is a Mamas and Dadas game darling .", "words": ["game", "is", "and", "this", "a"]}, {"sent": "and will we put this game back as well ?", "words": ["put", "game", "back", "we", "and", "this", "will"]}, {"sent": "is that a game ?", "words": ["game", "that", "a", "is"]}, {"sent": "were going to play this big game here .", "words": ["game", "to", "play", "here", "this", "big", "were"]}, {"sent": "careful because youre on the other game now .", "words": ["the", "game", "other", "careful", "on", "because"]}, {"sent": "this is that game you like isnt it ?", "words": ["game", "is", "like", "you", "this", "it", "that"]}, {"sent": "is it a running game ?", "words": ["game", "it", "a", "is"]}, {"sent": "these puzzles dont fit too well because this game has been used a lot .", "words": ["these", "game", "a lot", "fit", "dont", "this", "too", "because", "a"]}, {"sent": "this game ?", "words": ["this", "game"]}, {"sent": "oh see that basketball game last night ?", "words": ["see", "game", "that", "last"]}, {"sent": "is the sitting down game all done now ?", "words": ["the", "all", "game", "is", "down"]}, {"sent": "you this letterbox game with Nana and Granddad cant you ?", "words": ["game", "with", "and", "you", "this"]}, {"sent": "youre going to get out your game ?", "words": ["game", "your", "to", "get", "out"]}, {"sent": "well play another game .", "words": ["another", "game", "play"]}, {"sent": "yeah thats the game .", "words": ["the", "game"]}, {"sent": "were not playing that silly game .", "words": ["that", "were", "not", "game"]}, {"sent": "its a little game .", "words": ["game", "little", "a"]}, {"sent": "is this a game ?", "words": ["this", "game", "a", "is"]}, {"sent": "what kind of game do do you wanna play ?", "words": ["what", "game", "do", "of", "play", "you", "wanna"]}, {"sent": "partners partners partners partners heres a little a little game xxx partners .", "words": ["game", "little", "a"]}, {"sent": "but if we play this game then we should put them over there so we dont hafta move the I sofa everytime .", "words": ["the", "game", "put", "if", "we", "so", "play", "them", "this", "there", "dont", "sofa", "then", "over", "but"]}, {"sent": "your favorite game .", "words": ["game", "your"]}, {"sent": "I think youre playing a game .", "words": ["game", "think", "a"]}, {"sent": "uhhuh Ive got a game down here .", "words": ["here", "game", "down", "a"]}, {"sent": "I thought thats the game were playing hm .", "words": ["the", "game", "were"]}, {"sent": "what you dont like this game do you ?", "words": ["what", "game", "do", "like", "you", "dont", "this"]}, {"sent": "this game you pick up one picture this ones an umbrella and you hafta find another umbrella .", "words": ["up", "game", "pick", "picture", "you", "and", "this", "an", "another", "find"]}, {"sent": "are we going to the football game with Mommy ?", "words": ["the", "game", "we", "to", "with", "are"]}, {"sent": "no lets play the game .", "words": ["the", "game", "play"]}, {"sent": "one game .", "words": ["game"]}, {"sent": "do you wanna do a memory game ?", "words": ["do", "game", "you", "wanna", "a"]}, {"sent": "is that your game ?", "words": ["your", "game", "that", "is"]}, {"sent": "youre gonna clean up the bread first and thats on the floor and the yogurt and then after that well clean up the sentences game .", "words": ["clean", "the", "bread", "up", "game", "first", "yogurt", "and", "on", "then", "that"]}, {"sent": "were going to play a game .", "words": ["game", "to", "play", "were", "a"]}, {"sent": "different game .", "words": ["game"]}, {"sent": "hm maybe its from another game .", "words": ["another", "game"]}, {"sent": "thats true you got the red and yellow fish for your fish game at aunties house .", "words": ["the", "red", "game", "for", "your", "and", "you", "at", "house", "fish", "yellow"]}, {"sent": "youll hafta play a game and use something in there .", "words": ["game", "play", "and", "there", "in", "a"]}, {"sent": "wed better play a different game now .", "words": ["better", "game", "play", "a"]}, {"sent": "who made all this mess Fraser with the felt game ?", "words": ["the", "all", "game", "with", "this", "who"]}, {"sent": "lets play a game with more talking in it .", "words": ["game", "play", "with", "more", "in", "it", "a"]}, {"sent": "thats a whole new game though .", "words": ["game", "new", "a"]}, {"sent": "oh you like this game .", "words": ["this", "game", "like", "you"]}, {"sent": "I think were going to play another game now .", "words": ["think", "game", "to", "play", "another", "were"]}, {"sent": "Im looking for the game .", "words": ["the", "game", "for"]}, {"sent": "Thomas have you played this game before ?", "words": ["have", "this", "game", "you"]}, {"sent": "why dont you come in here and well play the sentences game ?", "words": ["the", "game", "play", "you", "here", "dont", "in", "and", "why"]}, {"sent": "dont think you can play your sound game with Caroline .", "words": ["can", "think", "game", "your", "play", "with", "you", "dont"]}, {"sent": "one game to love .", "words": ["to", "game", "love"]}, {"sent": "what game ?", "words": ["what", "game"]}, {"sent": "now the first game were going to play today .", "words": ["the", "game", "to", "first", "play", "were"]}, {"sent": "um she has a CuriousGeorge matching game that she really likes too .", "words": ["game", "too", "that", "she", "a"]}, {"sent": "a game .", "words": ["game", "a"]}, {"sent": "theres still a maze game .", "words": ["game", "a"]}, {"sent": "we have one more game which I hafta get .", "words": ["game", "we", "have", "get", "more", "which"]}, {"sent": "and you stay with mommy and play a game too .", "words": ["game", "play", "with", "you", "and", "too", "stay", "a"]}, {"sent": "I probably could have got another game .", "words": ["have", "could", "another", "game"]}, {"sent": "the odd game is only played with a little person .", "words": ["the", "game", "is", "with", "little", "a"]}, {"sent": "sing me Take me out to the baseball game .", "words": ["the", "game", "to", "sing", "out", "me"]}, {"sent": "what game did we play here ?", "words": ["what", "game", "we", "play", "here", "did"]}, {"sent": "xxx I know xxx yes its a very common game I suppose its a very obvious one and its what we do all through our lives isnt it trying to make people do what we want them to do .", "words": ["what", "game", "do", "all", "we", "to", "them", "and", "our", "it", "make", "a"]}, {"sent": "so weve been playing this kindof fun game at home thats called the magic dice game .", "words": ["the", "game", "so", "this", "at", "home"]}, {"sent": "well tell Ryan how you useta play the chase game .", "words": ["the", "how", "game", "play", "you", "chase"]}, {"sent": "shall we do a different game ?", "words": ["do", "game", "a", "we"]}, {"sent": "lets put it in the game okay ?", "words": ["the", "put", "game", "in", "it"]}, {"sent": "you dont wanna play that game anymore ?", "words": ["game", "play", "you", "dont", "wanna", "that"]}, {"sent": "Sarah I have a game I want you to play .", "words": ["game", "to", "play", "you", "have", "a"]}, {"sent": "were playing a game .", "words": ["game", "were", "a"]}, {"sent": "that looks like a game .", "words": ["game", "that", "like", "a"]}, {"sent": "game .", "words": ["game"]}, {"sent": "do you wanna do another game now ?", "words": ["do", "game", "you", "another", "wanna"]}, {"sent": "I dont like that game Lara .", "words": ["dont", "game", "that", "like"]}, {"sent": "I dont understand this game Sid .", "words": ["dont", "game", "this"]}, {"sent": "Ill give you it in the game .", "words": ["the", "game", "you", "in", "it", "give"]}, {"sent": "I made up that game Jason .", "words": ["up", "game", "that"]}, {"sent": "its a game .", "words": ["game", "a"]}, {"sent": "its like a what doesnt belong game .", "words": ["what", "game", "like", "a"]}, {"sent": "lets play another game .", "words": ["another", "game", "play"]}, {"sent": "Ill show you what game Im going to play .", "words": ["what", "game", "show", "to", "play", "you"]}, {"sent": "and then you come back and well have another little game .", "words": ["game", "back", "you", "have", "and", "another", "little", "then"]}, {"sent": "the monkeys climbing game is fun theyre swinging toandfro kitten builds some sandcastles standing in a .", "words": ["the", "game", "is", "in", "some", "a"]}, {"sent": "a baseball game .", "words": ["game", "a"]}, {"sent": "its a game on that board .", "words": ["on", "game", "that", "a"]}, {"sent": "Ive given the game away .", "words": ["the", "game", "away"]}, {"sent": "what kind of game is that ?", "words": ["what", "game", "is", "of", "that"]}, {"sent": "thats a bit noisy that game uhn isnt it ?", "words": ["game", "noisy", "it", "that", "a"]}, {"sent": "wow what a great game .", "words": ["what", "game", "a"]}, {"sent": "okay Ill put the game away and you xxx cuddle up with your mom so shell tell you a story .", "words": ["the", "up", "put", "game", "your", "so", "with", "and", "you", "away", "story", "a"]}, {"sent": "youre playing a new game ?", "words": ["game", "new", "a"]}, {"sent": "lets make them into pairs and we can start a game .", "words": ["can", "game", "into", "we", "them", "and", "make", "a"]}, {"sent": "if the weathers good were going to have a football game Sunday .", "words": ["the", "game", "if", "to", "have", "good", "were", "a"]}, {"sent": "what does he needta clean Aran ?", "words": ["clean", "what", "does", "he"]}, {"sent": "youd better clean up your clothes because I bought you three new short sleeve shirts just a little while ago and you have the one you have on .", "words": ["clean", "better", "up", "the", "your", "you", "little", "and", "have", "on", "new", "because", "a"]}, {"sent": "nice and clean ?", "words": ["clean", "nice", "and"]}, {"sent": "hes got to clean it ?", "words": ["clean", "to", "it"]}, {"sent": "oh Thomas it takes me ages to clean up after a meal doesnt it ?", "words": ["up", "clean", "to", "it", "me", "a"]}, {"sent": "are you sure this street is clean ?", "words": ["clean", "is", "you", "this", "are", "street"]}, {"sent": "to clean .", "words": ["clean", "to"]}, {"sent": "well yours are all clean .", "words": ["clean", "all", "are"]}, {"sent": "want me clean it up ?", "words": ["clean", "me", "it", "up"]}, {"sent": "clean that up .", "words": ["clean", "up", "that"]}, {"sent": "its time to clean up so we can play something else .", "words": ["clean", "up", "can", "we", "to", "so", "play"]}, {"sent": "clean the table .", "words": ["clean", "the", "table"]}, {"sent": "see if you can find a clean one Shem can you find a clean little dress for the baby ?", "words": ["can", "clean", "the", "for", "if", "you", "little", "find", "see", "dress", "a"]}, {"sent": "do you have to clean bathrooms ?", "words": ["clean", "do", "to", "you", "have"]}, {"sent": "lets get you a clean pookie first .", "words": ["clean", "first", "you", "get", "a"]}, {"sent": "time to clean up .", "words": ["clean", "to", "up"]}, {"sent": "do you needta clean your nose Fraser ?", "words": ["clean", "do", "your", "you", "nose"]}, {"sent": "you going to clean the cream off ?", "words": ["clean", "the", "off", "to", "you"]}, {"sent": "lets put a clean vest on you .", "words": ["clean", "put", "you", "on", "a"]}, {"sent": "it is a clean box but its still not nice to lick is it ?", "words": ["clean", "box", "is", "to", "nice", "it", "lick", "not", "but", "a"]}, {"sent": "clean up that mess .", "words": ["clean", "up", "that"]}, {"sent": "now shes got a nice clean bottom .", "words": ["clean", "nice", "bottom", "a"]}, {"sent": "why do you wanna give your teeth a clean ?", "words": ["clean", "do", "your", "you", "wanna", "give", "why", "a"]}, {"sent": "she can cat cats clean themselves .", "words": ["can", "cat", "she", "clean"]}, {"sent": "all clean .", "words": ["clean", "all"]}, {"sent": "because itll start to smell if you dont clean it properly wont it ?", "words": ["clean", "if", "to", "you", "dont", "it", "because"]}, {"sent": "mostly we just needta clean them .", "words": ["clean", "them", "we"]}, {"sent": "and then Laura doesnt like to hafta clean it up either .", "words": ["clean", "up", "to", "like", "and", "it", "then"]}, {"sent": "would you like the clean dishes ?", "words": ["the", "clean", "like", "you", "would"]}, {"sent": "and what do we do do we clean it ?", "words": ["clean", "what", "do", "we", "and", "it"]}, {"sent": "poor Piglet doesnt wanna be clean .", "words": ["clean", "be", "wanna", "poor"]}, {"sent": "what do they do after they get all clean ?", "words": ["clean", "what", "do", "all", "get", "they"]}, {"sent": "lets clean your nose a big bubble .", "words": ["clean", "your", "nose", "big", "a"]}, {"sent": "you must have a clean and cut pear .", "words": ["clean", "cut", "and", "have", "you", "a"]}, {"sent": "Mommy hasta get her hand all clean .", "words": ["clean", "all", "hand", "get", "her"]}, {"sent": "you want a clean diaper on .", "words": ["clean", "diaper", "you", "on", "a"]}, {"sent": "we havent got any clean that fit .", "words": ["clean", "we", "fit", "any", "that"]}, {"sent": "you clean up for you Mum .", "words": ["clean", "up", "for", "you"]}, {"sent": "I already helped you some now you clean up the rest .", "words": ["clean", "the", "up", "you", "some"]}, {"sent": "lets go up and get a clean clean clean clean clean clean clean clean clean diaper .", "words": ["up", "clean", "go", "diaper", "and", "get", "a"]}, {"sent": "were going to clean it off first and then well put everything back in the house okay ?", "words": ["clean", "off", "put", "the", "back", "to", "first", "and", "in", "it", "house", "then", "were"]}, {"sent": "what about your other hand is your other hand clean ?", "words": ["clean", "what", "your", "hand", "is", "other", "about"]}, {"sent": "and its clean .", "words": ["clean", "and"]}, {"sent": "I like floors clean .", "words": ["clean", "like"]}, {"sent": "now the tub was all clean but her dress was a mess .", "words": ["the", "all", "clean", "dress", "was", "her", "but", "a"]}, {"sent": "Lisas all clean and tidy there .", "words": ["clean", "all", "there", "and"]}, {"sent": "and clean clothes on .", "words": ["clean", "on", "and"]}, {"sent": "he looks very clean .", "words": ["clean", "he"]}, {"sent": "theyll clean up dont worry .", "words": ["clean", "dont", "up"]}, {"sent": "your hands are actually quite clean Babyoh .", "words": ["clean", "are", "your"]}, {"sent": "clean up clean up everybody do your share .", "words": ["clean", "do", "up", "your", "share"]}, {"sent": "it smells like lovely clean lovely clean fresh smell .", "words": ["clean", "it", "like"]}, {"sent": "Im trying to clean off the darn marker .", "words": ["clean", "to", "off", "the"]}, {"sent": "Im going to just sit here and clean your shoes .", "words": ["clean", "your", "to", "and", "here", "sit"]}, {"sent": "can you please help clean up sweetie ?", "words": ["can", "up", "help", "clean", "you"]}, {"sent": "a little bit on the clean side just a little bit .", "words": ["the", "clean", "little", "on", "a"]}, {"sent": "we needta keep things very clean for our visitors tomorrow .", "words": ["clean", "our", "for", "we"]}, {"sent": "help momma clean up .", "words": ["clean", "help", "up"]}, {"sent": "is it clean ?", "words": ["clean", "it", "is"]}, {"sent": "is he going home for some clean water ?", "words": ["clean", "for", "is", "he", "water", "some", "home"]}, {"sent": "clean up clean up this is the way we clean up our kitchen .", "words": ["clean", "the", "up", "is", "we", "our", "this", "kitchen"]}, {"sent": "so Ill just put away some clean dishes for a minute .", "words": ["clean", "put", "for", "so", "away", "some", "a"]}, {"sent": "mommys gonna clean up then .", "words": ["clean", "then", "up"]}, {"sent": "lets clean up then if youre all done .", "words": ["clean", "all", "up", "if", "then"]}, {"sent": "clean Laura .", "words": ["clean"]}, {"sent": "well its just to keep my hands clean .", "words": ["clean", "to", "my"]}, {"sent": "or are you gonna be real clean ?", "words": ["clean", "be", "are", "you"]}, {"sent": "clean it off good boy .", "words": ["clean", "it", "off", "good"]}, {"sent": "its terrible to get clean .", "words": ["get", "to", "clean"]}, {"sent": "its gonna be so clean .", "words": ["clean", "be", "so"]}, {"sent": "all clean .", "words": ["clean", "all"]}, {"sent": "they useta make her clean all the time clean the house and everything .", "words": ["clean", "all", "the", "and", "they", "make", "house", "her"]}, {"sent": "if all your toys were magnetic you wouldnt hafta clean your room .", "words": ["clean", "all", "your", "if", "you", "room", "were"]}, {"sent": "I asked for a nice clean job .", "words": ["nice", "for", "clean", "a"]}, {"sent": "oh is that all washed nice and clean ?", "words": ["clean", "all", "is", "nice", "and", "that"]}, {"sent": "and put clean clothes on .", "words": ["clean", "put", "on", "and"]}, {"sent": "clean the dishes for mommy ?", "words": ["clean", "the", "for"]}, {"sent": "lovely clean bottom .", "words": ["clean", "bottom"]}, {"sent": "are you gonna clean your own face ?", "words": ["clean", "your", "face", "you", "are"]}, {"sent": "because I would be I would not be happy to hafta clean that up .", "words": ["clean", "up", "happy", "be", "to", "would", "that", "not", "because"]}, {"sent": "mommys gotta clean your books up .", "words": ["clean", "up", "your"]}, {"sent": "yeah clean the table .", "words": ["clean", "the", "table"]}, {"sent": "I think we should probably get something to clean that up with dont you think .", "words": ["clean", "up", "think", "we", "to", "with", "you", "get", "dont", "that"]}, {"sent": "oh nice and clean for the weekend .", "words": ["clean", "the", "for", "nice", "and"]}, {"sent": "dont you like to have a clean face ?", "words": ["clean", "face", "to", "like", "you", "have", "dont", "a"]}, {"sent": "clean as I can get it .", "words": ["clean", "it", "can", "get"]}, {"sent": "Mommyll clean that up .", "words": ["clean", "up", "that"]}, {"sent": "its where you clean your teeth isnt it ?", "words": ["where", "clean", "your", "you", "it"]}, {"sent": "shall I get the carpet shampoo out and you can clean up all the dirty patches ?", "words": ["the", "can", "clean", "up", "all", "you", "get", "and", "dirty", "out"]}, {"sent": "no you better clean this mess up .", "words": ["clean", "better", "up", "you", "this"]}, {"sent": "lovely and clean .", "words": ["clean", "and"]}, {"sent": "okay youve been very patient booboo let me clean you up .", "words": ["clean", "me", "up", "you"]}, {"sent": "what do you clean your nose with ?", "words": ["clean", "what", "do", "your", "with", "you", "nose"]}, {"sent": "no its fake these toys arent clean .", "words": ["clean", "these"]}, {"sent": "weve got one going up the ladders to clean the front of the house .", "words": ["up", "the", "clean", "of", "to", "house"]}, {"sent": "wipe your face all off clean up .", "words": ["off", "all", "clean", "up", "your", "face", "wipe"]}, {"sent": "can you go clean up the tuna salad ?", "words": ["can", "up", "clean", "the", "go", "you", "tuna"]}, {"sent": "can I clean your hand ?", "words": ["can", "your", "clean", "hand"]}, {"sent": "James wanted to get some biscuits from the table .", "words": ["the", "table", "to", "get", "some"]}, {"sent": "is she gonna sleep in the diaper change table ?", "words": ["the", "table", "is", "sleep", "diaper", "in", "she"]}, {"sent": "lets see lets get the table the table and the chairs are ready .", "words": ["the", "table", "and", "get", "are", "see"]}, {"sent": "its actually your moisturizing cream that youre just banging on the table arent you ?", "words": ["the", "your", "table", "you", "on", "that"]}, {"sent": "Lola looked at the table .", "words": ["the", "table", "at"]}, {"sent": "dont iron the table .", "words": ["the", "dont", "table"]}, {"sent": "well if you keep banging it on the table Thomas the lid will come off the juice will spill and Mummy will be cross .", "words": ["the", "off", "if", "table", "be", "will", "you", "and", "it", "spill", "on", "juice"]}, {"sent": "please may I leave the table .", "words": ["the", "table"]}, {"sent": "and you I think you said he doesnt have um d he has some things like this on the changing table but .", "words": ["the", "think", "table", "like", "you", "have", "and", "this", "he", "on", "some", "but"]}, {"sent": "Nomi put them on the table .", "words": ["the", "put", "table", "them", "on"]}, {"sent": "sit on your table and chairs .", "words": ["your", "table", "and", "on", "sit"]}, {"sent": "we should get your table for your plasticine .", "words": ["for", "your", "table", "we", "get"]}, {"sent": "you wanna have a cracker on the table ?", "words": ["the", "table", "cracker", "you", "have", "wanna", "on", "a"]}, {"sent": "is the table moving up and down ?", "words": ["the", "up", "table", "is", "and", "down"]}, {"sent": "under the table ?", "words": ["the", "under", "table"]}, {"sent": "huh you know who else is sitting at a chair at the table honey ?", "words": ["the", "table", "is", "chair", "you", "at", "who", "a"]}, {"sent": "I will take the table .", "words": ["will", "the", "take", "table"]}, {"sent": "there it goes under the table .", "words": ["the", "table", "there", "under", "it"]}, {"sent": "who else sits at the table ?", "words": ["the", "at", "table", "who"]}, {"sent": "it has reminded you that you like your trains on the highchair table when youre eating .", "words": ["the", "your", "table", "when", "like", "you", "it", "on", "that"]}, {"sent": "itll snap better on the table .", "words": ["on", "better", "table", "the"]}, {"sent": "get your table out .", "words": ["get", "your", "table", "out"]}, {"sent": "its over by the table .", "words": ["the", "over", "table", "by"]}, {"sent": "Ive seen him on the small table there .", "words": ["the", "table", "there", "him", "on"]}, {"sent": "Im I make a blue table .", "words": ["blue", "make", "table", "a"]}, {"sent": "are you wiping the table ?", "words": ["the", "are", "table", "you"]}, {"sent": "what is your foot doing on the table ?", "words": ["the", "what", "your", "table", "is", "foot", "on"]}, {"sent": "oh Jillian Ernie cant see any of the things that are on the table .", "words": ["the", "table", "of", "any", "are", "on", "see", "that"]}, {"sent": "hes bringing flowers pretty flowers to the table .", "words": ["the", "to", "pretty", "table"]}, {"sent": "why dont you draw on the table .", "words": ["the", "table", "draw", "you", "dont", "on", "why"]}, {"sent": "why are you going under the table ?", "words": ["the", "table", "you", "under", "are", "why"]}, {"sent": "table ?", "words": ["table"]}, {"sent": "Sarah dont pull on the table .", "words": ["the", "table", "pull", "dont", "on"]}, {"sent": "please go to the table .", "words": ["the", "to", "table", "go"]}, {"sent": "your favorite thing to beat on the table huh ?", "words": ["the", "your", "table", "to", "on"]}, {"sent": "not on the table .", "words": ["on", "the", "not", "table"]}, {"sent": "yeah thats a picnic table .", "words": ["table", "a"]}, {"sent": "I can get the table for it .", "words": ["can", "the", "for", "table", "get", "it"]}, {"sent": "silly table .", "words": ["table"]}, {"sent": "my look at all the things on this table .", "words": ["the", "all", "my", "look", "table", "this", "at", "on"]}, {"sent": "you mustnt kick the table .", "words": ["the", "kick", "table", "you"]}, {"sent": "Ive told you about kicking the table .", "words": ["the", "table", "about", "you"]}, {"sent": "put the cookies back on the table so Ursula and Cromer can have some .", "words": ["the", "can", "put", "table", "back", "so", "and", "have", "on", "some"]}, {"sent": "will you come on ta the table please .", "words": ["the", "table", "you", "on", "will"]}, {"sent": "xxx xxx he turned the table xxx look at this little midget .", "words": ["the", "look", "table", "little", "this", "he", "at"]}, {"sent": "James wanted to get some biscuits from the table .", "words": ["the", "table", "to", "get", "some"]}, {"sent": "not on the table because if you fall off youre going to hurt yourself arent you ?", "words": ["the", "off", "yourself", "table", "if", "hurt", "to", "fall", "you", "on", "not", "because"]}, {"sent": "why dont you take all the cups out and put them on the table ?", "words": ["the", "all", "put", "table", "them", "you", "and", "dont", "out", "on", "take", "why"]}, {"sent": "that pens on that big table .", "words": ["on", "big", "that", "table"]}, {"sent": "what do you need when you set the table ?", "words": ["the", "what", "do", "need", "table", "you", "when"]}, {"sent": "thats my table lamp .", "words": ["my", "lamp", "table"]}, {"sent": "put it on the table .", "words": ["the", "put", "table", "it", "on"]}, {"sent": "oh table yeah .", "words": ["table"]}, {"sent": "want me to put it on the table ?", "words": ["the", "put", "table", "to", "it", "on", "me"]}, {"sent": "so are you bringing your table up to sit next to me ?", "words": ["up", "your", "table", "so", "to", "you", "are", "me", "sit"]}, {"sent": "are you going to clean off the table ?", "words": ["clean", "off", "the", "table", "to", "you", "are"]}, {"sent": "put it over the ah on the table .", "words": ["the", "put", "table", "it", "on", "over"]}, {"sent": "oh my gosh ut Jessicas sitting in a chair at the table .", "words": ["the", "my", "table", "chair", "in", "at", "a"]}, {"sent": "Wendy Muck Dizzy and Rolly had decorated a table and covered it with cakes and presents .", "words": ["table", "with", "and", "it", "a"]}, {"sent": "April I have them out here on the table .", "words": ["the", "table", "them", "have", "here", "out", "on"]}, {"sent": "can we pretend thats a table instead ?", "words": ["can", "table", "we", "pretend", "a"]}, {"sent": "so we hadta take the table out didnt we ?", "words": ["the", "table", "we", "so", "out", "take"]}, {"sent": "now you across the table on the table .", "words": ["the", "on", "table", "you"]}, {"sent": "go back to the table .", "words": ["the", "table", "back", "go", "to"]}, {"sent": "and wheres the table go ?", "words": ["the", "go", "table", "and"]}, {"sent": "for our table .", "words": ["for", "table", "our"]}, {"sent": "you see we have a bunch of things on the table okay ?", "words": ["the", "table", "we", "of", "you", "have", "on", "see", "a"]}, {"sent": "you dont want a grubby table for tea time .", "words": ["for", "table", "you", "dont", "a"]}, {"sent": "its over there by the table .", "words": ["the", "table", "by", "there", "over"]}, {"sent": "oh you wanna go to the other table .", "words": ["the", "table", "go", "to", "you", "other", "wanna"]}, {"sent": "make it go on the table .", "words": ["the", "table", "go", "it", "on", "make"]}, {"sent": "dont push it off the table I said .", "words": ["off", "the", "push", "table", "dont", "it"]}, {"sent": "whatre you doing under the table ?", "words": ["the", "under", "table", "you"]}, {"sent": "you dont cut the table .", "words": ["the", "cut", "table", "you", "dont"]}, {"sent": "okay scoot over to the table .", "words": ["the", "to", "over", "table"]}, {"sent": "xxx the table .", "words": ["the", "table"]}, {"sent": "Mummy doesnt like you putting crumbs from the table onto the floor .", "words": ["the", "like", "table", "you"]}, {"sent": "put them on the table .", "words": ["the", "put", "table", "them", "on"]}, {"sent": "put it on the table .", "words": ["the", "put", "table", "it", "on"]}, {"sent": "oh youre using the table as a little seat are you ?", "words": ["the", "table", "little", "you", "are", "a"]}, {"sent": "you can leave the table over there now Thomas .", "words": ["can", "the", "table", "you", "there", "over"]}, {"sent": "on the table .", "words": ["on", "the", "table"]}, {"sent": "put the box back on the changing table .", "words": ["the", "put", "box", "table", "back", "on"]}, {"sent": "on the table ?", "words": ["on", "the", "table"]}, {"sent": "if you wanna stay at the big table with me then you hafta have big table grownup manners .", "words": ["the", "if", "table", "me", "with", "you", "have", "wanna", "at", "stay", "then", "big"]}, {"sent": "making the table bigger ?", "words": ["the", "table"]}, {"sent": "the last three Gummies on the coffee table .", "words": ["the", "table", "last", "coffee", "on"]}, {"sent": "and the guys are like put the guys like heres a table .", "words": ["the", "put", "table", "like", "and", "are", "a"]}, {"sent": "you went under the table didnt you ?", "words": ["the", "under", "table", "you"]}, {"sent": "youre moving the whole table now .", "words": ["the", "table"]}, {"sent": "now the table drives me xxx .", "words": ["the", "me", "table"]}, {"sent": "Ill have that before I get down from the table .", "words": ["the", "table", "have", "get", "down", "that"]}, {"sent": "you did find it on the table .", "words": ["the", "table", "you", "it", "did", "find", "on"]}, {"sent": "okay and this goes on the kitchen table .", "words": ["the", "table", "and", "this", "kitchen", "on"]}, {"sent": "it is a little nice table .", "words": ["table", "is", "nice", "little", "it", "a"]}, {"sent": "teaching in in school how to eat on the table ?", "words": ["the", "how", "table", "to", "school", "eat", "in", "on"]}, {"sent": "are they eating at the table ?", "words": ["the", "table", "they", "are", "at"]}, {"sent": "please put it down on the table so you dont make a big mess because we hafta clean it up not you .", "words": ["the", "clean", "put", "up", "table", "we", "so", "you", "dont", "big", "it", "on", "down", "make", "not", "because", "a"]}, {"sent": "sit up at the table sit up at that chair .", "words": ["up", "the", "table", "chair", "at", "sit", "that"]}, {"sent": "shall we play hide and seek ?", "words": ["play", "and", "hide", "we"]}, {"sent": "hide again .", "words": ["hide"]}, {"sent": "well hafta hide it from the mice .", "words": ["the", "it", "hide"]}, {"sent": "let us hide the little mommy person .", "words": ["the", "us", "hide", "little"]}, {"sent": "do you like hide and seek ?", "words": ["do", "hide", "like", "and", "you"]}, {"sent": "hide .", "words": ["hide"]}, {"sent": "do you wanna hide him ?", "words": ["do", "hide", "you", "him", "wanna"]}, {"sent": "hide .", "words": ["hide"]}, {"sent": "I dont want them to hide from me .", "words": ["hide", "to", "them", "dont", "me"]}, {"sent": "we could hide the Littlepeople in the cup .", "words": ["the", "cup", "hide", "we", "in", "could"]}, {"sent": "should we maybe hide over here huh ?", "words": ["here", "over", "hide", "we"]}, {"sent": "right you hide in the box .", "words": ["the", "box", "hide", "you", "in"]}, {"sent": "hide him somewhere different .", "words": ["him", "hide"]}, {"sent": "hide ?", "words": ["hide"]}, {"sent": "hide it under a bushel .", "words": ["under", "it", "hide", "a"]}, {"sent": "lets hide over on this part .", "words": ["on", "this", "over", "hide"]}, {"sent": "shall we hide one more ?", "words": ["more", "hide", "we"]}, {"sent": "and then well hide the eight just like at Cotuit Nursery School .", "words": ["the", "hide", "like", "and", "at", "then"]}, {"sent": "hes going Im gonna go and hide .", "words": ["hide", "and", "go"]}, {"sent": "lets hide your feet .", "words": ["your", "hide"]}, {"sent": "dont play hide and seek with anyone other than .", "words": ["hide", "play", "with", "and", "other", "dont"]}, {"sent": "she likes playing hide and seek .", "words": ["she", "hide", "and"]}, {"sent": "now lets make a little ball and hide it inside here .", "words": ["hide", "inside", "and", "ball", "little", "here", "it", "make", "a"]}, {"sent": "do you wanna hide ?", "words": ["do", "hide", "wanna", "you"]}, {"sent": "you wanna play that game where you hide Emily in the hat ?", "words": ["where", "the", "game", "hide", "play", "you", "in", "wanna", "hat", "that"]}, {"sent": "hide ?", "words": ["hide"]}, {"sent": "h is for hide and seek .", "words": ["for", "and", "hide", "is"]}, {"sent": "we hafta hide the toothpaste now .", "words": ["the", "hide", "we"]}, {"sent": "what you hide ?", "words": ["what", "hide", "you"]}, {"sent": "why do you wanna hide it ?", "words": ["do", "hide", "you", "it", "wanna", "why"]}, {"sent": "youre hide youre hiding from Jwww ?", "words": ["hide"]}, {"sent": "lets lets hide them again .", "words": ["them", "hide"]}, {"sent": "why dont you hide ?", "words": ["dont", "hide", "why", "you"]}, {"sent": "I figured maybe we can hide in here .", "words": ["can", "hide", "we", "here", "in"]}, {"sent": "you wanna run and hide in your room ?", "words": ["your", "hide", "and", "you", "run", "in", "room", "wanna"]}, {"sent": "lets try to hide from me .", "words": ["hide", "to", "me", "try", "try to"]}, {"sent": "okay Im going to hide .", "words": ["to", "hide"]}, {"sent": "shall Mummy find shall Mummy hide him ?", "words": ["him", "hide", "find"]}, {"sent": "and then Im going to hide them .", "words": ["hide", "to", "them", "and", "then"]}, {"sent": "and well hide the pen while your gone .", "words": ["the", "your", "hide", "and", "pen"]}, {"sent": "Ill hide while you climb .", "words": ["climb", "hide", "you"]}, {"sent": "xxx hide and Im going to count to ten .", "words": ["to", "hide", "and"]}, {"sent": "so that he can hide explained dragonfly .", "words": ["can", "hide", "so", "he", "that"]}, {"sent": "does your cat hide in the cushions ?", "words": ["the", "does", "your", "hide", "cat", "in"]}, {"sent": "let us see if we can hide her .", "words": ["can", "if", "hide", "we", "us", "see", "her"]}, {"sent": "but they dont hide ?", "words": ["dont", "they", "hide", "but"]}, {"sent": "I didnt hide you away did I ?", "words": ["away", "did", "hide", "you"]}, {"sent": "run and hide .", "words": ["run", "hide", "and"]}, {"sent": "hide me .", "words": ["me", "hide"]}, {"sent": "go and hide .", "words": ["hide", "and", "go"]}, {"sent": "cricket pretends to be a leaf so then he can hide .", "words": ["can", "hide", "to", "be", "so", "he", "then", "a"]}, {"sent": "playing hide and seek .", "words": ["hide", "and"]}, {"sent": "do you ever hide ?", "words": ["do", "hide", "you"]}, {"sent": "are you gonna hide her ?", "words": ["are", "her", "hide", "you"]}, {"sent": "did you hide one over here ?", "words": ["hide", "you", "here", "did", "over"]}, {"sent": "shall we hide ?", "words": ["hide", "we"]}, {"sent": "hide hide .", "words": ["hide"]}, {"sent": "I dont wanna hide in your underpants thank you .", "words": ["your", "hide", "you", "dont", "in", "underpants", "wanna"]}, {"sent": "you always hide in the same place .", "words": ["the", "hide", "you", "in", "same"]}, {"sent": "you want the car to hide behind there ?", "words": ["the", "hide", "car", "to", "behind", "you", "there"]}, {"sent": "I know you always hide everything under there xxx .", "words": ["under", "there", "hide", "you"]}, {"sent": "kittys trying to hide from you Henry .", "words": ["to", "hide", "you"]}, {"sent": "youre gonna hide them ?", "words": ["them", "hide"]}, {"sent": "are you gonna hide in there ?", "words": ["hide", "you", "there", "in", "are"]}, {"sent": "is it playing hide and seek ?", "words": ["it", "and", "hide", "is"]}, {"sent": "lets hide through here Thomas .", "words": ["here", "hide"]}, {"sent": "hide hide ?", "words": ["hide"]}, {"sent": "the kids hide ?", "words": ["the", "hide"]}, {"sent": "they can hide in the silo ?", "words": ["can", "the", "hide", "they", "in"]}, {"sent": "hide where is the red one ?", "words": ["where", "the", "red", "hide", "is"]}, {"sent": "not hide him in the front room .", "words": ["the", "hide", "in", "him", "room", "not"]}, {"sent": "you go and take CocotheClown and hide him .", "words": ["hide", "go", "and", "you", "him", "take"]}, {"sent": "whos looking if we both hide ?", "words": ["if", "hide", "we"]}, {"sent": "hide the beanbag or hide you ?", "words": ["the", "hide", "you"]}, {"sent": "are you trying to hide ?", "words": ["to", "are", "hide", "you"]}, {"sent": "can I hide with you ?", "words": ["can", "with", "hide", "you"]}, {"sent": "they hide .", "words": ["they", "hide"]}, {"sent": "shall we hide ?", "words": ["hide", "we"]}, {"sent": "hide and seek .", "words": ["hide", "and"]}, {"sent": "hes gonna go and hide in the tree .", "words": ["the", "tree", "hide", "go", "and", "in"]}, {"sent": "theyre playing hide and seek arent they ?", "words": ["they", "hide", "and"]}, {"sent": "aw he is gonna hide her .", "words": ["her", "he", "hide", "is"]}, {"sent": "hes going to hide it somewhere .", "words": ["to", "it", "hide"]}, {"sent": "yeah do you wanna go and hide with AuntieLwww ?", "words": ["do", "hide", "go", "with", "and", "you", "wanna"]}, {"sent": "but how do you how does everyone else get a turn to hide ?", "words": ["do", "how", "does", "hide", "to", "you", "get", "but", "a"]}, {"sent": "I dont think she can hide under that .", "words": ["can", "think", "hide", "dont", "under", "that", "she"]}, {"sent": "do they hide on your nose ?", "words": ["do", "your", "hide", "they", "on", "nose"]}, {"sent": "where am I going to hide ?", "words": ["where", "to", "am", "hide"]}, {"sent": "weve been playing hide and seek .", "words": ["hide", "and"]}, {"sent": "Porker managed to scuttle out of the back door and hide behind a tree .", "words": ["the", "tree", "back", "hide", "of", "to", "behind", "and", "out", "a", "door"]}, {"sent": "did she hide something in there ?", "words": ["hide", "there", "in", "did", "she"]}, {"sent": "hide Coco .", "words": ["hide"]}, {"sent": "why do you wanna hide ?", "words": ["do", "hide", "you", "wanna", "why"]}, {"sent": "you hide you .", "words": ["hide", "you"]}, {"sent": "I didnt I didnt hide them .", "words": ["them", "hide"]}, {"sent": "well hide up here so you cant find us .", "words": ["up", "hide", "so", "you", "here", "us", "find"]}, {"sent": "do you play hide and seek with Daddy ?", "words": ["do", "hide", "play", "with", "and", "you"]}, {"sent": "arent you gonna hide someone there ?", "words": ["there", "hide", "you"]}, {"sent": "what are you going to hide for ?", "words": ["what", "for", "hide", "to", "you", "are"]}, {"sent": "we havent looked out the window today yet have we ?", "words": ["the", "we", "have", "out", "window"]}, {"sent": "no because the little girl wanted to make a snowman and grandma grandma looked out of the window and said ooh when I was little it useta be snowy at Christmas and we made snowmen .", "words": ["the", "when", "we", "of", "to", "was", "be", "little", "and", "snowman", "it", "out", "at", "window", "make", "because", "a"]}, {"sent": "can you can you see out of this window the window cleaner up the ladder there ?", "words": ["can", "the", "up", "of", "you", "this", "there", "out", "ladder", "see", "window"]}, {"sent": "we look out the window .", "words": ["the", "we", "out", "look", "window"]}, {"sent": "go look out the window .", "words": ["the", "go", "out", "look", "window"]}, {"sent": "I looked out the window be .", "words": ["the", "be", "window", "out"]}, {"sent": "choose a window to climb into and hop through when you arrive .", "words": ["into", "to", "and", "you", "window", "climb", "when", "a"]}, {"sent": "its on the its xxx in front of the window on the shelf .", "words": ["the", "of", "in", "on", "window"]}, {"sent": "lets have a look out your window first .", "words": ["your", "first", "have", "out", "look", "window", "a"]}, {"sent": "hes painting the window frames .", "words": ["the", "window"]}, {"sent": "going to the other window upstairs .", "words": ["the", "to", "window", "other"]}, {"sent": "there she is looking out the window .", "words": ["the", "is", "there", "out", "window", "she"]}, {"sent": "do you wanna look out the window ?", "words": ["the", "do", "you", "out", "look", "wanna", "window"]}, {"sent": "like a window cleaner ?", "words": ["window", "like", "a"]}, {"sent": "it was going up right in front of our back window .", "words": ["up", "back", "of", "was", "our", "in", "it", "window"]}, {"sent": "what was she doing looking out the window ?", "words": ["the", "what", "was", "out", "window", "she"]}, {"sent": "youre gonna hit it through the window .", "words": ["the", "it", "window", "hit"]}, {"sent": "it is not for wiping the window with .", "words": ["the", "for", "is", "with", "it", "window", "not"]}, {"sent": "you give me the car you hafta climb through the window .", "words": ["the", "car", "you", "window", "me", "give", "climb"]}, {"sent": "oh you going to put the train on the window ledge as well ?", "words": ["the", "put", "train", "to", "you", "on", "window"]}, {"sent": "hes got his head out the window so he can see whats going on .", "words": ["the", "can", "so", "head", "on", "he", "out", "his", "see", "window"]}, {"sent": "but he plays with his shadow in the window and in the mirror so .", "words": ["the", "so", "with", "and", "in", "he", "his", "window", "but"]}, {"sent": "and whats that in the window ?", "words": ["the", "and", "in", "window", "that"]}, {"sent": "Im gonna kick that window .", "words": ["window", "kick", "that"]}, {"sent": "could you do us a favor please and just look out of the win the bedroom window and see if you can see that dustbin lorry ?", "words": ["the", "do", "can", "bedroom", "if", "of", "you", "and", "us", "window", "out", "look", "see", "could", "that", "a"]}, {"sent": "it just went past the window didnt it ?", "words": ["the", "window", "it"]}, {"sent": "Mummy has just had a look out of the bedroom window .", "words": ["the", "bedroom", "of", "out", "look", "window", "a"]}, {"sent": "Purdie has Purdie has jumped into the window and shes looking out of the window .", "words": ["the", "into", "of", "and", "out", "window"]}, {"sent": "the window .", "words": ["the", "window"]}, {"sent": "remember the one we had last year who comes and begs at the window ?", "words": ["the", "we", "last", "and", "at", "who", "window"]}, {"sent": "come and look out the window .", "words": ["the", "and", "out", "look", "window"]}, {"sent": "you look out the window and what do you see when you look out the window ?", "words": ["the", "what", "do", "you", "and", "out", "look", "see", "window", "when"]}, {"sent": "out of that window .", "words": ["of", "window", "that", "out"]}, {"sent": "that dog looking out the window .", "words": ["the", "out", "window", "that", "dog"]}, {"sent": "is she looking out the window ?", "words": ["the", "is", "out", "window", "she"]}, {"sent": "it is a little window isnt it ?", "words": ["is", "little", "it", "window", "a"]}, {"sent": "fell out the window ?", "words": ["the", "window", "out"]}, {"sent": "she ah looking out the window .", "words": ["the", "window", "she", "out"]}, {"sent": "is brother in the window ?", "words": ["the", "window", "in", "is"]}, {"sent": "go and look out the window .", "words": ["the", "go", "and", "out", "look", "window"]}, {"sent": "and she can see aeroplanes from her window ?", "words": ["can", "and", "see", "window", "her", "she"]}, {"sent": "look whos on the front window ledge now .", "words": ["on", "the", "window", "look"]}, {"sent": "if you look out the other window maybe youll see it .", "words": ["the", "if", "you", "other", "it", "out", "look", "see", "window"]}, {"sent": "she has jumped down from the window .", "words": ["the", "window", "down", "she"]}, {"sent": "window window .", "words": ["window"]}, {"sent": "can you just look at the window a minute for me ?", "words": ["can", "the", "for", "you", "at", "look", "me", "window", "a"]}, {"sent": "I put that on me over a tshirt and stood at that window xxx out in the back garden .", "words": ["the", "over", "put", "back", "and", "in", "garden", "out", "at", "on", "me", "window", "that", "a"]}, {"sent": "hes out the window we could just see him out the window look out the window .", "words": ["the", "we", "him", "out", "look", "see", "window", "could"]}, {"sent": "dont dont get any nearer or youll fall out of the window .", "words": ["the", "of", "any", "fall", "get", "dont", "out", "window"]}, {"sent": "no I open xxx the window so I can hear Sarah when she wakes up .", "words": ["the", "can", "up", "open", "so", "hear", "window", "when", "she"]}, {"sent": "look out of the window .", "words": ["the", "of", "out", "look", "window"]}, {"sent": "because youre not climbing in the window .", "words": ["the", "in", "window", "not", "because"]}, {"sent": "and whats that in the window ?", "words": ["the", "and", "in", "window", "that"]}, {"sent": "well its usually two pounds twenty for the window cleaner .", "words": ["the", "window", "for"]}, {"sent": "see what we can see from your window .", "words": ["can", "what", "your", "we", "see", "window"]}, {"sent": "dont knock on the window Adam .", "words": ["the", "dont", "knock", "on", "window"]}, {"sent": "its got a window .", "words": ["window", "a"]}, {"sent": "near the window tree .", "words": ["the", "tree", "window"]}, {"sent": "and he opened his car window didnt he ?", "words": ["car", "and", "he", "his", "window"]}, {"sent": "fell out the window uhn didnt he ?", "words": ["the", "window", "he", "out"]}, {"sent": "I dont know whether you banged the window but the alarm for the house went off yesterday morning didnt it ?", "words": ["the", "off", "for", "you", "dont", "it", "house", "window", "but"]}, {"sent": "here www window .", "words": ["here", "window"]}, {"sent": "now Mummys going to find the window cleaner with a cup of tea .", "words": ["the", "cup", "of", "to", "with", "find", "window", "a"]}, {"sent": "and she can look outof the window .", "words": ["can", "the", "and", "look", "window", "she"]}, {"sent": "yeah hes looking out the window isnt he and he wants to go and play on his sledge .", "words": ["the", "go", "to", "play", "and", "his", "he", "out", "on", "window"]}, {"sent": "thats the window in the bathroom .", "words": ["the", "in", "window", "bathroom"]}, {"sent": "window .", "words": ["window"]}, {"sent": "do you think that window was a good idea ?", "words": ["do", "think", "was", "you", "good", "window", "that", "a"]}, {"sent": "and she threw them out of the window .", "words": ["the", "of", "them", "and", "out", "window", "she"]}, {"sent": "window ledge .", "words": ["window"]}, {"sent": "what did they see when they looked out the window ?", "words": ["the", "what", "they", "did", "out", "see", "window", "when"]}, {"sent": "did you look out the window when we were on the airplane ?", "words": ["the", "we", "airplane", "you", "did", "out", "look", "on", "window", "when", "were"]}, {"sent": "lets see if we see a bird out this window .", "words": ["if", "we", "bird", "this", "out", "see", "window", "a"]}, {"sent": "you know what else we could use is this window open .", "words": ["what", "we", "is", "open", "you", "window", "this", "could"]}, {"sent": "it will be not on the window becauseof you .", "words": ["the", "be", "you", "on", "it", "will", "window", "not"]}, {"sent": "thats the window .", "words": ["the", "window"]}, {"sent": "room for one more window .", "words": ["window", "more", "room", "for"]}, {"sent": "youre smashing the window ?", "words": ["the", "window"]}, {"sent": "what can those elephants see when they look out the window ?", "words": ["can", "what", "the", "those", "they", "out", "look", "see", "window", "when"]}, {"sent": "youre going to play with the racing car on the window ledge .", "words": ["the", "to", "car", "play", "with", "on", "window"]}, {"sent": "uhhuh do you want me to open your window ?", "words": ["do", "your", "open", "to", "you", "me", "window"]}, {"sent": "thats the bank thats the drivethrough window at the bank that gives you money .", "words": ["the", "money", "you", "at", "window", "that"]}, {"sent": "you going to the window ?", "words": ["the", "to", "window", "you"]}, {"sent": "were looking out the window arent we ?", "words": ["the", "we", "out", "window", "were"]}, {"sent": "window .", "words": ["window"]}, {"sent": "you were too slow getting up to the window .", "words": ["up", "the", "to", "slow", "you", "too", "window", "were"]}, {"sent": "outof the window .", "words": ["the", "window"]}, {"sent": "and the cats asleep under the window .", "words": ["the", "and", "asleep", "under", "window"]}, {"sent": "thats what they give people to jump in if theyre up high in a window .", "words": ["up", "what", "if", "to", "window", "jump", "they", "in", "high", "give", "a"]}, {"sent": "stop tapping on the window Graham .", "words": ["on", "the", "window", "stop"]}, {"sent": "I can see an aeroplane out the window .", "words": ["can", "the", "an", "out", "see", "window"]}, {"sent": "window yes look this little baby has a nappie on .", "words": ["little", "this", "look", "on", "window", "a"]}, {"sent": "Mummys plants have grown a lot since we last looked in the window havent they ?", "words": ["the", "we", "a lot", "last", "have", "they", "in", "window", "a"]}, {"sent": "that one was almost a circle window .", "words": ["was", "window", "that", "a"]}, {"sent": "shall we open a window ?", "words": ["open", "window", "a", "we"]}, {"sent": "youre talking about the window ledge and the helicopters .", "words": ["the", "window", "about", "and"]}, {"sent": "what can we see in the window ?", "words": ["can", "what", "the", "we", "in", "see", "window"]}, {"sent": "thats a window .", "words": ["window", "a"]}, {"sent": "you want me to pull them out Honey so your hands dont get all dirty ?", "words": ["all", "your", "to", "so", "them", "pull", "you", "dont", "get", "dirty", "out", "me"]}, {"sent": "can you say no dirty schmutz honey ?", "words": ["can", "say", "dirty", "you"]}, {"sent": "that is pretty dirty .", "words": ["pretty", "that", "dirty", "is"]}, {"sent": "thankyou thats that ole dirty teddy .", "words": ["that", "dirty"]}, {"sent": "she thinks that babys got a dirty diaper .", "words": ["diaper", "dirty", "that", "she", "a"]}, {"sent": "you dirty little girl .", "words": ["dirty", "little", "you"]}, {"sent": "thats dirty Sarah .", "words": ["dirty"]}, {"sent": "Andys got a dirty nappie .", "words": ["dirty", "a"]}, {"sent": "why does it make them dirty ?", "words": ["does", "them", "it", "dirty", "make", "why"]}, {"sent": "dirty food .", "words": ["food", "dirty"]}, {"sent": "is it dirty ?", "words": ["it", "dirty", "is"]}, {"sent": "the dirty taxi is growing more and more angry and lotsof black smoke appears .", "words": ["the", "is", "and", "black", "more", "dirty"]}, {"sent": "this you dont like my dirty hands do you ?", "words": ["do", "my", "like", "you", "dont", "this", "dirty"]}, {"sent": "Ive actually just looked at my shoes and realized how dirty they are .", "words": ["my", "how", "and", "they", "are", "dirty", "at"]}, {"sent": "whats the difference between dirty greasy and clean greasy ?", "words": ["the", "dirty", "clean", "and"]}, {"sent": "well just get your shirt dirty thats okay .", "words": ["get", "shirt", "your", "dirty"]}, {"sent": "lotsof dirty dishes .", "words": ["dirty"]}, {"sent": "shes very dirty .", "words": ["dirty"]}, {"sent": "is that dirty ?", "words": ["that", "dirty", "is"]}, {"sent": "shes not dirty but you do hafta wash your hands when youve touched her Thomas .", "words": ["do", "your", "when", "her", "you", "dirty", "wash", "not", "but"]}, {"sent": "because youre dirty .", "words": ["dirty", "because"]}, {"sent": "dirty police dogs ?", "words": ["dirty"]}, {"sent": "because theyre dirty .", "words": ["dirty", "because"]}, {"sent": "thats the dirty nappie .", "words": ["the", "dirty"]}, {"sent": "a bit dirty ?", "words": ["dirty", "a"]}, {"sent": "uh put them in the dirty clothes .", "words": ["the", "put", "them", "in", "dirty"]}, {"sent": "its a bit dirty .", "words": ["dirty", "a"]}, {"sent": "yesterday he was dirty .", "words": ["was", "he", "dirty"]}, {"sent": "we have dirty .", "words": ["have", "dirty", "we"]}, {"sent": "thats dirty .", "words": ["dirty"]}, {"sent": "was your shirt dirty .", "words": ["was", "your", "dirty", "shirt"]}, {"sent": "dirty yeah .", "words": ["dirty"]}, {"sent": "yeah he is dirty now hes dirty so whats he gonna go home and do ?", "words": ["do", "go", "is", "so", "and", "he", "dirty", "home"]}, {"sent": "take it out your mouth because its all dirty .", "words": ["all", "your", "mouth", "it", "dirty", "out", "take", "because"]}, {"sent": "oh youre dirty Thomas .", "words": ["dirty"]}, {"sent": "because they get dirty .", "words": ["get", "they", "dirty", "because"]}, {"sent": "oh dont do that its dirty .", "words": ["dont", "do", "that", "dirty"]}, {"sent": "xxx your hands werent dirty .", "words": ["your", "dirty"]}, {"sent": "a dirty shirt is that what he said ?", "words": ["what", "is", "shirt", "he", "dirty", "that", "a"]}, {"sent": "is it a very very dirty nappie ?", "words": ["it", "dirty", "a", "is"]}, {"sent": "oh its a bit dirty isnt it ?", "words": ["it", "dirty", "a"]}, {"sent": "and that dirty thumb oh my gosh .", "words": ["my", "that", "dirty", "and"]}, {"sent": "theyre dirty .", "words": ["dirty"]}, {"sent": "howcome ya get everything so dirty ?", "words": ["get", "so", "dirty"]}, {"sent": "that is all the dirty clothes that is all the dirty clothes .", "words": ["the", "all", "is", "dirty", "that"]}, {"sent": "dirty .", "words": ["dirty"]}, {"sent": "theyre not that dirty are they ?", "words": ["they", "are", "dirty", "that", "not"]}, {"sent": "but because youre quite dirty I didnt wanna scrub a lot with cream .", "words": ["a lot", "with", "dirty", "wanna", "but", "because", "a"]}, {"sent": "its dirty .", "words": ["dirty"]}, {"sent": "eating dirty sausage ?", "words": ["dirty"]}, {"sent": "Garys got a dirty face .", "words": ["face", "dirty", "a"]}, {"sent": "youll get it dirty .", "words": ["get", "it", "dirty"]}, {"sent": "how did your hands get so dirty ?", "words": ["how", "your", "so", "get", "did", "dirty"]}, {"sent": "all your tables dirty huh .", "words": ["all", "your", "dirty"]}, {"sent": "whats dirty ?", "words": ["dirty"]}, {"sent": "very dirty .", "words": ["dirty"]}, {"sent": "dirty .", "words": ["dirty"]}, {"sent": "and this little piggy went whee whee whee all the way up his dirty legs and knees .", "words": ["the", "all", "up", "little", "and", "this", "dirty", "his"]}, {"sent": "is that how you got your dirty face ?", "words": ["how", "your", "face", "is", "you", "dirty", "that"]}, {"sent": "no leave it its dirty .", "words": ["it", "dirty"]}, {"sent": "get the dirty dirty dishes outof the picture oh .", "words": ["get", "the", "picture", "dirty"]}, {"sent": "it is dirty its filthy water .", "words": ["water", "it", "dirty", "is"]}, {"sent": "dirty .", "words": ["dirty"]}, {"sent": "very dirty truck .", "words": ["truck", "dirty"]}, {"sent": "itll get dirty .", "words": ["get", "dirty"]}, {"sent": "because you dont wanna get the carpet dirty ?", "words": ["the", "you", "get", "dont", "dirty", "wanna", "because"]}, {"sent": "your shirts all dirty .", "words": ["all", "your", "dirty"]}, {"sent": "theyre all dirty arent they ?", "words": ["all", "dirty", "they"]}, {"sent": "inside the xxx the whales mouth is dirty and green and slimy .", "words": ["the", "is", "inside", "mouth", "and", "dirty", "green"]}, {"sent": "is it a a dirty crayon crayon ?", "words": ["is", "crayon", "it", "dirty", "a"]}, {"sent": "kind of dirty ?", "words": ["of", "dirty"]}, {"sent": "but you have dirty hands .", "words": ["have", "dirty", "but", "you"]}, {"sent": "yeah dont get your hands dirty .", "words": ["get", "dont", "your", "dirty"]}, {"sent": "howd you get all dirty ?", "words": ["get", "all", "dirty", "you"]}, {"sent": "quite a dirty truck .", "words": ["truck", "dirty", "a"]}, {"sent": "oops dirty now .", "words": ["dirty"]}, {"sent": "yeah its dirty but its okay .", "words": ["dirty", "but"]}, {"sent": "because thats a dirty saw .", "words": ["dirty", "because", "a"]}, {"sent": "dirty dirty rubbish bags .", "words": ["dirty"]}, {"sent": "ripped his trousers and got very dirty .", "words": ["his", "dirty", "and"]}, {"sent": "theyre very dirty xxx .", "words": ["dirty"]}, {"sent": "its dirty .", "words": ["dirty"]}, {"sent": "I think you held her when she when your hands were dirty .", "words": ["think", "your", "when", "you", "dirty", "her", "she", "were"]}, {"sent": "theyre not dirty theyve just got soap suds on them .", "words": ["them", "soap", "dirty", "on", "not"]}, {"sent": "now that you got all dirty .", "words": ["all", "that", "dirty", "you"]}, {"sent": "um well not dirty luggage .", "words": ["dirty", "not"]}, {"sent": "it has got dirty wheels .", "words": ["it", "dirty"]}, {"sent": "this is a very dirty strap here .", "words": ["is", "here", "this", "dirty", "a"]}, {"sent": "those are your dirty diapers .", "words": ["your", "are", "those", "dirty"]}, {"sent": "because theyre so dirty .", "words": ["so", "dirty", "because"]}, {"sent": "all dirty .", "words": ["all", "dirty"]}, {"sent": "because itll get dirty .", "words": ["get", "dirty", "because"]}, {"sent": "dirty hands ?", "words": ["dirty"]}, {"sent": "Esther will kill you if you get them dirty ?", "words": ["if", "them", "you", "get", "dirty", "will"]}, {"sent": "I think dirty .", "words": ["think", "dirty"]}, {"sent": "Thomas dirty .", "words": ["dirty"]}, {"sent": "its dirty right now .", "words": ["dirty"]}, {"sent": "so we can pretend thats our house right ?", "words": ["can", "we", "so", "our", "house", "pretend"]}, {"sent": "you dont you dont eat anything with that because its just pretend one .", "words": ["with", "you", "eat", "dont", "pretend", "that", "because"]}, {"sent": "thats pretend pie .", "words": ["pretend"]}, {"sent": "pretend you killed me again .", "words": ["me", "pretend", "you"]}, {"sent": "pretend .", "words": ["pretend"]}, {"sent": "just pretend .", "words": ["pretend"]}, {"sent": "its a pretend store .", "words": ["store", "pretend", "a"]}, {"sent": "theres no pennies in there I know you hafta pretend .", "words": ["pretend", "there", "in", "you"]}, {"sent": "you just hafta pretend Fraser .", "words": ["pretend", "you"]}, {"sent": "well pretend youre the postman and you hafta take it to the peoples home .", "words": ["the", "to", "you", "and", "it", "pretend", "take", "home"]}, {"sent": "he was doing pretend drinking ?", "words": ["was", "pretend", "he"]}, {"sent": "pretend one .", "words": ["pretend"]}, {"sent": "pretend Im in the car now .", "words": ["the", "in", "pretend", "car"]}, {"sent": "no we dont have pretend food at home we have real food and he likes to eat .", "words": ["we", "to", "and", "have", "dont", "food", "eat", "he", "at", "pretend", "home"]}, {"sent": "here we can pretend this is orange juice .", "words": ["can", "we", "is", "here", "this", "orange", "pretend", "juice"]}, {"sent": "shall we pretend shall we pretend its a pool and the whales swimming in there ?", "words": ["the", "we", "pool", "and", "there", "in", "pretend", "a"]}, {"sent": "do you ever pretend to feed them or anything ?", "words": ["do", "to", "them", "you", "feed", "pretend"]}, {"sent": "a pretend bottle ?", "words": ["pretend", "bottle", "a"]}, {"sent": "its its like a pretend animal .", "words": ["pretend", "animal", "like", "a"]}, {"sent": "or she has um she has youknow like a bunch of kids songs and one of thems like a song youknow all on the alphabet and another ones all songs with numbers and shell sort of pretend to sing along with them .", "words": ["the", "all", "of", "to", "with", "like", "them", "and", "another", "sing", "on", "pretend", "she", "a"]}, {"sent": "pretend she Mickey Mouse now you show me how you turn the light on .", "words": ["the", "how", "show", "light", "you", "on", "me", "pretend", "she"]}, {"sent": "you can pretend its a candle on top .", "words": ["can", "you", "on", "pretend", "a"]}, {"sent": "so Ill pretend to be asleep .", "words": ["to", "so", "be", "asleep", "pretend"]}, {"sent": "pretend .", "words": ["pretend"]}, {"sent": "and pretend that you took off your mask .", "words": ["off", "your", "you", "and", "pretend", "that"]}, {"sent": "can you say pretend ?", "words": ["can", "pretend", "say", "you"]}, {"sent": "all right lets pretend .", "words": ["all", "pretend"]}, {"sent": "we could though we could like pretend that theyre like a family or something .", "words": ["we", "like", "pretend", "that", "a", "could"]}, {"sent": "we eat the proper ones but you dont eat pretend ones .", "words": ["the", "we", "you", "eat", "dont", "pretend", "but"]}, {"sent": "here we can pretend this is another sidewalk over here .", "words": ["can", "sidewalk", "we", "is", "here", "this", "another", "pretend", "over"]}, {"sent": "Im not having a bad behavior even in pretend play .", "words": ["play", "bad", "in", "pretend", "not", "a"]}, {"sent": "pretend I didnt have him .", "words": ["have", "pretend", "him"]}, {"sent": "you should sit in your chair and build a pretend wall sothat the giant cant see you .", "words": ["the", "your", "chair", "you", "and", "in", "build", "pretend", "see", "sit", "a"]}, {"sent": "okay Ill buy this this is a pretend its a pretend leash okay .", "words": ["is", "this", "pretend", "buy", "a"]}, {"sent": "pretend to lick it okay ?", "words": ["to", "pretend", "it", "lick"]}, {"sent": "pretend .", "words": ["pretend"]}, {"sent": "then we can pretend that this is the pool .", "words": ["can", "the", "we", "is", "pool", "this", "pretend", "then", "that"]}, {"sent": "do you ever play with pretend food at home ?", "words": ["do", "play", "with", "you", "food", "at", "pretend", "home"]}, {"sent": "youre going to pretend the dustbin bags on the road .", "words": ["the", "to", "pretend", "on"]}, {"sent": "tell me a story pretend like hes our daddy uh oh what does our daddy do ?", "words": ["what", "story", "does", "do", "like", "our", "me", "pretend", "a"]}, {"sent": "pretend I hafta share mine okay ?", "words": ["mine", "pretend", "share"]}, {"sent": "he does a lot of pretend play with baby tuck tuck but not dressing .", "words": ["does", "of", "a lot", "play", "with", "he", "pretend", "not", "but", "a"]}, {"sent": "now you can pretend .", "words": ["can", "pretend", "you"]}, {"sent": "pretend .", "words": ["pretend"]}, {"sent": "can he pretend hes having some ?", "words": ["can", "some", "pretend", "he"]}, {"sent": "yeah yeah its pretend .", "words": ["pretend"]}, {"sent": "pretend lunch .", "words": ["pretend"]}, {"sent": "hell pretend to put her helmet on .", "words": ["put", "to", "on", "pretend", "her"]}, {"sent": "that is a pretend sandbox xxx .", "words": ["sandbox", "is", "pretend", "that", "a"]}, {"sent": "she had a pretend wash in the bath .", "words": ["the", "in", "pretend", "wash", "she", "a"]}, {"sent": "let us pretend to have juice .", "words": ["to", "have", "us", "pretend", "juice"]}, {"sent": "no pretend you were too old enough and you couldnt be a zoomer and sing the zoomer song .", "words": ["old", "the", "be", "you", "and", "sing", "too", "pretend", "were", "a"]}, {"sent": "just pretend that you have coffee in there .", "words": ["you", "have", "there", "in", "coffee", "pretend", "that"]}, {"sent": "you wanna pretend were firefighters ?", "words": ["pretend", "were", "wanna", "you"]}, {"sent": "you havent got a crane but youve got toys that you pretend are cranes .", "words": ["you", "are", "pretend", "that", "but", "a"]}, {"sent": "thats a pretend cookie .", "words": ["cookie", "pretend", "a"]}, {"sent": "when you pretend it is right ?", "words": ["is", "you", "it", "pretend", "when"]}, {"sent": "my pretend cup of tea we can put away .", "words": ["cup", "can", "my", "put", "we", "of", "away", "pretend"]}, {"sent": "thats pretend .", "words": ["pretend"]}, {"sent": "thats just pretend .", "words": ["pretend"]}, {"sent": "you know that is pretend let us on neighborhood pretend .", "words": ["is", "you", "us", "on", "pretend", "that"]}, {"sent": "um xxx she kept giving me the juice to drink and pretend with her I mean she recognized the icecream right away which is a lot about what we feed her .", "words": ["the", "what", "drink", "about", "me", "juice", "which", "with", "away", "pretend", "she", "is", "a lot", "to", "and", "feed", "a", "we", "her"]}, {"sent": "we can pretend that they have joined up .", "words": ["can", "up", "we", "have", "they", "pretend", "that"]}, {"sent": "pretend youre like this like this guy .", "words": ["this", "pretend", "like"]}, {"sent": "um lets pretend he works at the grocery store .", "words": ["the", "store", "he", "at", "pretend"]}, {"sent": "we can pretend like were eating .", "words": ["can", "we", "like", "pretend", "were"]}, {"sent": "just pretend .", "words": ["pretend"]}, {"sent": "pretend I was a zoomer .", "words": ["was", "pretend", "a"]}, {"sent": "all right Ill pretend .", "words": ["all", "pretend"]}, {"sent": "just pretend .", "words": ["pretend"]}, {"sent": "see this a m like a a baking pan so we can pretend that were making cupcakes .", "words": ["can", "we", "so", "like", "this", "see", "pretend", "that", "were", "a"]}, {"sent": "but we could pretend it was bubblegum or xxx olu .", "words": ["we", "was", "it", "pretend", "but", "could"]}, {"sent": "can you pretend this is a spoon ?", "words": ["can", "is", "you", "this", "spoon", "pretend", "a"]}, {"sent": "yeah lets just pretend .", "words": ["pretend"]}, {"sent": "pretend like youre cooking .", "words": ["pretend", "like"]}, {"sent": "you just pretend .", "words": ["pretend", "you"]}, {"sent": "pretend this was a big kid .", "words": ["was", "this", "pretend", "big", "a"]}, {"sent": "well what what could we pretend is a watermelon ?", "words": ["what", "is", "we", "pretend", "a", "could"]}, {"sent": "pretend Abi .", "words": ["pretend"]}, {"sent": "thats right we had a pretend dog what color was the pretend dog we had ?", "words": ["the", "what", "we", "was", "pretend", "dog", "a"]}, {"sent": "okay look pretend this is the school bus .", "words": ["the", "is", "school", "this", "look", "pretend", "bus"]}, {"sent": "lets pretend call Nwww .", "words": ["pretend"]}, {"sent": "its pretend those are all pretend so we dont eat them okay ?", "words": ["all", "those", "we", "so", "them", "eat", "dont", "are", "pretend"]}, {"sent": "pretend .", "words": ["pretend"]}, {"sent": "pretend youre going like that xxx .", "words": ["pretend", "that", "like"]}, {"sent": "no lets pretend we broke it .", "words": ["pretend", "it", "we"]}, {"sent": "are you going to pretend to phone .", "words": ["to", "pretend", "are", "you"]}, {"sent": "they had a pretend .", "words": ["they", "pretend", "a"]}, {"sent": "you were playing with the pretend birthday cake and candles this morning werent you ?", "words": ["the", "cake", "with", "you", "and", "this", "pretend", "were"]}, {"sent": "lets give the animals some pretend rockets .", "words": ["the", "some", "give", "pretend"]}, {"sent": "you can look maybe in there and pretend over there .", "words": ["can", "you", "and", "there", "in", "look", "pretend", "over"]}, {"sent": "if you wanna you can pretend this is a table if you wanna a round table .", "words": ["can", "if", "table", "is", "you", "this", "wanna", "pretend", "a"]}, {"sent": "just pretend dont just go like this .", "words": ["go", "like", "dont", "this", "pretend"]}, {"sent": "why dont we pretend those rocks are the cookies ?", "words": ["the", "those", "we", "dont", "are", "pretend", "why"]}, {"sent": "then you can pretend like youre making more .", "words": ["can", "like", "you", "more", "pretend", "then"]}, {"sent": "but thats just pretend .", "words": ["pretend", "but"]}, {"sent": "shall we pretend its cheese then ?", "words": ["pretend", "cheese", "then", "we"]}, {"sent": "you wanna try it again like pretend he keeps falling in and we say you gotta be super careful tortoise .", "words": ["we", "be", "say", "like", "you", "and", "in", "it", "he", "wanna", "careful", "pretend", "try"]}, {"sent": "shall we pretend were going to Sainsburys and buy some bananas .", "words": ["we", "to", "and", "some", "pretend", "were", "buy"]}, {"sent": "er no shall we pretend ?", "words": ["pretend", "we"]}, {"sent": "and shes never really been a huge kind of stuffed animal fan .", "words": ["of", "animal", "a", "and"]}, {"sent": "Daddy you wanna be an animal ?", "words": ["be", "you", "an", "animal", "wanna"]}, {"sent": "animal .", "words": ["animal"]}, {"sent": "but what kind of animal is it ?", "words": ["what", "is", "of", "animal", "it", "but"]}, {"sent": "animal .", "words": ["animal"]}, {"sent": "here is an animal cracker outof your Easter basket .", "words": ["your", "is", "cracker", "here", "an", "animal", "basket"]}, {"sent": "whats that little animal doing ?", "words": ["animal", "that", "little"]}, {"sent": "do you think he recognizes the names or any of the like food items or the utensils or any of the animal names yet ?", "words": ["the", "do", "think", "of", "any", "like", "you", "food", "animal", "he"]}, {"sent": "its an animal .", "words": ["an", "animal"]}, {"sent": "its an animal going to the theater .", "words": ["the", "to", "an", "animal"]}, {"sent": "you get the squeaky animal books then .", "words": ["the", "you", "get", "animal", "then"]}, {"sent": "because there isnt an animal that begins with x so what theyve done is theyve got an animal that ends in x .", "words": ["what", "is", "so", "with", "there", "an", "animal", "in", "that", "because"]}, {"sent": "but when the giant sloth was on the ground its claws forced the animal to walk on the sides of its feet making movement difficult .", "words": ["the", "of", "was", "to", "walk", "animal", "on", "when", "but"]}, {"sent": "animal friends .", "words": ["animal"]}, {"sent": "lets make an animal maybe .", "words": ["an", "animal", "make"]}, {"sent": "um and now and now I dont think he knows what that animal is because I think its a made up animal .", "words": ["up", "what", "think", "is", "and", "dont", "animal", "he", "that", "because", "a"]}, {"sent": "they even offered to play a wild animal game if he would prefer .", "words": ["game", "if", "to", "play", "they", "animal", "he", "would", "a"]}, {"sent": "is called animal nursery right ?", "words": ["animal", "is"]}, {"sent": "and I tell you what when they come up and if Mia do does that for you you go and point to the you can go and point to the animal you made .", "words": ["up", "what", "do", "does", "for", "if", "the", "go", "can", "to", "you", "and", "they", "that", "animal", "when"]}, {"sent": "now ElsieGrandma hasnt got an animal .", "words": ["an", "animal"]}, {"sent": "she sleeps with an animal but .", "words": ["with", "an", "animal", "she", "but"]}, {"sent": "and the same with any other animal .", "words": ["the", "any", "with", "and", "other", "animal", "same"]}, {"sent": "what furry animal gave you a kiss at toddler time ?", "words": ["what", "you", "animal", "at", "kiss", "a"]}, {"sent": "what kind of animal do you like the least ?", "words": ["the", "what", "do", "of", "like", "you", "animal"]}, {"sent": "wanna read your animal kisses book ?", "words": ["your", "animal", "read", "wanna", "book"]}, {"sent": "sortof its like a half stuffed animal half blanket thing that he sleeps with youknow and some things like that that are kindof preferred but .", "words": ["with", "like", "and", "animal", "he", "are", "blanket", "some", "that", "but", "a"]}, {"sent": "you think animal would get mad ?", "words": ["think", "mad", "you", "get", "animal", "would"]}, {"sent": "which animal ?", "words": ["animal", "which"]}, {"sent": "and then we can you can choose an animal .", "words": ["can", "we", "you", "and", "an", "animal", "then"]}, {"sent": "why dont we get one of your stuffed animals to be and animal in the ocean when the oil spills ?", "words": ["the", "your", "we", "of", "to", "be", "and", "get", "dont", "in", "animal", "when", "why"]}, {"sent": "I dont really do any other animal noises I dont think .", "words": ["do", "think", "any", "other", "dont", "animal"]}, {"sent": "do you keep her engaged by animals like animal sounds .", "words": ["do", "like", "by", "you", "animal", "her"]}, {"sent": "what animal is that ?", "words": ["what", "animal", "that", "is"]}, {"sent": "no thats not an animal .", "words": ["an", "animal", "not"]}, {"sent": "oh I went to the animal fair the birds and the bees were there .", "words": ["the", "to", "and", "there", "animal", "were"]}, {"sent": "what what another animal says ?", "words": ["another", "what", "animal"]}, {"sent": "right you make that animal then .", "words": ["you", "animal", "make", "then", "that"]}, {"sent": "is a walrus a big animal or a little animal ?", "words": ["is", "little", "animal", "big", "a"]}, {"sent": "shall we put another animal on or are we not allowed to do that either ?", "words": ["do", "put", "we", "to", "another", "animal", "are", "on", "that", "not"]}, {"sent": "and the letters spell out what the animal is .", "words": ["the", "what", "is", "and", "animal", "out"]}, {"sent": "do you know what animal milk comes from ?", "words": ["do", "what", "you", "animal", "milk"]}, {"sent": "animal yes .", "words": ["animal"]}, {"sent": "heres your animal book .", "words": ["book", "animal", "your"]}, {"sent": "whos this animal ?", "words": ["this", "animal"]}, {"sent": "what animal does that look like ?", "words": ["what", "does", "like", "animal", "look", "that"]}, {"sent": "that big animal outof the zoo ?", "words": ["the", "zoo", "that", "animal", "big"]}, {"sent": "what kind of animal ?", "words": ["of", "what", "animal"]}, {"sent": "do you wanna do the animal puzzle ?", "words": ["the", "do", "you", "animal", "puzzle", "wanna"]}, {"sent": "your best animal friend .", "words": ["animal", "your"]}, {"sent": "what animal do you think ?", "words": ["what", "do", "think", "you", "animal"]}, {"sent": "um but yeah she knows a lot of animal noises and she she just really likes animals so .", "words": ["of", "so", "a lot", "and", "animal", "she", "but", "a"]}, {"sent": "the animal one ?", "words": ["the", "animal"]}, {"sent": "a lamb is a very tame animal .", "words": ["lamb", "animal", "is", "a"]}, {"sent": "what animal did you hold ?", "words": ["what", "hold", "you", "animal", "did"]}, {"sent": "I think I before you were talking about animal names a little bit does that these types toys she looked at .", "words": ["does", "think", "these", "you", "little", "that", "animal", "about", "at", "were", "she", "a"]}, {"sent": "what animal is Maisy Fraser ?", "words": ["what", "animal", "is"]}, {"sent": "tell me what is that what kind of little thing is that what is that thats an animal isnt it ?", "words": ["what", "is", "of", "little", "an", "animal", "it", "me", "that"]}, {"sent": "what animal is that ?", "words": ["what", "animal", "that", "is"]}, {"sent": "he never made it to the last page of the farm animal one but .", "words": ["the", "of", "to", "last", "it", "he", "animal", "but"]}, {"sent": "is there an animal ?", "words": ["there", "an", "animal", "is"]}, {"sent": "we saw a bear and what animal did we see ?", "words": ["what", "we", "bear", "and", "animal", "did", "see", "a"]}, {"sent": "which animal do you like best ?", "words": ["do", "like", "you", "animal", "which"]}, {"sent": "whats that animal ?", "words": ["animal", "that"]}, {"sent": "her favorite animal .", "words": ["animal", "her"]}, {"sent": "its your farm animal books isnt it ?", "words": ["it", "animal", "your"]}, {"sent": "are you going to put an animal in the back ?", "words": ["the", "put", "back", "to", "you", "an", "are", "animal", "in"]}, {"sent": "uhhuh thats a animal .", "words": ["animal", "a"]}, {"sent": "where where will we find the animal nursery ?", "words": ["where", "the", "we", "animal", "find", "will"]}, {"sent": "well it says animal hut .", "words": ["it", "animal"]}, {"sent": "what kind of animal is that ?", "words": ["what", "is", "of", "animal", "that"]}, {"sent": "I think I saw your animal cards in the living room is that where they are ?", "words": ["the", "where", "think", "your", "is", "living room", "they", "in", "animal", "room", "are", "that"]}, {"sent": "oh wheres the other farm animal ?", "words": ["the", "animal", "other"]}, {"sent": "animal .", "words": ["animal"]}, {"sent": "this animal likes wallowing in the mud .", "words": ["the", "this", "in", "animal"]}, {"sent": "a barn animal which is a .", "words": ["animal", "which", "is", "a"]}, {"sent": "and piggy said oh animal oh animal could I have a lick of your icecream cone .", "words": ["your", "of", "and", "have", "animal", "could", "lick", "a"]}, {"sent": "animal sounds zoo .", "words": ["zoo", "animal"]}, {"sent": "animal peek through there .", "words": ["there", "animal"]}, {"sent": "yeah but shes making some of the no some of the animal noises right ?", "words": ["the", "of", "animal", "some", "but"]}, {"sent": "are you gonna put an animal inside the cage ?", "words": ["the", "put", "inside", "you", "an", "are", "animal"]}, {"sent": "nine but whats the animal ?", "words": ["the", "animal", "but"]}, {"sent": "thats a big animal isnt it ?", "words": ["it", "animal", "big", "a"]}, {"sent": "what kind of animal is that ?", "words": ["what", "is", "of", "animal", "that"]}, {"sent": "letters on the back of the animal cards .", "words": ["the", "back", "of", "animal", "on"]}, {"sent": "what kind of animal ?", "words": ["of", "what", "animal"]}, {"sent": "oh have you picked up the animal book ?", "words": ["the", "up", "you", "have", "animal", "book"]}, {"sent": "what animal does that look like ?", "words": ["what", "does", "like", "animal", "look", "that"]}, {"sent": "some some type of stuffed animal something that has buttons and snaps and all kinds of things that you can practice .", "words": ["can", "all", "of", "and", "you", "animal", "some", "that"]}, {"sent": "Cody loved to play with his animal friends .", "words": ["to", "play", "with", "animal", "his"]}, {"sent": "what is that animal ?", "words": ["what", "animal", "that", "is"]}, {"sent": "you looking for an animal ?", "words": ["an", "for", "animal", "you"]}, {"sent": "does that look like an animal ?", "words": ["does", "like", "an", "animal", "look", "that"]}, {"sent": "whats this animal ?", "words": ["this", "animal"]}, {"sent": "shall we try one more peculiar game with this animal whatever he is ?", "words": ["game", "we", "is", "with", "this", "more", "animal", "he", "try"]}, {"sent": "it was an animal who was it ?", "words": ["was", "an", "it", "animal", "who"]}, {"sent": "were there I I um I think I heard animal names and sounds but I wasnt sure if they were these animals I .", "words": ["these", "think", "if", "and", "there", "they", "animal", "were", "but"]}, {"sent": "or a dog or any type of animal you wanna look after it .", "words": ["of", "any", "you", "animal", "it", "wanna", "look", "dog", "a"]}, {"sent": "what kind of animal .", "words": ["of", "what", "animal"]}, {"sent": "its a funny animal .", "words": ["animal", "a"]}, {"sent": "um does she is do you think shes familiar with any animal names ?", "words": ["does", "do", "think", "is", "with", "any", "you", "animal", "she"]}, {"sent": "theres a monkey .", "words": ["monkey", "a"]}, {"sent": "wheres the monkey ?", "words": ["the", "monkey"]}, {"sent": "thats nice for the monkey .", "words": ["the", "monkey", "nice", "for"]}, {"sent": "oh theres a monkey .", "words": ["monkey", "a"]}, {"sent": "this is a monkey .", "words": ["this", "monkey", "a", "is"]}, {"sent": "happy birthday to you you live in a zoo you smell like a monkey and you look like one too .", "words": ["zoo", "happy", "to", "monkey", "like", "you", "and", "in", "look", "too", "a"]}, {"sent": "monkey dunkie dunkies ?", "words": ["monkey"]}, {"sent": "look at the monkey .", "words": ["the", "monkey", "at", "look"]}, {"sent": "are you going to wave to the monkey ?", "words": ["the", "to", "monkey", "you", "are"]}, {"sent": "where is the monkey ?", "words": ["where", "the", "monkey", "is"]}, {"sent": "the monkey ?", "words": ["the", "monkey"]}, {"sent": "oh you gave your monkey a kiss .", "words": ["your", "monkey", "you", "kiss", "a"]}, {"sent": "and the crocodile swam out in the river with the monkey .", "words": ["the", "monkey", "with", "and", "in", "out"]}, {"sent": "is the monkey a crazy driver ?", "words": ["the", "monkey", "a", "is"]}, {"sent": "no monkey ?", "words": ["monkey"]}, {"sent": "theres the monkey .", "words": ["the", "monkey"]}, {"sent": "its in here isnt it the monkey ?", "words": ["the", "monkey", "here", "in", "it"]}, {"sent": "wheres the monkey ?", "words": ["the", "monkey"]}, {"sent": "but that monkey doesnt wanna let him go does he ?", "words": ["does", "go", "monkey", "him", "he", "wanna", "that", "but"]}, {"sent": "say monkey .", "words": ["monkey", "say"]}, {"sent": "the monkey can go on that .", "words": ["the", "can", "go", "monkey", "on", "that"]}, {"sent": "what is this Mommy monkey doing ?", "words": ["what", "monkey", "this", "is"]}, {"sent": "wheres the monkey sitting ?", "words": ["the", "monkey"]}, {"sent": "the monkey caught the weasel .", "words": ["the", "monkey"]}, {"sent": "uhhuh weve got a lion monkey tiger pretty parrots and a happy hippo and whats this ?", "words": ["happy", "monkey", "and", "this", "pretty", "tiger", "lion", "a"]}, {"sent": "Sams a monkey on Spot .", "words": ["on", "monkey", "a"]}, {"sent": "and what kind of fruit do you like to eat monkey ?", "words": ["what", "do", "of", "to", "monkey", "like", "you", "and", "eat"]}, {"sent": "monkey .", "words": ["monkey"]}, {"sent": "do you see a monkey in there ?", "words": ["do", "monkey", "you", "there", "in", "see", "a"]}, {"sent": "are you going to sing the monkey song for us ?", "words": ["the", "for", "to", "monkey", "you", "sing", "us", "are"]}, {"sent": "what noise does a monkey make ?", "words": ["what", "does", "monkey", "make", "a"]}, {"sent": "monkey .", "words": ["monkey"]}, {"sent": "is that for the monkey ?", "words": ["the", "for", "is", "monkey", "that"]}, {"sent": "is it a monkey ?", "words": ["monkey", "it", "a", "is"]}, {"sent": "weve got a monkey puppet havent we ?", "words": ["monkey", "we", "a"]}, {"sent": "I dont know where your monkey is .", "words": ["where", "your", "is", "monkey", "dont"]}, {"sent": "is that the monkey ?", "words": ["the", "monkey", "that", "is"]}, {"sent": "can you get some tea for the monkey and the bear ?", "words": ["can", "the", "for", "monkey", "you", "get", "and", "bear", "some"]}, {"sent": "down the monkey path and onto BurnishLane .", "words": ["the", "monkey", "down", "and"]}, {"sent": "look at the monkey hanging off .", "words": ["the", "off", "look", "monkey", "at"]}, {"sent": "the monkey isnt it ?", "words": ["the", "monkey", "it"]}, {"sent": "where is the monkey ?", "words": ["where", "the", "monkey", "is"]}, {"sent": "how how long have you been a monkey ?", "words": ["how", "monkey", "long", "you", "have", "a"]}, {"sent": "monkey .", "words": ["monkey"]}, {"sent": "one when day when all of these animals were out on this big farm along came a big brown monkey who jumped right in to the middle of the farm and said .", "words": ["the", "all", "these", "brown", "of", "to", "monkey", "and", "this", "big", "in", "out", "who", "on", "when", "were", "a"]}, {"sent": "are you a monkey ?", "words": ["monkey", "are", "a", "you"]}, {"sent": "thats monkey .", "words": ["monkey"]}, {"sent": "and what does the monkey eat ?", "words": ["the", "what", "does", "monkey", "and", "eat"]}, {"sent": "well if we go down the monkey path .", "words": ["the", "if", "go", "we", "monkey", "down"]}, {"sent": "yeah you could say monkey .", "words": ["monkey", "could", "say", "you"]}, {"sent": "can you say monkey ?", "words": ["can", "monkey", "say", "you"]}, {"sent": "no no no its a monkey eating a banana in a closet .", "words": ["monkey", "closet", "banana", "in", "a"]}, {"sent": "Boots is the monkey isnt he ?", "words": ["the", "monkey", "he", "is"]}, {"sent": "so the monkey .", "words": ["the", "so", "monkey"]}, {"sent": "hes holding a monkey .", "words": ["monkey", "a"]}, {"sent": "oh that snow monkey mightve been the daddy snow monkey .", "words": ["the", "monkey", "snow", "that"]}, {"sent": "patty cake was a little monkey that had her arm broken .", "words": ["cake", "was", "monkey", "her", "little", "broken", "arm", "that", "a"]}, {"sent": "what does a monkey say ?", "words": ["what", "does", "monkey", "say", "a"]}, {"sent": "you want the monkey you want me to feed the monkey too ?", "words": ["the", "to", "monkey", "you", "feed", "me", "too"]}, {"sent": "monkey done a handstand .", "words": ["monkey", "a"]}, {"sent": "and a monkey .", "words": ["monkey", "a", "and"]}, {"sent": "where is the monkey ?", "words": ["where", "the", "monkey", "is"]}, {"sent": "one day all of the animals were on the farm when along came a big brown monkey and he jumped right into the middle of the farm and said .", "words": ["the", "all", "when", "into", "brown", "of", "monkey", "and", "big", "he", "on", "were", "a"]}, {"sent": "is that the way the monkey goes ?", "words": ["the", "monkey", "that", "is"]}, {"sent": "the monkey ?", "words": ["the", "monkey"]}, {"sent": "you saw a monkey at the zoo ?", "words": ["the", "zoo", "monkey", "you", "at", "a"]}, {"sent": "the monkey he got drunk .", "words": ["the", "monkey", "he"]}, {"sent": "is the monkey hiding under the foot .", "words": ["the", "is", "monkey", "under", "foot"]}, {"sent": "its like the monkey on on the Pgtips advertisement .", "words": ["the", "monkey", "on", "like"]}, {"sent": "those are your monkey rings .", "words": ["your", "monkey", "are", "those"]}, {"sent": "lets see Elmer doesnt have any monkey friends honey .", "words": ["have", "see", "monkey", "any"]}, {"sent": "wheres the monkey .", "words": ["the", "monkey"]}, {"sent": "for the monkey ?", "words": ["the", "monkey", "for"]}, {"sent": "the monkey climbed down the tree .", "words": ["the", "tree", "monkey", "down"]}, {"sent": "monkey uhoh .", "words": ["monkey"]}, {"sent": "trying to rule the monkey tribe from my monkey zone .", "words": ["the", "to", "monkey", "my"]}, {"sent": "I think hes a cheeky monkey thats what he is .", "words": ["what", "think", "is", "monkey", "he", "a"]}, {"sent": "what does a monkey say ?", "words": ["what", "does", "monkey", "say", "a"]}, {"sent": "oh we can have the monkey .", "words": ["can", "the", "we", "monkey", "have"]}, {"sent": "what does the monkey do in the zoo ?", "words": ["the", "what", "does", "do", "zoo", "monkey", "in"]}, {"sent": "Ill do what the monkey did .", "words": ["the", "do", "what", "monkey", "did"]}, {"sent": "the man saw a blue monkey a blue monkey .", "words": ["the", "monkey", "blue", "a"]}, {"sent": "and then the monkey swings on it .", "words": ["the", "monkey", "and", "it", "on", "then"]}, {"sent": "a monkey .", "words": ["monkey", "a"]}, {"sent": "the monkey is under arrest ?", "words": ["the", "under", "monkey", "is"]}, {"sent": "can you see a monkey ?", "words": ["can", "monkey", "you", "see", "a"]}, {"sent": "monkey food ?", "words": ["monkey", "food"]}, {"sent": "oh youre a monkey .", "words": ["monkey", "a"]}, {"sent": "what the gru the monkey ?", "words": ["the", "what", "monkey"]}, {"sent": "give the monkey the cake .", "words": ["the", "give", "monkey", "cake"]}, {"sent": "monkey .", "words": ["monkey"]}, {"sent": "hes going to drive the monkey about is he ?", "words": ["the", "is", "to", "monkey", "drive", "he", "about"]}, {"sent": "a monkey screams and two cubs scramble for cover .", "words": ["for", "monkey", "and", "cover", "a"]}, {"sent": "monkey .", "words": ["monkey"]}, {"sent": "moneys Mommys little monkey ?", "words": ["monkey", "little"]}, {"sent": "cheeky monkey .", "words": ["monkey"]}, {"sent": "does he want the little monkey ?", "words": ["the", "does", "monkey", "little", "he"]}, {"sent": "look at this a monkey riding a bicycle .", "words": ["look", "monkey", "bicycle", "this", "at", "a"]}, {"sent": "one day all the animals were out in the yard and a monkey jumped right into the middle of the farm .", "words": ["the", "all", "into", "of", "monkey", "and", "in", "out", "were", "a"]}, {"sent": "no not that chair .", "words": ["that", "not", "chair"]}, {"sent": "let Daddy pull up a chair .", "words": ["up", "chair", "a", "pull"]}, {"sent": "chair chair .", "words": ["chair"]}, {"sent": "and after he left presents presents in the rocking chair and in your bedroom he put the fire guard back didnt he ?", "words": ["the", "put", "your", "back", "rocking chair", "chair", "and", "in", "he", "bedroom"]}, {"sent": "well uh theres your chair .", "words": ["your", "chair"]}, {"sent": "which chair ?", "words": ["which", "chair"]}, {"sent": "you sit on a chair .", "words": ["chair", "you", "on", "sit", "a"]}, {"sent": "Thomas are you going to go and sit in your chair please ?", "words": ["your", "go", "to", "chair", "and", "you", "in", "are", "sit"]}, {"sent": "Doug found the chair .", "words": ["the", "chair"]}, {"sent": "Mommy get your chair and put you in your chair .", "words": ["put", "your", "chair", "and", "get", "you", "in"]}, {"sent": "you put dolly on the chair .", "words": ["the", "put", "chair", "you", "on"]}, {"sent": "you wanna sit on the rocking chair with me well read a book ?", "words": ["the", "book", "rocking chair", "with", "chair", "you", "read", "wanna", "on", "me", "sit", "a"]}, {"sent": "which chair ?", "words": ["which", "chair"]}, {"sent": "wheres the chair you have ?", "words": ["the", "have", "chair", "you"]}, {"sent": "you dont use a high chair .", "words": ["chair", "you", "high chair", "dont", "high", "a"]}, {"sent": "heres a chair heres something .", "words": ["chair", "a"]}, {"sent": "Chrissy want ta sit in the chair ?", "words": ["the", "in", "sit", "chair"]}, {"sent": "I need a chair .", "words": ["need", "chair", "a"]}, {"sent": "have you seen how her tails hanging off that chair ?", "words": ["off", "how", "chair", "you", "have", "that", "her"]}, {"sent": "yeah shes locked in the chair its okay sweetheart .", "words": ["the", "in", "chair"]}, {"sent": "thats the talking chair ?", "words": ["the", "chair"]}, {"sent": "Frasers in the chair .", "words": ["the", "in", "chair"]}, {"sent": "mamma said someone has been sitting in my chair too .", "words": ["chair", "in", "my", "too"]}, {"sent": "that is what you say when you swing Daddys swivel chair .", "words": ["what", "when", "is", "swing", "say", "chair", "you", "that"]}, {"sent": "you stay on your chair and play with your toys .", "words": ["your", "play", "with", "chair", "you", "and", "on", "stay"]}, {"sent": "chair and the ball are the same color arent they ?", "words": ["the", "chair", "and", "ball", "they", "are", "same"]}, {"sent": "the dog can sit in the chair ?", "words": ["the", "can", "chair", "in", "sit", "dog"]}, {"sent": "okay the bears can sit down in the chair .", "words": ["the", "can", "chair", "in", "sit", "down"]}, {"sent": "who sits on that chair ?", "words": ["on", "that", "chair", "who"]}, {"sent": "dont jump on the chair please Thomas .", "words": ["the", "chair", "dont", "jump", "on"]}, {"sent": "no just scoot the chair over .", "words": ["the", "over", "chair"]}, {"sent": "I dont know whose turn it is to sit in the high chair .", "words": ["the", "is", "to", "chair", "high chair", "dont", "in", "it", "high", "sit"]}, {"sent": "like a chair .", "words": ["like", "chair", "a"]}, {"sent": "you fell off your chair ?", "words": ["off", "your", "chair", "you"]}, {"sent": "so she climbed out of that rocking chair .", "words": ["of", "so", "rocking chair", "chair", "out", "that", "she"]}, {"sent": "you cant have the push chair .", "words": ["the", "push", "chair", "you", "have"]}, {"sent": "Thomas would you like to get in your chair please ?", "words": ["your", "to", "like", "chair", "you", "get", "in", "would"]}, {"sent": "on the chair ?", "words": ["on", "the", "chair"]}, {"sent": "you cant sit in her chair while shes there .", "words": ["chair", "you", "there", "in", "sit", "her"]}, {"sent": "you wanna sit in a chair and play with the playdough ?", "words": ["the", "play", "with", "chair", "you", "and", "in", "wanna", "sit", "a"]}, {"sent": "here sit in this little blowup chair okay ?", "words": ["chair", "little", "here", "this", "in", "sit"]}, {"sent": "why a trying to turn me in this chair Remi ?", "words": ["to", "chair", "this", "in", "me", "why", "a"]}, {"sent": "well why dont we sit you in your chair then if youre gonna eat some ?", "words": ["your", "if", "we", "then", "chair", "you", "eat", "dont", "in", "some", "sit", "why"]}, {"sent": "on your chair .", "words": ["on", "your", "chair"]}, {"sent": "this chair .", "words": ["this", "chair"]}, {"sent": "go on which chair would she sit on uhn dyou think ?", "words": ["think", "go", "chair", "would", "on", "sit", "which", "she"]}, {"sent": "you wanna sit on that little baby chair ?", "words": ["chair", "you", "little", "wanna", "on", "sit", "that"]}, {"sent": "stuck in my chair at noon .", "words": ["my", "chair", "in", "stuck", "at"]}, {"sent": "the cloth on the back of the rocking chair please .", "words": ["the", "back", "of", "rocking chair", "chair", "on"]}, {"sent": "let us rock this rocking chair .", "words": ["rocking chair", "chair", "us", "this", "rock"]}, {"sent": "youre gonna come in here and Im gonna put you in your chair .", "words": ["put", "your", "chair", "you", "here", "and", "in"]}, {"sent": "you get down off the chair and Ill pass you your pie to go and put in the oven .", "words": ["the", "off", "put", "your", "go", "to", "chair", "you", "get", "and", "in", "down", "oven"]}, {"sent": "so I thought the best thing was to put them up there because you useta sit in your high chair round here .", "words": ["the", "up", "put", "your", "so", "was", "to", "them", "chair", "you", "here", "there", "in", "high chair", "high", "sit", "because"]}, {"sent": "chair and pear rhymes .", "words": ["chair", "and"]}, {"sent": "and Daddys chair ?", "words": ["chair", "and"]}, {"sent": "Peter you shouldnt have broken that chair that wasnt very nice .", "words": ["nice", "chair", "broken", "have", "you", "that"]}, {"sent": "and youre in the other chair .", "words": ["the", "chair", "other", "and", "in"]}, {"sent": "looks like a little tiny piece of a chair .", "words": ["tiny", "of", "like", "chair", "little", "a"]}, {"sent": "you wanna sit in your chair again ?", "words": ["your", "chair", "you", "in", "wanna", "sit"]}, {"sent": "well Ill organize some juice and you needta see down in a chair .", "words": ["down", "chair", "you", "and", "in", "see", "juice", "a", "some"]}, {"sent": "okay heres a chair .", "words": ["chair", "a"]}, {"sent": "no thats okay you dont hafta say sorry to the chair .", "words": ["the", "to", "say", "chair", "you", "dont"]}, {"sent": "you just sit in your chair .", "words": ["your", "chair", "you", "in", "sit"]}, {"sent": "well if you dont if youre not going to kneel youre not coming on the chair at all .", "words": ["the", "all", "if", "to", "chair", "you", "dont", "at", "on", "not"]}, {"sent": "put him on the chair .", "words": ["the", "put", "chair", "him", "on"]}, {"sent": "come and sit on my chair .", "words": ["my", "chair", "and", "on", "sit"]}, {"sent": "oh somebodys been sitting in my chair .", "words": ["chair", "in", "my"]}, {"sent": "can you pull your chair over then too ?", "words": ["can", "your", "chair", "pull", "you", "too", "then", "over"]}, {"sent": "sitting in my high chair feed me soon .", "words": ["my", "chair", "high", "high chair", "in", "feed", "me"]}, {"sent": "theres one under the other chair .", "words": ["the", "under", "chair", "other"]}, {"sent": "shes fallen off her chair and her books have allgone up in the air .", "words": ["off", "up", "the", "chair", "and", "have", "in", "her"]}, {"sent": "hes only got one chair left .", "words": ["chair"]}, {"sent": "can you bring your chair over ?", "words": ["can", "your", "bring", "chair", "you", "over"]}, {"sent": "in your usual chair or do you wanna sit at this table today ?", "words": ["do", "your", "table", "chair", "you", "this", "in", "wanna", "at", "sit"]}, {"sent": "you have a high chair at home .", "words": ["chair", "you", "have", "high chair", "at", "high", "home", "a"]}, {"sent": "wanna put the chair beside the table ?", "words": ["the", "put", "table", "beside", "chair", "wanna"]}, {"sent": "the chair is sad .", "words": ["the", "sad", "chair", "is"]}, {"sent": "a chair .", "words": ["chair", "a"]}, {"sent": "so we put the television there what about the chair ?", "words": ["the", "what", "put", "we", "so", "chair", "there", "about"]}, {"sent": "theres one in the rocking chair and her little babys on her lap .", "words": ["the", "rocking chair", "chair", "and", "little", "in", "on", "her"]}, {"sent": "Ill get a big chair .", "words": ["get", "big", "chair", "a"]}, {"sent": "well come and sit down on your chair then .", "words": ["your", "then", "chair", "and", "on", "sit", "down"]}, {"sent": "can you pull a chair up as well Lara ?", "words": ["can", "up", "chair", "pull", "you", "a"]}, {"sent": "Ill put you in your chair .", "words": ["put", "your", "chair", "you", "in"]}, {"sent": "sits up straight in his chair .", "words": ["up", "his", "in", "chair"]}, {"sent": "sit on the chair .", "words": ["on", "the", "sit", "chair"]}, {"sent": "let go of the chair .", "words": ["of", "the", "chair", "go"]}, {"sent": "like mothers chair yeah mother foxs chair .", "words": ["like", "chair"]}, {"sent": "and another chair .", "words": ["another", "chair", "and"]}, {"sent": "get up in the chair and sit down .", "words": ["the", "up", "chair", "and", "get", "in", "sit", "down"]}, {"sent": "youre stuck on the chair ?", "words": ["on", "the", "stuck", "chair"]}, {"sent": "you want Mama to sit in the chair with William ?", "words": ["the", "to", "with", "chair", "you", "in", "sit"]}, {"sent": "thats a chair .", "words": ["chair", "a"]}, {"sent": "you hafta put him in different chair somewhere .", "words": ["put", "chair", "you", "in", "him"]}, {"sent": "for his chair .", "words": ["his", "for", "chair"]}, {"sent": "Ive moved the chair that Nana normally sits in and thats where Po was .", "words": ["the", "where", "was", "chair", "and", "in", "that"]}, {"sent": "here is your little rocking chair .", "words": ["your", "is", "rocking chair", "chair", "little", "here"]}, {"sent": "wheres your chair ?", "words": ["your", "chair"]}, {"sent": "you wanna sit in that chair ?", "words": ["chair", "you", "in", "wanna", "sit", "that"]}, {"sent": "sitting in the high chair big chair my chair sitting in the chair banging my spoon .", "words": ["the", "my", "chair", "high chair", "in", "spoon", "high", "big"]}, {"sent": "okay Ill put that arm in .", "words": ["in", "put", "that", "arm"]}, {"sent": "David your arm is dirty .", "words": ["your", "dirty", "arm", "is"]}, {"sent": "my sleeve dangling down and my arm sticking out .", "words": ["my", "and", "out", "arm", "down"]}, {"sent": "one arm .", "words": ["arm"]}, {"sent": "under your other arm .", "words": ["under", "your", "arm", "other"]}, {"sent": "I thought you broke your arm .", "words": ["your", "arm", "you"]}, {"sent": "oh theres teddys arm .", "words": ["arm"]}, {"sent": "with her arm ?", "words": ["her", "with", "arm"]}, {"sent": "Ive brought my teddybear to see you because hes got a poorly arm and a poorly leg .", "words": ["my", "to", "teddybear", "you", "and", "leg", "arm", "see", "because", "a"]}, {"sent": "and the other is theres a wooden arm here that is in your way .", "words": ["the", "your", "is", "other", "and", "here", "in", "arm", "that", "a"]}, {"sent": "you making the hole with your arm ?", "words": ["the", "your", "with", "you", "arm"]}, {"sent": "Ive got a sore arm .", "words": ["arm", "a"]}, {"sent": "didnt put the arm on yet .", "words": ["the", "put", "on", "arm"]}, {"sent": "you put your arm in because you washed it .", "words": ["put", "your", "you", "in", "it", "arm", "because"]}, {"sent": "oh youve put your arm up it now and youre wearing it like a sleeve .", "words": ["up", "put", "your", "like", "and", "it", "arm", "a"]}, {"sent": "give me your arm .", "words": ["me", "give", "your", "arm"]}, {"sent": "it was a ghost in your dream t pulled the arm off your raptor .", "words": ["the", "off", "your", "was", "in", "it", "arm", "a"]}, {"sent": "this one here is holding a baby in her arm .", "words": ["is", "here", "this", "in", "arm", "her", "a"]}, {"sent": "meanwhile outside in the gloomy street the strong arm of the robbers were working a treat .", "words": ["the", "of", "outside", "in", "arm", "street", "were", "a"]}, {"sent": "William push your arm through please .", "words": ["your", "push", "arm"]}, {"sent": "what was she doing when she broke her arm ?", "words": ["what", "when", "was", "arm", "her", "she"]}, {"sent": "let me have a look at your arm .", "words": ["your", "have", "at", "arm", "look", "me", "a"]}, {"sent": "why are you hitting my arm ?", "words": ["my", "you", "are", "arm", "why"]}, {"sent": "lets put your arm in .", "words": ["in", "put", "your", "arm"]}, {"sent": "you just needta put his arm up thats it .", "words": ["up", "put", "you", "it", "arm", "his"]}, {"sent": "a little girl broke her arm yesterday .", "words": ["arm", "her", "little", "a"]}, {"sent": "now you do the other arm take the other arm off .", "words": ["the", "do", "off", "other", "you", "arm", "take"]}, {"sent": "did he bite your arm or something ?", "words": ["your", "he", "did", "arm", "bite"]}, {"sent": "did she hurt her arm ?", "words": ["hurt", "did", "arm", "her", "she"]}, {"sent": "did his arm come off ?", "words": ["his", "did", "arm", "off"]}, {"sent": "arm in .", "words": ["in", "arm"]}, {"sent": "what did Ivy do to her arm ?", "words": ["what", "do", "to", "did", "arm", "her"]}, {"sent": "does he need medicine for his leg poorly leg and poorly arm ?", "words": ["does", "for", "need", "medicine", "and", "he", "leg", "arm", "his"]}, {"sent": "look at Conner I did on your arm .", "words": ["look", "your", "did", "arm", "at", "on"]}, {"sent": "did you ever break an arm or a leg ?", "words": ["you", "an", "did", "arm", "leg", "break", "a"]}, {"sent": "my arm is too big .", "words": ["my", "is", "arm", "too", "big"]}, {"sent": "move your arm .", "words": ["your", "arm"]}, {"sent": "and if you jump on and if your arm too the if your arm to and the and maybe Robbie who jumped to .", "words": ["the", "your", "if", "to", "you", "and", "jump", "arm", "too", "on", "who"]}, {"sent": "then he tucked it comfortably under his arm and stomped down the path towards Piglets .", "words": ["the", "and", "under", "it", "he", "arm", "his", "then", "down"]}, {"sent": "just drive right over the arm .", "words": ["drive", "the", "over", "arm"]}, {"sent": "I hurt my arm when I was playing .", "words": ["my", "hurt", "was", "arm", "when"]}, {"sent": "put it on Nevilles arm .", "words": ["on", "put", "it", "arm"]}, {"sent": "youve gotta do your arm .", "words": ["do", "your", "arm"]}, {"sent": "because I scratch my arm ?", "words": ["arm", "my", "because"]}, {"sent": "right arm through there .", "words": ["there", "arm"]}, {"sent": "arm in .", "words": ["in", "arm"]}, {"sent": "see earlier on I was standing it was like elec somebody giving me electric shocks up my fingers right up my arm .", "words": ["up", "my", "was", "me", "like", "it", "arm", "on", "see"]}, {"sent": "broke her arm .", "words": ["her", "arm"]}, {"sent": "okay put the arm through this hole and the other arm through this hole .", "words": ["the", "put", "and", "other", "this", "arm"]}, {"sent": "can you put one arm through there ?", "words": ["can", "put", "you", "there", "arm"]}, {"sent": "Ill do one arm you do the other arm .", "words": ["the", "do", "other", "you", "arm"]}, {"sent": "you wear my arm out .", "words": ["out", "my", "arm", "you"]}, {"sent": "lets see whether theres an arm in there to put on this little man .", "words": ["put", "to", "little", "there", "an", "in", "this", "arm", "on", "see"]}, {"sent": "I think she has hurt her arm dont you ?", "words": ["think", "hurt", "you", "dont", "arm", "her", "she"]}, {"sent": "hand and an arm .", "words": ["arm", "an", "and", "hand"]}, {"sent": "arm .", "words": ["arm"]}, {"sent": "wheres his arm ?", "words": ["his", "arm"]}, {"sent": "which arm is this ?", "words": ["this", "which", "arm", "is"]}, {"sent": "he caught his arm on it .", "words": ["on", "it", "he", "arm", "his"]}, {"sent": "it thought your arm was a tree .", "words": ["tree", "your", "was", "it", "arm", "a"]}, {"sent": "Im threading the arm .", "words": ["the", "arm"]}, {"sent": "put her arm in then .", "words": ["put", "in", "arm", "then", "her"]}, {"sent": "hes got such a tiny arm aswell .", "words": ["tiny", "arm", "a"]}, {"sent": "one arm .", "words": ["arm"]}, {"sent": "and the right arm .", "words": ["the", "arm", "and"]}, {"sent": "there is his other arm .", "words": ["is", "other", "there", "arm", "his"]}, {"sent": "are you kissing Mummys arm ?", "words": ["are", "arm", "you"]}, {"sent": "take his other arm out .", "words": ["other", "out", "arm", "his", "take"]}, {"sent": "no hes just resting his arm .", "words": ["his", "arm"]}, {"sent": "I might as well move this so you dont put your arm on it there .", "words": ["put", "your", "so", "you", "dont", "this", "it", "there", "arm", "on"]}, {"sent": "and thread that arm through there .", "words": ["there", "that", "arm", "and"]}, {"sent": "I hope youre not standing on Amys arm .", "words": ["on", "not", "arm"]}, {"sent": "this arm .", "words": ["this", "arm"]}, {"sent": "you make an arm .", "words": ["an", "make", "arm", "you"]}, {"sent": "if you reached your arm down far enough .", "words": ["your", "if", "you", "arm", "down"]}, {"sent": "isnt it a tasty and delicious arm ?", "words": ["arm", "it", "and", "a"]}, {"sent": "that goes on your arm .", "words": ["on", "that", "arm", "your"]}, {"sent": "thats one arm .", "words": ["arm"]}, {"sent": "you wanna rest your head on Mummys arm .", "words": ["your", "head", "you", "wanna", "arm", "on"]}, {"sent": "okay wheres your other arm ?", "words": ["your", "arm", "other"]}, {"sent": "her arm what ?", "words": ["what", "her", "arm"]}, {"sent": "one arm .", "words": ["arm"]}, {"sent": "whats underneath that arm ?", "words": ["that", "arm"]}, {"sent": "now we gonna dry this arm .", "words": ["this", "dry", "arm", "we"]}, {"sent": "you hurt your arm ?", "words": ["hurt", "your", "arm", "you"]}, {"sent": "theres a big scratch down my arm and the only thing I can think of is that perhaps Ive picked Purdie up and she has scratched me .", "words": ["the", "can", "my", "think", "up", "is", "of", "and", "that", "arm", "me", "down", "big", "she", "a"]}, {"sent": "can you thread her left arm ?", "words": ["can", "her", "arm", "you"]}, {"sent": "anyway your sister broke her arm ?", "words": ["her", "your", "arm"]}, {"sent": "and sure enough xxx he couldnt move his arm .", "words": ["his", "he", "arm", "and"]}, {"sent": "because now I mean I can kind of move my arm a little more now .", "words": ["can", "my", "of", "little", "more", "arm", "because", "a"]}, {"sent": "put your arm around her and love her .", "words": ["put", "your", "and", "arm", "around", "love", "her"]}, {"sent": "did you bang your arm ?", "words": ["your", "did", "arm", "you"]}, {"sent": "on his arm .", "words": ["on", "his", "arm"]}, {"sent": "are you going to pick her arm up and dry her arm ?", "words": ["up", "pick", "to", "dry", "you", "and", "are", "arm", "her"]}, {"sent": "and said Ive got a sore arm youknow .", "words": ["arm", "a", "and"]}, {"sent": "other arm .", "words": ["arm", "other"]}, {"sent": "you bumped your arm ?", "words": ["your", "arm", "you"]}, {"sent": "you have a bandaid on your arm .", "words": ["your", "you", "have", "arm", "on", "a"]}, {"sent": "but her arm doesnt come off pet .", "words": ["off", "arm", "her", "but"]}, {"sent": "you hurt yourself ?", "words": ["hurt", "yourself", "you"]}, {"sent": "I hurt myself .", "words": ["hurt", "myself"]}, {"sent": "did I hurt ?", "words": ["hurt", "did"]}, {"sent": "watch you dont hurt yourself pumpkin .", "words": ["watch", "yourself", "hurt", "you", "dont", "pumpkin"]}, {"sent": "have you hurt your foot ?", "words": ["your", "hurt", "you", "have", "foot"]}, {"sent": "that is how youre going to get hurt dont swing .", "words": ["how", "is", "hurt", "to", "swing", "get", "dont", "that"]}, {"sent": "its gonna hurt .", "words": ["hurt"]}, {"sent": "that hurt .", "words": ["hurt", "that"]}, {"sent": "does your throat hurt ?", "words": ["hurt", "does", "your"]}, {"sent": "youre gonna hurt AuntCarey .", "words": ["hurt"]}, {"sent": "dont hurt yourself television man .", "words": ["hurt", "dont", "yourself"]}, {"sent": "did that hurt ?", "words": ["hurt", "that", "did"]}, {"sent": "all right were gonna put you somewhere where no one can hurt you .", "words": ["where", "all", "put", "can", "hurt", "you", "were"]}, {"sent": "where does it hurt .", "words": ["where", "does", "it", "hurt"]}, {"sent": "has hurt his teeth .", "words": ["hurt", "his"]}, {"sent": "youre the one that hurt your head .", "words": ["the", "your", "hurt", "head", "that"]}, {"sent": "dont do that Fraser because youll hurt yourself or youll hurt Mummy doing that .", "words": ["do", "yourself", "hurt", "dont", "that", "because"]}, {"sent": "I think you probably could do it but it would hurt .", "words": ["do", "think", "hurt", "you", "it", "would", "could", "but"]}, {"sent": "have you hurt your arm ?", "words": ["your", "hurt", "you", "have", "arm"]}, {"sent": "where did you hurt yourself ?", "words": ["where", "yourself", "hurt", "you", "did"]}, {"sent": "youll hurt Lala .", "words": ["hurt"]}, {"sent": "they make your throat hurt ?", "words": ["your", "they", "make", "hurt"]}, {"sent": "gonna hurt your mouth .", "words": ["hurt", "mouth", "your"]}, {"sent": "how did the teddy bear hurt his bottom ?", "words": ["the", "how", "bottom", "hurt", "bear", "did", "his"]}, {"sent": "you might hurt yourself .", "words": ["hurt", "yourself", "you"]}, {"sent": "and hurt yourself .", "words": ["hurt", "yourself", "and"]}, {"sent": "does he think youre gonna hurt the policeman ?", "words": ["the", "does", "think", "hurt", "he"]}, {"sent": "you know if you fall and hurt yourself Caroline and I arent going to pick you up .", "words": ["up", "yourself", "if", "hurt", "to", "pick", "fall", "and", "you"]}, {"sent": "yes Sue got hurt oven .", "words": ["hurt", "oven"]}, {"sent": "doesnt hurt .", "words": ["hurt"]}, {"sent": "does your ear hurt ?", "words": ["hurt", "does", "your", "ear"]}, {"sent": "you might hurt the doggie .", "words": ["hurt", "the", "you"]}, {"sent": "does it hurt ?", "words": ["hurt", "does", "it"]}, {"sent": "otherwise if you put the chair on my foot it would hurt wouldnt it ?", "words": ["the", "put", "my", "if", "hurt", "chair", "you", "foot", "it", "would", "on"]}, {"sent": "and she hurt her nose ?", "words": ["hurt", "and", "her", "she", "nose"]}, {"sent": "I hurt your fingers ?", "words": ["hurt", "your"]}, {"sent": "oh did it hurt ?", "words": ["hurt", "it", "did"]}, {"sent": "yeah I told you it will hurt .", "words": ["will", "it", "hurt", "you"]}, {"sent": "oh that hurt .", "words": ["hurt", "that"]}, {"sent": "doctor doesnt hurt .", "words": ["hurt"]}, {"sent": "oh that might hurt your eye .", "words": ["hurt", "that", "eye", "your"]}, {"sent": "does it still hurt ?", "words": ["hurt", "does", "it"]}, {"sent": "shouldnt hurt your eyes should you ?", "words": ["hurt", "your", "you"]}, {"sent": "it doesnt hurt if I dont hafta chase you does it ?", "words": ["does", "if", "hurt", "you", "dont", "it", "chase"]}, {"sent": "who hurt your arm at school buddy ?", "words": ["your", "hurt", "school", "arm", "at", "who"]}, {"sent": "any any second now and youre going to tumble outof that and hurt yourself .", "words": ["yourself", "hurt", "to", "any", "and", "that"]}, {"sent": "do you think he has hurt his knee ?", "words": ["do", "think", "knee", "hurt", "you", "he", "his"]}, {"sent": "no it doesnt hurt them ?", "words": ["hurt", "it", "them"]}, {"sent": "you hurt your socks ?", "words": ["hurt", "your", "you"]}, {"sent": "careful the baby xxx hurt yourself .", "words": ["the", "yourself", "hurt", "careful"]}, {"sent": "does anything hurt ?", "words": ["hurt", "does"]}, {"sent": "you know Teddys hurt his foot .", "words": ["hurt", "his", "foot", "you"]}, {"sent": "they were such big dogs you would think that they could hurt each other .", "words": ["think", "each", "hurt", "you", "other", "they", "that", "would", "could", "big", "were"]}, {"sent": "it wont hurt .", "words": ["hurt", "it"]}, {"sent": "would you not want the ambulance coming to come to check I was alright and not hurt ?", "words": ["the", "hurt", "to", "was", "you", "and", "would", "not"]}, {"sent": "that would hurt wouldnt it ?", "words": ["hurt", "it", "that", "would"]}, {"sent": "oh you hurt your finger ?", "words": ["hurt", "finger", "your", "you"]}, {"sent": "youre gonna hurt the horse .", "words": ["hurt", "the", "horse"]}, {"sent": "shall I move the baby tiger so it doesnt get hurt .", "words": ["the", "hurt", "so", "get", "it", "tiger"]}, {"sent": "how does it hurt ?", "words": ["hurt", "does", "how", "it"]}, {"sent": "did you hurt your foot ?", "words": ["your", "hurt", "you", "foot", "did"]}, {"sent": "you hurt your finger ?", "words": ["hurt", "finger", "your", "you"]}, {"sent": "mind yourself because you might hurt yourself .", "words": ["hurt", "yourself", "because", "you"]}, {"sent": "Ma he hurt my toe .", "words": ["hurt", "my", "toe", "he"]}, {"sent": "you know what we better take care of Mommys hurt .", "words": ["better", "what", "we", "of", "hurt", "you", "take"]}, {"sent": "what did you hurt ?", "words": ["hurt", "what", "did", "you"]}, {"sent": "if it hit you like that it will hurt wont it ?", "words": ["if", "hurt", "like", "you", "it", "will", "hit", "that"]}, {"sent": "has it hurt your hand ?", "words": ["hurt", "it", "your", "hand"]}, {"sent": "has he hurt himself ?", "words": ["hurt", "he"]}, {"sent": "theyre really hurt uhn arent they ?", "words": ["hurt", "they"]}, {"sent": "why does he hurt ?", "words": ["hurt", "does", "he", "why"]}, {"sent": "does it hurt or does it tickle ?", "words": ["hurt", "does", "it", "tickle"]}, {"sent": "you xxx hurt your fingers there you go .", "words": ["your", "go", "hurt", "you", "there"]}, {"sent": "youll hurt yourself .", "words": ["hurt", "yourself"]}, {"sent": "Im not hurt .", "words": ["hurt", "not"]}, {"sent": "you know I hurt my foot too yesterday I stepped on a bee .", "words": ["my", "hurt", "bee", "you", "foot", "too", "on", "a"]}, {"sent": "no I dont want you to hurt any be people because theres too many games with all these people getting hurt .", "words": ["all", "these", "hurt", "to", "be", "any", "with", "you", "dont", "too", "because"]}, {"sent": "did you hurt yourself ?", "words": ["hurt", "yourself", "did", "you"]}, {"sent": "do they hurt your feet ?", "words": ["hurt", "do", "your", "they"]}, {"sent": "our friend Jims hurt himself .", "words": ["hurt", "our"]}, {"sent": "have you hurt your foot again ?", "words": ["your", "hurt", "you", "have", "foot"]}, {"sent": "youll hurt her .", "words": ["hurt", "her"]}, {"sent": "but you might hurt mommy if you throw it .", "words": ["throw", "if", "hurt", "you", "it", "but"]}, {"sent": "and it hurt and you were crying but you stopped crying almost right away .", "words": ["hurt", "you", "and", "it", "away", "were", "but"]}, {"sent": "I wanna know why its always this left knee that you hurt .", "words": ["knee", "hurt", "you", "this", "wanna", "that", "why"]}, {"sent": "does it hurt ?", "words": ["hurt", "does", "it"]}, {"sent": "Ive hurt my arm .", "words": ["hurt", "my", "arm"]}, {"sent": "stop that you dont pull her head back like that it will hurt her .", "words": ["back", "hurt", "head", "stop", "her", "like", "pull", "you", "dont", "it", "will", "that"]}, {"sent": "does it hurt your eyes ?", "words": ["hurt", "does", "it", "your"]}, {"sent": "youll hurt your fingers .", "words": ["hurt", "your"]}, {"sent": "watch dont hurt your face .", "words": ["watch", "your", "face", "hurt", "dont"]}, {"sent": "but it wont hurt for very long okay now wait one minute .", "words": ["wait", "for", "hurt", "long", "it", "but"]}, {"sent": "that is okay it wont hurt you .", "words": ["is", "hurt", "you", "it", "that"]}, {"sent": "you hurt your finger ?", "words": ["hurt", "finger", "your", "you"]}, {"sent": "itll hurt .", "words": ["hurt"]}, {"sent": "does your face hurt ?", "words": ["hurt", "does", "your", "face"]}, {"sent": "that hurt now .", "words": ["hurt", "that"]}, {"sent": "its got its head in the sand .", "words": ["the", "in", "head"]}, {"sent": "thats a funny funny head .", "words": ["head", "a"]}, {"sent": "you bang your head so hard ?", "words": ["your", "so", "head", "you", "hard"]}, {"sent": "and his head .", "words": ["his", "head", "and"]}, {"sent": "turn round the head ?", "words": ["the", "head"]}, {"sent": "lemme see your head .", "words": ["see", "head", "your"]}, {"sent": "tickle tickle on the head .", "words": ["on", "the", "head", "tickle"]}, {"sent": "Dipsys got a long thin antenna on his head hasnt he ?", "words": ["head", "long", "his", "he", "on", "a"]}, {"sent": "I dont know no head .", "words": ["dont", "head"]}, {"sent": "whats that man got on his head ?", "words": ["on", "his", "head", "that"]}, {"sent": "theyre all going to fall on your head .", "words": ["all", "your", "to", "head", "fall", "on"]}, {"sent": "if they were on my head .", "words": ["my", "if", "head", "they", "on", "were"]}, {"sent": "whoops now youve bumped your head .", "words": ["head", "your"]}, {"sent": "what did you do with the WinniethePooh rabbit head ?", "words": ["the", "what", "do", "head", "with", "you", "did"]}, {"sent": "here put your head up on silly goose .", "words": ["up", "put", "your", "head", "here", "on", "goose"]}, {"sent": "oh watch your head .", "words": ["head", "watch", "your"]}, {"sent": "you bumped your head ?", "words": ["head", "your", "you"]}, {"sent": "aw no the horsies fell on its head again .", "words": ["the", "on", "head"]}, {"sent": "okay watch your head watch your head .", "words": ["head", "watch", "your"]}, {"sent": "bite the babys head .", "words": ["bite", "the", "head"]}, {"sent": "why did you tell your Mummy you bumped your head ?", "words": ["your", "head", "you", "did", "why"]}, {"sent": "because if you dont have a head your eyes and your nose and your mouth will fall on the ground right ?", "words": ["the", "your", "if", "nose", "head", "mouth", "fall", "you", "have", "dont", "and", "on", "will", "because", "a"]}, {"sent": "wheres his head ?", "words": ["his", "head"]}, {"sent": "you bumped your head didnt you ?", "words": ["head", "your", "you"]}, {"sent": "youre gonna screw my head off ?", "words": ["off", "my", "head"]}, {"sent": "wheres his head gone ?", "words": ["his", "head"]}, {"sent": "what did you do to his head ?", "words": ["what", "do", "to", "head", "you", "did", "his"]}, {"sent": "head up .", "words": ["up", "head"]}, {"sent": "the head fell off .", "words": ["the", "head", "off"]}, {"sent": "just put her head back .", "words": ["put", "her", "head", "back"]}, {"sent": "no hitting your head .", "words": ["head", "your"]}, {"sent": "lets take the box of his head and see if hell go away from us .", "words": ["the", "box", "if", "go", "of", "head", "and", "us", "away", "his", "see", "take"]}, {"sent": "push down on my engine drivers head said Thomas .", "words": ["my", "push", "head", "on", "down"]}, {"sent": "he doesnt wanna put your dirty cowboy hat on his head .", "words": ["put", "your", "head", "his", "on", "he", "dirty", "wanna", "hat"]}, {"sent": "theres his head .", "words": ["his", "head"]}, {"sent": "oh she hit her head but she didnt bang it .", "words": ["head", "it", "hit", "her", "she", "but"]}, {"sent": "haha thats too little for your head .", "words": ["for", "your", "head", "little", "too"]}, {"sent": "now pull your head in .", "words": ["in", "head", "your", "pull"]}, {"sent": "oh is it going on her head ?", "words": ["is", "head", "it", "on", "her"]}, {"sent": "so we could get your head out .", "words": ["your", "we", "so", "head", "get", "out", "could"]}, {"sent": "you just bumped his head .", "words": ["his", "head", "you"]}, {"sent": "is it on your head ?", "words": ["your", "is", "head", "it", "on"]}, {"sent": "head and shoulders .", "words": ["head", "and"]}, {"sent": "another head ?", "words": ["another", "head"]}, {"sent": "mister potato head with a red nose and a moustache .", "words": ["red", "nose", "head", "with", "and", "potato", "a"]}, {"sent": "Ill get your head salty then well do the other hand .", "words": ["the", "do", "your", "hand", "head", "other", "get", "then"]}, {"sent": "big head and funny body .", "words": ["head", "big", "and"]}, {"sent": "hes got a screwdriver on his head .", "words": ["on", "his", "head", "a"]}, {"sent": "oh your washing your head .", "words": ["head", "your"]}, {"sent": "my head is turning around I hafta leave my friends in Kingston town .", "words": ["my", "is", "head", "in", "around"]}, {"sent": "now Noddys head .", "words": ["head"]}, {"sent": "now turn the head .", "words": ["the", "head"]}, {"sent": "no thats xxx part of the potato head thats the pieces .", "words": ["of", "the", "head", "potato"]}, {"sent": "that feel funny on your head ?", "words": ["on", "head", "that", "your"]}, {"sent": "your head is itchy ?", "words": ["head", "your", "is"]}, {"sent": "are you going to put Thomass hat on Mummys head ?", "words": ["put", "to", "head", "you", "on", "are", "hat"]}, {"sent": "does it hafta go over your head Lara ?", "words": ["does", "your", "go", "head", "it", "over"]}, {"sent": "well put him to bed the sleepy head and well look for the bear in the morning .", "words": ["the", "put", "for", "to", "head", "bed", "and", "bear", "him", "in", "look", "sleepy"]}, {"sent": "mind your head .", "words": ["head", "your"]}, {"sent": "has he hurt his head ?", "words": ["hurt", "his", "head", "he"]}, {"sent": "little wock head .", "words": ["head", "little"]}, {"sent": "I think its a head .", "words": ["think", "head", "a"]}, {"sent": "sitting by the front door shaking your head real hard ?", "words": ["the", "your", "head", "by", "hard", "door"]}, {"sent": "shall I take the head off ?", "words": ["the", "head", "take", "off"]}, {"sent": "and Marky and he was petting Markys head and said .", "words": ["was", "head", "he", "and"]}, {"sent": "have you put the basket back on your head ?", "words": ["the", "put", "your", "back", "head", "you", "have", "basket", "on"]}, {"sent": "hes got a pair of great big horns on his head .", "words": ["of", "head", "on", "his", "big", "a"]}, {"sent": "now put it down next to your little head .", "words": ["put", "your", "to", "head", "little", "it", "down"]}, {"sent": "I hope it doesnt because if it does itll hit us on the head wont it ?", "words": ["the", "does", "if", "head", "us", "it", "on", "hit", "because"]}, {"sent": "get your head .", "words": ["get", "head", "your"]}, {"sent": "okay lets head over here .", "words": ["here", "head", "over"]}, {"sent": "pound this guy on the head .", "words": ["on", "this", "head", "the"]}, {"sent": "you bumped your head on the lampshade ?", "words": ["the", "your", "head", "you", "on"]}, {"sent": "Daddy was bumping his head .", "words": ["his", "was", "head"]}, {"sent": "can you make his head turn round .", "words": ["can", "head", "you", "his", "make"]}, {"sent": "thats her head thats the top of her head .", "words": ["the", "of", "head", "her"]}, {"sent": "look that the babys behind the laundry she has socks on her head .", "words": ["the", "head", "behind", "her", "look", "on", "that", "she"]}, {"sent": "put it on your head I guess you could .", "words": ["put", "your", "head", "you", "it", "on", "could"]}, {"sent": "see I got a two so I get a head .", "words": ["so", "head", "get", "see", "a"]}, {"sent": "it went backwards over your head .", "words": ["your", "head", "it", "over"]}, {"sent": "our Conor had it on his head .", "words": ["head", "our", "his", "it", "on"]}, {"sent": "assoonas we get this over your head youll be able to .", "words": ["your", "we", "be", "head", "to", "get", "this", "over"]}, {"sent": "he bumped his head .", "words": ["his", "head", "he"]}, {"sent": "Teddys got a heart in his head .", "words": ["his", "in", "head", "a"]}, {"sent": "over his head and then what ?", "words": ["what", "head", "and", "his", "then", "over"]}, {"sent": "take the box off your head .", "words": ["the", "off", "box", "your", "head", "take"]}, {"sent": "thats band around his head .", "words": ["his", "head", "around"]}, {"sent": "oh who did he drop on his head ?", "words": ["head", "his", "he", "drop", "did", "who", "on"]}, {"sent": "nod your head .", "words": ["head", "your"]}, {"sent": "Gails got her head stuck .", "words": ["head", "stuck", "her"]}, {"sent": "eat the boys head .", "words": ["eat", "the", "head"]}, {"sent": "hes got hes standing on the head of the fireengine .", "words": ["on", "the", "head", "of"]}, {"sent": "okay put his head in there .", "words": ["put", "head", "there", "in", "his"]}, {"sent": "his front feet well part his head on one side and thought about things .", "words": ["head", "and", "on", "about", "his"]}, {"sent": "not on my head .", "words": ["on", "my", "head", "not"]}, {"sent": "put the cow on Warrens head .", "words": ["the", "put", "head", "on", "cow"]}, {"sent": "whose head is on the stamp ?", "words": ["on", "the", "head", "is"]}, {"sent": "I hope that if I bang my head against the wall hard enough it will help me think of a story .", "words": ["the", "my", "help", "think", "if", "of", "head", "it", "hard", "will", "me", "story", "that", "a"]}, {"sent": "how did you hurt your head ?", "words": ["how", "your", "hurt", "head", "you", "did"]}, {"sent": "big head .", "words": ["head", "big"]}, {"sent": "whos behind the laundry ?", "words": ["the", "behind"]}, {"sent": "is he sitting behind the horse in the trailer ?", "words": ["the", "is", "behind", "in", "he", "horse"]}, {"sent": "you think theyre behind that door ?", "words": ["think", "behind", "you", "that", "door"]}, {"sent": "it looks like hes doing what behind there ?", "words": ["what", "behind", "like", "there", "it"]}, {"sent": "youre trying to get the post from behind my back .", "words": ["the", "my", "back", "to", "behind", "get"]}, {"sent": "whos behind the laundry ?", "words": ["the", "behind"]}, {"sent": "not behind the door .", "words": ["the", "behind", "not", "door"]}, {"sent": "one thing I hafta teach you is how to clean behind stoves and clean behind furniture .", "words": ["clean", "how", "is", "to", "behind", "you", "and"]}, {"sent": "theres one behind you aswell by the chips .", "words": ["the", "behind", "by", "you"]}, {"sent": "look behind you .", "words": ["behind", "you", "look"]}, {"sent": "oh look behind you .", "words": ["behind", "you", "look"]}, {"sent": "hiding behind a ?", "words": ["behind", "a"]}, {"sent": "I think shes left threads behind .", "words": ["think", "behind"]}, {"sent": "she was hiding behind the ball .", "words": ["the", "was", "behind", "ball", "she"]}, {"sent": "whos behind the laundry where are you .", "words": ["the", "where", "behind", "you", "are"]}, {"sent": "behind you there it is .", "words": ["is", "behind", "you", "there", "it"]}, {"sent": "I think theres one behind the sofa .", "words": ["the", "sofa", "think", "behind"]}, {"sent": "line up behind me .", "words": ["up", "me", "behind"]}, {"sent": "whose behind the teddy bear ?", "words": ["the", "behind", "bear"]}, {"sent": "did you look behind you ?", "words": ["look", "behind", "did", "you"]}, {"sent": "are you behind the ball ?", "words": ["the", "behind", "you", "ball", "are"]}, {"sent": "one two okay so Im behind you right ?", "words": ["so", "behind", "you"]}, {"sent": "and the other one was when they were stuck behind somebody that doesnt seem to know theyre there and to get out of the way .", "words": ["the", "of", "was", "to", "behind", "other", "and", "they", "that", "there", "stuck", "get", "out", "when", "were"]}, {"sent": "and theres another hamburger behind ya .", "words": ["another", "hamburger", "behind", "and"]}, {"sent": "by the time Tomcat turned back home six happy cats were trotting along behind him .", "words": ["the", "happy", "back", "behind", "by", "him", "were", "home"]}, {"sent": "whos behind the curtain ?", "words": ["the", "behind"]}, {"sent": "Mamas gonna hide behind the chair .", "words": ["the", "chair", "behind", "hide"]}, {"sent": "hang it on the line behind you .", "words": ["the", "behind", "you", "it", "on"]}, {"sent": "behind the street ?", "words": ["the", "street", "behind"]}, {"sent": "behind Fraser .", "words": ["behind"]}, {"sent": "whats behind there Liz ?", "words": ["there", "behind"]}, {"sent": "behind the door ?", "words": ["the", "behind", "door"]}, {"sent": "hes going behind the house .", "words": ["the", "house", "behind"]}, {"sent": "should it be in front or behind ?", "words": ["be", "in", "it", "behind"]}, {"sent": "theres some pieces behind you .", "words": ["some", "behind", "you"]}, {"sent": "I think it must be in the field behind the hedge .", "words": ["the", "think", "be", "behind", "in", "it"]}, {"sent": "are you behind the ball ?", "words": ["the", "behind", "you", "ball", "are"]}, {"sent": "must be his little house behind the wall .", "words": ["the", "be", "behind", "little", "house", "his"]}, {"sent": "is the building site behind me ?", "words": ["the", "me", "behind", "is"]}, {"sent": "maybe shes behind the settee .", "words": ["the", "behind"]}, {"sent": "Im putting a cushion behind now .", "words": ["behind", "a"]}, {"sent": "not behind your scarf .", "words": ["your", "scarf", "behind", "not"]}, {"sent": "theres lotsof space behind the house .", "words": ["the", "house", "behind"]}, {"sent": "whos behind the gate ?", "words": ["the", "behind"]}, {"sent": "he jumped down behind you .", "words": ["down", "he", "you", "behind"]}, {"sent": "is it behind you ?", "words": ["it", "behind", "you", "is"]}, {"sent": "can you find one of those with Katie and PoPat standing behind her ?", "words": ["can", "those", "of", "with", "behind", "you", "and", "find", "her"]}, {"sent": "is she behind the tellie ?", "words": ["the", "behind", "she", "is"]}, {"sent": "so the balloon followed the bus followed behind the bus .", "words": ["the", "so", "behind", "balloon", "bus"]}, {"sent": "it was behind you .", "words": ["was", "it", "behind", "you"]}, {"sent": "behind those ?", "words": ["behind", "those"]}, {"sent": "not behind you ?", "words": ["behind", "not", "you"]}, {"sent": "and the balloon followed behind the bus .", "words": ["the", "behind", "and", "balloon", "bus"]}, {"sent": "thats Rabbit behind five .", "words": ["behind"]}, {"sent": "look another one is stuck behind the lorry .", "words": ["the", "is", "behind", "another", "stuck", "look"]}, {"sent": "whats behind you ?", "words": ["behind", "you"]}, {"sent": "did you show Helen the little girl on your shirt thats behind the door ?", "words": ["the", "your", "show", "shirt", "behind", "little", "you", "did", "on", "door"]}, {"sent": "what animals are outside behind fences ?", "words": ["what", "are", "outside", "behind"]}, {"sent": "what do ya think is behind the curtain Madelyn ?", "words": ["the", "what", "do", "think", "is", "behind"]}, {"sent": "are you behind the ball ?", "words": ["the", "behind", "you", "ball", "are"]}, {"sent": "you put them in behind the ear like that .", "words": ["the", "put", "them", "behind", "like", "you", "in", "ear", "that"]}, {"sent": "hes behind the door .", "words": ["the", "behind", "door"]}, {"sent": "is baby behind the teddy bear ?", "words": ["the", "bear", "behind", "is"]}, {"sent": "whos behind the curtains ?", "words": ["the", "behind"]}, {"sent": "hes behind you .", "words": ["behind", "you"]}, {"sent": "I think the wee babys pram must have got left behind at home .", "words": ["the", "think", "behind", "have", "at", "home"]}, {"sent": "are you behind the ball ?", "words": ["the", "behind", "you", "ball", "are"]}, {"sent": "but your hands are have gone behind your potty .", "words": ["your", "behind", "have", "are", "potty", "but"]}, {"sent": "dont go behind there .", "words": ["dont", "behind", "there", "go"]}, {"sent": "its not behind that ear .", "words": ["that", "behind", "not", "ear"]}, {"sent": "I bet hes hiding behind that curtain .", "words": ["that", "behind"]}, {"sent": "maybe behind the sofa .", "words": ["the", "sofa", "behind"]}, {"sent": "go behind him like that ?", "words": ["go", "behind", "like", "him", "that"]}, {"sent": "hes hiding behind the tree isnt he ?", "words": ["the", "he", "tree", "behind"]}, {"sent": "behind the glasses ?", "words": ["the", "behind", "glasses"]}, {"sent": "Im just gonna pass this behind you .", "words": ["this", "behind", "you"]}, {"sent": "ah its right behind you there .", "words": ["there", "behind", "you"]}, {"sent": "whos behind the teddybear ?", "words": ["the", "behind", "teddybear"]}, {"sent": "shes still behind the wall there .", "words": ["the", "there", "behind"]}, {"sent": "and a t whos behind the teddy bear ?", "words": ["the", "behind", "bear", "and", "a"]}, {"sent": "the hammers behind you .", "words": ["the", "behind", "you"]}, {"sent": "is there someone behind the other side of the curtain ?", "words": ["the", "is", "of", "behind", "other", "there"]}, {"sent": "stand over here behind the table .", "words": ["the", "table", "stand", "behind", "here", "over"]}, {"sent": "what they said was it must have been hiding behind the other one so .", "words": ["the", "what", "was", "so", "behind", "other", "have", "they", "it"]}, {"sent": "comere just hook this to your behind .", "words": ["to", "this", "your", "behind"]}, {"sent": "so shes gonna be left behind .", "words": ["so", "be", "behind"]}, {"sent": "ah whose behind the laundry ?", "words": ["the", "behind"]}, {"sent": "whos behind the teddy bear ?", "words": ["the", "behind", "bear"]}, {"sent": "doesnt he wanna have a little drive behind the wheel first ?", "words": ["the", "first", "behind", "little", "have", "drive", "he", "wanna", "a"]}, {"sent": "behind you .", "words": ["behind", "you"]}, {"sent": "how about behind xxx the brown cart ?", "words": ["the", "how", "brown", "behind", "about"]}, {"sent": "behind a tree .", "words": ["tree", "behind", "a"]}, {"sent": "I think you put too many cars behind it Aran .", "words": ["think", "put", "behind", "you", "it", "too"]}, {"sent": "shall we leave you behind ?", "words": ["behind", "you", "we"]}, {"sent": "bet hes behind that rocking chair .", "words": ["that", "rocking chair", "behind", "chair"]}, {"sent": "are you behind the dog ?", "words": ["the", "behind", "you", "are", "dog"]}, {"sent": "whos behind the ball ?", "words": ["the", "ball", "behind"]}, {"sent": "you can have your book behind here .", "words": ["can", "your", "behind", "you", "have", "here", "book"]}, {"sent": "and three PuddleDucks came along the har high road marching one behind the other and doing the goosestep .", "words": ["the", "behind", "other", "and", "high"]}, {"sent": "a doll a dragon and doggie .", "words": ["and", "doll", "a"]}, {"sent": "all the accessories and everything that goes with the doll is free ?", "words": ["the", "all", "is", "with", "doll", "and", "that"]}, {"sent": "this little girls got a Cindy doll .", "words": ["this", "doll", "a", "little"]}, {"sent": "did the doll enjoy that ?", "words": ["the", "that", "did", "doll"]}, {"sent": "are you laughing at the baby doll ?", "words": ["the", "doll", "you", "are", "at"]}, {"sent": "and then Ill think well have a look at your little doll .", "words": ["think", "your", "doll", "and", "have", "little", "at", "look", "then", "a"]}, {"sent": "and then we kinda went over to the doll lost interest in the doll real fast .", "words": ["the", "fast", "we", "to", "doll", "and", "in", "then", "over"]}, {"sent": "um he has a baby doll at home .", "words": ["doll", "he", "at", "a", "home"]}, {"sent": "thats a little doll too .", "words": ["doll", "too", "little", "a"]}, {"sent": "got the doll ?", "words": ["the", "doll"]}, {"sent": "must be from your doll your dolls hair .", "words": ["be", "hair", "your", "doll"]}, {"sent": "so what is the little doll made outof ?", "words": ["the", "what", "is", "so", "doll", "little"]}, {"sent": "playing with a doll right now in the bathroom .", "words": ["the", "with", "doll", "in", "bathroom", "a"]}, {"sent": "thats a very delicate doll .", "words": ["doll", "a"]}, {"sent": "okay Ill pick the doll house up .", "words": ["the", "up", "pick", "doll", "house"]}, {"sent": "now put the doll in .", "words": ["the", "in", "put", "doll"]}, {"sent": "oh that was the doll yeah .", "words": ["the", "was", "that", "doll"]}, {"sent": "you see the big baby doll no no no no let go of the cord .", "words": ["the", "go", "of", "doll", "you", "see", "big"]}, {"sent": "yeah we needta get something to put all your doll clothes in dont we Catherine ?", "words": ["all", "put", "your", "we", "to", "doll", "get", "dont", "in"]}, {"sent": "you hafta you know the doll which MrsCwww bought you ?", "words": ["the", "which", "doll", "you"]}, {"sent": "its not Dominics doll either .", "words": ["not", "doll"]}, {"sent": "oh the doll is gonna sleep in Naimas bed I see .", "words": ["the", "is", "sleep", "bed", "doll", "in", "see"]}, {"sent": "xxx the doctor doll .", "words": ["the", "doll"]}, {"sent": "well hafta bring the doll back to robin .", "words": ["the", "back", "to", "bring", "doll"]}, {"sent": "hey ya got a good snoopy doll there to cut out .", "words": ["cut", "to", "doll", "there", "good", "out", "a"]}, {"sent": "och the wee doll .", "words": ["the", "doll"]}, {"sent": "um we have a doll .", "words": ["have", "doll", "a", "we"]}, {"sent": "youre gonna go doll shopping ?", "words": ["doll", "go"]}, {"sent": "this girl is putting a doll in her pram .", "words": ["is", "doll", "this", "in", "her", "a"]}, {"sent": "doll .", "words": ["doll"]}, {"sent": "no it goes on the doll huh ?", "words": ["on", "the", "it", "doll"]}, {"sent": "I never heard of a doll corner .", "words": ["of", "doll", "a"]}, {"sent": "I dont shes never really seen a doll .", "words": ["dont", "doll", "a"]}, {"sent": "she loves this baby doll .", "words": ["this", "she", "doll"]}, {"sent": "and look theres a doll .", "words": ["doll", "look", "a", "and"]}, {"sent": "the doll xxx foot .", "words": ["the", "foot", "doll"]}, {"sent": "show Elizabeth the doll .", "words": ["show", "the", "doll"]}, {"sent": "bring the doll to me .", "words": ["the", "to", "bring", "doll", "me"]}, {"sent": "see the RaggedyAnn doll ?", "words": ["the", "see", "doll"]}, {"sent": "see the doll ?", "words": ["the", "see", "doll"]}, {"sent": "are you putting bricks on the doll ?", "words": ["the", "doll", "you", "are", "on"]}, {"sent": "oh youve got the doll out .", "words": ["the", "out", "doll"]}, {"sent": "whose doll is that ?", "words": ["that", "doll", "is"]}, {"sent": "I think this one is too small for that doll .", "words": ["think", "for", "is", "doll", "this", "too", "that"]}, {"sent": "and the only doll that we have at home so far is a stuffed animal doll and she hasnt showed a lot of interest so it was interesting to see her responding to the hard eyes and everything every time I put it down she seemed to want it back .", "words": ["the", "put", "of", "hard", "have", "it", "down", "that", "she", "home", "is", "a lot", "so", "was", "to", "doll", "and", "every", "see", "a", "back", "we", "animal", "at", "her"]}, {"sent": "thats a doll yeah .", "words": ["doll", "a"]}, {"sent": "do we needta get you a baby doll ?", "words": ["do", "we", "doll", "you", "get", "a"]}, {"sent": "heres the doll .", "words": ["the", "doll"]}, {"sent": "the little girls playing with her cat pretending her cat is a doll .", "words": ["the", "is", "with", "doll", "little", "cat", "her", "a"]}, {"sent": "but possession seems to be more important than actually doing anything with the doll .", "words": ["the", "to", "be", "with", "doll", "more", "but"]}, {"sent": "is it a doll ?", "words": ["doll", "it", "a", "is"]}, {"sent": "would you like to take your doll to Maggies house today ?", "words": ["your", "to", "like", "doll", "you", "house", "would", "take"]}, {"sent": "she doesnt unless there is something attached to the baby doll .", "words": ["the", "is", "to", "doll", "there", "she"]}, {"sent": "a little bit to the doll .", "words": ["the", "to", "doll", "little", "a"]}, {"sent": "shall I put a dress on the doll so she doesnt get cold ?", "words": ["the", "put", "cold", "so", "doll", "get", "on", "dress", "she", "a"]}, {"sent": "and there was that high chair and the wee doll .", "words": ["the", "was", "chair", "and", "doll", "there", "high chair", "high", "that"]}, {"sent": "because the doll is in there and the doll is talking .", "words": ["the", "is", "doll", "and", "there", "in", "because"]}, {"sent": "and how about the baby doll ?", "words": ["the", "how", "doll", "and", "about"]}, {"sent": "the doll .", "words": ["the", "doll"]}, {"sent": "why dont ya finish giving your doll a bath ?", "words": ["your", "finish", "doll", "dont", "why", "a"]}, {"sent": "oh I think robin wants to keep her doll dressed .", "words": ["to", "think", "her", "doll"]}, {"sent": "you discovered the baby doll .", "words": ["the", "doll", "you"]}, {"sent": "a very pretty doll .", "words": ["pretty", "doll", "a"]}, {"sent": "imagine a real baby instead of a doll .", "words": ["of", "doll", "a"]}, {"sent": "okay lets put it in the doll house .", "words": ["the", "put", "doll", "in", "it", "house"]}, {"sent": "he looks like that baby doll with the black hair that sticks straight up dont he Megan ?", "words": ["the", "up", "with", "like", "doll", "black", "dont", "he", "hair", "that"]}, {"sent": "its a baby doll .", "words": ["doll", "a"]}, {"sent": "would you like to dress up the doll ?", "words": ["up", "the", "to", "like", "doll", "you", "would", "dress"]}, {"sent": "you were in the doll yesterday ?", "words": ["the", "doll", "you", "in", "were"]}, {"sent": "that doll .", "words": ["that", "doll"]}, {"sent": "she usually calls her doll a baby .", "words": ["her", "she", "doll", "a"]}, {"sent": "your bottle with the doll ?", "words": ["the", "your", "with", "doll", "bottle"]}, {"sent": "what else does the daddy doll have ?", "words": ["the", "what", "does", "doll", "have"]}, {"sent": "mummy put it back on your doll ?", "words": ["put", "your", "back", "doll", "it", "on"]}, {"sent": "and clothes for the doll arent there ?", "words": ["the", "for", "doll", "and", "there"]}, {"sent": "would the little doll like to go for a ride on her horse ?", "words": ["the", "for", "go", "to", "her", "like", "doll", "little", "horse", "would", "on", "ride", "a"]}, {"sent": "baby doll .", "words": ["doll"]}, {"sent": "shes putting a doll in her pram .", "words": ["in", "her", "doll", "a"]}, {"sent": "and the doll .", "words": ["the", "doll", "and"]}, {"sent": "there is a doll over here who needs some clothes .", "words": ["is", "doll", "here", "there", "who", "some", "over", "a"]}, {"sent": "hey look at doll .", "words": ["at", "doll", "look"]}, {"sent": "well take the Mama doll and the baby doll .", "words": ["the", "take", "doll", "and"]}, {"sent": "do you not remember the doll that weve had since you were a little boy ?", "words": ["the", "do", "doll", "you", "little", "that", "not", "were", "a"]}, {"sent": "another doll ?", "words": ["another", "doll"]}, {"sent": "you wanna put the backpack on the doll ?", "words": ["the", "put", "doll", "you", "wanna", "on"]}, {"sent": "you see the baby doll ?", "words": ["the", "see", "doll", "you"]}, {"sent": "wheres your Grover doll ?", "words": ["your", "doll"]}, {"sent": "it isnt a doll at all honey .", "words": ["all", "doll", "it", "at", "a"]}, {"sent": "do you want to come and play with the doll ?", "words": ["the", "do", "to", "play", "with", "doll", "and", "you", "want to"]}, {"sent": "did he play with the doll ?", "words": ["the", "play", "with", "doll", "he", "did"]}, {"sent": "is she a nice doll ?", "words": ["is", "nice", "doll", "she", "a"]}, {"sent": "yeah thats her doll .", "words": ["her", "doll"]}, {"sent": "does this go with the doll ?", "words": ["the", "does", "go", "with", "doll", "this"]}, {"sent": "or do you hafta buy the doll with it ?", "words": ["the", "do", "with", "doll", "you", "it", "buy"]}, {"sent": "Tammy put that baby doll down so you can eat .", "words": ["can", "put", "so", "doll", "you", "eat", "down", "that"]}, {"sent": "thats no doll .", "words": ["doll"]}, {"sent": "its a doll .", "words": ["doll", "a"]}, {"sent": "wheres the doll wheres the doll ?", "words": ["the", "doll"]}, {"sent": "I think the doll I think you oughta take the dolly shopping .", "words": ["the", "think", "doll", "you", "take"]}, {"sent": "yeah he didnt seem to have much of an interest in the doll so .", "words": ["the", "of", "to", "so", "doll", "have", "an", "in", "he", "much"]}, {"sent": "you hate everybody you even hate me .", "words": ["me", "hate", "you"]}, {"sent": "you hate pits ?", "words": ["hate", "you"]}, {"sent": "I hate the thought of that .", "words": ["the", "of", "that", "hate"]}, {"sent": "I hate cleaning two .", "words": ["hate"]}, {"sent": "you dont hate pink .", "words": ["dont", "hate", "you"]}, {"sent": "does hate mean you like or you dont like butter ?", "words": ["does", "hate", "like", "you", "dont", "butter"]}, {"sent": "I hate them .", "words": ["hate", "them"]}, {"sent": "I hate crusts .", "words": ["hate"]}, {"sent": "well I hate like to heck to have you feed me .", "words": ["to", "hate", "like", "you", "have", "feed", "me"]}, {"sent": "why do you hate it ?", "words": ["do", "hate", "you", "it", "why"]}, {"sent": "I hate that toy .", "words": ["toy", "that", "hate"]}, {"sent": "I hate baloney and swiss cheese .", "words": ["cheese", "hate", "and"]}, {"sent": "oh and I hate to tell him but his pants are on backwards and hes gonna get made fun of so we better fix that .", "words": ["better", "we", "of", "to", "so", "fix", "hate", "and", "get", "him", "are", "pants", "on", "his", "that", "but"]}, {"sent": "you dont like that you hate it when we tell you no dont you ?", "words": ["when", "we", "hate", "like", "you", "dont", "it", "that"]}, {"sent": "I hate that toy Henry .", "words": ["toy", "that", "hate"]}, {"sent": "och I hate being in .", "words": ["in", "hate"]}, {"sent": "its work how I hate it Id much rather play .", "words": ["how", "work", "play", "hate", "it", "much"]}, {"sent": "I hate it when you wash my face and and rub too hard .", "words": ["my", "face", "hate", "wash", "you", "and", "it", "hard", "too", "when"]}, {"sent": "I hate the Rangers .", "words": ["the", "hate"]}, {"sent": "you hate to eat hm ?", "words": ["eat", "to", "hate", "you"]}, {"sent": "I hate peanutbutter .", "words": ["hate"]}, {"sent": "we hate Tony .", "words": ["hate", "we"]}, {"sent": "I hate pictures .", "words": ["hate"]}, {"sent": "because you hate putting them in xxx .", "words": ["them", "hate", "you", "in", "because"]}, {"sent": "I even hate the smell of it .", "words": ["the", "of", "it", "hate"]}, {"sent": "aye you hate them .", "words": ["them", "hate", "you"]}, {"sent": "Id hate to meet up with him in the forest .", "words": ["up", "the", "to", "hate", "with", "him", "in"]}, {"sent": "and you hate people singing dont you ?", "words": ["dont", "hate", "you", "and"]}, {"sent": "I hate it when you get soap in my eyes .", "words": ["my", "hate", "soap", "you", "get", "in", "it", "when"]}, {"sent": "oh I love raw carrots but I hate them cooked .", "words": ["hate", "them", "carrots", "love", "but"]}, {"sent": "I hate the Audubon .", "words": ["the", "hate"]}, {"sent": "err I hate freckles .", "words": ["hate"]}, {"sent": "I hate .", "words": ["hate"]}, {"sent": "you dont hate it at all .", "words": ["all", "hate", "you", "dont", "it", "at"]}, {"sent": "I am the queen said Lily and I hate Julius .", "words": ["the", "hate", "am", "and"]}, {"sent": "they dont hate being your toys .", "words": ["dont", "they", "hate", "your"]}, {"sent": "I hate to say but we are were like what can .", "words": ["can", "what", "we", "to", "hate", "say", "like", "are", "were", "but"]}, {"sent": "from sitting just sitting here day after day its work how I hate it Id much rather .", "words": ["how", "work", "hate", "here", "it", "much"]}, {"sent": "um because hes eighteen months and I hate bottles .", "words": ["hate", "because", "and"]}, {"sent": "I hate junk .", "words": ["hate"]}, {"sent": "hate that .", "words": ["that", "hate"]}, {"sent": "hate it ?", "words": ["it", "hate"]}, {"sent": "I hate it when you make me stop playing and Im not through yet .", "words": ["hate", "stop", "you", "and", "it", "make", "me", "when", "not"]}, {"sent": "uh I hate that sound .", "words": ["that", "hate"]}, {"sent": "yeah I know how much you hate breakfast .", "words": ["hate", "how", "much", "you"]}, {"sent": "I hate it when you make me stay still and not wiggle .", "words": ["stay", "hate", "you", "and", "it", "make", "me", "when", "not"]}, {"sent": "but I hate the skin .", "words": ["the", "hate", "but"]}, {"sent": "what else do you hate ?", "words": ["what", "do", "hate", "you"]}, {"sent": "I hate I hate john too .", "words": ["hate", "too"]}, {"sent": "you didnt hate it the other day when you had it .", "words": ["the", "hate", "other", "you", "it", "when"]}, {"sent": "I hate it when you comb my hair when it gets I hate it when you comb my hair when it has tangles in it .", "words": ["my", "comb", "hate", "you", "in", "it", "hair", "when"]}, {"sent": "oh I hate xxx .", "words": ["hate"]}, {"sent": "hate to see you if you lived in Alaska .", "words": ["if", "to", "hate", "you", "in", "see"]}, {"sent": "what else do you hate ?", "words": ["what", "do", "hate", "you"]}, {"sent": "I hate it .", "words": ["it", "hate"]}, {"sent": "which ones do you hate ?", "words": ["hate", "do", "which", "you"]}, {"sent": "you hate that .", "words": ["that", "hate", "you"]}, {"sent": "it was a long time for her to go back to college but she useta hate xxx go to college .", "words": ["for", "back", "go", "was", "to", "hate", "long", "it", "her", "she", "but", "a"]}, {"sent": "wow Id hate to see one of those wouldnt you ?", "words": ["those", "of", "to", "hate", "you", "see"]}, {"sent": "I hate football .", "words": ["hate"]}, {"sent": "and for Johnathan I hate that snow suit thing when it when its cold out .", "words": ["for", "when", "snow", "hate", "and", "it", "out", "that", "cold"]}, {"sent": "I hate coat .", "words": ["hate", "coat"]}, {"sent": "you hate it when we say dont mess up Cindys work ?", "words": ["up", "work", "we", "hate", "say", "you", "dont", "it", "when"]}, {"sent": "I hate dat ?", "words": ["hate"]}, {"sent": "well how do you know you hate it ?", "words": ["do", "how", "hate", "you", "it"]}, {"sent": "I hate I hate .", "words": ["hate"]}, {"sent": "I hate to disappoint you but I think thats it .", "words": ["think", "to", "hate", "you", "it", "but"]}, {"sent": "do you have chai tea or just usual because I hate to use this up on xxx .", "words": ["up", "do", "to", "hate", "you", "have", "this", "on", "because"]}, {"sent": "xxx I hate a filthy place .", "words": ["hate", "a"]}, {"sent": "hate like iceskate ?", "words": ["hate", "like"]}, {"sent": "I hate Thursdays .", "words": ["hate"]}, {"sent": "boy Henry I sure hate that toy .", "words": ["toy", "that", "hate"]}, {"sent": "I hate you .", "words": ["hate", "you"]}, {"sent": "oh I sound so young I hate my voice on tape haha .", "words": ["my", "so", "hate", "tape", "on"]}, {"sent": "I hate them .", "words": ["hate", "them"]}, {"sent": "because they hate you ?", "words": ["they", "hate", "because", "you"]}, {"sent": "the thing I hate worse is having a sore nose .", "words": ["the", "is", "hate", "a", "nose"]}, {"sent": "I hate that .", "words": ["that", "hate"]}, {"sent": "you hate that .", "words": ["that", "hate", "you"]}, {"sent": "you hate it when I get mad at you ?", "words": ["mad", "hate", "you", "get", "it", "at", "when"]}, {"sent": "I hate that stage Tanya .", "words": ["that", "hate"]}, {"sent": "theres plenty of room if you wanna put the camera here I mean I hate to keep making you move it but hes not .", "words": ["the", "put", "camera", "if", "of", "to", "hate", "you", "here", "room", "it", "wanna", "not", "but"]}, {"sent": "you do not hate that one very .", "words": ["do", "hate", "you", "that", "not"]}, {"sent": "I hate to admit it .", "words": ["to", "it", "hate"]}, {"sent": "you hate when theyre stacked .", "words": ["hate", "when", "you"]}, {"sent": "I hate that .", "words": ["that", "hate"]}, {"sent": "you hate girls ?", "words": ["hate", "you"]}, {"sent": "I hate to see people get in trouble .", "words": ["to", "hate", "get", "in", "see"]}, {"sent": "when I go to peoples house and people get in trouble I hate that .", "words": ["go", "to", "hate", "and", "get", "in", "that", "house", "when"]}, {"sent": "I hate to see that on her face .", "words": ["face", "to", "hate", "her", "on", "see", "that"]}, {"sent": "I hate it .", "words": ["it", "hate"]}, {"sent": "and I hate it .", "words": ["it", "hate", "and"]}, {"sent": "I hate coming in here .", "words": ["here", "in", "hate"]}, {"sent": "I hate going to bed dont you ?", "words": ["to", "hate", "bed", "you", "dont"]}, {"sent": "oh I hate it when they have really small neck holes .", "words": ["hate", "have", "they", "it", "when"]}, {"sent": "I hate rats .", "words": ["hate"]}, {"sent": "hate to disappoint you kiddo but theyre just ordinary biscuits .", "words": ["to", "hate", "but", "you"]}, {"sent": "I hate it .", "words": ["it", "hate"]}, {"sent": "no I hate bacon .", "words": ["hate"]}, {"sent": "no you cant wipe them in the living room .", "words": ["the", "living room", "them", "you", "in", "room", "wipe"]}, {"sent": "is he going to sweep the kitchen or is he going to sweep up here in the living room or is he going to sweep up here in the bedroom ?", "words": ["the", "up", "is", "to", "living room", "sweep", "here", "in", "room", "he", "kitchen", "bedroom"]}, {"sent": "is that the living room that the mommys in ?", "words": ["the", "is", "living room", "in", "room", "that"]}, {"sent": "could it be the living room where everybody watches tv ?", "words": ["the", "where", "be", "living room", "tv", "it", "room", "could"]}, {"sent": "why do you hafta go open the living room doors ?", "words": ["the", "do", "go", "open", "living room", "you", "room", "why"]}, {"sent": "in the living room our thing is off .", "words": ["the", "off", "is", "living room", "our", "in", "room"]}, {"sent": "living room .", "words": ["living room", "room"]}, {"sent": "I wanted to now if you wanted to carry the microphone into the living room .", "words": ["the", "if", "into", "to", "living room", "you", "carry", "room"]}, {"sent": "the one in the living room you were talking about ?", "words": ["the", "living room", "you", "in", "room", "about", "were"]}, {"sent": "Mama she cant go to heres her living room at ?", "words": ["go", "to", "living room", "room", "at", "her", "she"]}, {"sent": "why dont well put it back over here in the living room so Lois can see you make a car okay ?", "words": ["the", "can", "put", "back", "so", "car", "living room", "you", "here", "dont", "in", "it", "room", "make", "see", "over", "why", "a"]}, {"sent": "and thats she wasnt really crawling yet so I moved it from the kitchen to the living room .", "words": ["the", "so", "to", "living room", "and", "it", "room", "kitchen", "she"]}, {"sent": "would that be fun to have a picture of an ice cream cone in your living room ?", "words": ["ice cream", "your", "of", "to", "be", "picture", "living room", "have", "an", "in", "room", "would", "ice", "that", "a"]}, {"sent": "did you tell Edna what was on the wall in the living room last night ?", "words": ["the", "what", "was", "last", "living room", "you", "in", "room", "did", "on"]}, {"sent": "you getting all subtle about going off to the living room .", "words": ["off", "all", "the", "to", "living room", "you", "room", "about"]}, {"sent": "Elmos playhouse has a living room and a dining room and a kitchen .", "words": ["living room", "and", "room", "kitchen", "a"]}, {"sent": "thats the living room .", "words": ["the", "living room", "room"]}, {"sent": "hm I hear a funny noise in the living room I wonder what that noise is ?", "words": ["the", "what", "is", "living room", "hear", "in", "room", "that", "a"]}, {"sent": "dont take the birds into the living room .", "words": ["the", "into", "living room", "dont", "room", "take"]}, {"sent": "I dont know where her living room is .", "words": ["where", "is", "living room", "dont", "room", "her"]}, {"sent": "okay lets go in the living room .", "words": ["the", "go", "living room", "in", "room"]}, {"sent": "can you guess up you know what thats their living room .", "words": ["can", "what", "up", "their", "living room", "you", "room"]}, {"sent": "in the living room .", "words": ["the", "in", "room", "living room"]}, {"sent": "that magnets in the living room ?", "words": ["the", "living room", "in", "room", "that"]}, {"sent": "you can sit wherever you want maybe in the living room or up here whatever you feel more comfortable .", "words": ["can", "the", "up", "living room", "you", "here", "in", "room", "more", "sit"]}, {"sent": "you belong in the living room .", "words": ["the", "living room", "you", "in", "room"]}, {"sent": "one pretzel and then you go in the living room okay ?", "words": ["the", "go", "pretzel", "living room", "you", "and", "in", "room", "then"]}, {"sent": "go and put it in the living room .", "words": ["the", "put", "go", "living room", "and", "in", "it", "room"]}, {"sent": "xxx oo go in the living room and play .", "words": ["the", "go", "play", "living room", "and", "in", "room"]}, {"sent": "lets go back in the living room okay .", "words": ["the", "back", "go", "living room", "in", "room"]}, {"sent": "his friends sitting in the living room all alone waiting for the baby yeah .", "words": ["the", "all", "for", "living room", "in", "room", "his"]}, {"sent": "Father Cat vacuums the living room .", "words": ["the", "living room", "room"]}, {"sent": "you want me to make a big rug for the living room and another big rug for out in the hallway ?", "words": ["the", "for", "to", "living room", "you", "and", "another", "big", "room", "in", "out", "me", "make", "a"]}, {"sent": "in my pocketbook out in the living room .", "words": ["the", "my", "living room", "in", "room", "out"]}, {"sent": "does it go in the living room or the bedroom ?", "words": ["the", "does", "go", "living room", "in", "it", "room", "bedroom"]}, {"sent": "how about in the living room ?", "words": ["the", "how", "living room", "in", "room", "about"]}, {"sent": "do you remember the picture thats in the living room ?", "words": ["the", "do", "picture", "living room", "you", "in", "room"]}, {"sent": "in the living room ?", "words": ["the", "in", "room", "living room"]}, {"sent": "whose living room ?", "words": ["living room", "room"]}, {"sent": "lets go out into the living room .", "words": ["the", "into", "go", "living room", "room", "out"]}, {"sent": "then well come back and straighten up the living room .", "words": ["up", "the", "back", "living room", "and", "room", "then"]}, {"sent": "our televisions in the living room thats right .", "words": ["the", "living room", "our", "in", "room"]}, {"sent": "where are you going where you staying in here green room living room ?", "words": ["where", "living room", "you", "here", "in", "are", "room", "green"]}, {"sent": "is it in the living room ?", "words": ["the", "is", "living room", "in", "it", "room"]}, {"sent": "you put the tiger in the living room ?", "words": ["the", "put", "living room", "you", "in", "room", "tiger"]}, {"sent": "because Im going to want my wee living room done up .", "words": ["up", "my", "to", "living room", "room", "because"]}, {"sent": "we both xxx in the living room there .", "words": ["the", "we", "living room", "there", "in", "room"]}, {"sent": "good very good okay well make this a living room then .", "words": ["living room", "this", "room", "good", "then", "make", "a"]}, {"sent": "in Ninas room in Mommys room or in the living room ?", "words": ["the", "in", "room", "living room"]}, {"sent": "I think its probably either in the living room or in Mommy and Daddys room .", "words": ["the", "think", "living room", "and", "in", "room"]}, {"sent": "its a couch like we have in our living room .", "words": ["we", "living room", "couch", "like", "our", "have", "in", "room", "a"]}, {"sent": "okay hold on with both hands both hands thats right xxx now take it to the living room thats right .", "words": ["the", "hold", "to", "living room", "with", "it", "room", "on", "take"]}, {"sent": "what else should go in the living room besides the sofa ?", "words": ["the", "what", "go", "living room", "in", "room", "sofa"]}, {"sent": "in the living room ?", "words": ["the", "in", "room", "living room"]}, {"sent": "in their living room .", "words": ["in", "room", "living room", "their"]}, {"sent": "in the living room .", "words": ["the", "in", "room", "living room"]}, {"sent": "the one in the back on the living room ?", "words": ["the", "back", "living room", "in", "room", "on"]}, {"sent": "Ive come to take you back in the living room .", "words": ["the", "back", "to", "living room", "you", "in", "room", "take"]}, {"sent": "Daddy is gonna go back living room and sit down .", "words": ["back", "go", "is", "living room", "and", "room", "sit", "down"]}, {"sent": "walk in her living room and say shit .", "words": ["living room", "her", "say", "and", "in", "room", "walk"]}, {"sent": "they go into the living room and papa bear sits in his chair and he starts to rock .", "words": ["the", "into", "go", "to", "living room", "chair", "and", "bear", "they", "in", "room", "he", "his", "rock"]}, {"sent": "you mean the man thats sitting in the living room ?", "words": ["the", "living room", "you", "in", "room"]}, {"sent": "until you should eat in the living room .", "words": ["the", "living room", "you", "eat", "in", "room"]}, {"sent": "this is why my living room stay so clean because they dont be in here .", "words": ["clean", "my", "is", "so", "be", "living room", "because", "here", "this", "they", "room", "dont", "in", "stay", "why"]}, {"sent": "I think its in the living room isnt it ?", "words": ["the", "think", "living room", "in", "room", "it"]}, {"sent": "you dont play football in the living room .", "words": ["the", "play", "living room", "you", "dont", "in", "room"]}, {"sent": "shall I bring Jennifer out to the uh living room ?", "words": ["the", "to", "living room", "bring", "room", "out"]}, {"sent": "take it to the living room .", "words": ["the", "to", "living room", "room", "it", "take"]}, {"sent": "lets go back in the living room fix the guitar in there .", "words": ["the", "back", "go", "fix", "living room", "there", "in", "room"]}, {"sent": "Matthew could you go into the living room with those ?", "words": ["the", "those", "into", "go", "living room", "with", "you", "room", "could"]}, {"sent": "oi in the living room come on .", "words": ["the", "living room", "in", "room", "on"]}, {"sent": "you can d do the living room while youre at it .", "words": ["can", "do", "the", "living room", "you", "room", "it", "at"]}, {"sent": "um we dont we dont have a living room but we could move our this and then we wouldnt we could move our table downstairs in the kitchen .", "words": ["the", "table", "we", "living room", "then", "our", "have", "dont", "this", "room", "and", "in", "kitchen", "could", "but", "a"]}, {"sent": "would you like to go back in the living room ?", "words": ["the", "back", "go", "to", "living room", "like", "you", "in", "room", "would"]}, {"sent": "see its the living room .", "words": ["the", "see", "living room", "room"]}, {"sent": "they set up a tent in the living room and they were sleeping in the tent .", "words": ["up", "the", "living room", "and", "they", "in", "room", "were", "a"]}, {"sent": "and heres the living room .", "words": ["the", "living room", "room", "and"]}, {"sent": "her father and I are both have like multiple laptops in the living room and so shes constantly scooting over to where like the power adapters are so .", "words": ["the", "where", "so", "to", "living room", "like", "and", "have", "in", "are", "room", "over", "her"]}, {"sent": "Pete dont lets not eat in the living room okay ?", "words": ["the", "living room", "eat", "dont", "in", "room", "not"]}, {"sent": "yeah lets leave it in the living room .", "words": ["the", "living room", "in", "it", "room"]}, {"sent": "okay wanna play here or in the living room ?", "words": ["the", "play", "living room", "here", "in", "room", "wanna"]}, {"sent": "ready to go back to the living room ?", "words": ["the", "back", "go", "to", "living room", "room"]}, {"sent": "Fabians going to the living room to play with .", "words": ["the", "to", "play", "with", "living room", "room"]}, {"sent": "whos in the living room ?", "words": ["the", "in", "room", "living room"]}, {"sent": "Im gonna get dinner ready so you play in the living room .", "words": ["the", "so", "play", "living room", "you", "get", "in", "room"]}, {"sent": "want ta go back in the living room to read the book ?", "words": ["the", "back", "go", "to", "living room", "in", "room", "read", "book"]}, {"sent": "oh in the living room ?", "words": ["the", "in", "room", "living room"]}, {"sent": "go out in the living room and write on the paper .", "words": ["the", "go", "paper", "living room", "write", "and", "in", "room", "out", "on"]}, {"sent": "and living room .", "words": ["living room", "room", "and"]}, {"sent": "xxx Mommy says that Pippo cant be far away and that they should look in the living room and thats where Pippo has been all the time .", "words": ["the", "where", "all", "be", "living room", "and", "they", "in", "away", "room", "look", "that"]}, {"sent": "we could make this the living room .", "words": ["the", "we", "living room", "this", "room", "could", "make"]}, {"sent": "and this is the living room .", "words": ["the", "is", "living room", "and", "this", "room"]}, {"sent": "lets take the fire engine in the living room .", "words": ["the", "living room", "in", "room", "take"]}, {"sent": "we are gonna walk back in the living room .", "words": ["the", "back", "we", "living room", "in", "are", "room", "walk"]}, {"sent": "back in the living room .", "words": ["the", "back", "living room", "in", "room"]}, {"sent": "do you wanna go in the living room and play with legos for a little while ?", "words": ["the", "do", "for", "go", "play", "with", "living room", "you", "and", "little", "in", "room", "wanna", "a"]}, {"sent": "they never in the living room anyway .", "words": ["the", "living room", "they", "in", "room"]}, {"sent": "Im going in the living room for awhile .", "words": ["the", "for", "living room", "in", "room"]}, {"sent": "do you wanna go in the living room and swing up ?", "words": ["the", "do", "up", "go", "swing", "living room", "you", "and", "in", "room", "wanna"]}, {"sent": "see that living room Cathy ?", "words": ["see", "living room", "room", "that"]}, {"sent": "listen .", "words": ["listen"]}, {"sent": "are you gonna listen ?", "words": ["are", "listen", "you"]}, {"sent": "because listen .", "words": ["listen", "because"]}, {"sent": "listen to Jenell listen .", "words": ["to", "listen"]}, {"sent": "because its what you listen to people talking with .", "words": ["what", "listen", "to", "with", "you", "because"]}, {"sent": "listen .", "words": ["listen"]}, {"sent": "listen Carl .", "words": ["listen"]}, {"sent": "nobody xxx nobody listen ?", "words": ["listen"]}, {"sent": "put your tongue in and listen .", "words": ["put", "your", "listen", "and", "in", "tongue"]}, {"sent": "well listen when am I seeing you again ?", "words": ["when", "am", "listen", "you"]}, {"sent": "listen .", "words": ["listen"]}, {"sent": "but we say listen because we listen to them .", "words": ["listen", "we", "to", "say", "them", "but", "because"]}, {"sent": "listen do you hear that ?", "words": ["do", "listen", "hear", "you", "that"]}, {"sent": "listen .", "words": ["listen"]}, {"sent": "now listen you cant the teacher said listen to me .", "words": ["the", "listen", "to", "you", "me"]}, {"sent": "listen .", "words": ["listen"]}, {"sent": "Martys gonna come out and say Oliver please put the microphone on I wanna listen to you .", "words": ["the", "put", "listen", "to", "say", "and", "you", "out", "wanna", "on"]}, {"sent": "listen .", "words": ["listen"]}, {"sent": "you fix your pants first then you can listen .", "words": ["can", "your", "listen", "fix", "first", "you", "pants", "then"]}, {"sent": "have a little listen and if its breaking up and if the sounds not properly uh .", "words": ["up", "the", "if", "listen", "and", "have", "little", "not", "a"]}, {"sent": "listen listen .", "words": ["listen"]}, {"sent": "listen .", "words": ["listen"]}, {"sent": "listen listen .", "words": ["listen"]}, {"sent": "did she listen to your chest ?", "words": ["your", "listen", "to", "did", "she"]}, {"sent": "pitter patter pitter patter listen to the rain .", "words": ["the", "to", "rain", "listen"]}, {"sent": "listen carefully .", "words": ["listen"]}, {"sent": "I dont listen to whining .", "words": ["to", "dont", "listen"]}, {"sent": "I know you dont want to listen to Gigis phone go beep beep beep .", "words": ["listen", "go", "to", "you", "dont", "want to"]}, {"sent": "Baxter listen is this your bear ?", "words": ["your", "listen", "is", "bear", "this"]}, {"sent": "yes listen to the noise .", "words": ["the", "to", "listen"]}, {"sent": "not like RiceKrispies when you listen is it ?", "words": ["listen", "is", "like", "you", "it", "when", "not"]}, {"sent": "well listen Pete .", "words": ["listen"]}, {"sent": "now listen .", "words": ["listen"]}, {"sent": "Laura listen .", "words": ["listen"]}, {"sent": "good morning said wasp politely listen .", "words": ["listen", "good"]}, {"sent": "let me have a listen .", "words": ["have", "me", "listen", "a"]}, {"sent": "listen I wanna talk to you .", "words": ["listen", "to", "you", "talk", "wanna"]}, {"sent": "you let people listen to it ?", "words": ["to", "it", "listen", "you"]}, {"sent": "but listen .", "words": ["listen", "but"]}, {"sent": "are you sure you wanna listen to it ?", "words": ["listen", "to", "you", "are", "it", "wanna"]}, {"sent": "lets just mix the milk in a little bit and then youll be able to listen to them .", "words": ["the", "listen", "be", "to", "them", "little", "and", "in", "milk", "then", "a"]}, {"sent": "listen .", "words": ["listen"]}, {"sent": "listen you know when your mommy take a picture of you she put you in a picture frame .", "words": ["put", "your", "when", "listen", "of", "picture", "you", "in", "take", "she", "a"]}, {"sent": "listen come see Daddy .", "words": ["see", "listen"]}, {"sent": "you hafta listen now .", "words": ["listen", "you"]}, {"sent": "listen Steven the playdough is for everyone to share .", "words": ["the", "for", "share", "listen", "is", "to"]}, {"sent": "and when they listen they can hear the fog horn and that tells them oh theres land there we better stay away .", "words": ["can", "the", "better", "listen", "we", "them", "and", "hear", "they", "that", "there", "away", "stay", "when"]}, {"sent": "you wanna listen and be part of it you can .", "words": ["can", "listen", "of", "be", "and", "you", "it", "wanna"]}, {"sent": "see what kind of story they might like to listen to .", "words": ["what", "listen", "of", "to", "like", "they", "see", "story"]}, {"sent": "listen to your Mummy and Daddy too much there .", "words": ["your", "listen", "to", "and", "there", "too", "much"]}, {"sent": "no listen we gotta start thinking about going to school Bobby .", "words": ["listen", "we", "to", "school", "about"]}, {"sent": "and you listen to it ?", "words": ["listen", "to", "you", "and", "it"]}, {"sent": "but listen lets talk about that book that you used to love that you .", "words": ["listen", "to", "you", "about", "talk", "book", "love", "that", "but"]}, {"sent": "Brodie listen Brodie listen .", "words": ["listen"]}, {"sent": "lets listen for the morning dove again .", "words": ["the", "for", "listen"]}, {"sent": "listen listen listen .", "words": ["listen"]}, {"sent": "what do you listen to sometimes before you go to bed ?", "words": ["what", "do", "listen", "go", "to", "bed", "you"]}, {"sent": "all I can suggest is that one day when AuntieMabels on and its the episode with the dentist we listen very carefully .", "words": ["can", "all", "the", "when", "listen", "is", "we", "with", "and", "on", "that"]}, {"sent": "alright listen I got you dont throw this you know you dont hafta throw balls around the house .", "words": ["the", "throw", "listen", "you", "dont", "this", "house", "around"]}, {"sent": "listen .", "words": ["listen"]}, {"sent": "listen let us get a dry diaper on you first .", "words": ["listen", "diaper", "dry", "first", "you", "get", "us", "on", "a"]}, {"sent": "we can listen to Lara on the tape and Ill prove it .", "words": ["can", "the", "listen", "we", "to", "tape", "and", "it", "on"]}, {"sent": "hey listen listen .", "words": ["listen"]}, {"sent": "listen .", "words": ["listen"]}, {"sent": "ooh listen .", "words": ["listen"]}, {"sent": "now listen .", "words": ["listen"]}, {"sent": "okay lets try to listen to more thing till mommy loses her mind .", "words": ["listen", "to", "her", "more", "try", "try to"]}, {"sent": "listen .", "words": ["listen"]}, {"sent": "listen .", "words": ["listen"]}, {"sent": "do you wanna listen ?", "words": ["do", "listen", "wanna", "you"]}, {"sent": "but we can listen to it later .", "words": ["can", "listen", "we", "to", "it", "but"]}, {"sent": "listen .", "words": ["listen"]}, {"sent": "did you listen to them ?", "words": ["listen", "to", "them", "you", "did"]}, {"sent": "you can listen to it later on .", "words": ["can", "listen", "to", "you", "it", "on"]}, {"sent": "listen to daddy ?", "words": ["to", "listen"]}, {"sent": "sit up for me and then you can listen to it .", "words": ["up", "can", "for", "listen", "to", "then", "and", "you", "it", "me", "sit"]}, {"sent": "listen .", "words": ["listen"]}, {"sent": "but his baba wont listen .", "words": ["his", "listen", "but"]}, {"sent": "listen .", "words": ["listen"]}, {"sent": "no kicking listen .", "words": ["listen"]}, {"sent": "if you listen very carefully you can hear something that sounds like the ocean .", "words": ["can", "the", "if", "listen", "like", "hear", "you", "that"]}, {"sent": "listen right now youre not getting dessert right now its going to be a while before its ready .", "words": ["listen", "to", "be", "not", "a"]}, {"sent": "listen .", "words": ["listen"]}, {"sent": "to listen to stories .", "words": ["to", "listen"]}, {"sent": "well lets listen well hear .", "words": ["listen", "hear"]}, {"sent": "listen to what Daddys saying .", "words": ["to", "what", "listen"]}, {"sent": "listen .", "words": ["listen"]}, {"sent": "xxx up there listen to me .", "words": ["up", "listen", "to", "there", "me"]}, {"sent": "listen give Daddy a kiss .", "words": ["give", "kiss", "listen", "a"]}, {"sent": "do you wanna listen to me with this ?", "words": ["do", "listen", "to", "with", "you", "this", "wanna", "me"]}, {"sent": "listen listen .", "words": ["listen"]}, {"sent": "listen .", "words": ["listen"]}, {"sent": "shes not shes not stopping stopping to listen to you telling her that you like her .", "words": ["listen", "to", "like", "you", "that", "her", "not"]}, {"sent": "listen to me for a minute .", "words": ["for", "listen", "to", "me", "a"]}, {"sent": "why dont you listen to me ?", "words": ["listen", "to", "you", "dont", "me", "why"]}, {"sent": "listen .", "words": ["listen"]}, {"sent": "otherwise Ill get cross with you because you said you were going to listen to me .", "words": ["listen", "to", "with", "you", "get", "me", "were", "because"]}, {"sent": "listen .", "words": ["listen"]}, {"sent": "listen darling can I tell you something ?", "words": ["can", "listen", "you"]}, {"sent": "red juice .", "words": ["red", "juice"]}, {"sent": "red ya .", "words": ["red"]}, {"sent": "its a red dog thats right .", "words": ["red", "dog", "a"]}, {"sent": "red blue .", "words": ["red", "blue"]}, {"sent": "put your red one in .", "words": ["red", "in", "put", "your"]}, {"sent": "red train .", "words": ["red", "train"]}, {"sent": "xxx red one missing .", "words": ["red"]}, {"sent": "and Ill hold the red one for you theres the blue one .", "words": ["the", "red", "for", "hold", "and", "you", "blue"]}, {"sent": "and theres only one red engine .", "words": ["red", "and"]}, {"sent": "that is the red one .", "words": ["the", "red", "that", "is"]}, {"sent": "and one red light there ?", "words": ["red", "light", "there", "and"]}, {"sent": "you called them Po buss because Po from Teletubbies was red and you couldnt say red .", "words": ["red", "was", "them", "say", "you", "and", "because"]}, {"sent": "they look very nice theyre red white and blue .", "words": ["red", "nice", "and", "blue", "they", "look", "white"]}, {"sent": "color with red .", "words": ["red", "with"]}, {"sent": "the conductor brings the baton down and the hall which is as wide and long as a red velvet football field .", "words": ["the", "red", "is", "long", "and", "down", "which", "a"]}, {"sent": "thats red .", "words": ["red"]}, {"sent": "red what colors the stop sign ?", "words": ["red", "what", "stop", "the"]}, {"sent": "or is it red ?", "words": ["red", "it", "is"]}, {"sent": "all red or different colors ?", "words": ["red", "all"]}, {"sent": "red boots ?", "words": ["red", "boots"]}, {"sent": "and well put in brackets po because thats what you call red .", "words": ["red", "what", "put", "and", "you", "in", "because"]}, {"sent": "Rudolph the red nosed reindeer ?", "words": ["the", "red"]}, {"sent": "brown orange green yellow black red blue pink .", "words": ["red", "brown", "blue", "black", "orange", "green", "yellow"]}, {"sent": "you want a red heart ?", "words": ["red", "a", "you"]}, {"sent": "mm red .", "words": ["red"]}, {"sent": "why is that red ?", "words": ["red", "that", "why", "is"]}, {"sent": "and the leaves are all brown and red and yellow and golden arent they ?", "words": ["the", "all", "red", "brown", "and", "they", "are", "yellow"]}, {"sent": "red bird red bird what do you see ?", "words": ["red", "what", "do", "bird", "you", "see"]}, {"sent": "thats not red .", "words": ["red", "not"]}, {"sent": "that ones red .", "words": ["red", "that"]}, {"sent": "the red one .", "words": ["the", "red"]}, {"sent": "thats a red light .", "words": ["red", "light", "a"]}, {"sent": "shall we put your red car .", "words": ["red", "put", "your", "we", "car"]}, {"sent": "no it is a different color than red .", "words": ["red", "it", "a", "is"]}, {"sent": "the last carriage is an elephant on a red background .", "words": ["the", "red", "is", "last", "elephant", "an", "on", "a"]}, {"sent": "look here comes the red car .", "words": ["the", "red", "car", "here", "look"]}, {"sent": "its red .", "words": ["red"]}, {"sent": "got red .", "words": ["red"]}, {"sent": "now it says color the triangles red .", "words": ["the", "it", "red"]}, {"sent": "Im gonna roll the red ball .", "words": ["the", "ball", "red"]}, {"sent": "hes a red doggy ?", "words": ["red", "a"]}, {"sent": "and the red balloon .", "words": ["the", "red", "balloon", "and"]}, {"sent": "we should put that red one away someplace else because it really doesnt belong with those others .", "words": ["red", "put", "those", "we", "with", "away", "it", "that", "because"]}, {"sent": "could I put some red in ?", "words": ["red", "put", "in", "some", "could"]}, {"sent": "in a store with a big red k .", "words": ["red", "store", "with", "in", "big", "a"]}, {"sent": "one morning little red riding hoods mother asked her to take a basket of fruit to her Grandma who was in bed and not feeling very well .", "words": ["red", "of", "to", "was", "her", "bed", "little", "and", "in", "basket", "who", "take", "not", "a"]}, {"sent": "wheres the red one go ?", "words": ["the", "red", "go"]}, {"sent": "theres another red one .", "words": ["another", "red"]}, {"sent": "dyou wanna open up this red ?", "words": ["up", "red", "open", "this", "wanna"]}, {"sent": "does everything look red ?", "words": ["red", "does", "look"]}, {"sent": "a red onell do .", "words": ["red", "do", "a"]}, {"sent": "so we have a lot of red and blue things .", "words": ["red", "we", "of", "so", "a lot", "and", "have", "blue", "a"]}, {"sent": "can you see the red pot ?", "words": ["can", "the", "red", "you", "see"]}, {"sent": "it was yellow and red wasnt it ?", "words": ["red", "was", "and", "it", "yellow"]}, {"sent": "oh I dont think itll break but how about we put it back underneath the table in the red bin ?", "words": ["the", "red", "think", "how", "put", "table", "back", "we", "dont", "in", "it", "about", "break", "but"]}, {"sent": "Mary had a red dress .", "words": ["red", "dress", "a"]}, {"sent": "red .", "words": ["red"]}, {"sent": "okay see she has white shoes with red tassels on them .", "words": ["red", "with", "them", "on", "see", "white", "she"]}, {"sent": "do you think this is her little red hat ?", "words": ["red", "do", "think", "is", "you", "little", "this", "hat", "her"]}, {"sent": "two red roses .", "words": ["red"]}, {"sent": "looks a bit like a red apple though uhn doesnt it ?", "words": ["red", "like", "it", "apple", "a"]}, {"sent": "blue yellow green red white .", "words": ["red", "blue", "green", "yellow", "white"]}, {"sent": "red for pink .", "words": ["red", "for"]}, {"sent": "red lights and blue lights .", "words": ["red", "blue", "and"]}, {"sent": "can you find a red crayon ?", "words": ["can", "red", "crayon", "you", "find", "a"]}, {"sent": "a red door .", "words": ["red", "door", "a"]}, {"sent": "the red one has gone .", "words": ["the", "red"]}, {"sent": "red .", "words": ["red"]}, {"sent": "and whats red yellow and green ?", "words": ["red", "yellow", "green", "and"]}, {"sent": "red milk ?", "words": ["red", "milk"]}, {"sent": "well we dont have any white shoes but we have red shoes .", "words": ["red", "we", "any", "have", "dont", "white", "but"]}, {"sent": "well the it was a red van that said ParcelForce wasnt it ?", "words": ["the", "red", "was", "it", "that", "a"]}, {"sent": "it is your red .", "words": ["red", "it", "your", "is"]}, {"sent": "youve used all the red ones .", "words": ["the", "all", "red"]}, {"sent": "heres the red .", "words": ["the", "red"]}, {"sent": "put them in your red car .", "words": ["red", "put", "your", "car", "them", "in"]}, {"sent": "red tulips .", "words": ["red"]}, {"sent": "its red .", "words": ["red"]}, {"sent": "and the wolf replied its little red riding hood Grandma .", "words": ["the", "red", "wolf", "little", "and"]}, {"sent": "I gonna make a red piglet whos gonna eat up the icecream cone .", "words": ["red", "up", "the", "eat", "make", "a"]}, {"sent": "how about red ?", "words": ["red", "how", "about"]}, {"sent": "you hafta check that theres a green light an orange light and a red light dont you ?", "words": ["red", "light", "you", "and", "dont", "an", "orange", "green", "that", "a"]}, {"sent": "do you like his red car ?", "words": ["red", "do", "car", "like", "you", "his"]}, {"sent": "red pants .", "words": ["red", "pants"]}, {"sent": "one side of your face is very red Thomas .", "words": ["red", "your", "face", "is", "of"]}, {"sent": "yeah thats a red rose .", "words": ["red", "a"]}, {"sent": "red ?", "words": ["red"]}, {"sent": "this red heart means were xxx .", "words": ["red", "this", "were"]}, {"sent": "where do you think the other red one is ?", "words": ["where", "do", "think", "the", "red", "is", "you", "other"]}, {"sent": "oh hm see red peppers and green peppers .", "words": ["red", "see", "green", "and"]}, {"sent": "so shall we put a red .", "words": ["red", "put", "we", "so", "a"]}, {"sent": "because look hes got a red tummy .", "words": ["red", "look", "tummy", "because", "a"]}, {"sent": "well you find the red one .", "words": ["the", "red", "find", "you"]}, {"sent": "can you find a red star ?", "words": ["can", "red", "star", "you", "find", "a"]}, {"sent": "well you get some red wine and you drop one of these .", "words": ["red", "these", "of", "and", "get", "you", "drop", "some"]}, {"sent": "is that Annas car the red one ?", "words": ["the", "red", "is", "car", "that"]}, {"sent": "red this is a red one .", "words": ["red", "this", "a", "is"]}, {"sent": "it goes in the red one .", "words": ["the", "in", "it", "red"]}, {"sent": "red brick .", "words": ["red"]}, {"sent": "there was an old black cat .", "words": ["old", "was", "black", "there", "an", "cat"]}, {"sent": "how old are you ?", "words": ["old", "how", "are", "you"]}, {"sent": "an old bottle that you insisted we brought back from the restaurant with us the cafe and somehow its ended up in the packaging with the nativity set .", "words": ["old", "the", "up", "back", "we", "with", "you", "and", "us", "an", "in", "bottle", "that"]}, {"sent": "oh and heres a trash can because we hafta get rid of the old diaper .", "words": ["can", "the", "old", "we", "of", "diaper", "trash", "and", "get", "because", "a"]}, {"sent": "its a big wagon wheel it looks very old .", "words": ["old", "it", "big", "a"]}, {"sent": "he called the old woman .", "words": ["the", "he", "old"]}, {"sent": "just an old wood stove .", "words": ["old", "an", "stove"]}, {"sent": "you have one old message .", "words": ["have", "old", "you"]}, {"sent": "its too old for you .", "words": ["old", "for", "you", "too"]}, {"sent": "the cat is old and slow .", "words": ["the", "old", "is", "slow", "and", "cat"]}, {"sent": "there might be an old pair of trousers in there .", "words": ["old", "of", "be", "there", "an", "in"]}, {"sent": "Ive a couple of old bowls that would do salad if you want them .", "words": ["old", "do", "if", "of", "them", "you", "would", "that", "a"]}, {"sent": "well my mummie lent me an old one she had .", "words": ["old", "my", "an", "me", "she"]}, {"sent": "those are Deedees old shoes .", "words": ["old", "are", "those"]}, {"sent": "Daddys getting old uhhuh .", "words": ["old"]}, {"sent": "silly old pudding isnt he ?", "words": ["old", "pudding", "he"]}, {"sent": "Im eight years old and my best friend look at all the necklaces shes wearing .", "words": ["old", "all", "my", "the", "and", "at", "look"]}, {"sent": "so old .", "words": ["old", "so"]}, {"sent": "an old fashioned radio .", "words": ["old", "an", "radio"]}, {"sent": "old and new .", "words": ["old", "new", "and"]}, {"sent": "youre glad they did that because it was an old fashioned police station with beds with cover ?", "words": ["old", "was", "with", "they", "an", "it", "did", "cover", "that", "because"]}, {"sent": "how old are you Gabriel ?", "words": ["old", "how", "are", "you"]}, {"sent": "old Macdonald had a farm ee ai eeai .", "words": ["old", "a"]}, {"sent": "oh theyre old .", "words": ["old"]}, {"sent": "do you know how old she is ?", "words": ["old", "do", "how", "is", "you", "she"]}, {"sent": "and there by the river was the sly old fox .", "words": ["the", "old", "was", "by", "and", "there"]}, {"sent": "youre two years old arent you ?", "words": ["old", "you"]}, {"sent": "its a very old one .", "words": ["old", "a"]}, {"sent": "old man is .", "words": ["old", "is"]}, {"sent": "all weve got to do now is put all these old leaves into the bin .", "words": ["old", "all", "do", "put", "these", "into", "the", "is", "to"]}, {"sent": "oh poor old you .", "words": ["old", "you", "poor"]}, {"sent": "poohcheesy old feet .", "words": ["old"]}, {"sent": "no not until you get to be an old man .", "words": ["old", "to", "be", "you", "get", "an", "not"]}, {"sent": "your daddy says Adams bikes an old wreck .", "words": ["old", "an", "your"]}, {"sent": "xxx the old one is good .", "words": ["the", "old", "good", "is"]}, {"sent": "I ate the pot and the gruel and the old woman too .", "words": ["the", "too", "old", "and"]}, {"sent": "how old is the horsie ?", "words": ["old", "the", "how", "is"]}, {"sent": "thats how big Amys hand was at six months and how big her foot was a six months old .", "words": ["old", "how", "hand", "was", "her", "and", "foot", "at", "big", "a"]}, {"sent": "your old Moms gonna build a tower .", "words": ["old", "your", "build", "a"]}, {"sent": "and how old was Thomas last week ?", "words": ["old", "how", "was", "last", "and"]}, {"sent": "how old are you ?", "words": ["old", "how", "are", "you"]}, {"sent": "around and around old Joe xxx round and around I say round and around old Joe barn I dont have time to stay .", "words": ["old", "to", "say", "and", "have", "dont", "around", "stay"]}, {"sent": "one quiet old lady whispering hush .", "words": ["old", "quiet"]}, {"sent": "somebodys made it old ?", "words": ["old", "it"]}, {"sent": "like the grand old Duke of York .", "words": ["the", "of", "old", "like"]}, {"sent": "Zoey okay and how old is Zoey ?", "words": ["old", "how", "is", "and"]}, {"sent": "a two year old a five year old and a seven year old to .", "words": ["old", "to", "and", "a"]}, {"sent": "to a very special three year old Thomas .", "words": ["old", "to", "a"]}, {"sent": "old times there are not forgotten .", "words": ["old", "there", "are", "not"]}, {"sent": "how old are you ?", "words": ["old", "how", "are", "you"]}, {"sent": "you old water head .", "words": ["old", "head", "water", "you"]}, {"sent": "just shoving him in any old how wont mean that hes going to stand up you know .", "words": ["old", "up", "how", "to", "stand", "any", "you", "in", "him", "that"]}, {"sent": "old hat .", "words": ["old", "hat"]}, {"sent": "do you know how old your sister is ?", "words": ["old", "do", "how", "your", "is", "you"]}, {"sent": "two years old .", "words": ["old"]}, {"sent": "thats pretty old theres another purple over here see it ?", "words": ["old", "here", "another", "it", "pretty", "see", "over"]}, {"sent": "when you were about a year old .", "words": ["old", "you", "about", "when", "were", "a"]}, {"sent": "well thats your old changing mat .", "words": ["old", "your"]}, {"sent": "I told you it was a old cookie .", "words": ["old", "cookie", "was", "you", "it", "a"]}, {"sent": "perhaps its a little bit old for you still .", "words": ["old", "for", "you", "little", "a"]}, {"sent": "three years old ?", "words": ["old"]}, {"sent": "and he saw the old castle .", "words": ["the", "he", "old", "and"]}, {"sent": "with a knickknack paddywack give a dog a bone this old man came rolling home .", "words": ["old", "with", "this", "home", "give", "dog", "a"]}, {"sent": "how old are you ?", "words": ["old", "how", "are", "you"]}, {"sent": "theyre old curtains .", "words": ["old"]}, {"sent": "a little a old school oven .", "words": ["old", "school", "little", "oven", "a"]}, {"sent": "do you know how old you are ?", "words": ["old", "do", "how", "you", "are"]}, {"sent": "poor old them .", "words": ["old", "them", "poor"]}, {"sent": "how old is mummy ?", "words": ["old", "how", "is"]}, {"sent": "dis xxx man this is an old game but it lits .", "words": ["old", "game", "is", "this", "an", "it", "but"]}, {"sent": "they were a week old and one died .", "words": ["old", "and", "they", "were", "a"]}, {"sent": "really old .", "words": ["old"]}, {"sent": "how old are you ?", "words": ["old", "how", "are", "you"]}, {"sent": "how old will you be on your birthday Nicole ?", "words": ["old", "how", "your", "be", "you", "on", "will"]}, {"sent": "poor old daddy .", "words": ["old", "poor"]}, {"sent": "whatever old lady whispering hush .", "words": ["old"]}, {"sent": "silly old Rabbit .", "words": ["old"]}, {"sent": "he plays one he plays one on his old drum xxx .", "words": ["on", "his", "he", "old"]}, {"sent": "three and a half years old .", "words": ["old", "a", "and"]}, {"sent": "these are very old fashioned trains arent they ?", "words": ["old", "these", "are", "they"]}, {"sent": "amongst all the old dark trees .", "words": ["the", "all", "old", "dark"]}, {"sent": "why dont we play the old lady and the alligator ?", "words": ["the", "old", "we", "play", "and", "dont", "alligator", "why"]}, {"sent": "how old are you today Laura ?", "words": ["old", "how", "are", "you"]}, {"sent": "now thats twice as big and old as you .", "words": ["old", "big", "you", "and"]}, {"sent": "do you think its a new one or part of an old one ?", "words": ["old", "do", "think", "of", "you", "an", "new", "a"]}, {"sent": "well Im not a big old lady so thats alright .", "words": ["old", "so", "big", "not", "a"]}, {"sent": "have you told Jeannine how old you are ?", "words": ["old", "how", "you", "have", "are"]}, {"sent": "when Mummys teddy bear wasnt quite so old teddy uh Thomas .", "words": ["old", "so", "when", "bear"]}, {"sent": "youve made a rare old mess .", "words": ["old", "a"]}, {"sent": "thats an old one somebody gave us .", "words": ["old", "us", "an"]}, {"sent": "and here comes the tractor afterwards bringing poor old Harry .", "words": ["the", "old", "poor", "here", "and", "tractor"]}, {"sent": "isnt Daddy old ?", "words": ["old"]}, {"sent": "here lets see if any of the other ones work because this one this is also one that xxx it just got old for some reason at the store .", "words": ["the", "old", "for", "work", "if", "is", "of", "store", "any", "other", "here", "this", "it", "at", "see", "that", "because", "some"]}, {"sent": "got all Hannahs old rubber pants out .", "words": ["old", "all", "pants", "out"]}, {"sent": "this is an old book .", "words": ["old", "is", "this", "an", "book"]}, {"sent": "xxx old duckling .", "words": ["old"]}, {"sent": "now shes picking up the scraps from an old magazine .", "words": ["up", "the", "an", "old"]}, {"sent": "silly old monkey .", "words": ["old", "monkey"]}, {"sent": "so she just stood in the canal and then pretty soon an old raft came along .", "words": ["the", "old", "so", "and", "in", "an", "pretty", "then", "she"]}, {"sent": "and how old are you gonna be ?", "words": ["old", "how", "be", "you", "and", "are"]}, {"sent": "I am going to put you to work today .", "words": ["put", "work", "to", "you", "am"]}, {"sent": "I am afraid Ive been nothing but trouble for you lately P S .", "words": ["for", "am", "but", "you"]}, {"sent": "I am making piles .", "words": ["am"]}, {"sent": "I am I am going to put the dry macaroni in the pot yeah .", "words": ["the", "put", "to", "dry", "in", "am"]}, {"sent": "why am I doing it very hard ?", "words": ["hard", "it", "am", "why"]}, {"sent": "I didnt even think of that because I was like oh man Im gonna be away all weekend how am I gonna get anything done xxx .", "words": ["all", "think", "how", "of", "was", "be", "like", "get", "away", "am", "that", "because"]}, {"sent": "so am I how am I sposta call you Thomas ?", "words": ["so", "how", "am", "you"]}, {"sent": "I am ten .", "words": ["am"]}, {"sent": "I am ?", "words": ["am"]}, {"sent": "I Grover the football player am gonna run for the football on Sesame Street Cyclone team .", "words": ["the", "for", "run", "am", "on"]}, {"sent": "I know thats what I am .", "words": ["what", "am"]}, {"sent": "I am ?", "words": ["am"]}, {"sent": "how am I sposta ow ?", "words": ["how", "am"]}, {"sent": "what am I doing right now ?", "words": ["what", "am"]}, {"sent": "a little bit am I ?", "words": ["am", "little", "a"]}, {"sent": "am I right is it you ?", "words": ["it", "am", "you", "is"]}, {"sent": "I am not .", "words": ["not", "am"]}, {"sent": "I am not waking you up because you are not sleeping .", "words": ["up", "you", "are", "am", "not", "because"]}, {"sent": "Im not really going to eat it am I ?", "words": ["to", "eat", "it", "am", "not"]}, {"sent": "I am a Darlik .", "words": ["am", "a"]}, {"sent": "*AD1: I am f two .", "words": ["am"]}, {"sent": "I am .", "words": ["am"]}, {"sent": "I am too .", "words": ["am", "too"]}, {"sent": "am I your friend and your sweetheart ?", "words": ["your", "am", "and"]}, {"sent": "I am so proud I can dress myself .", "words": ["can", "so", "am", "myself", "dress"]}, {"sent": "here I am said the dinosaur .", "words": ["here", "the", "am"]}, {"sent": "I am cleaning you up .", "words": ["up", "am", "you"]}, {"sent": "who am I ?", "words": ["am", "who"]}, {"sent": "I am not going to do the piggies now .", "words": ["the", "do", "to", "am", "not"]}, {"sent": "youre on candid camera here so am I .", "words": ["camera", "so", "here", "am", "on"]}, {"sent": "I am not .", "words": ["not", "am"]}, {"sent": "Im the fireman on the train am I ?", "words": ["the", "on", "am", "train"]}, {"sent": "I am doing it .", "words": ["it", "am"]}, {"sent": "tsh too noisy I am enjoying my quiet with no big kids .", "words": ["my", "with", "noisy", "am", "quiet", "too", "big"]}, {"sent": "am I still the queen ?", "words": ["the", "am"]}, {"sent": "what am I doing now ?", "words": ["what", "am"]}, {"sent": "well why am I sitting over here ?", "words": ["here", "over", "am", "why"]}, {"sent": "I am the Im Im a baby brother .", "words": ["the", "am", "a"]}, {"sent": "am I ?", "words": ["am"]}, {"sent": "am I very small ?", "words": ["am"]}, {"sent": "what am I gonna do with it ?", "words": ["what", "do", "with", "it", "am"]}, {"sent": "oh am I a baby ?", "words": ["am", "a"]}, {"sent": "yeah I am .", "words": ["am"]}, {"sent": "I am very happy .", "words": ["am", "happy"]}, {"sent": "okay am I good to go ?", "words": ["to", "am", "good", "go"]}, {"sent": "what are you doing what am I doing ?", "words": ["what", "are", "am", "you"]}, {"sent": "am I ?", "words": ["am"]}, {"sent": "I probably am .", "words": ["am"]}, {"sent": "I am packing .", "words": ["am"]}, {"sent": "I am going to miss you all very very much .", "words": ["all", "to", "you", "am", "much"]}, {"sent": "what am I gonna put him in .", "words": ["what", "put", "in", "him", "am"]}, {"sent": "I am sitting where I xxx .", "words": ["where", "am"]}, {"sent": "tell me again what am I doing ?", "words": ["me", "what", "am"]}, {"sent": "I am cheeky arent I ?", "words": ["am"]}, {"sent": "so am I .", "words": ["so", "am"]}, {"sent": "am I ?", "words": ["am"]}, {"sent": "I am off again .", "words": ["off", "am"]}, {"sent": "when I am four Ill ride a big .", "words": ["when", "ride", "am", "big", "a"]}, {"sent": "I am .", "words": ["am"]}, {"sent": "like um to to ride uh I am riding like I am riding a bicycle .", "words": ["to", "like", "bicycle", "am", "ride", "a"]}, {"sent": "I am a policewoman with a job to do .", "words": ["do", "to", "with", "am", "a"]}, {"sent": "am I meeting you at the theater ?", "words": ["the", "at", "am", "you"]}, {"sent": "I am getting another book .", "words": ["another", "book", "am"]}, {"sent": "am I Sally ?", "words": ["am"]}, {"sent": "here I am .", "words": ["here", "am"]}, {"sent": "Bernice am I looking at the cards .", "words": ["the", "am", "at"]}, {"sent": "who am I ?", "words": ["am", "who"]}, {"sent": "am I talking to myself ?", "words": ["to", "am", "myself"]}, {"sent": "I am sorry darling .", "words": ["am"]}, {"sent": "which man am I hiding ?", "words": ["which", "am"]}, {"sent": "here I am .", "words": ["here", "am"]}, {"sent": "I am a little bunny .", "words": ["bunny", "am", "little", "a"]}, {"sent": "oh what am I getting ?", "words": ["what", "am"]}, {"sent": "what am I sposta do with all these huh ?", "words": ["all", "what", "do", "these", "with", "am"]}, {"sent": "which one am I missing ?", "words": ["which", "am"]}, {"sent": "I am being the doctor .", "words": ["the", "am"]}, {"sent": "I am the queen she told Vik .", "words": ["the", "am", "she"]}, {"sent": "oh Im MissusGoggins now am I ?", "words": ["am"]}, {"sent": "am I missing pages ?", "words": ["am"]}, {"sent": "what am I what am I drawing now ?", "words": ["what", "am"]}, {"sent": "I am find her .", "words": ["her", "am", "find"]}, {"sent": "just am .", "words": ["am"]}, {"sent": "where am I going to sit ?", "words": ["where", "to", "sit", "am"]}, {"sent": "I always am .", "words": ["am"]}, {"sent": "I am feeling that .", "words": ["that", "am"]}, {"sent": "what am I called then ?", "words": ["what", "then", "am"]}, {"sent": "what am I doing with them ?", "words": ["what", "with", "am", "them"]}, {"sent": "am I going to the party with you ?", "words": ["the", "to", "with", "party", "you", "am"]}, {"sent": "for now I am a beautiful butterfly after all .", "words": ["butterfly", "all", "for", "am", "a"]}, {"sent": "am I a funny booby ?", "words": ["am", "a"]}, {"sent": "am I WyeBird ?", "words": ["am"]}, {"sent": "I am happy .", "words": ["am", "happy"]}, {"sent": "I am .", "words": ["am"]}, {"sent": "Im a dinosaur am I ?", "words": ["am", "a"]}, {"sent": "why am I a beany beany beany ?", "words": ["am", "why", "a"]}, {"sent": "am I in the way ?", "words": ["the", "in", "am"]}, {"sent": "I am I am making a tunnel .", "words": ["am", "a"]}, {"sent": "I am Spiderman Im Spiderman .", "words": ["am"]}, {"sent": "am I lost and alone ?", "words": ["am", "and"]}, {"sent": "clock .", "words": ["clock"]}, {"sent": "ticktock thats a clock .", "words": ["clock", "a"]}, {"sent": "I want you to stop knocking that clock off of here .", "words": ["off", "clock", "of", "to", "stop", "you", "here", "that"]}, {"sent": "on top of the clock .", "words": ["on", "of", "clock", "the"]}, {"sent": "he can sound like a clock .", "words": ["can", "clock", "like", "he", "a"]}, {"sent": "and a clock .", "words": ["clock", "a", "and"]}, {"sent": "lets have a look at the clock .", "words": ["the", "clock", "have", "at", "look", "a"]}, {"sent": "see the clock ?", "words": ["the", "see", "clock"]}, {"sent": "where does the clock work ?", "words": ["where", "does", "the", "work", "clock"]}, {"sent": "he can sound like a clock .", "words": ["can", "clock", "like", "he", "a"]}, {"sent": "an alarm clock ?", "words": ["an", "clock"]}, {"sent": "wheres the clock ?", "words": ["the", "clock"]}, {"sent": "look at the hands its a clock ?", "words": ["the", "look", "clock", "at", "a"]}, {"sent": "they made a clock .", "words": ["clock", "they", "a"]}, {"sent": "and played with the clock a little bit .", "words": ["the", "clock", "with", "and", "little", "a"]}, {"sent": "yeah first of all theres a big clock in the middle .", "words": ["the", "all", "clock", "of", "first", "in", "big", "a"]}, {"sent": "a clock .", "words": ["clock", "a"]}, {"sent": "and that clock .", "words": ["clock", "that", "and"]}, {"sent": "look Lucas look honey this is a clock .", "words": ["clock", "is", "this", "look", "a"]}, {"sent": "can you see the hands moving on the clock ?", "words": ["can", "the", "clock", "you", "on", "see"]}, {"sent": "clock ?", "words": ["clock"]}, {"sent": "Tweenie clock whir whir stop .", "words": ["stop", "clock"]}, {"sent": "on the clock face ?", "words": ["on", "the", "face", "clock"]}, {"sent": "this clock has eyes they go up and down dont they ?", "words": ["up", "go", "clock", "and", "this", "they", "dont", "down"]}, {"sent": "yeah you bring the cards and Mamall play with the clock .", "words": ["the", "clock", "play", "with", "bring", "and", "you"]}, {"sent": "the clock ?", "words": ["the", "clock"]}, {"sent": "clock .", "words": ["clock"]}, {"sent": "clock .", "words": ["clock"]}, {"sent": "because it was quarter past five this morning when our alarm clock went off .", "words": ["off", "clock", "was", "our", "this", "it", "when", "because"]}, {"sent": "and a clock you like the clock again ?", "words": ["the", "clock", "like", "you", "and", "a"]}, {"sent": "you know that clock in Mikes room .", "words": ["clock", "you", "in", "room", "that"]}, {"sent": "is that clock ?", "words": ["clock", "that", "is"]}, {"sent": "clock what would you do with the clock ?", "words": ["the", "what", "do", "clock", "with", "you", "would"]}, {"sent": "Grannys Grannys clock .", "words": ["clock"]}, {"sent": "clock ?", "words": ["clock"]}, {"sent": "put a clock in the middle .", "words": ["the", "put", "clock", "in", "a"]}, {"sent": "a clock yeah .", "words": ["clock", "a"]}, {"sent": "you want that clock ?", "words": ["clock", "that", "you"]}, {"sent": "the Tweenies clock where will it stop ?", "words": ["the", "where", "clock", "stop", "it", "will"]}, {"sent": "you remember the clock at home ?", "words": ["the", "clock", "you", "at", "home"]}, {"sent": "oh its a clock .", "words": ["clock", "a"]}, {"sent": "the clock I dont .", "words": ["the", "dont", "clock"]}, {"sent": "clock .", "words": ["clock"]}, {"sent": "the jigsaw clock ?", "words": ["the", "clock"]}, {"sent": "it reminds me of the Tesco clock tower does that Thomas .", "words": ["the", "does", "clock", "of", "it", "me", "that"]}, {"sent": "excuse me clock .", "words": ["me", "clock"]}, {"sent": "hah thats a clock .", "words": ["clock", "a"]}, {"sent": "yeah the clock and Humptydumpty are missing .", "words": ["the", "are", "and", "clock"]}, {"sent": "yeah that goes on top of the clock I think .", "words": ["the", "think", "clock", "of", "on", "that"]}, {"sent": "thats a boy holding a clock .", "words": ["clock", "a"]}, {"sent": "yup that was a clock .", "words": ["clock", "was", "that", "a"]}, {"sent": "how does the clock .", "words": ["the", "does", "how", "clock"]}, {"sent": "yes so well put the clock upstairs with the bed .", "words": ["the", "put", "clock", "so", "with", "bed"]}, {"sent": "you want the clock ?", "words": ["the", "clock", "you"]}, {"sent": "but either way it has got a spire and a clock .", "words": ["clock", "and", "it", "but", "a"]}, {"sent": "the clock struck .", "words": ["the", "clock"]}, {"sent": "do you want this one about the clock ?", "words": ["the", "do", "clock", "you", "this", "about"]}, {"sent": "you want Daddy to do the clock ?", "words": ["the", "do", "clock", "to", "you"]}, {"sent": "oh you found the clock .", "words": ["the", "clock", "you"]}, {"sent": "I dont know if well fit a cow on the clock .", "words": ["the", "if", "clock", "fit", "dont", "on", "cow", "a"]}, {"sent": "oh a little clock there .", "words": ["clock", "there", "little", "a"]}, {"sent": "and the clock is interesting .", "words": ["the", "clock", "is", "and"]}, {"sent": "ooh heres the clock .", "words": ["the", "clock"]}, {"sent": "yes clock .", "words": ["clock"]}, {"sent": "see the clock ?", "words": ["the", "see", "clock"]}, {"sent": "clock .", "words": ["clock"]}, {"sent": "xxx the clock .", "words": ["the", "clock"]}, {"sent": "clock .", "words": ["clock"]}, {"sent": "a clock .", "words": ["clock", "a"]}, {"sent": "just as he gets into bed the alarm clock goes off .", "words": ["the", "off", "into", "clock", "bed", "he"]}, {"sent": "your clock ?", "words": ["your", "clock"]}, {"sent": "over the clock ?", "words": ["the", "over", "clock"]}, {"sent": "its a clock .", "words": ["clock", "a"]}, {"sent": "now lets get the clock at the right time .", "words": ["get", "the", "at", "clock"]}, {"sent": "say byebye clock .", "words": ["say", "clock"]}, {"sent": "and what does that Church clock say ?", "words": ["what", "does", "clock", "say", "and", "that"]}, {"sent": "I think we hafta do minor repairs on the happy clock here .", "words": ["the", "do", "think", "happy", "we", "clock", "here", "on"]}, {"sent": "but yeah then he went for the clock and then .", "words": ["the", "for", "clock", "and", "he", "then", "but"]}, {"sent": "the clock mhm .", "words": ["the", "clock"]}, {"sent": "clock ?", "words": ["clock"]}, {"sent": "of the clock ?", "words": ["of", "the", "clock"]}, {"sent": "ah that is a clock yeah .", "words": ["clock", "that", "a", "is"]}, {"sent": "a clock .", "words": ["clock", "a"]}, {"sent": "she picked up the clock .", "words": ["up", "the", "she", "clock"]}, {"sent": "yay its our clock .", "words": ["clock", "our"]}, {"sent": "and whats what can you see on there can you see a clock ?", "words": ["can", "what", "clock", "you", "and", "there", "on", "see", "a"]}, {"sent": "oh she have another clock .", "words": ["have", "she", "another", "clock"]}, {"sent": "Mummys alarm clock when it comes on its the radio isnt it ?", "words": ["the", "clock", "radio", "it", "on", "when"]}, {"sent": "and hes also saying clock properly now .", "words": ["clock", "and"]}, {"sent": "its a clock .", "words": ["clock", "a"]}, {"sent": "um can we leave that clock alone ?", "words": ["can", "clock", "that", "we"]}, {"sent": "hes holding a clock .", "words": ["clock", "a"]}, {"sent": "lets put the clock under the table .", "words": ["the", "put", "table", "clock", "under"]}, {"sent": "just like a clock ?", "words": ["clock", "like", "a"]}, {"sent": "that is a nice clock Abe .", "words": ["clock", "is", "nice", "that", "a"]}, {"sent": "the mans got to come and mend that clock hasnt he soon ?", "words": ["the", "clock", "to", "and", "he", "that"]}, {"sent": "wheres the clock ?", "words": ["the", "clock"]}, {"sent": "draw a clock cake this time .", "words": ["cake", "clock", "draw", "this", "a"]}, {"sent": "or hell find the clock and say clock and then hell kind of wait for me to find it .", "words": ["the", "wait", "for", "clock", "of", "to", "say", "and", "it", "find", "me", "then"]}, {"sent": "watch .", "words": ["watch"]}, {"sent": "watch your .", "words": ["watch", "your"]}, {"sent": "watch watch watch watch watch .", "words": ["watch"]}, {"sent": "xxx Lara and I will watch Toddy .", "words": ["will", "watch", "and"]}, {"sent": "turn this now wait a minute watch .", "words": ["this", "wait", "watch", "a"]}, {"sent": "watch the painting cant she ?", "words": ["the", "watch", "she"]}, {"sent": "just you watch this .", "words": ["this", "watch", "you"]}, {"sent": "when did you watch it ?", "words": ["watch", "you", "it", "did", "when"]}, {"sent": "didnt you watch any car programs then ?", "words": ["watch", "car", "any", "you", "then"]}, {"sent": "watch this .", "words": ["this", "watch"]}, {"sent": "watch your head .", "words": ["head", "watch", "your"]}, {"sent": "mhm watch out for Jenny .", "words": ["watch", "for", "out"]}, {"sent": "you can watch me .", "words": ["can", "me", "watch", "you"]}, {"sent": "have you watch this ?", "words": ["have", "this", "watch", "you"]}, {"sent": "watch hell move .", "words": ["watch"]}, {"sent": "watch .", "words": ["watch"]}, {"sent": "watch your feet .", "words": ["watch", "your"]}, {"sent": "watch you ?", "words": ["watch", "you"]}, {"sent": "I wanna watch Lucy .", "words": ["watch", "wanna"]}, {"sent": "watch .", "words": ["watch"]}, {"sent": "watch out for the .", "words": ["the", "watch", "for", "out"]}, {"sent": "watch out Dipsy ?", "words": ["watch", "out"]}, {"sent": "uh Alex watch your head xxx please watch your head .", "words": ["head", "watch", "your"]}, {"sent": "he wants to watch Bert .", "words": ["to", "watch", "he"]}, {"sent": "I wanna watch television .", "words": ["watch", "wanna"]}, {"sent": "Alexander watch .", "words": ["watch"]}, {"sent": "watch .", "words": ["watch"]}, {"sent": "watch the gum .", "words": ["the", "watch", "gum"]}, {"sent": "watch it then .", "words": ["it", "then", "watch"]}, {"sent": "for see when hes in your arms you hafta watch anything that he can grab .", "words": ["can", "for", "your", "watch", "you", "in", "that", "he", "see", "when"]}, {"sent": "watch it .", "words": ["it", "watch"]}, {"sent": "then you watch the news right ?", "words": ["the", "then", "watch", "you"]}, {"sent": "but I think its nice for you to go and watch other children playing isnt it ?", "words": ["think", "for", "watch", "go", "to", "nice", "you", "and", "other", "it", "but"]}, {"sent": "watch you dont break it darling .", "words": ["watch", "you", "dont", "it", "break"]}, {"sent": "watch this .", "words": ["this", "watch"]}, {"sent": "thats a childs watch .", "words": ["watch", "a"]}, {"sent": "you can watch thirteen .", "words": ["can", "watch", "you"]}, {"sent": "now you watch your feet .", "words": ["watch", "your", "you"]}, {"sent": "you watch the Thundercats ?", "words": ["the", "watch", "you"]}, {"sent": "watch this .", "words": ["this", "watch"]}, {"sent": "watch you dont hit your head on the chair darling .", "words": ["the", "watch", "your", "head", "chair", "you", "dont", "on", "hit"]}, {"sent": "wanna lets put him up there and watch him go for a ride ?", "words": ["up", "put", "watch", "for", "go", "and", "there", "him", "wanna", "ride", "a"]}, {"sent": "Ill watch you and tell you what to draw .", "words": ["what", "watch", "to", "draw", "and", "you"]}, {"sent": "here push the ball down with your hand and watch what happens .", "words": ["the", "what", "watch", "push", "your", "hand", "with", "and", "here", "ball", "down"]}, {"sent": "are you going to watch ?", "words": ["watch", "to", "are", "you"]}, {"sent": "squirt of that and you watch what happens now .", "words": ["what", "watch", "of", "you", "and", "that"]}, {"sent": "watch your head .", "words": ["head", "watch", "your"]}, {"sent": "xxx watch out sweet stuff watch your foot .", "words": ["foot", "watch", "your", "out"]}, {"sent": "okay watch .", "words": ["watch"]}, {"sent": "yeah I think she does watch alot like my husband usually puts her up on the counter with him as he cooks kind of shows her .", "words": ["up", "does", "think", "watch", "my", "the", "of", "with", "like", "him", "he", "on", "her", "she"]}, {"sent": "Mama watch this car .", "words": ["this", "watch", "car"]}, {"sent": "watch .", "words": ["watch"]}, {"sent": "watch .", "words": ["watch"]}, {"sent": "watch Daddy .", "words": ["watch"]}, {"sent": "and you can perhaps sit and watch Pippin or Po while Im doing that .", "words": ["can", "watch", "you", "and", "sit", "that"]}, {"sent": "watch what mommy is doing .", "words": ["what", "watch", "is"]}, {"sent": "watch watch carefully .", "words": ["watch"]}, {"sent": "Ill watch you .", "words": ["watch", "you"]}, {"sent": "watch .", "words": ["watch"]}, {"sent": "Summer you watch that .", "words": ["watch", "that", "you"]}, {"sent": "whats the whats the lady that we watch that cooks ?", "words": ["the", "watch", "that", "we"]}, {"sent": "Dexter watch this .", "words": ["this", "watch"]}, {"sent": "well just watch for a minute Boneeno .", "words": ["watch", "for", "a"]}, {"sent": "watch .", "words": ["watch"]}, {"sent": "you want watch ?", "words": ["watch", "you"]}, {"sent": "Mama watch .", "words": ["watch"]}, {"sent": "tonight do you wanna watch the football ?", "words": ["the", "do", "watch", "you", "wanna"]}, {"sent": "like this watch .", "words": ["this", "watch", "like"]}, {"sent": "watch the lead .", "words": ["the", "watch"]}, {"sent": "xxx shes been in that barrel a long time must be pretty tired there the cowll watch her .", "words": ["the", "watch", "be", "her", "long", "tired", "there", "in", "pretty", "that", "a"]}, {"sent": "you started wanting to sit an hour to watch television havent you ?", "words": ["watch", "to", "you", "an", "sit"]}, {"sent": "xxx were gonna go inside and watch something um .", "words": ["watch", "go", "inside", "and", "were"]}, {"sent": "what do you want Mummy to watch it oh right .", "words": ["what", "do", "watch", "to", "you", "it"]}, {"sent": "you watch .", "words": ["watch", "you"]}, {"sent": "you going to come and watch it ?", "words": ["watch", "to", "and", "you", "it"]}, {"sent": "would you like to go watch Daddy play ball ?", "words": ["watch", "go", "to", "play", "like", "you", "ball", "would"]}, {"sent": "now watch this .", "words": ["this", "watch"]}, {"sent": "come draw out here sothat I can watch you .", "words": ["can", "watch", "draw", "you", "here", "out"]}, {"sent": "what play did you watch ?", "words": ["what", "watch", "play", "you", "did"]}, {"sent": "watch the cord .", "words": ["the", "watch"]}, {"sent": "watch this .", "words": ["this", "watch"]}, {"sent": "oh heres a watch .", "words": ["watch", "a"]}, {"sent": "no were not gonna watch Barney again .", "words": ["watch", "were", "not"]}, {"sent": "whats a watch whatd you do with it ?", "words": ["do", "watch", "with", "you", "it", "a"]}, {"sent": "sure you wanna come watch me play ?", "words": ["watch", "play", "you", "wanna", "me"]}, {"sent": "if I could watch you as an old man tottering round with a walking stick .", "words": ["old", "stick", "watch", "if", "with", "you", "an", "could", "a"]}, {"sent": "what did we watch last night did we watch did we watch Sleeping Beauty last night ?", "words": ["what", "watch", "we", "last", "did"]}, {"sent": "because youve got all those cups to watch wash havent you ?", "words": ["all", "watch", "those", "to", "you", "wash", "because"]}, {"sent": "watch .", "words": ["watch"]}, {"sent": "watch this Wes .", "words": ["this", "watch"]}, {"sent": "hes got to be busy outside our street youve got to watch it really .", "words": ["watch", "to", "be", "outside", "our", "it", "street"]}, {"sent": "*SI1: ah lets watch that one ?", "words": ["watch", "that"]}, {"sent": "what did you watch ?", "words": ["what", "watch", "did", "you"]}, {"sent": "this is kinda neat watch .", "words": ["this", "watch", "is"]}, {"sent": "watch .", "words": ["watch"]}, {"sent": "are you going to watch me put them in now ?", "words": ["put", "watch", "to", "them", "you", "in", "are", "me"]}, {"sent": "can she sit on the bench and watch the kids play ?", "words": ["can", "the", "watch", "play", "and", "bench", "on", "sit", "she"]}, {"sent": "ah they like to watch the fire engine .", "words": ["the", "watch", "to", "like", "they"]}, {"sent": "watch .", "words": ["watch"]}, {"sent": "if I hold her there and you pull the string .", "words": ["the", "hold", "if", "and", "you", "there", "pull", "her"]}, {"sent": "um but she also really likes um anything thats soft that she can hold herself .", "words": ["can", "soft", "hold", "that", "she", "but"]}, {"sent": "do you wanna hold your microphone Honey ?", "words": ["do", "your", "hold", "you", "wanna"]}, {"sent": "I hold the baby .", "words": ["the", "hold"]}, {"sent": "you hold it like that .", "words": ["hold", "like", "you", "it", "that"]}, {"sent": "that is is that how you hold your pencil ?", "words": ["how", "your", "hold", "is", "pencil", "you", "that"]}, {"sent": "where you hold down somebodys feet and they get up .", "words": ["where", "up", "hold", "and", "you", "they", "get", "down"]}, {"sent": "well you hafta hold this train right here .", "words": ["hold", "train", "you", "here", "this"]}, {"sent": "do you wanna hold ?", "words": ["do", "hold", "wanna", "you"]}, {"sent": "yeah well we dont wanna hold that honey .", "words": ["hold", "we", "dont", "wanna", "that"]}, {"sent": "hold your belly in .", "words": ["in", "your", "hold"]}, {"sent": "if you hold it it wont spill .", "words": ["hold", "if", "you", "it", "spill"]}, {"sent": "hold baby .", "words": ["hold"]}, {"sent": "hold still .", "words": ["hold"]}, {"sent": "you can hold on to me .", "words": ["can", "hold", "to", "you", "on", "me"]}, {"sent": "shall I hold the doll while you put her pants on .", "words": ["the", "put", "hold", "doll", "you", "pants", "on", "her"]}, {"sent": "you wanna hold it .", "words": ["it", "hold", "wanna", "you"]}, {"sent": "you hold it there now .", "words": ["there", "it", "hold", "you"]}, {"sent": "hold tight BooBoos .", "words": ["hold"]}, {"sent": "hey hey you hold these Violet Violet you hold these look .", "words": ["these", "look", "hold", "you"]}, {"sent": "hold onto your baby .", "words": ["your", "hold"]}, {"sent": "you wanna hold the baby ?", "words": ["the", "hold", "wanna", "you"]}, {"sent": "hold on .", "words": ["on", "hold"]}, {"sent": "you can hold that one .", "words": ["can", "that", "hold", "you"]}, {"sent": "you want me to hold it up ?", "words": ["up", "hold", "to", "you", "it", "me"]}, {"sent": "want mommy to hold this .", "words": ["to", "hold", "this"]}, {"sent": "well go very slowly so you can go without any hold on to your hands .", "words": ["can", "your", "hold", "go", "so", "to", "any", "you", "on"]}, {"sent": "hold up .", "words": ["up", "hold"]}, {"sent": "okay you hold your box .", "words": ["box", "your", "hold", "you"]}, {"sent": "do you want his hold him ?", "words": ["do", "hold", "you", "him", "his"]}, {"sent": "Ill hold him down .", "words": ["him", "down", "hold"]}, {"sent": "oh hold on .", "words": ["on", "hold"]}, {"sent": "I dont hafta hold it up .", "words": ["up", "dont", "it", "hold"]}, {"sent": "you can hold the pink one .", "words": ["can", "the", "hold", "you"]}, {"sent": "dont hold it with your teeth Thomas .", "words": ["your", "hold", "with", "dont", "it"]}, {"sent": "shall Mummy hold the tractor and trailer ?", "words": ["the", "tractor", "hold", "and"]}, {"sent": "I think he loved to chew on it and hold it and .", "words": ["think", "hold", "to", "and", "it", "he", "on"]}, {"sent": "hold on there you .", "words": ["on", "there", "hold", "you"]}, {"sent": "can you hold the scissors properly darling .", "words": ["can", "the", "hold", "scissors", "you"]}, {"sent": "hold on .", "words": ["on", "hold"]}, {"sent": "no its not very strong well Ill hold it then .", "words": ["hold", "it", "then", "not"]}, {"sent": "I know you get cross sometimes when were walking and Mummy keeps saying hold onto my hand and dont go near the road .", "words": ["the", "my", "hold", "hand", "go", "and", "get", "you", "dont", "when", "were"]}, {"sent": "okay then you just need to hold it .", "words": ["need", "hold", "need to", "to", "you", "it", "then"]}, {"sent": "so you want me to hold it for you ?", "words": ["for", "hold", "to", "so", "you", "it", "me"]}, {"sent": "they wear skirts and they hold what ?", "words": ["they", "what", "hold", "and"]}, {"sent": "there ya go that is how you hold them .", "words": ["how", "hold", "is", "go", "them", "you", "there", "that"]}, {"sent": "why dont you hold it then .", "words": ["hold", "you", "dont", "it", "then", "why"]}, {"sent": "why do you hafta hold one ?", "words": ["do", "hold", "why", "you"]}, {"sent": "that is not how you hold that .", "words": ["how", "hold", "is", "you", "that", "not"]}, {"sent": "hold on momma gotta put some frosting on it .", "words": ["put", "hold", "it", "on", "some"]}, {"sent": "okay hold on .", "words": ["on", "hold"]}, {"sent": "okay hold on not all of them .", "words": ["all", "hold", "of", "them", "on", "not"]}, {"sent": "oh she wants to hold a piece of mail ?", "words": ["hold", "of", "to", "she", "a"]}, {"sent": "hold still one more minute .", "words": ["more", "hold"]}, {"sent": "hold on a minute said daddy laughing we dont know those things yet we can only guess itll be a surprise a few days later mike had another surprise aunt pat came to visit shes going to help us take care of the baby said daddy at supper mike asked wholl bring the baby ?", "words": ["can", "the", "help", "those", "hold", "we", "of", "be", "to", "bring", "another", "dont", "us", "at", "on", "take", "a"]}, {"sent": "well hold the microphone .", "words": ["the", "hold"]}, {"sent": "I can hold your drink if you want .", "words": ["can", "your", "hold", "drink", "if", "you"]}, {"sent": "hold it at the bottom .", "words": ["the", "bottom", "hold", "it", "at"]}, {"sent": "you hold it .", "words": ["it", "hold", "you"]}, {"sent": "hold up .", "words": ["up", "hold"]}, {"sent": "hold that up like that ?", "words": ["up", "that", "hold", "like"]}, {"sent": "I wanna hold him .", "words": ["him", "hold", "wanna"]}, {"sent": "well if I should come across it Ill hold it for you .", "words": ["for", "hold", "if", "you", "it"]}, {"sent": "they dont have a really good place to hold on a rattle I guess .", "words": ["hold", "to", "have", "dont", "they", "good", "on", "a"]}, {"sent": "here momma hold it do you wanna say hello ?", "words": ["do", "hold", "say", "you", "here", "it", "wanna"]}, {"sent": "youre gonna hold it ?", "words": ["it", "hold"]}, {"sent": "you hold it with two hands .", "words": ["it", "with", "hold", "you"]}, {"sent": "hold on there you go .", "words": ["hold", "go", "you", "there", "on"]}, {"sent": "you you hold the pencil properly first .", "words": ["the", "hold", "first", "pencil", "you"]}, {"sent": "Max hold on xxx .", "words": ["on", "hold"]}, {"sent": "maybe he can hold on .", "words": ["can", "on", "he", "hold"]}, {"sent": "okay hold on lemme put this over here .", "words": ["put", "hold", "here", "this", "on", "over"]}, {"sent": "gotta hold this pen up for the ink to come out .", "words": ["up", "the", "for", "hold", "to", "this", "out", "pen"]}, {"sent": "hold the bag ?", "words": ["the", "hold"]}, {"sent": "if hed yeah if he didnt hold on to that branch he mightve falled down right ?", "words": ["hold", "if", "to", "he", "on", "down", "that"]}, {"sent": "theres another m and whats this hold on .", "words": ["hold", "and", "another", "this", "on"]}, {"sent": "hold on tight .", "words": ["on", "hold"]}, {"sent": "hold on .", "words": ["on", "hold"]}, {"sent": "so hold that .", "words": ["so", "that", "hold"]}, {"sent": "hold on .", "words": ["on", "hold"]}, {"sent": "hold your pen properly .", "words": ["your", "hold", "pen"]}, {"sent": "you hold my toast .", "words": ["my", "hold", "toast", "you"]}, {"sent": "wanna hold it for a little bit ?", "words": ["for", "hold", "little", "it", "wanna", "a"]}, {"sent": "you wanna hold them ?", "words": ["them", "hold", "wanna", "you"]}, {"sent": "alright hold your bucket and Ill get you some vegetables .", "words": ["your", "hold", "bucket", "and", "get", "you", "some"]}, {"sent": "you want me to hold her ?", "words": ["hold", "to", "you", "me", "her"]}, {"sent": "hold the cutter .", "words": ["the", "hold"]}, {"sent": "you think it would hold on there ?", "words": ["think", "hold", "you", "there", "it", "would", "on"]}, {"sent": "okay xxx hold the door please thankyou xxx hold it .", "words": ["the", "it", "hold", "door"]}, {"sent": "hold on .", "words": ["on", "hold"]}, {"sent": "hold the doggie ?", "words": ["the", "hold"]}, {"sent": "hold it gently .", "words": ["it", "hold"]}, {"sent": "can you hold it too .", "words": ["can", "hold", "you", "it", "too"]}, {"sent": "well you sit on that side and Ill Ill help her to hold on this side .", "words": ["help", "hold", "to", "her", "and", "you", "this", "on", "sit", "that"]}, {"sent": "maybe I put it in some funny place hold on .", "words": ["put", "hold", "in", "it", "on", "some"]}, {"sent": "you want Mommy to hold eat it ?", "words": ["hold", "to", "you", "eat", "it"]}, {"sent": "hold your pen nicely .", "words": ["your", "hold", "pen"]}, {"sent": "Ill hold the button up on this side you put it through .", "words": ["the", "up", "put", "hold", "you", "this", "it", "on", "button"]}, {"sent": "is he gonna hold that one ?", "words": ["that", "he", "hold", "is"]}, {"sent": "hold on .", "words": ["on", "hold"]}, {"sent": "gonna make a cake ?", "words": ["make", "cake", "a"]}, {"sent": "can you see theres a cake .", "words": ["can", "cake", "you", "see", "a"]}, {"sent": "Ive got the cake .", "words": ["the", "cake"]}, {"sent": "right theyre all cake look it this is a oh thats cherry cake and a chocolate cake aw whats he doing ?", "words": ["all", "cake", "is", "chocolate", "and", "this", "it", "he", "look", "a"]}, {"sent": "and this is a little book called its my birthday and its all about making a birthday cake .", "words": ["all", "my", "cake", "is", "little", "and", "this", "about", "book", "a"]}, {"sent": "shall I get the cake down ?", "words": ["get", "the", "down", "cake"]}, {"sent": "and cake mix .", "words": ["cake", "and"]}, {"sent": "does the cake go in there ?", "words": ["the", "does", "cake", "go", "there", "in"]}, {"sent": "that is a birthday cake .", "words": ["that", "cake", "a", "is"]}, {"sent": "theres a cake .", "words": ["cake", "a"]}, {"sent": "spoon is very good for cake .", "words": ["for", "cake", "is", "spoon", "good"]}, {"sent": "a piece of cake .", "words": ["of", "cake", "a"]}, {"sent": "Id like to know what kind of cake youre going to make here .", "words": ["what", "cake", "of", "to", "like", "here", "make"]}, {"sent": "was it a small birthday cake ?", "words": ["was", "it", "cake", "a"]}, {"sent": "we saw a spot cake didnt we ?", "words": ["cake", "a", "we"]}, {"sent": "its always fun to get a present all for yourself and your cake all for yourself .", "words": ["all", "for", "yourself", "your", "present", "cake", "to", "and", "get", "a"]}, {"sent": "hes got some cake and crispies .", "words": ["some", "cake", "and"]}, {"sent": "see she is cutting the cake and giving the cake .", "words": ["the", "cake", "is", "and", "see", "she"]}, {"sent": "Ive never heard of a little girl eating radishes and cake .", "words": ["cake", "of", "and", "little", "a"]}, {"sent": "wanna play patty cake ?", "words": ["play", "cake", "wanna"]}, {"sent": "is that Mummys cake ?", "words": ["that", "cake", "is"]}, {"sent": "where did you make this cake ?", "words": ["where", "cake", "you", "this", "did", "make"]}, {"sent": "does she get cake before she eats her dirt ?", "words": ["does", "cake", "get", "her", "she"]}, {"sent": "chocolate mushroom cake that isnt it ?", "words": ["it", "chocolate", "that", "cake"]}, {"sent": "cake please .", "words": ["cake"]}, {"sent": "what about a cake ?", "words": ["what", "cake", "about", "a"]}, {"sent": "what happened with that birthday cake ?", "words": ["what", "that", "with", "cake"]}, {"sent": "like a wooden birthday cake set .", "words": ["like", "cake", "a"]}, {"sent": "oh okay well lets get the cake done .", "words": ["get", "the", "cake"]}, {"sent": "not a birthday cake .", "words": ["not", "cake", "a"]}, {"sent": "is the cake for me ?", "words": ["the", "for", "cake", "is", "me"]}, {"sent": "Ive got a cake yes .", "words": ["cake", "a"]}, {"sent": "oh birthday cake .", "words": ["cake"]}, {"sent": "is it chocolate cake ?", "words": ["chocolate", "it", "cake", "is"]}, {"sent": "its fun to have fun but you hafta know how I can hold up the cup and the milk and the cake .", "words": ["can", "up", "how", "the", "cup", "hold", "cake", "to", "you", "have", "and", "milk", "but"]}, {"sent": "a rice cake .", "words": ["cake", "a"]}, {"sent": "did you want some cake ?", "words": ["some", "did", "cake", "you"]}, {"sent": "are you going to have cake ?", "words": ["cake", "to", "you", "have", "are"]}, {"sent": "look at the cake Luna .", "words": ["the", "at", "cake", "look"]}, {"sent": "a candle and a cake .", "words": ["cake", "and", "a"]}, {"sent": "a cake ?", "words": ["cake", "a"]}, {"sent": "you gonna sing happybirthday to your cake ?", "words": ["your", "cake", "to", "you", "sing"]}, {"sent": "I think chocolate cake is my favorite dessert .", "words": ["my", "think", "cake", "is", "chocolate"]}, {"sent": "do you like to eat cake ?", "words": ["do", "cake", "to", "like", "you", "eat"]}, {"sent": "bake me a cake .", "words": ["me", "cake", "a"]}, {"sent": "building another cake .", "words": ["another", "cake"]}, {"sent": "Max made a caterpillar ooze cake with earthworm icing .", "words": ["with", "cake", "a"]}, {"sent": "mummie is making a cake .", "words": ["cake", "a", "is"]}, {"sent": "so Grandma got a little cup cake didnt she ?", "words": ["cup", "cake", "so", "little", "she", "a"]}, {"sent": "what else was left of the cake ?", "words": ["the", "what", "cake", "of", "was"]}, {"sent": "when she was away the three beggars decided to bake her a cake .", "words": ["the", "cake", "was", "to", "her", "away", "when", "she", "a"]}, {"sent": "oh you like cake dont you ?", "words": ["dont", "like", "cake", "you"]}, {"sent": "patty cake is a monkey right .", "words": ["monkey", "a", "cake", "is"]}, {"sent": "oh we havent got any chocolate cake .", "words": ["chocolate", "any", "cake", "we"]}, {"sent": "cake a cake .", "words": ["cake", "a"]}, {"sent": "is that a cake ?", "words": ["that", "cake", "a", "is"]}, {"sent": "what a baby cake a cup cake ?", "words": ["cup", "what", "cake", "a"]}, {"sent": "well he kept taking little pieces of cake right and then I kept giving you some more and more and more .", "words": ["cake", "of", "and", "little", "you", "more", "he", "some", "then"]}, {"sent": "do you know how to do patty cake ?", "words": ["do", "how", "cake", "to", "you"]}, {"sent": "a blue cake .", "words": ["blue", "cake", "a"]}, {"sent": "Mama will make a nice cake and then you can cut a piece of cake .", "words": ["can", "cut", "cake", "of", "nice", "and", "you", "will", "then", "make", "a"]}, {"sent": "you keep asking for this blue cake dont you ?", "words": ["for", "cake", "blue", "you", "this", "dont"]}, {"sent": "and whos going to bake your cake ?", "words": ["to", "your", "cake", "and"]}, {"sent": "xxx that cake xxx that cake he wanted xxx .", "words": ["he", "that", "cake"]}, {"sent": "or a BluesClues birthday cake ?", "words": ["cake", "a"]}, {"sent": "how about some pretend cake ?", "words": ["how", "cake", "about", "some", "pretend"]}, {"sent": "what cake would you .", "words": ["what", "you", "cake", "would"]}, {"sent": "candles on the cake .", "words": ["on", "the", "cake"]}, {"sent": "it was a nice chocolate cake wasnt it ?", "words": ["cake", "was", "nice", "chocolate", "it", "a"]}, {"sent": "and do you remember the cake we had John ?", "words": ["the", "do", "cake", "we", "you", "and"]}, {"sent": "you got oh they lit the cake didnt the candles on the cake didnt they ?", "words": ["the", "cake", "you", "they", "on"]}, {"sent": "you put coffee and cake two pounds fifty .", "words": ["put", "cake", "and", "you", "coffee"]}, {"sent": "she wants the cake .", "words": ["the", "she", "cake"]}, {"sent": "and theres a football match cake .", "words": ["cake", "a", "and"]}, {"sent": "a fervop cake .", "words": ["cake", "a"]}, {"sent": "a birthday cake .", "words": ["cake", "a"]}, {"sent": "yummy cake .", "words": ["cake"]}, {"sent": "I made a wish cake .", "words": ["wish", "cake", "a"]}, {"sent": "patty cake patty cake .", "words": ["cake"]}, {"sent": "you could draw this cake .", "words": ["cake", "draw", "you", "this", "could"]}, {"sent": "what kind of cake do you want then ?", "words": ["what", "do", "cake", "of", "you", "then"]}, {"sent": "whose cake is it ?", "words": ["it", "cake", "is"]}, {"sent": "he made a cake ?", "words": ["he", "cake", "a"]}, {"sent": "some cake tins here look .", "words": ["here", "some", "cake", "look"]}, {"sent": "FiddlerCat played Mellason sang and in the middle of the floor danced Jerry and Tuppy each with a big crumb of cake in his paw .", "words": ["the", "each", "cake", "of", "with", "and", "in", "his", "big", "a"]}, {"sent": "patty cake patty cake bakers man .", "words": ["cake"]}, {"sent": "drizzling it onto the cake .", "words": ["the", "it", "cake"]}, {"sent": "because thats the proper way to eat cake .", "words": ["the", "cake", "to", "eat", "because"]}, {"sent": "and Im not waiting up on a cake .", "words": ["up", "cake", "and", "on", "not", "a"]}, {"sent": "you dont stand on the cake .", "words": ["the", "cake", "stand", "you", "dont", "on"]}, {"sent": "and what did he have candles on the cake ?", "words": ["the", "what", "cake", "and", "have", "he", "did", "on"]}, {"sent": "you ever heard of patty cake ?", "words": ["of", "cake", "you"]}, {"sent": "whos going to eat that cake ?", "words": ["eat", "to", "that", "cake"]}, {"sent": "oh thats a nice cake .", "words": ["nice", "cake", "a"]}, {"sent": "bake a cake as fast as you can .", "words": ["can", "fast", "cake", "you", "a"]}, {"sent": "patty cake patty cake .", "words": ["cake"]}, {"sent": "bake me a cake as fast as you can .", "words": ["can", "fast", "cake", "you", "me", "a"]}, {"sent": "cake ?", "words": ["cake"]}, {"sent": "and what sort of cake did you want when we got home ?", "words": ["what", "cake", "we", "of", "and", "you", "did", "when", "home"]}, {"sent": "youre gonna have a delicious taste of that cake .", "words": ["cake", "of", "taste", "have", "that", "a"]}, {"sent": "mainly meat .", "words": ["meat"]}, {"sent": "hes separating his bones from his meat as he always does .", "words": ["his", "does", "he", "meat"]}, {"sent": "and then when its cooled down you hold it with your your skewer like that and then you can cut slices of meat off .", "words": ["can", "off", "your", "hold", "cut", "of", "with", "like", "meat", "you", "and", "that", "it", "then", "down", "when"]}, {"sent": "what kind of meat ?", "words": ["of", "what", "meat"]}, {"sent": "well just put some meat put some meat in the oven .", "words": ["the", "put", "meat", "in", "some", "oven"]}, {"sent": "oh and theyre not meat eaters ?", "words": ["not", "meat", "and"]}, {"sent": "meat .", "words": ["meat"]}, {"sent": "he had this in this in the mystery meat the keys .", "words": ["the", "meat", "this", "in", "he", "keys"]}, {"sent": "meat ?", "words": ["meat"]}, {"sent": "do you need some meat ?", "words": ["do", "need", "meat", "you", "some"]}, {"sent": "may I have some meat please ?", "words": ["have", "some", "meat"]}, {"sent": "look she pick out duh meat I xxx meat see they .", "words": ["pick", "meat", "they", "out", "look", "see", "she"]}, {"sent": "oh I like my meat at night .", "words": ["my", "like", "meat", "at"]}, {"sent": "or mystery meat ?", "words": ["meat"]}, {"sent": "go and eat your meat balls Bobby because they gonna get cold .", "words": ["your", "go", "meat", "and", "eat", "they", "get", "cold", "because"]}, {"sent": "who wants more meat ?", "words": ["more", "meat", "who"]}, {"sent": "that was meat wasnt it ?", "words": ["was", "it", "that", "meat"]}, {"sent": "you want some meat Summer ?", "words": ["some", "meat", "you"]}, {"sent": "alright give her the meat and then she can take the pea from her .", "words": ["the", "can", "then", "meat", "and", "take", "give", "her", "she"]}, {"sent": "you didnt have any meat ?", "words": ["have", "any", "meat", "you"]}, {"sent": "you pour the coffee on the meat .", "words": ["the", "pour", "meat", "you", "coffee", "on"]}, {"sent": "its not meat .", "words": ["not", "meat"]}, {"sent": "little weird cuts of meat like that which they have .", "words": ["of", "like", "meat", "little", "which", "they", "have", "that"]}, {"sent": "some kind of meat maybe ?", "words": ["of", "some", "meat"]}, {"sent": "and meat .", "words": ["meat", "and"]}, {"sent": "I think theyd like meat better .", "words": ["better", "think", "like", "meat"]}, {"sent": "so I remember what youre saying so its actually you have almost the word right its not meat eater its meteor meteor shower .", "words": ["the", "what", "shower", "so", "meat", "you", "have", "not"]}, {"sent": "theres the old meat stick we were looking for .", "words": ["the", "old", "stick", "for", "we", "meat", "were"]}, {"sent": "he cuts the meat .", "words": ["the", "he", "meat"]}, {"sent": "okay put put the meat in your plate .", "words": ["the", "put", "your", "plate", "meat", "in"]}, {"sent": "meat and pops ?", "words": ["meat", "and"]}, {"sent": "I like some meat please .", "words": ["some", "like", "meat"]}, {"sent": "sitting on the the meat .", "words": ["on", "the", "meat"]}, {"sent": "the butchers is for sausages and meat and bacon .", "words": ["the", "for", "is", "meat", "and"]}, {"sent": "we had hot sauce on that meat and you didnt eat a piece of it .", "words": ["sauce", "we", "of", "meat", "and", "you", "eat", "hot", "it", "on", "that", "a"]}, {"sent": "does he like meat ?", "words": ["does", "he", "like", "meat"]}, {"sent": "want some more meat ?", "words": ["some", "more", "meat"]}, {"sent": "god what happened to this meat loaf ?", "words": ["to", "what", "this", "meat"]}, {"sent": "you want meat ?", "words": ["meat", "you"]}, {"sent": "meats meat .", "words": ["meat"]}, {"sent": "how many pounds of meat two ?", "words": ["of", "how", "meat"]}, {"sent": "do you like meat ?", "words": ["do", "like", "meat", "you"]}, {"sent": "does that meat taste good ?", "words": ["does", "taste", "meat", "good", "that"]}, {"sent": "I think its eating meat .", "words": ["think", "meat"]}, {"sent": "weve got some bread and some flesh colored meat .", "words": ["some", "bread", "meat", "and"]}, {"sent": "and more meat .", "words": ["more", "meat", "and"]}, {"sent": "xxx take the cheese and the meat .", "words": ["the", "meat", "and", "take", "cheese"]}, {"sent": "uh she eats meat but not lambchops exactly .", "words": ["but", "not", "she", "meat"]}, {"sent": "I really dont know which ones are plant eaters and which ones are the meat eaters .", "words": ["the", "meat", "and", "dont", "are", "plant", "which"]}, {"sent": "you like the meat ?", "words": ["the", "like", "meat", "you"]}, {"sent": "lets find some meat .", "words": ["some", "meat", "find"]}, {"sent": "that is a ferocious meat eater .", "words": ["meat", "that", "a", "is"]}, {"sent": "and then something like meat in the middle .", "words": ["the", "like", "meat", "and", "in", "then"]}, {"sent": "were there any meat eaters at the dinosaur museum ?", "words": ["the", "any", "meat", "there", "at", "were"]}, {"sent": "yes meat again .", "words": ["meat"]}, {"sent": "want one more bite of meat ?", "words": ["bite", "of", "more", "meat"]}, {"sent": "heres some meat and cheese some lettuce pickles tomatoes .", "words": ["some", "cheese", "meat", "and"]}, {"sent": "what about your meat ?", "words": ["what", "your", "about", "meat"]}, {"sent": "hes a funny elephant if he eats meat .", "words": ["if", "meat", "elephant", "he", "a"]}, {"sent": "then they xxx Elizabeth can I have more meat ?", "words": ["can", "meat", "have", "they", "more", "then"]}, {"sent": "that meat greasy .", "words": ["that", "meat"]}, {"sent": "xxx its got bones and a piece of meat on it .", "words": ["of", "meat", "and", "it", "on", "a"]}, {"sent": "move your fork Derwood Ill put a little bit on this piece of meat right here .", "words": ["put", "your", "of", "meat", "little", "here", "this", "fork", "on", "a"]}, {"sent": "look at the meat .", "words": ["the", "at", "meat", "look"]}, {"sent": "can you get me some bread to go with our lunch meat ?", "words": ["can", "bread", "go", "to", "with", "meat", "you", "get", "our", "me", "some"]}, {"sent": "Purdie has some meat doesnt she ?", "words": ["some", "she", "meat"]}, {"sent": "is that meat on it ?", "words": ["is", "meat", "it", "on", "that"]}, {"sent": "the meat ready yet ?", "words": ["the", "meat"]}, {"sent": "I think you put some meat in the middle and its sposta make a hamburger .", "words": ["the", "think", "put", "meat", "you", "and", "hamburger", "in", "some", "make", "a"]}, {"sent": "is he eating meat ?", "words": ["he", "meat", "is"]}, {"sent": "er heres some meat .", "words": ["some", "meat"]}, {"sent": "oh meat .", "words": ["meat"]}, {"sent": "that meat is greasy .", "words": ["that", "meat", "is"]}, {"sent": "you dont want the meat eaters to hold you ?", "words": ["the", "hold", "to", "meat", "you", "dont"]}, {"sent": "can you get the kitty cat some meat ?", "words": ["can", "the", "meat", "you", "get", "cat", "kitty", "some"]}, {"sent": "meat ?", "words": ["meat"]}, {"sent": "you dont eat any meat .", "words": ["any", "meat", "you", "eat", "dont"]}, {"sent": "you want meat ?", "words": ["meat", "you"]}, {"sent": "tigers eat um meat and things like that .", "words": ["like", "meat", "and", "eat", "that"]}, {"sent": "meat ?", "words": ["meat"]}, {"sent": "is there meat that goes in the middle of that Anna ?", "words": ["the", "is", "of", "meat", "there", "in", "that"]}, {"sent": "oh the butcher men with the meat .", "words": ["the", "with", "meat"]}, {"sent": "where did he get meat ?", "words": ["where", "meat", "get", "he", "did"]}, {"sent": "now we have meat and rice and peas left .", "words": ["we", "peas", "meat", "and", "have"]}, {"sent": "you willnt eat meat will you ?", "words": ["eat", "will", "meat", "you"]}, {"sent": "yummy with meat .", "words": ["with", "meat"]}, {"sent": "shes got other meat but the same as you apart from that .", "words": ["the", "meat", "other", "you", "same", "that", "but"]}, {"sent": "hes cutting meat .", "words": ["meat"]}, {"sent": "is there a meat ?", "words": ["there", "meat", "a", "is"]}, {"sent": "xxx you want a piece of meat ?", "words": ["of", "meat", "a", "you"]}, {"sent": "Ill have a little more meat .", "words": ["meat", "little", "have", "more", "a"]}, {"sent": "I dunno what this is this must be some type of meat .", "words": ["what", "is", "of", "be", "meat", "this", "some"]}, {"sent": "its its a meat skewer .", "words": ["meat", "a"]}, {"sent": "*SI2: Im going get some more meat .", "words": ["get", "some", "more", "meat"]}, {"sent": "Ill give you a couple of pieces of meat inbetween rob okay ?", "words": ["of", "meat", "you", "give", "a"]}, {"sent": "do you want the baby meat .", "words": ["the", "do", "meat", "you"]}, {"sent": "you dont want the meat balls .", "words": ["the", "dont", "meat", "you"]}, {"sent": "thats how people turn into vegetarians because meat looks like that .", "words": ["how", "into", "like", "meat", "that", "because"]}, {"sent": "because they eat sometimes they they have alotof meat xxx around it but theres a big bone but deres a big bone dis has meat on the bone .", "words": ["the", "meat", "eat", "they", "have", "it", "but", "around", "on", "big", "because", "a"]}, {"sent": "i .", "words": ["i"]}, {"sent": "i .", "words": ["i"]}, {"sent": "a e i o u .", "words": ["i", "a"]}, {"sent": "i .", "words": ["i"]}, {"sent": "thats it i .", "words": ["i", "it"]}, {"sent": "when i is Kevins moving day going to be earlier ?", "words": ["i", "is", "to", "be", "when"]}, {"sent": "mm that looks like one i and another little i .", "words": ["i", "like", "and", "another", "little", "that"]}, {"sent": "i .", "words": ["i"]}, {"sent": "f i v e thats the long five .", "words": ["the", "i", "long"]}, {"sent": "i is for ice cream .", "words": ["ice cream", "i", "for", "is", "ice"]}, {"sent": "i .", "words": ["i"]}, {"sent": "Daddy made an n he made an i .", "words": ["an", "i", "he"]}, {"sent": "and you know he could use an h a i r c u t .", "words": ["i", "you", "and", "an", "he", "could", "a"]}, {"sent": "can you make an i ?", "words": ["can", "i", "you", "an", "make"]}, {"sent": "n a i m a .", "words": ["i", "a"]}, {"sent": "i for insect .", "words": ["i", "for"]}, {"sent": "*SI1: tell us some stuff that you do i tell us some stuff that you do there .", "words": ["do", "i", "you", "us", "there", "some", "that"]}, {"sent": "then it looks like two t if you erase the bottom of the i .", "words": ["the", "i", "bottom", "if", "of", "like", "you", "it", "then"]}, {"sent": "a b c d e f g h i j k l .", "words": ["i", "a"]}, {"sent": "do another i like that okay .", "words": ["do", "i", "like", "another", "that"]}, {"sent": "b i n g o b i n g o .", "words": ["i"]}, {"sent": "i .", "words": ["i"]}, {"sent": "oh lookit heres an i .", "words": ["an", "i"]}, {"sent": "see a i anywhere ?", "words": ["see", "i", "a"]}, {"sent": "oh sorry no i .", "words": ["i"]}, {"sent": "what the heck is i ?", "words": ["the", "what", "i", "is"]}, {"sent": "then i .", "words": ["then", "i"]}, {"sent": "now i .", "words": ["i"]}, {"sent": "i .", "words": ["i"]}, {"sent": "a b c d e f g h i j k l m n o p q r s t u v w x y z .", "words": ["i", "a"]}, {"sent": "wheres the small i ?", "words": ["the", "i"]}, {"sent": "next to the i .", "words": ["the", "to", "i"]}, {"sent": "i .", "words": ["i"]}, {"sent": "now Im gonna show you the i .", "words": ["show", "the", "i", "you"]}, {"sent": "i .", "words": ["i"]}, {"sent": "xxx e i e i o .", "words": ["i"]}, {"sent": "a b c d e f g h i .", "words": ["i", "a"]}, {"sent": "i ?", "words": ["i"]}, {"sent": "i .", "words": ["i"]}, {"sent": "wanna see how to do i ?", "words": ["do", "how", "i", "to", "wanna", "see"]}, {"sent": "xxx l i l y Lily .", "words": ["i"]}, {"sent": "xxx tip them out i this isnt working very well .", "words": ["this", "i", "them", "out"]}, {"sent": "make an i .", "words": ["an", "i", "make"]}, {"sent": "i .", "words": ["i"]}, {"sent": "m i s s i s s i p p i .", "words": ["i"]}, {"sent": "um was it only language based or was some of it intelligence i q stuff too ?", "words": ["i", "of", "was", "it", "too", "some"]}, {"sent": "h i j k l m n o p .", "words": ["i"]}, {"sent": "oh the e i e i o .", "words": ["the", "i"]}, {"sent": "oh I know what i is .", "words": ["what", "i", "is"]}, {"sent": "a l i a h Eleanor .", "words": ["i", "a"]}, {"sent": "i .", "words": ["i"]}, {"sent": "Americas a m e r i c a apostrophe s .", "words": ["i", "a"]}, {"sent": "and an i .", "words": ["an", "i", "and"]}, {"sent": "friends is the hard one f r i e n d s .", "words": ["the", "i", "hard", "is"]}, {"sent": "the i will stay on .", "words": ["the", "i", "on", "will", "stay"]}, {"sent": "i m .", "words": ["i"]}, {"sent": "like when we play it back when we have the e i s button .", "words": ["the", "i", "back", "we", "button", "play", "like", "have", "it", "when"]}, {"sent": "i .", "words": ["i"]}, {"sent": "i .", "words": ["i"]}, {"sent": "then we have the i and then we need an e right there .", "words": ["the", "i", "need", "we", "and", "have", "there", "an", "then"]}, {"sent": "i n g o .", "words": ["i"]}, {"sent": "next to the i .", "words": ["the", "to", "i"]}, {"sent": "i .", "words": ["i"]}, {"sent": "show me where i is .", "words": ["where", "i", "is", "show", "me"]}, {"sent": "i .", "words": ["i"]}, {"sent": "you need the middle bit with a i on it .", "words": ["the", "i", "need", "with", "you", "it", "on", "a"]}, {"sent": "i to me it looked like prickly heat .", "words": ["i", "to", "like", "it", "me"]}, {"sent": "a e i o u .", "words": ["i", "a"]}, {"sent": "i .", "words": ["i"]}, {"sent": "two i .", "words": ["i"]}, {"sent": "i .", "words": ["i"]}, {"sent": "this is participant five five seven three i h and this is there first visit .", "words": ["i", "is", "first", "and", "this", "there"]}, {"sent": "do the i beside the .", "words": ["the", "do", "i", "beside"]}, {"sent": "it is i .", "words": ["i", "it", "is"]}, {"sent": "r i d e ?", "words": ["i"]}, {"sent": "xxx an i .", "words": ["an", "i"]}, {"sent": "can I can do i .", "words": ["can", "do", "i"]}, {"sent": "you know how to say i .", "words": ["how", "i", "to", "say", "you"]}, {"sent": "no thats an i Im sorry .", "words": ["an", "i"]}, {"sent": "no thats not an i .", "words": ["an", "i", "not"]}, {"sent": "a b c d e h i j k l m n o p q r s t u v w x y z .", "words": ["i", "a"]}, {"sent": "e i e i o .", "words": ["i"]}, {"sent": "h i j k l m n o p .", "words": ["i"]}, {"sent": "m i c k e y m o u s e .", "words": ["i"]}, {"sent": "I think you have a pink i .", "words": ["think", "i", "you", "have", "a"]}, {"sent": "sorry i s .", "words": ["i"]}, {"sent": "e l i z a b e t h Ewww .", "words": ["i", "a"]}, {"sent": "h i j k l m n o p .", "words": ["i"]}, {"sent": "is that an i ?", "words": ["an", "i", "that", "is"]}, {"sent": "he I guess they sing e i e i o at school .", "words": ["i", "school", "sing", "they", "he", "at"]}, {"sent": "i n .", "words": ["i"]}, {"sent": "h i j k l m n o p .", "words": ["i"]}, {"sent": "take away the g i and make him an ant .", "words": ["the", "i", "and", "him", "away", "make", "an", "ant", "take"]}, {"sent": "h i j k l m n o p .", "words": ["i"]}, {"sent": "a l i a Eleanor .", "words": ["i", "a"]}, {"sent": "i for Ivor .", "words": ["i", "for"]}, {"sent": "i .", "words": ["i"]}, {"sent": "h i .", "words": ["i"]}, {"sent": "well you havent got to that letter yet have you because it thats h i j k .", "words": ["i", "to", "you", "have", "it", "that", "because"]}, {"sent": "put them back on this nice coffee tray .", "words": ["put", "back", "nice", "them", "this", "coffee", "on", "tray"]}, {"sent": "you usually go into the restaurant for a cup of coffee or a drink of milk dont you ?", "words": ["the", "cup", "for", "into", "drink", "go", "of", "you", "dont", "coffee", "milk", "a"]}, {"sent": "is the coffee cold .", "words": ["the", "coffee", "cold", "is"]}, {"sent": "want more coffee John ?", "words": ["more", "coffee"]}, {"sent": "what coffee ?", "words": ["what", "coffee"]}, {"sent": "have you seen a a coffee machine like that ?", "words": ["like", "you", "have", "coffee", "that", "a"]}, {"sent": "let us get Daddys coffee and Ill take you to swing .", "words": ["swing", "to", "and", "get", "us", "you", "coffee", "take"]}, {"sent": "coffee youre not having coffee .", "words": ["coffee", "not"]}, {"sent": "you made a good cup of coffee there .", "words": ["cup", "of", "you", "there", "coffee", "good", "a"]}, {"sent": "seal wants a coffee ?", "words": ["coffee", "a"]}, {"sent": "coffee .", "words": ["coffee"]}, {"sent": "I know you said there are things like coffee and sugar to make coffee but do you mean youre a its more like a cafe where we can make coffee and sugar ?", "words": ["where", "do", "can", "we", "to", "like", "you", "and", "there", "more", "are", "coffee", "make", "but", "a"]}, {"sent": "coffee coffee .", "words": ["coffee"]}, {"sent": "hot coffee okay .", "words": ["hot", "coffee"]}, {"sent": "can I have a coffee for me please ?", "words": ["can", "for", "have", "coffee", "me", "a"]}, {"sent": "he never lets me have lunch in peace in school always come on come to the store with me lets go there go dere Im eating my lunch and of a sudden a roach runs across the counter no he was eating his lunch in the in the coffee shop xxx you know they gotta a supply ?", "words": ["the", "go", "of", "school", "me", "store", "with", "have", "there", "on", "to", "was", "and", "you", "a", "my", "they", "in", "coffee", "he", "his"]}, {"sent": "yeah coffee outof that pot there .", "words": ["there", "coffee", "that"]}, {"sent": "just coffee and biscuits .", "words": ["coffee", "and"]}, {"sent": "Daddy put hot coffee in the refrigerator .", "words": ["the", "put", "refrigerator", "hot", "coffee", "in"]}, {"sent": "coffee .", "words": ["coffee"]}, {"sent": "mm maybe we should offer those workers a cup of of coffee or tea so they can have their break .", "words": ["cup", "can", "those", "their", "we", "of", "so", "have", "they", "coffee", "break", "a"]}, {"sent": "I want some coffee .", "words": ["some", "coffee"]}, {"sent": "does Johnny drink coffee .", "words": ["does", "coffee", "drink"]}, {"sent": "okay coffee outof the way .", "words": ["the", "coffee"]}, {"sent": "it has coffee in it .", "words": ["in", "it", "coffee"]}, {"sent": "is it alright if he drinks his coffee now ?", "words": ["if", "is", "it", "he", "coffee", "his"]}, {"sent": "lets make another coffee cup here .", "words": ["cup", "another", "here", "coffee", "make"]}, {"sent": "and whilst your Mummy makes the coffee are you gonna write your name for me ?", "words": ["the", "for", "your", "write", "and", "you", "coffee", "are", "me"]}, {"sent": "Mommy hasta eat something too and she hasta make herself a pot of coffee .", "words": ["of", "and", "eat", "coffee", "too", "make", "she", "a"]}, {"sent": "okay Mommyll drink her coffee .", "words": ["coffee", "her", "drink"]}, {"sent": "give grandma her coffee .", "words": ["give", "coffee", "her"]}, {"sent": "coffee cups .", "words": ["coffee"]}, {"sent": "oh youll have the police after you actually mister coffee bean man .", "words": ["have", "the", "coffee", "you"]}, {"sent": "maybe the kittycat wants some coffee ?", "words": ["the", "some", "coffee"]}, {"sent": "youre playing that coffee game again ?", "words": ["game", "coffee", "that"]}, {"sent": "thats my coffee .", "words": ["my", "coffee"]}, {"sent": "next to the coffee machine .", "words": ["the", "to", "coffee"]}, {"sent": "do you like coffee ?", "words": ["do", "coffee", "like", "you"]}, {"sent": "pretend coffee like cups ?", "words": ["pretend", "coffee", "like"]}, {"sent": "yes hot coffee is good .", "words": ["hot", "coffee", "good", "is"]}, {"sent": "because shes drinking her coffee .", "words": ["coffee", "her", "because"]}, {"sent": "do you like coffee ?", "words": ["do", "coffee", "like", "you"]}, {"sent": "give the baby some coffee .", "words": ["the", "some", "give", "coffee"]}, {"sent": "oh I dont think you can have coffee .", "words": ["can", "think", "you", "have", "dont", "coffee"]}, {"sent": "yes thats Daddys coffee .", "words": ["coffee"]}, {"sent": "mind my coffee .", "words": ["my", "coffee"]}, {"sent": "coffee .", "words": ["coffee"]}, {"sent": "thats two big cups of coffee youve just drunk .", "words": ["of", "coffee", "big"]}, {"sent": "and a cup of coffee .", "words": ["cup", "of", "and", "coffee", "a"]}, {"sent": "coffee maybe .", "words": ["coffee"]}, {"sent": "see Frasers not drinking his coffee that way .", "words": ["coffee", "his", "see", "that", "not"]}, {"sent": "daddy would like that coffee .", "words": ["coffee", "that", "like", "would"]}, {"sent": "daddy brought me some coffee .", "words": ["me", "coffee", "some"]}, {"sent": "or youre turning coffee beans into instant powder or something ?", "words": ["coffee", "into", "beans"]}, {"sent": "tea and coffee and orange juice and buttermilk .", "words": ["juice", "coffee", "orange", "and"]}, {"sent": "Im drinking my coffee .", "words": ["my", "coffee"]}, {"sent": "I think I need some more coffee .", "words": ["think", "need", "more", "coffee", "some"]}, {"sent": "thankyou for the coffee .", "words": ["the", "for", "coffee"]}, {"sent": "can I have some coffee please ?", "words": ["can", "some", "have", "coffee"]}, {"sent": "thats Courtneys coffee and you stay outof it .", "words": ["you", "and", "coffee", "it", "stay"]}, {"sent": "thats Mamas coffee .", "words": ["coffee"]}, {"sent": "oh can I have my coffee now ?", "words": ["can", "have", "coffee", "my"]}, {"sent": "you said coffee she said Rwwws group .", "words": ["coffee", "she", "you"]}, {"sent": "can you keep the coffee in the refrigerator ?", "words": ["can", "the", "refrigerator", "you", "in", "coffee"]}, {"sent": "but I see coffee .", "words": ["see", "coffee", "but"]}, {"sent": "do you like coffee Laura ?", "words": ["do", "coffee", "like", "you"]}, {"sent": "cup of coffee please .", "words": ["cup", "of", "coffee"]}, {"sent": "are you making a cup of coffee for me ?", "words": ["cup", "for", "of", "you", "are", "coffee", "me", "a"]}, {"sent": "would the elephant like some coffee ?", "words": ["the", "like", "elephant", "coffee", "would", "some"]}, {"sent": "thank you but I think Id like to have some coffee .", "words": ["think", "to", "like", "you", "have", "coffee", "some", "but"]}, {"sent": "in the coffee ?", "words": ["the", "in", "coffee"]}, {"sent": "coffee and a CocaCola and chips please .", "words": ["coffee", "a", "and"]}, {"sent": "who wants coffee ?", "words": ["coffee", "who"]}, {"sent": "do you want coffee ?", "words": ["do", "coffee", "you"]}, {"sent": "mhm coffee for Fraser .", "words": ["coffee", "for"]}, {"sent": "did you ask him if hed like some coffee ?", "words": ["if", "like", "you", "him", "coffee", "did", "some"]}, {"sent": "you going to put some xxx coffee .", "words": ["put", "to", "you", "coffee", "some"]}, {"sent": "yes hes drinking his coffee .", "words": ["his", "coffee"]}, {"sent": "is it tea or coffee Thomas ?", "words": ["it", "coffee", "is"]}, {"sent": "shes eating coffee as a matter of fact .", "words": ["of", "coffee", "a"]}, {"sent": "for buying that coffee ?", "words": ["for", "that", "coffee"]}, {"sent": "oh look hes fallen in my coffee .", "words": ["in", "coffee", "my", "look"]}, {"sent": "and Mummy is going to make herself a cup of coffee .", "words": ["cup", "is", "of", "to", "and", "coffee", "make", "a"]}, {"sent": "its coffee .", "words": ["coffee"]}, {"sent": "theres a coffee cup is there a cup in there we can pour coffee ?", "words": ["cup", "can", "is", "we", "pour", "there", "in", "coffee", "a"]}, {"sent": "mouses drink coffee ?", "words": ["coffee", "drink"]}, {"sent": "but I thought you were making coffee and tea ?", "words": ["and", "you", "coffee", "were", "but"]}, {"sent": "you wanna put them back on Daddys coffee tree ?", "words": ["put", "tree", "back", "them", "you", "coffee", "wanna", "on"]}, {"sent": "whos the coffee for ?", "words": ["the", "coffee", "for"]}, {"sent": "I dont drink coffee but .", "words": ["dont", "but", "coffee", "drink"]}, {"sent": "coffee there ?", "words": ["there", "coffee"]}, {"sent": "I wonder if he found any coffee at the store .", "words": ["the", "if", "store", "any", "coffee", "he", "at"]}, {"sent": "please Nomi can I have some coffee ?", "words": ["can", "some", "have", "coffee"]}, {"sent": "mommy wants no more coffee .", "words": ["more", "coffee"]}, {"sent": "are you gonna drink coffee ?", "words": ["are", "coffee", "drink", "you"]}, {"sent": "coffee ?", "words": ["coffee"]}, {"sent": "well I need a coffee pot for my coffee .", "words": ["my", "for", "need", "coffee", "a"]}, {"sent": "its not rock like a stone .", "words": ["like", "stone", "rock", "not", "a"]}, {"sent": "but listen if everybody here went to b and q and took a tiny stone .", "words": ["tiny", "if", "listen", "to", "and", "here", "stone", "but", "a"]}, {"sent": "theres a stone inside .", "words": ["inside", "stone", "a"]}, {"sent": "whats a stone doing there ?", "words": ["there", "stone", "a"]}, {"sent": "and it might only be a little little stone .", "words": ["be", "little", "and", "it", "stone", "a"]}, {"sent": "stone fly .", "words": ["stone"]}, {"sent": "are you getting your pebble stone back ?", "words": ["your", "back", "you", "are", "stone"]}, {"sent": "all twisted loops onto a wooden stone .", "words": ["all", "stone", "a"]}, {"sent": "he has a stone in his shoe .", "words": ["shoe", "in", "he", "stone", "his", "a"]}, {"sent": "no you dont eat the stone .", "words": ["the", "you", "eat", "dont", "stone"]}, {"sent": "where oh where could your stone be ?", "words": ["where", "your", "be", "stone", "could"]}, {"sent": "perfect stone he hands me .", "words": ["me", "he", "stone"]}, {"sent": "is he chopping the stone ?", "words": ["the", "he", "stone", "is"]}, {"sent": "and with the cherries we hafta take the stone .", "words": ["the", "we", "with", "and", "stone", "take"]}, {"sent": "and theyll only be a stone left in the middle .", "words": ["the", "be", "and", "in", "stone", "a"]}, {"sent": "you mustnt throw the throw the stone .", "words": ["the", "stone", "throw", "you"]}, {"sent": "you fell last night and you banged your head on a paving stone .", "words": ["your", "last", "head", "and", "you", "stone", "on", "a"]}, {"sent": "this is the harbor marked by a huge stone where first steps were taken to chart the unknown .", "words": ["the", "where", "is", "to", "first", "by", "this", "stone", "were", "a"]}, {"sent": "a stone yuck .", "words": ["stone", "a"]}, {"sent": "rough stone .", "words": ["stone"]}, {"sent": "a big stone .", "words": ["big", "stone", "a"]}, {"sent": "look at the stone path .", "words": ["the", "at", "stone", "look"]}, {"sent": "grey stone .", "words": ["stone"]}, {"sent": "there was a stone in the bag bully kicked .", "words": ["the", "was", "there", "in", "stone", "a"]}, {"sent": "we just we just eat the cherry and dont eat the stone .", "words": ["the", "we", "and", "eat", "dont", "stone"]}, {"sent": "picking a big stone up .", "words": ["up", "big", "stone", "a"]}, {"sent": "you want to sit down on that stone ?", "words": ["to", "you", "want to", "stone", "on", "sit", "down", "that"]}, {"sent": "oh do we see an old stone wall while we were driving ?", "words": ["old", "do", "we", "an", "stone", "see", "were"]}, {"sent": "bye bye stone .", "words": ["stone"]}, {"sent": "you see a lot of dry stone walls in Yorkshire .", "words": ["of", "a lot", "dry", "you", "in", "stone", "see", "a"]}, {"sent": "well you owe me four pence please because you said the stone cost a penny and I gave you five pence .", "words": ["the", "you", "and", "penny", "stone", "me", "because", "a"]}, {"sent": "mhm a big stone .", "words": ["big", "stone", "a"]}, {"sent": "all along the meadow where the cows grazed and the horses ran there was an old stone wall .", "words": ["the", "all", "where", "old", "was", "and", "there", "an", "stone"]}, {"sent": "a stone ?", "words": ["stone", "a"]}, {"sent": "mind your toes with that stone .", "words": ["that", "your", "with", "stone"]}, {"sent": "a stone wall ?", "words": ["stone", "a"]}, {"sent": "there you are little stone .", "words": ["little", "you", "there", "are", "stone"]}, {"sent": "theres your stone .", "words": ["your", "stone"]}, {"sent": "a stone hopped in my shoe .", "words": ["my", "shoe", "in", "stone", "a"]}, {"sent": "mm do you think we should make stone soup one day ?", "words": ["do", "think", "we", "you", "stone", "soup", "make"]}, {"sent": "here is a stone .", "words": ["here", "stone", "a", "is"]}, {"sent": "but today said Siew his face as bright as the moon we will show them how to make stone soup .", "words": ["the", "how", "moon", "face", "we", "show", "to", "will", "them", "soup", "stone", "his", "make", "but"]}, {"sent": "I think thats the stone for the pizza .", "words": ["the", "think", "for", "pizza", "stone"]}, {"sent": "this little stone .", "words": ["this", "stone", "little"]}, {"sent": "this stone is smooth .", "words": ["this", "is", "stone"]}, {"sent": "that stone has letters .", "words": ["that", "stone"]}, {"sent": "this duck is standing on a stone .", "words": ["is", "this", "on", "stone", "duck", "a"]}, {"sent": "grandma the stone came from grandma and grandpas gallery actually yeah .", "words": ["the", "and", "stone"]}, {"sent": "weve got a stone tortoise .", "words": ["stone", "a"]}, {"sent": "tread on the stepping stone instead .", "words": ["on", "the", "stone"]}, {"sent": "hes the only one who can take Excaliber out of the stone right ?", "words": ["the", "can", "stone", "of", "out", "who", "take"]}, {"sent": "theres a stone with a doggy on .", "words": ["on", "stone", "with", "a"]}, {"sent": "what a stone ladder ?", "words": ["what", "stone", "ladder", "a"]}, {"sent": "were making stone soup and we need three round smooth stones said Siew .", "words": ["need", "we", "and", "stone", "soup", "were"]}, {"sent": "because he sticks the he doesnt mean to but the sword goes back into the stone right ?", "words": ["the", "into", "back", "to", "because", "he", "stone", "but"]}, {"sent": "see the stone ?", "words": ["the", "see", "stone"]}, {"sent": "one big stone .", "words": ["big", "stone"]}, {"sent": "theres silly me talking about domes on steam engines and youre just talking about a stone that youve put in the smoke stack .", "words": ["the", "put", "and", "in", "about", "stone", "on", "me", "that", "a"]}, {"sent": "a stone spreader spread the stones evenly over the road bed .", "words": ["the", "bed", "stone", "over", "a"]}, {"sent": "and you were eating them quicker than I could get the stone out .", "words": ["the", "them", "you", "and", "get", "out", "stone", "could", "were"]}, {"sent": "weve used all the stones to make a stone wall .", "words": ["the", "all", "to", "stone", "make", "a"]}, {"sent": "there is a stone over here .", "words": ["is", "here", "there", "stone", "over", "a"]}, {"sent": "oh and another stone .", "words": ["another", "stone", "and"]}, {"sent": "this is a beautiful shiny stone .", "words": ["this", "stone", "a", "is"]}, {"sent": "oh the wee smiling stone youre right you do have a book called the wee smiling stone .", "words": ["the", "do", "you", "have", "stone", "book", "a"]}, {"sent": "now here is a stone ?", "words": ["here", "stone", "a", "is"]}, {"sent": "and what about this stone ?", "words": ["what", "and", "this", "about", "stone"]}, {"sent": "so maybe when its was growing right downward it may of hit a stone .", "words": ["of", "was", "so", "it", "stone", "hit", "when", "a"]}, {"sent": "what sort of stone ?", "words": ["of", "what", "stone"]}, {"sent": "you just eat around the stone .", "words": ["the", "stone", "you", "eat", "around"]}, {"sent": "this would be a great thing to do today theres a book called stone soup which is a really good book .", "words": ["do", "stone", "is", "to", "be", "soup", "this", "good", "would", "book", "which", "a"]}, {"sent": "well I see that theyve put a little stone rabbit next to him now .", "words": ["put", "to", "little", "him", "stone", "see", "that", "a"]}, {"sent": "thats a stone or a rock .", "words": ["rock", "stone", "a"]}, {"sent": "a stone wow .", "words": ["stone", "a"]}, {"sent": "thats the stepping stone .", "words": ["the", "stone"]}, {"sent": "its like a little stone isnt it ?", "words": ["like", "little", "it", "stone", "a"]}, {"sent": "happy stone xxx .", "words": ["happy", "stone"]}, {"sent": "its a round stone isnt it ?", "words": ["it", "stone", "a"]}, {"sent": "and docked in the harbor marked by a huge stone .", "words": ["the", "by", "and", "in", "stone", "a"]}, {"sent": "it dropped a stone didnt it to break it open .", "words": ["open", "to", "it", "stone", "break", "a"]}, {"sent": "it is considered real neat to have an undressed rough stone with one polished face on it .", "words": ["face", "is", "to", "with", "have", "an", "it", "stone", "on"]}, {"sent": "its not the stone .", "words": ["the", "not", "stone"]}, {"sent": "the dark red stone .", "words": ["the", "red", "stone", "dark"]}, {"sent": "this stone or this shell ?", "words": ["this", "stone"]}, {"sent": "theyve put a little stone rabbit next to the garden gnome .", "words": ["the", "put", "to", "little", "garden", "stone", "a"]}, {"sent": "this is the Mayflower ship in full sail that weathered the rough seas the wind and the hail and docked in the harbor marked by a huge stone .", "words": ["the", "full", "is", "and", "by", "this", "in", "stone", "that", "a", "wind"]}, {"sent": "stone wall .", "words": ["stone"]}, {"sent": "thats a stone wall yes .", "words": ["stone", "a"]}, {"sent": "if youd four stone off thered be nothing left of you .", "words": ["off", "if", "of", "be", "you", "stone"]}, {"sent": "now just slow down because youre eating them quicker than I can stone them .", "words": ["can", "them", "slow", "stone", "down", "because"]}, {"sent": "shadows on the stone .", "words": ["on", "the", "stone"]}, {"sent": "close your eyes said Frederick as he climbed on a big stone I send you the rays of sun .", "words": ["the", "your", "of", "sun", "you", "he", "stone", "on", "close", "big", "a"]}, {"sent": "big stone wall .", "words": ["big", "stone"]}, {"sent": "bump your head on that stone .", "words": ["bump", "your", "head", "stone", "on", "that"]}, {"sent": "it was done in that rough stone .", "words": ["was", "in", "it", "stone", "that"]}, {"sent": "so it had ta go tsh tsh tsh tsh tsh tsh tsh tsh tsh around the stone .", "words": ["the", "stone", "go", "so", "it", "around"]}, {"sent": "can you get a stone and stick it on Sunday please .", "words": ["can", "stick", "you", "get", "and", "it", "stone", "on", "a"]}, {"sent": "stone soup yummy .", "words": ["soup", "stone"]}, {"sent": "it has got all bits of stone and pebbles stuck to the outside of the church .", "words": ["the", "all", "of", "to", "outside", "church", "and", "it", "stuck", "stone"]}, {"sent": "who the guy on tv ?", "words": ["the", "tv", "on", "who"]}, {"sent": "Conners by the tv thats not fair .", "words": ["the", "tv", "not", "by"]}, {"sent": "would you um will you call me up when she comes on tv so I can see what she looks like ?", "words": ["up", "can", "what", "see", "so", "like", "you", "on", "tv", "would", "will", "me", "when", "she"]}, {"sent": "do you ever watch sesame street on tv ?", "words": ["do", "watch", "you", "tv", "on", "street"]}, {"sent": "thats like a tv huh ?", "words": ["tv", "like", "a"]}, {"sent": "did you ever seen one of these planes Parson on tv ?", "words": ["these", "of", "you", "tv", "did", "on"]}, {"sent": "its in Longport right next to the tv .", "words": ["the", "to", "in", "tv"]}, {"sent": "why is it tv time ?", "words": ["tv", "it", "why", "is"]}, {"sent": "take a bath watch tv ?", "words": ["tv", "watch", "take", "a"]}, {"sent": "here Johnny do you watch tv ?", "words": ["do", "watch", "you", "here", "tv"]}, {"sent": "that can be your tv .", "words": ["can", "your", "be", "tv", "that"]}, {"sent": "I dont see him on tv any more .", "words": ["any", "dont", "him", "tv", "more", "on", "see"]}, {"sent": "you watch tv with me ?", "words": ["watch", "with", "you", "tv", "me"]}, {"sent": "a tv .", "words": ["tv", "a"]}, {"sent": "like you saw on tv yesterday .", "words": ["on", "tv", "like", "you"]}, {"sent": "was on the tv ?", "words": ["on", "was", "tv", "the"]}, {"sent": "whos watching tv ?", "words": ["tv"]}, {"sent": "Ma you should buy a color tv .", "words": ["tv", "buy", "a", "you"]}, {"sent": "supposedly its a tv show .", "words": ["show", "tv", "a"]}, {"sent": "there is the tv .", "words": ["the", "there", "tv", "is"]}, {"sent": "show me that tv .", "words": ["show", "me", "tv", "that"]}, {"sent": "the tv a second .", "words": ["the", "tv", "a"]}, {"sent": "they have a new tv show .", "words": ["show", "have", "they", "tv", "new", "a"]}, {"sent": "tv .", "words": ["tv"]}, {"sent": "okay sleep over by the tv ?", "words": ["the", "sleep", "by", "tv", "over"]}, {"sent": "yeah it was on the tv .", "words": ["the", "was", "tv", "it", "on"]}, {"sent": "did you see it on the little tv or the really big tv ?", "words": ["the", "you", "little", "tv", "it", "did", "on", "see", "big"]}, {"sent": "dont hit the tv .", "words": ["the", "dont", "tv", "hit"]}, {"sent": "go go play with your tv .", "words": ["your", "go", "play", "with", "tv"]}, {"sent": "can I turn the tv on ?", "words": ["can", "the", "tv", "on"]}, {"sent": "the tv .", "words": ["the", "tv"]}, {"sent": "and the tv fell off the table ?", "words": ["the", "off", "table", "and", "tv"]}, {"sent": "Harley do you have a favorite tv program ?", "words": ["do", "you", "have", "tv", "a"]}, {"sent": "what do you like to watch on tv ?", "words": ["what", "do", "watch", "to", "like", "you", "tv", "on"]}, {"sent": "you watching tv again ?", "words": ["tv", "you"]}, {"sent": "do you mind if we look at tv um um awhile the test is .", "words": ["the", "do", "if", "we", "is", "you", "tv", "at", "look"]}, {"sent": "tv yes .", "words": ["tv"]}, {"sent": "but whats your favorite show to watch on tv ?", "words": ["watch", "your", "show", "to", "tv", "on", "but"]}, {"sent": "are you watching tv Xavier ?", "words": ["tv", "are", "you"]}, {"sent": "on tv .", "words": ["on", "tv"]}, {"sent": "do you wanna watch tv ?", "words": ["do", "watch", "you", "tv", "wanna"]}, {"sent": "can you sit in that chair and watch tv ?", "words": ["can", "watch", "chair", "you", "and", "in", "tv", "sit", "that"]}, {"sent": "because shes shes a lawyer and she went back to work after a couple months and said to the nanny like I dont want him watching any tv .", "words": ["the", "work", "back", "to", "any", "like", "and", "dont", "him", "tv", "she", "because", "a"]}, {"sent": "no no tv no tv .", "words": ["tv"]}, {"sent": "so we can only have a snack with them when were watching them on tv .", "words": ["can", "we", "so", "with", "them", "have", "tv", "on", "when", "were", "a"]}, {"sent": "because theyre sitting in front of the tv .", "words": ["the", "of", "in", "tv", "because"]}, {"sent": "tv .", "words": ["tv"]}, {"sent": "now can you tell me what did Peppapig do to the tv ?", "words": ["can", "what", "do", "the", "to", "you", "tv", "did", "me"]}, {"sent": "look Wal tv ?", "words": ["tv", "look"]}, {"sent": "did that tv explode ?", "words": ["tv", "that", "did"]}, {"sent": "top of the tv .", "words": ["of", "the", "tv"]}, {"sent": "and whats that on the tv ?", "words": ["the", "and", "tv", "on", "that"]}, {"sent": "oh I watched tv .", "words": ["tv"]}, {"sent": "and there was nothing on tv .", "words": ["was", "and", "there", "tv", "on"]}, {"sent": "what did we see on uh tv for a movie last night when you wanted to go to bed ?", "words": ["what", "for", "we", "go", "to", "last", "bed", "you", "tv", "did", "on", "see", "when", "a"]}, {"sent": "they didnt even watch tv .", "words": ["they", "tv", "watch"]}, {"sent": "you always watch him on tv ?", "words": ["watch", "you", "tv", "him", "on"]}, {"sent": "so she decides to turn on the tv .", "words": ["the", "to", "so", "tv", "on", "she"]}, {"sent": "no tv .", "words": ["tv"]}, {"sent": "yeah youre gonna be on tv again .", "words": ["on", "be", "tv"]}, {"sent": "who tell you to look tru the mirror to see tv ?", "words": ["the", "see", "to", "you", "tv", "look", "who"]}, {"sent": "yeah that is a tv .", "words": ["tv", "that", "a", "is"]}, {"sent": "come on pick that fork up Samantha Samantha pick you fork up or I shut the tv off .", "words": ["up", "the", "off", "pick", "you", "tv", "fork", "on", "that"]}, {"sent": "why you turn on that tv ?", "words": ["you", "tv", "on", "that", "why"]}, {"sent": "and were all sitting about watching tv or playing about .", "words": ["all", "and", "tv", "about", "were"]}, {"sent": "did you hear it on the tv ?", "words": ["the", "hear", "you", "tv", "it", "did", "on"]}, {"sent": "a tv ?", "words": ["tv", "a"]}, {"sent": "no tv with this vcr .", "words": ["this", "tv", "with"]}, {"sent": "do I have a tv ?", "words": ["have", "do", "tv", "a"]}, {"sent": "so if she normally watches tv .", "words": ["so", "tv", "if", "she"]}, {"sent": "no William you dont go behind the tv .", "words": ["the", "go", "behind", "you", "dont", "tv"]}, {"sent": "this is Et watching tv .", "words": ["this", "tv", "is"]}, {"sent": "youre gonna watch tv ?", "words": ["tv", "watch"]}, {"sent": "I dont let him watch tv .", "words": ["dont", "him", "watch", "tv"]}, {"sent": "I saw dat on tv .", "words": ["on", "tv"]}, {"sent": "tv ?", "words": ["tv"]}, {"sent": "dont show it on tv ?", "words": ["show", "dont", "tv", "it", "on"]}, {"sent": "are you gonna read books or watch tv ?", "words": ["watch", "you", "tv", "are", "read"]}, {"sent": "wanna go play ball with mommy on tv ?", "words": ["go", "play", "with", "ball", "tv", "wanna", "on"]}, {"sent": "you know what this is a tv news helicopter .", "words": ["what", "helicopter", "is", "you", "this", "tv", "a"]}, {"sent": "thats xxx tv .", "words": ["tv"]}, {"sent": "the tv is not on .", "words": ["the", "is", "tv", "on", "not"]}, {"sent": "we can watch tv when Helen and Tina hafta go home .", "words": ["can", "watch", "we", "go", "and", "tv", "when", "home"]}, {"sent": "wanna go in the tv room and read some books ?", "words": ["the", "go", "and", "tv", "in", "room", "read", "wanna", "some"]}, {"sent": "playing with your tv .", "words": ["your", "tv", "with"]}, {"sent": "there is the tv .", "words": ["the", "there", "tv", "is"]}, {"sent": "howcome it was on tv ?", "words": ["on", "was", "tv", "it"]}, {"sent": "whats wrong with the tv ?", "words": ["the", "tv", "with"]}, {"sent": "but yeah hes hes familiar with most of these animals either from books or possibly seeing them on tv .", "words": ["these", "of", "with", "them", "tv", "on", "but"]}, {"sent": "give you a lasagna head you wanna watch tv while you eat ?", "words": ["watch", "head", "you", "eat", "tv", "wanna", "give", "a"]}, {"sent": "if you if you dont Im gonna shut that tv off .", "words": ["off", "if", "you", "dont", "tv", "that"]}, {"sent": "one of the slides was it was on tv .", "words": ["the", "of", "was", "tv", "it", "on"]}, {"sent": "you learnt that at school or on tv ?", "words": ["school", "you", "tv", "at", "on", "that"]}, {"sent": "do you remember we went to the hospital last week and saw the wee baby on the tv screen when we were there ?", "words": ["the", "do", "we", "to", "last", "you", "and", "there", "tv", "on", "when", "were"]}, {"sent": "she says Im I sat up in the bedroom watching tv she says .", "words": ["up", "the", "in", "tv", "she", "bedroom"]}, {"sent": "tv off .", "words": ["off", "tv"]}, {"sent": "but tonight were going to watch a movie on tv .", "words": ["watch", "to", "tv", "on", "were", "but", "a"]}, {"sent": "hes watching tv who likes to watch tv ?", "words": ["to", "tv", "watch", "who"]}, {"sent": "whaddoo you wanna watch on tv ?", "words": ["watch", "you", "tv", "wanna", "on"]}, {"sent": "you put it in your mouth .", "words": ["put", "your", "mouth", "you", "in", "it"]}, {"sent": "youve got a bit of a hair in your mouth again .", "words": ["your", "of", "mouth", "in", "hair", "a"]}, {"sent": "and you see his mouth ?", "words": ["mouth", "you", "and", "his", "see"]}, {"sent": "how about we have a white mouth with a red tongue ?", "words": ["red", "how", "we", "mouth", "with", "have", "about", "tongue", "white", "a"]}, {"sent": "you couldnt sing properly with a boat in your mouth can you ?", "words": ["can", "your", "mouth", "with", "you", "sing", "in", "boat", "a"]}, {"sent": "heres his mouth .", "words": ["his", "mouth"]}, {"sent": "would you like to wipe your mouth ?", "words": ["your", "to", "mouth", "like", "you", "would", "wipe"]}, {"sent": "thats not his mouth .", "words": ["his", "mouth", "not"]}, {"sent": "everything goes in his mouth .", "words": ["his", "in", "mouth"]}, {"sent": "in the mouth .", "words": ["the", "in", "mouth"]}, {"sent": "no you cant have him in your mouth .", "words": ["your", "mouth", "you", "have", "in", "him"]}, {"sent": "big mouth .", "words": ["mouth", "big"]}, {"sent": "well stand far more chance of it going into your mouth wont we ?", "words": ["your", "into", "we", "of", "stand", "mouth", "more", "it"]}, {"sent": "oh mygoodness cover your mouth when you cough .", "words": ["your", "mouth", "you", "cover", "when"]}, {"sent": "youve made a big mouth have you ?", "words": ["mouth", "you", "have", "big", "a"]}, {"sent": "why dont you wipe your mouth off ?", "words": ["off", "your", "mouth", "you", "dont", "wipe", "why"]}, {"sent": "you not putting my finger in your mouth .", "words": ["my", "your", "mouth", "you", "finger", "in", "not"]}, {"sent": "dont sing with your mouth in your cup .", "words": ["cup", "your", "mouth", "with", "sing", "dont", "in"]}, {"sent": "take it outof your mouth please .", "words": ["your", "it", "take", "mouth"]}, {"sent": "where is Daddys mouth .", "words": ["where", "mouth", "is"]}, {"sent": "why do you think what youre not sposta talk with your mouth mouth full ?", "words": ["do", "think", "what", "your", "full", "mouth", "with", "you", "talk", "not", "why"]}, {"sent": "out of your mouth .", "words": ["of", "mouth", "your", "out"]}, {"sent": "all you hafta do is open your mouth and say good day .", "words": ["all", "do", "your", "is", "open", "mouth", "say", "you", "and", "good"]}, {"sent": "its good to put in his mouth and to chew on isnt it ?", "words": ["put", "to", "mouth", "and", "in", "on", "it", "good", "his"]}, {"sent": "you have a cough so maybe you shouldnt put your stuff in your mouth .", "words": ["put", "your", "so", "mouth", "you", "have", "in", "a"]}, {"sent": "he does he doesnt have a bandaid thats just his mouth thats how they draw his mouth .", "words": ["does", "how", "mouth", "draw", "have", "they", "he", "his", "a"]}, {"sent": "dont put the brush in your mouth .", "words": ["the", "put", "your", "brush", "mouth", "dont", "in"]}, {"sent": "wipe your mouth off .", "words": ["mouth", "wipe", "your", "off"]}, {"sent": "Doctor Ringler looked in your mouth .", "words": ["in", "mouth", "your"]}, {"sent": "dont put anything else in your mouth until you swallow whats in there .", "words": ["put", "your", "mouth", "you", "dont", "in", "there"]}, {"sent": "thats the babys mouth .", "words": ["the", "mouth"]}, {"sent": "why did you put it in your mouth ?", "words": ["put", "your", "mouth", "you", "in", "it", "did", "why"]}, {"sent": "mouth .", "words": ["mouth"]}, {"sent": "dont put it in your mouth .", "words": ["put", "your", "mouth", "dont", "in", "it"]}, {"sent": "right to your mouth .", "words": ["to", "mouth", "your"]}, {"sent": "put take that out of your mouth .", "words": ["put", "your", "of", "mouth", "out", "take", "that"]}, {"sent": "straight into the mouth huh ?", "words": ["the", "mouth", "into"]}, {"sent": "thats a good face now wheres her nose and mouth gonna be ?", "words": ["face", "be", "mouth", "and", "good", "her", "a", "nose"]}, {"sent": "mouth .", "words": ["mouth"]}, {"sent": "spoon goes in your mouth ?", "words": ["in", "mouth", "your", "spoon"]}, {"sent": "doesnt go in the mouth does it go on an ear ?", "words": ["the", "does", "go", "mouth", "in", "it", "an", "ear", "on"]}, {"sent": "Im I think I modeled putting the pizza in my mouth but then he was putting the cookies in his mouth over and over again Im pre I know Im he knows the word cookie so .", "words": ["the", "cookie", "think", "my", "was", "so", "mouth", "pizza", "and", "in", "he", "his", "then", "over", "but"]}, {"sent": "his mouth ?", "words": ["his", "mouth"]}, {"sent": "so he probably recognizes all the food here now whereas before I dont even Imean I think he recognized it before but he was more just interested in gnawing on it and putting it in his mouth so .", "words": ["the", "all", "think", "so", "was", "mouth", "and", "here", "dont", "food", "it", "he", "more", "in", "his", "on", "but"]}, {"sent": "I mean right now I think partial cause shes teething and probably probably the stage alot of things she just picks up thats looking at and then will put in her mouth so .", "words": ["the", "up", "think", "put", "of", "so", "mouth", "and", "in", "at", "will", "then", "her", "she"]}, {"sent": "no dont put that in your mouth .", "words": ["put", "your", "mouth", "dont", "in", "that"]}, {"sent": "her mouth yes .", "words": ["mouth", "her"]}, {"sent": "cover your mouth .", "words": ["mouth", "your", "cover"]}, {"sent": "that is his mouth I guess .", "words": ["his", "mouth", "that", "is"]}, {"sent": "straight to the mouth huh ?", "words": ["the", "to", "mouth"]}, {"sent": "wheres his mouth at ?", "words": ["his", "mouth", "at"]}, {"sent": "who lost their mouth ?", "words": ["mouth", "their", "who"]}, {"sent": "mouth nose eye eye teeth teeth teeth .", "words": ["mouth", "eye", "nose"]}, {"sent": "dont cry if it burns your mouth .", "words": ["your", "if", "cry", "mouth", "dont", "it"]}, {"sent": "careful those dont go too far in your mouth honey .", "words": ["those", "your", "go", "careful", "mouth", "dont", "in", "too"]}, {"sent": "dont put that in your mouth .", "words": ["put", "your", "mouth", "dont", "in", "that"]}, {"sent": "can I have a look in your mouth ?", "words": ["can", "your", "mouth", "have", "in", "look", "a"]}, {"sent": "how about you take the straw out of your mouth ?", "words": ["the", "how", "your", "of", "mouth", "you", "about", "out", "take"]}, {"sent": "not in your mouth .", "words": ["in", "mouth", "your", "not"]}, {"sent": "where is babys mouth ?", "words": ["where", "mouth", "is"]}, {"sent": "get your hands outof your mouth when you talk .", "words": ["your", "mouth", "you", "get", "talk", "when"]}, {"sent": "the mouth the mouth organ makes you laugh doesnt it ?", "words": ["the", "it", "mouth", "you"]}, {"sent": "she had a literally she had it in her mouth the whole time .", "words": ["the", "mouth", "in", "it", "her", "she", "a"]}, {"sent": "hes got a big mouth and sharp teeth .", "words": ["mouth", "big", "and", "a"]}, {"sent": "its hard to talk with your mouth full isnt ?", "words": ["your", "full", "to", "mouth", "with", "talk", "hard"]}, {"sent": "thats his mouth .", "words": ["his", "mouth"]}, {"sent": "I mean with books shell just eat she wants to put them in her mouth .", "words": ["put", "to", "mouth", "with", "them", "eat", "in", "her", "she"]}, {"sent": "not in your mouth .", "words": ["in", "mouth", "your", "not"]}, {"sent": "now the waffle might fit in your mouth if you wanna put something in your mouth huh .", "words": ["the", "put", "your", "if", "fit", "mouth", "you", "in", "wanna"]}, {"sent": "whats the matter with his mouth ?", "words": ["the", "his", "mouth", "with"]}, {"sent": "and this mouth ?", "words": ["this", "mouth", "and"]}, {"sent": "a little bit of a mouth .", "words": ["of", "mouth", "little", "a"]}, {"sent": "oh Thomas get it out of your mouth .", "words": ["your", "of", "mouth", "get", "it", "out"]}, {"sent": "Fraser dont put it in your mouth .", "words": ["put", "your", "mouth", "dont", "in", "it"]}, {"sent": "no take it outof your mouth please .", "words": ["your", "it", "take", "mouth"]}, {"sent": "his mouth is eating a yummy treat .", "words": ["his", "mouth", "a", "is"]}, {"sent": "are you trying to get that in your mouth ?", "words": ["your", "to", "mouth", "you", "get", "in", "are", "that"]}, {"sent": "talk with food in ya mouth .", "words": ["mouth", "with", "in", "food", "talk"]}, {"sent": "straight into the mouth huh .", "words": ["the", "mouth", "into"]}, {"sent": "not in your mouth .", "words": ["in", "mouth", "your", "not"]}, {"sent": "the fox jumped ashore opened his mouth snip .", "words": ["the", "his", "mouth"]}, {"sent": "take that outof your mouth .", "words": ["your", "that", "mouth", "take"]}, {"sent": "youve gone and got blooming bubbles in your mouth havent you ?", "words": ["your", "mouth", "and", "you", "in", "bubbles"]}, {"sent": "and heres a very pretty little mouth .", "words": ["mouth", "little", "and", "pretty", "a"]}, {"sent": "no dont put that in your mouth .", "words": ["put", "your", "mouth", "dont", "in", "that"]}, {"sent": "yes but your mouth is upside down .", "words": ["your", "is", "mouth", "down", "but"]}, {"sent": "should we put his mouth in first before we put his pipe in ?", "words": ["put", "we", "mouth", "first", "in", "his"]}, {"sent": "yes mouth .", "words": ["mouth"]}, {"sent": "does he he put everything in his mouth or ?", "words": ["does", "put", "mouth", "in", "he", "his"]}, {"sent": "can you cover your mouth please ?", "words": ["can", "your", "mouth", "you", "cover"]}, {"sent": "but you mustnt put it in your mouth .", "words": ["put", "your", "mouth", "you", "in", "it", "but"]}, {"sent": "oh a mouth .", "words": ["mouth", "a"]}, {"sent": "and so I think they they feel it and they put it in their mouth and theyre teething so that feels good to them .", "words": ["think", "put", "their", "so", "to", "mouth", "them", "and", "they", "in", "it", "good", "that"]}, {"sent": "is it time for everything to the mouth ?", "words": ["the", "for", "is", "to", "mouth", "it"]}, {"sent": "where is his mouth ?", "words": ["where", "his", "mouth", "is"]}, {"sent": "its got very sharp teeth and a very big mouth hasnt it ?", "words": ["mouth", "and", "it", "big", "a"]}, {"sent": "and um a lot of times hell uh put the feet or the hands in his mouth .", "words": ["the", "put", "of", "a lot", "mouth", "and", "in", "his", "a"]}, {"sent": "take it outof your mouth .", "words": ["your", "it", "take", "mouth"]}, {"sent": "can you put that in your mouth ?", "words": ["can", "put", "your", "mouth", "you", "in", "that"]}, {"sent": "you dont hit do you ?", "words": ["dont", "do", "hit", "you"]}, {"sent": "well you hit me and it hurt .", "words": ["hurt", "and", "you", "it", "me", "hit"]}, {"sent": "why does he hit the ball onto the fence ?", "words": ["the", "does", "ball", "he", "hit", "why"]}, {"sent": "no you didnt hit me .", "words": ["me", "hit", "you"]}, {"sent": "he drove through the wet road and hes hit a dump truck .", "words": ["the", "wet", "dump", "truck", "and", "he", "hit", "a"]}, {"sent": "what you hit me for ?", "words": ["what", "for", "you", "me", "hit"]}, {"sent": "time for a quick hit .", "words": ["for", "hit", "a"]}, {"sent": "and it has hit your bum when you stood up didnt you ?", "words": ["up", "your", "and", "you", "it", "hit", "when"]}, {"sent": "mommy hit the light .", "words": ["the", "hit", "light"]}, {"sent": "Dingo hit an egg van .", "words": ["an", "hit", "egg"]}, {"sent": "dont hit people .", "words": ["dont", "hit"]}, {"sent": "maybe if I roll this it might hit him .", "words": ["if", "this", "him", "it", "hit"]}, {"sent": "*SI1: nice hit .", "words": ["nice", "hit"]}, {"sent": "uhuh uhuh uhuh dont hit me Perri dont hit me Perri .", "words": ["dont", "me", "hit"]}, {"sent": "I wouldnt hit her .", "words": ["hit", "her"]}, {"sent": "now it can hit the road .", "words": ["can", "the", "it", "hit"]}, {"sent": "oh yeah people hit the puck with their stick and then what happens during the middle when its time when theyre taking a break ?", "words": ["the", "what", "stick", "break", "their", "with", "and", "then", "hit", "when", "a"]}, {"sent": "when she does it she t jams the food under the prong and she hit herself .", "words": ["the", "does", "and", "under", "food", "it", "hit", "when", "she"]}, {"sent": "I dont want to hit swing you on the table .", "words": ["the", "table", "swing", "to", "you", "dont", "want to", "on", "hit"]}, {"sent": "you wanna hit it into the fence ?", "words": ["the", "into", "you", "it", "wanna", "hit"]}, {"sent": "youd hit me on the head with it .", "words": ["the", "head", "with", "it", "on", "me", "hit"]}, {"sent": "dont hit your head .", "words": ["dont", "head", "hit", "your"]}, {"sent": "you cant hit it from there .", "words": ["it", "hit", "there", "you"]}, {"sent": "oh yes oh you dont hit Steven .", "words": ["dont", "hit", "you"]}, {"sent": "xxx hit him with duh puzzle .", "words": ["puzzle", "him", "hit", "with"]}, {"sent": "you mustnt hit anybody with a stick .", "words": ["stick", "with", "you", "hit", "a"]}, {"sent": "the things they will hit .", "words": ["the", "they", "will", "hit"]}, {"sent": "twang and it might hit you and then it hurts .", "words": ["you", "and", "it", "then", "hit"]}, {"sent": "oh my gosh we hit the chair .", "words": ["the", "my", "we", "chair", "hit"]}, {"sent": "you could hit .", "words": ["could", "hit", "you"]}, {"sent": "did daddy hit the sack yet ?", "words": ["the", "hit", "did"]}, {"sent": "do you want me to not hit ?", "words": ["do", "to", "you", "me", "hit", "not"]}, {"sent": "why you hit me ?", "words": ["me", "hit", "why", "you"]}, {"sent": "see if you can hit his mouth .", "words": ["can", "if", "mouth", "you", "his", "see", "hit"]}, {"sent": "okay sweetie pie lets not hit the book .", "words": ["the", "book", "hit", "not"]}, {"sent": "you hit me on the thumb .", "words": ["the", "you", "on", "me", "hit"]}, {"sent": "want mummy hit the ceiling with it ?", "words": ["the", "it", "hit", "with"]}, {"sent": "you hit who ?", "words": ["who", "hit", "you"]}, {"sent": "dont hit hug .", "words": ["hug", "dont", "hit"]}, {"sent": "why dont you try to hit the horse ?", "words": ["the", "to", "you", "dont", "horse", "hit", "try", "try to", "why"]}, {"sent": "Dougs gonna hit the chair .", "words": ["the", "hit", "chair"]}, {"sent": "oh no no dont hit anybody .", "words": ["dont", "hit"]}, {"sent": "I thought that that ladys going to pick her children from school and she reversed out of the drive and very nearly hit the car that couldnt get past because of the bin bag lorry .", "words": ["the", "pick", "to", "school", "of", "car", "and", "drive", "get", "that", "out", "hit", "her", "she", "because"]}, {"sent": "they were trying to hit you on the head were they ?", "words": ["the", "to", "head", "you", "they", "on", "hit", "were"]}, {"sent": "um he has a couple books that make noise and so hell just sit there and youknow hit the buttons and make noise .", "words": ["the", "so", "and", "there", "make", "he", "sit", "hit", "that", "a"]}, {"sent": "hit it harder .", "words": ["it", "hit"]}, {"sent": "you hit it back to me .", "words": ["back", "to", "you", "it", "me", "hit"]}, {"sent": "who hit you ?", "words": ["hit", "you", "who"]}, {"sent": "you hit your head ?", "words": ["head", "hit", "your", "you"]}, {"sent": "uhoh he has just hit Purdie .", "words": ["hit", "he"]}, {"sent": "I thought I would hit him for fif fifty cents at least .", "words": ["for", "would", "him", "at", "hit"]}, {"sent": "dont hit me mommy .", "words": ["dont", "me", "hit"]}, {"sent": "dont hit everything .", "words": ["dont", "hit"]}, {"sent": "you wouldnt like it if you were playing with somebody and they threw something and it hit you would you ?", "words": ["if", "with", "like", "you", "and", "they", "it", "would", "hit", "were"]}, {"sent": "oh great job you hit them all down .", "words": ["all", "hit", "them", "you", "down"]}, {"sent": "no you hit .", "words": ["hit", "you"]}, {"sent": "if she gets hit by a train shell be hurt .", "words": ["if", "train", "hurt", "be", "by", "hit", "she", "a"]}, {"sent": "her cousin hit her .", "words": ["hit", "her"]}, {"sent": "so what well hit Robbie ourselves .", "words": ["so", "what", "hit"]}, {"sent": "they hit the wheat with it they hit it like this sothat itll turn into flour so you can make pancakes with it .", "words": ["the", "can", "into", "so", "with", "like", "you", "they", "this", "it", "hit", "make"]}, {"sent": "hit that button .", "words": ["hit", "that", "button"]}, {"sent": "good hit .", "words": ["hit", "good"]}, {"sent": "dont hit yourself baby .", "words": ["yourself", "dont", "hit"]}, {"sent": "I didnt hit it hard .", "words": ["it", "hit", "hard"]}, {"sent": "no dont hit anybody with the stick .", "words": ["the", "stick", "with", "dont", "hit"]}, {"sent": "yeah hit the bat thats right .", "words": ["the", "hit", "bat"]}, {"sent": "right one car hit the other car and then the second car hit a third car .", "words": ["the", "car", "and", "other", "then", "hit", "a"]}, {"sent": "you hit yourself ?", "words": ["yourself", "hit", "you"]}, {"sent": "Chicon hit him .", "words": ["him", "hit"]}, {"sent": "yeah I hit it I hit it right back to you .", "words": ["back", "to", "you", "it", "hit"]}, {"sent": "Eleanor just hit the .", "words": ["the", "hit"]}, {"sent": "the car hit .", "words": ["the", "car", "hit"]}, {"sent": "you hit the bottom of the .", "words": ["the", "bottom", "of", "you", "hit"]}, {"sent": "good hit good hit .", "words": ["hit", "good"]}, {"sent": "let me move it over a tad for you so you dont hit the chair .", "words": ["the", "for", "so", "chair", "you", "dont", "it", "me", "hit", "over", "a"]}, {"sent": "xxx you better not hit me .", "words": ["better", "you", "me", "hit", "not"]}, {"sent": "no I dont wanna hit hard .", "words": ["hard", "dont", "hit", "wanna"]}, {"sent": "oh shouldnt hit the man .", "words": ["the", "hit"]}, {"sent": "no honey that really hurt it really hit my back .", "words": ["my", "back", "hurt", "it", "hit", "that"]}, {"sent": "well for making a gong sound you hit it and it goes bong .", "words": ["for", "you", "and", "it", "hit", "a"]}, {"sent": "you throw very well right okay this ones gonna hit your car okay .", "words": ["your", "throw", "car", "you", "this", "hit"]}, {"sent": "I know it almost hit you in the head ?", "words": ["the", "head", "you", "in", "it", "hit"]}, {"sent": "this kitchens a big hit isnt it ?", "words": ["this", "it", "hit", "big", "a"]}, {"sent": "somebody comes in and might hit the bed .", "words": ["the", "bed", "and", "in", "hit"]}, {"sent": "I was thinking about remember when the guy had the ball and he hit the pins .", "words": ["the", "was", "and", "ball", "he", "about", "hit", "when"]}, {"sent": "Im a hit you two times .", "words": ["hit", "you", "a"]}, {"sent": "I I hit it .", "words": ["it", "hit"]}, {"sent": "hit the red one again ?", "words": ["the", "hit", "red"]}, {"sent": "and you hit it .", "words": ["it", "hit", "you", "and"]}, {"sent": "oh but does daddy want you to hit people with it ?", "words": ["does", "to", "with", "you", "it", "hit", "but"]}, {"sent": "hah can you hit them together like with your sticks at home ?", "words": ["can", "your", "them", "like", "with", "you", "at", "hit", "home"]}, {"sent": "did I hit you foot ?", "words": ["foot", "hit", "did", "you"]}, {"sent": "Ill hit that old spot with this broom .", "words": ["old", "with", "this", "broom", "hit", "that"]}, {"sent": "and then the bottle and the sippycup were a big hit .", "words": ["the", "and", "big", "bottle", "then", "hit", "were", "a"]}, {"sent": "you hit the horses eye .", "words": ["the", "hit", "eye", "you"]}, {"sent": "hit the blue button .", "words": ["the", "hit", "button", "blue"]}, {"sent": "oh wow hon dont hit me .", "words": ["dont", "me", "hit"]}, {"sent": "did I give you a hit you too ?", "words": ["you", "did", "too", "give", "hit", "a"]}, {"sent": "to the park ?", "words": ["the", "to", "park"]}, {"sent": "you want the park next time ?", "words": ["the", "park", "you"]}, {"sent": "and have lunch in the park .", "words": ["the", "and", "have", "in", "park"]}, {"sent": "uh I think you maybe able to park right across the street .", "words": ["the", "think", "to", "you", "park", "street"]}, {"sent": "the park ?", "words": ["the", "park"]}, {"sent": "but when the Beehive closes were going to the big park where the swings and slide and the roundabout is .", "words": ["the", "where", "slide", "when", "is", "to", "and", "big", "park", "were", "but"]}, {"sent": "oh a park .", "words": ["park", "a"]}, {"sent": "were going to the park with Ewww .", "words": ["the", "to", "with", "park", "were"]}, {"sent": "is it from the park do you think ?", "words": ["the", "do", "think", "is", "you", "park", "it"]}, {"sent": "Beehive and the park .", "words": ["the", "park", "and"]}, {"sent": "hes gone to the park and what can he see ?", "words": ["the", "what", "can", "to", "and", "park", "he", "see"]}, {"sent": "remember you tripped on a tree root when we were walking in the park ?", "words": ["the", "tree", "we", "you", "in", "park", "on", "when", "were", "a"]}, {"sent": "which park ?", "words": ["park", "which"]}, {"sent": "no you dont park in the street .", "words": ["the", "you", "dont", "in", "park", "street"]}, {"sent": "he fell in the park .", "words": ["the", "in", "park", "he"]}, {"sent": "youre going to the park again ?", "words": ["the", "to", "park"]}, {"sent": "Lwww has gone on to the park J Fraser .", "words": ["on", "to", "park", "the"]}, {"sent": "not in the park .", "words": ["the", "in", "park", "not"]}, {"sent": "it might still be in the pushchair from when we came back from the park .", "words": ["the", "back", "we", "be", "in", "it", "park", "when"]}, {"sent": "we just do that at the park .", "words": ["the", "do", "we", "park", "at", "that"]}, {"sent": "tell me what you did at the park what did you do at the park ?", "words": ["the", "what", "do", "you", "park", "did", "at", "me"]}, {"sent": "in the park where skaters danced .", "words": ["the", "in", "park", "where"]}, {"sent": "what did she see when she went to the park ?", "words": ["the", "what", "to", "park", "did", "see", "when", "she"]}, {"sent": "where shall we park it ?", "words": ["where", "it", "park", "we"]}, {"sent": "do you remember going to the park ?", "words": ["the", "do", "to", "you", "park"]}, {"sent": "did we go to that big park ?", "words": ["go", "we", "to", "big", "park", "did", "that"]}, {"sent": "were going to go to the park with her .", "words": ["the", "go", "to", "with", "her", "park", "were"]}, {"sent": "in the park ?", "words": ["the", "in", "park"]}, {"sent": "in the city hall park they have some announcements and they happened to have a flier an advertisement under the glass thing .", "words": ["the", "to", "and", "have", "they", "in", "park", "an", "under", "glass", "some", "a"]}, {"sent": "whats he going to do at the park ?", "words": ["the", "do", "to", "park", "he", "at"]}, {"sent": "the park .", "words": ["the", "park"]}, {"sent": "oh whats at the park ?", "words": ["the", "park", "at"]}, {"sent": "to park at the top of your car park ?", "words": ["the", "your", "of", "to", "car", "park", "at"]}, {"sent": "come on park it up Thomas .", "words": ["on", "it", "park", "up"]}, {"sent": "to the park to the turtle park ?", "words": ["the", "to", "park", "turtle"]}, {"sent": "what did you do at the park ?", "words": ["the", "what", "do", "you", "park", "did", "at"]}, {"sent": "yes there was a big dog at the park wasnt it ?", "words": ["the", "was", "there", "park", "it", "at", "big", "dog", "a"]}, {"sent": "but hes been to Kinderfarm park .", "words": ["to", "park", "but"]}, {"sent": "theres the park over there .", "words": ["the", "there", "park", "over"]}, {"sent": "are you going to talk to Dimitra about when you saw her on Saturday in the park ?", "words": ["the", "to", "her", "you", "in", "are", "park", "about", "talk", "on", "when"]}, {"sent": "thats primrose park .", "words": ["park"]}, {"sent": "a zoo for the park keeper ?", "words": ["zoo", "the", "for", "park", "a"]}, {"sent": "if it doesnt rain well go to the park .", "words": ["the", "if", "rain", "go", "to", "it", "park"]}, {"sent": "you going to put that rubbish out the bin on the park now ?", "words": ["the", "put", "to", "you", "park", "out", "on", "that"]}, {"sent": "and youve been to the park .", "words": ["the", "to", "park", "and"]}, {"sent": "you can ask Dada to take you to the park .", "words": ["can", "the", "to", "you", "park", "take"]}, {"sent": "they gonna go to the car park and park that truck ?", "words": ["the", "go", "to", "car", "truck", "and", "they", "park", "that"]}, {"sent": "and what was at the park that was new that we were going to have a look at ?", "words": ["the", "what", "look", "we", "was", "to", "and", "have", "park", "at", "new", "that", "were", "a"]}, {"sent": "we went to the park and he was on the round a .", "words": ["the", "we", "to", "was", "and", "park", "he", "on", "a"]}, {"sent": "the vans in the car park this is the shop .", "words": ["the", "is", "car", "this", "in", "park"]}, {"sent": "that was in the little park wasnt it ?", "words": ["the", "was", "little", "in", "park", "it", "that"]}, {"sent": "where do you think you can park ?", "words": ["where", "do", "think", "can", "you", "park"]}, {"sent": "up the hill to the park .", "words": ["up", "the", "park", "to"]}, {"sent": "pretend theres a an absolutely enormous car park or plane port heliport or whatever theyre called .", "words": ["car", "an", "park", "pretend", "a"]}, {"sent": "opposite Whitworth park .", "words": ["park"]}, {"sent": "at the park ?", "words": ["the", "park", "at"]}, {"sent": "since you havent heard of patty cake we we gonna make a plan we gonna make plans for a trip to central park zoo .", "words": ["zoo", "for", "cake", "we", "of", "to", "you", "park", "make", "a"]}, {"sent": "xxx dat park dat little park ?", "words": ["park", "little"]}, {"sent": "lets get the lego train and we can park that we can park that on the floor beneath cant we ?", "words": ["the", "can", "we", "train", "and", "get", "park", "on", "that"]}, {"sent": "park yeah .", "words": ["park"]}, {"sent": "its like the zoo in central park where they they they left they took the animals outof the zoo and they moved them to other zoos .", "words": ["the", "zoo", "where", "to", "them", "like", "and", "other", "they", "in", "park"]}, {"sent": "to go to the park .", "words": ["the", "to", "park", "go"]}, {"sent": "are we going to go to a park later ?", "words": ["go", "we", "to", "are", "park", "a"]}, {"sent": "to the Flamingo park ?", "words": ["the", "to", "park"]}, {"sent": "youre a nice car park attendant arent you ?", "words": ["car", "nice", "you", "park", "a"]}, {"sent": "when weve been to the park today .", "words": ["the", "to", "park", "when"]}, {"sent": "to the where to the park ?", "words": ["the", "to", "park", "where"]}, {"sent": "because you willnt be able to find anywhere to park .", "words": ["to", "be", "you", "park", "find", "because"]}, {"sent": "finish your milk then well go to the park .", "words": ["the", "your", "go", "to", "finish", "park", "milk", "then"]}, {"sent": "Jwww likes playing football in the park doesnt he ?", "words": ["the", "in", "park", "he"]}, {"sent": "you bring them to the park .", "words": ["the", "to", "them", "bring", "you", "park"]}, {"sent": "in the park ?", "words": ["the", "in", "park"]}, {"sent": "and you were running around the park .", "words": ["the", "you", "and", "park", "around", "were"]}, {"sent": "they played in the park .", "words": ["the", "they", "in", "park"]}, {"sent": "remember when you were playing in the park you got all wet and muddy .", "words": ["the", "all", "wet", "you", "and", "in", "park", "when", "were"]}, {"sent": "park ?", "words": ["park"]}, {"sent": "well well park him here shall we ?", "words": ["here", "him", "park", "we"]}, {"sent": "we go to the Beehive and the park on Fridays .", "words": ["the", "go", "we", "to", "and", "park", "on"]}, {"sent": "and we go on long walks we go to the park a lot .", "words": ["the", "go", "we", "to", "a lot", "long", "and", "park", "on", "a"]}, {"sent": "its time to go to the park .", "words": ["the", "to", "park", "go"]}, {"sent": "well Jennifer and Spicey went to the park yesterday .", "words": ["the", "to", "park", "and"]}, {"sent": "were discussing this going to the park .", "words": ["the", "to", "this", "park", "were"]}, {"sent": "no hes just wearing shorts his legs are naked you have fun in the park ?", "words": ["the", "shorts", "you", "have", "in", "are", "park", "his"]}, {"sent": "which bits the park ?", "words": ["the", "park", "which"]}, {"sent": "and all the boys call him Mark park .", "words": ["the", "all", "and", "him", "park"]}, {"sent": "youve done that at the ball park havent you ?", "words": ["the", "you", "ball", "park", "at", "that"]}, {"sent": "what did you do at the park ?", "words": ["the", "what", "do", "you", "park", "did", "at"]}, {"sent": "autumn park ?", "words": ["park"]}, {"sent": "in the park .", "words": ["the", "in", "park"]}, {"sent": "the park .", "words": ["the", "park"]}, {"sent": "they go to a park .", "words": ["go", "to", "they", "park", "a"]}, {"sent": "we could go on a walk or go to a park maybe .", "words": ["go", "we", "to", "park", "on", "could", "walk", "a"]}, {"sent": "park .", "words": ["park"]}, {"sent": "you can run through the park cant you ?", "words": ["can", "the", "you", "run", "park"]}, {"sent": "but of course they could swim and one day they swam over to the park on the river xxx and there they met a policeman called Michael .", "words": ["the", "of", "to", "swim", "and", "they", "there", "park", "on", "could", "over", "but", "a"]}, {"sent": "and then we went to the park on our way home didnt we ?", "words": ["the", "we", "to", "and", "our", "park", "on", "then", "home"]}, {"sent": "whats your favorite thing on the park ?", "words": ["on", "the", "park", "your"]}, {"sent": "did they take you to the park ?", "words": ["the", "to", "you", "they", "park", "did", "take"]}, {"sent": "is it like the park ?", "words": ["the", "is", "like", "park", "it"]}, {"sent": "the park ?", "words": ["the", "park"]}, {"sent": "which one ?", "words": ["which"]}, {"sent": "you gotta ask me which type of sandwiches I want .", "words": ["of", "me", "which", "you"]}, {"sent": "right turn them all over then you can see which bit you want uhn cant you ?", "words": ["can", "all", "them", "which", "you", "see", "then", "over"]}, {"sent": "which book ?", "words": ["book", "which"]}, {"sent": "which color do you like ?", "words": ["do", "which", "like", "you"]}, {"sent": "which is which ?", "words": ["which", "is"]}, {"sent": "which which type of cheese do you have ?", "words": ["do", "of", "which", "you", "have", "cheese"]}, {"sent": "which break down lorry ?", "words": ["down", "break", "which"]}, {"sent": "tell me which one .", "words": ["me", "which"]}, {"sent": "so which one dyou wanna use ?", "words": ["so", "which", "wanna"]}, {"sent": "can you show me which is t t .", "words": ["can", "is", "show", "you", "me", "which"]}, {"sent": "and I cant remember which one now .", "words": ["which", "and"]}, {"sent": "youll like that better than his first idea which was that you guys were gonna play that chase game .", "words": ["better", "game", "was", "chase", "first", "play", "like", "which", "you", "his", "that", "were"]}, {"sent": "okay which one you gonna use ?", "words": ["which", "you"]}, {"sent": "no which ones that ?", "words": ["that", "which"]}, {"sent": "which red thing ?", "words": ["red", "which"]}, {"sent": "um and uh which book um would you child pref .", "words": ["you", "and", "would", "book", "which"]}, {"sent": "xxx which one is the .", "words": ["the", "which", "is"]}, {"sent": "which Grandma and Granddad is it ?", "words": ["it", "which", "is", "and"]}, {"sent": "which one ?", "words": ["which"]}, {"sent": "which one ?", "words": ["which"]}, {"sent": "which one ?", "words": ["which"]}, {"sent": "which is which I kind of thought was interesting .", "words": ["of", "was", "which", "is"]}, {"sent": "so which one do you want ?", "words": ["so", "do", "which", "you"]}, {"sent": "which one is blue ?", "words": ["blue", "which", "is"]}, {"sent": "which book do you wanna draw on ?", "words": ["do", "draw", "you", "on", "wanna", "book", "which"]}, {"sent": "which one do you like ?", "words": ["do", "which", "like", "you"]}, {"sent": "Carol which one is the baby ?", "words": ["the", "which", "is"]}, {"sent": "okay which one ?", "words": ["which"]}, {"sent": "which which shoe is missing ?", "words": ["which", "is", "shoe"]}, {"sent": "show Mummy which ones crying .", "words": ["show", "which"]}, {"sent": "which is like .", "words": ["which", "like", "is"]}, {"sent": "which ones b .", "words": ["which"]}, {"sent": "okay show me which finger .", "words": ["show", "me", "finger", "which"]}, {"sent": "thats a piece of a orange or a lemon cant quite tell which one .", "words": ["of", "orange", "which", "a"]}, {"sent": "which egg comes next ?", "words": ["which", "egg"]}, {"sent": "yeah which one shall we do ?", "words": ["do", "which", "we"]}, {"sent": "which one ?", "words": ["which"]}, {"sent": "now which one ?", "words": ["which"]}, {"sent": "and which is the left one ?", "words": ["the", "which", "is", "and"]}, {"sent": "on which side are you .", "words": ["on", "are", "which", "you"]}, {"sent": "I think that I might wanna buy one too but which one would you recommend which one ?", "words": ["think", "which", "would", "you", "wanna", "too", "that", "buy", "but"]}, {"sent": "which door ?", "words": ["which", "door"]}, {"sent": "and which store would that be ?", "words": ["store", "be", "which", "and", "would", "that"]}, {"sent": "which one of those things is your shoe ?", "words": ["your", "those", "is", "of", "shoe", "which"]}, {"sent": "which chair would you like to sit in ?", "words": ["to", "like", "chair", "you", "in", "would", "sit", "which"]}, {"sent": "which one is which ?", "words": ["which", "is"]}, {"sent": "which way round do you want it to go ?", "words": ["do", "go", "to", "you", "it", "which"]}, {"sent": "which one first ?", "words": ["first", "which"]}, {"sent": "which holiday do you like best ?", "words": ["do", "which", "like", "you"]}, {"sent": "then go finish our breakfast which is really the main thing on my mind .", "words": ["the", "my", "go", "is", "finish", "our", "on", "then", "which"]}, {"sent": "which ones different ?", "words": ["which"]}, {"sent": "which one ?", "words": ["which"]}, {"sent": "in which room ?", "words": ["in", "room", "which"]}, {"sent": "so which of the toys seem to interest Caroline the most ?", "words": ["the", "of", "so", "to", "which"]}, {"sent": "which guys .", "words": ["which"]}, {"sent": "which ones the biggest ?", "words": ["the", "which"]}, {"sent": "which one ?", "words": ["which"]}, {"sent": "which ones going to win this time ?", "words": ["to", "which", "this"]}, {"sent": "which ones which ?", "words": ["which"]}, {"sent": "which one did you like best ?", "words": ["which", "did", "like", "you"]}, {"sent": "which ones do you think hes more fascinated with or youve been working on with him ?", "words": ["do", "think", "with", "you", "more", "him", "on", "which"]}, {"sent": "when I uh we drove back from NewYork we hadta wait for an hour in line behind one car which had smashed into a signpost .", "words": ["wait", "for", "into", "back", "we", "car", "behind", "which", "an", "in", "when", "a"]}, {"sent": "you show me which one is all white .", "words": ["all", "is", "show", "you", "me", "white", "which"]}, {"sent": "which one now the square ?", "words": ["the", "which"]}, {"sent": "and which ones Eeyore ?", "words": ["which", "and"]}, {"sent": "so which is the which flag is the same as that ?", "words": ["the", "is", "so", "flag", "that", "same", "which"]}, {"sent": "which shape ?", "words": ["which"]}, {"sent": "which one ?", "words": ["which"]}, {"sent": "if you get them all mixed up you wont know in which bag they go in after uhn will you ?", "words": ["up", "all", "if", "go", "them", "you", "get", "they", "in", "will", "which"]}, {"sent": "first what first which bed does she get in ?", "words": ["what", "does", "first", "bed", "get", "in", "which", "she"]}, {"sent": "which ones do you want ?", "words": ["do", "which", "you"]}, {"sent": "xxx those are what we call twenties these ones now these are called two twos now which these ones these ones these are called fives and these are called you know what these are called these are called ones you know oh look .", "words": ["what", "these", "those", "we", "and", "you", "are", "look", "which"]}, {"sent": "which ?", "words": ["which"]}, {"sent": "which said English daffodils .", "words": ["which"]}, {"sent": "which princess ?", "words": ["which"]}, {"sent": "which bugs you wanna cook those or those ?", "words": ["those", "you", "cook", "wanna", "which"]}, {"sent": "because dyou remember which ones I have ?", "words": ["have", "which", "because"]}, {"sent": "which one ?", "words": ["which"]}, {"sent": "and theyre asking all sorts of questions which we hafta fill in together Thomas .", "words": ["all", "we", "of", "and", "in", "which"]}, {"sent": "which one ?", "words": ["which"]}, {"sent": "you want which job do I do ?", "words": ["do", "which", "you"]}, {"sent": "which season are we in now ?", "words": ["in", "are", "which", "we"]}, {"sent": "which ones are you going for then ?", "words": ["for", "you", "are", "then", "which"]}, {"sent": "were gonna havta brush your teeth which look pink from Pez now .", "words": ["your", "brush", "which", "look", "were"]}, {"sent": "and when you you were a baby which of course you were at the time of your christening when it was bought .", "words": ["the", "your", "when", "of", "was", "which", "you", "and", "it", "at", "were", "a"]}, {"sent": "and uh which is what I feed him .", "words": ["what", "is", "and", "him", "feed", "which"]}, {"sent": "which drink are you gonna choose ?", "words": ["are", "which", "drink", "you"]}, {"sent": "which railway station ?", "words": ["which"]}, {"sent": "which movie ?", "words": ["which"]}, {"sent": "which one shall we do next ?", "words": ["do", "which", "we"]}, {"sent": "these include the planets one of which is Earth where we live .", "words": ["the", "these", "where", "is", "of", "we", "which"]}, {"sent": "which which of these two teddy bears would you like to be BramwellBrown ?", "words": ["these", "of", "to", "be", "like", "you", "would", "which"]}, {"sent": "which ones orange soup ?", "words": ["soup", "orange", "which"]}, {"sent": "which house do you think the farmer lives in ?", "words": ["the", "do", "think", "you", "in", "house", "which"]}, {"sent": "okay which beans would you like to glue ?", "words": ["to", "like", "you", "beans", "would", "glue", "which"]}, {"sent": "which ones nicest ?", "words": ["which"]}, {"sent": "which other one ?", "words": ["which", "other"]}, {"sent": "which finger which finger hurts ?", "words": ["finger", "which"]}, {"sent": "and which one have we got left ?", "words": ["have", "which", "we", "and"]}, {"sent": "Id eat that myself .", "words": ["eat", "that", "myself"]}, {"sent": "Im really going to be ahead of myself .", "words": ["of", "to", "be", "myself"]}, {"sent": "I thought to myself .", "words": ["to", "myself"]}, {"sent": "xxx myself .", "words": ["myself"]}, {"sent": "I still can be driving all by myself .", "words": ["can", "all", "be", "by", "myself"]}, {"sent": "you just want me to make a nut outof myself .", "words": ["to", "you", "myself", "me", "make", "a"]}, {"sent": "youre giving me fifty pence to go and buy myself some lollies .", "words": ["go", "to", "and", "myself", "me", "buy", "some"]}, {"sent": "and I wouldnt put that pressure on myself .", "words": ["put", "and", "myself", "on", "that"]}, {"sent": "Im doing it all by myself youre right I am .", "words": ["all", "by", "it", "myself", "am"]}, {"sent": "yes but xxx you can have some Ill go and make myself another one .", "words": ["can", "go", "you", "have", "and", "another", "myself", "some", "make", "but"]}, {"sent": "no youre not there I guess Ill just hafta go upstairs to the playroom to find a toy to play with by myself .", "words": ["the", "go", "to", "play", "with", "by", "there", "toy", "myself", "find", "not", "a"]}, {"sent": "right right I dont think Id like myself .", "words": ["dont", "think", "like", "myself"]}, {"sent": "to myself .", "words": ["to", "myself"]}, {"sent": "well Im Im trying to make myself like it .", "words": ["to", "like", "it", "myself", "make"]}, {"sent": "I hafta figure it out all by myself .", "words": ["all", "by", "it", "myself", "out"]}, {"sent": "well I dont know too many myself .", "words": ["dont", "myself", "too"]}, {"sent": "Im doing this all by myself .", "words": ["all", "this", "myself", "by"]}, {"sent": "I wet myself with my soup .", "words": ["wet", "my", "with", "myself", "soup"]}, {"sent": "I can dress myself .", "words": ["can", "dress", "myself"]}, {"sent": "myself to some of those .", "words": ["those", "of", "to", "myself", "some"]}, {"sent": "can you hear all by myself ?", "words": ["can", "all", "hear", "by", "you", "myself"]}, {"sent": "going to reach over and help myself to this .", "words": ["help", "to", "and", "this", "myself", "over"]}, {"sent": "perhaps Ill just go and sit and eat my cake by myself then .", "words": ["my", "cake", "go", "then", "and", "eat", "by", "myself", "sit"]}, {"sent": "I was flying by myself for the very first time .", "words": ["the", "for", "was", "first", "by", "myself"]}, {"sent": "I will infest myself with xxx .", "words": ["will", "with", "myself"]}, {"sent": "uh I couldnt hear myself .", "words": ["myself", "hear"]}, {"sent": "look at me she said I can ride my bike all by myself .", "words": ["can", "all", "my", "look", "by", "myself", "at", "me", "ride", "she"]}, {"sent": "yeah I can do it by myself .", "words": ["can", "do", "by", "it", "myself"]}, {"sent": "and I heard myself .", "words": ["myself", "and"]}, {"sent": "just amusing myself now .", "words": ["myself"]}, {"sent": "Im scared of myself that would be pretty scary .", "words": ["of", "be", "scared", "pretty", "myself", "would", "that"]}, {"sent": "right well Ill read this book to myself .", "words": ["to", "this", "read", "myself", "book"]}, {"sent": "Jane Ive done it as many times myself .", "words": ["it", "myself"]}, {"sent": "I made those for you myself .", "words": ["for", "those", "myself", "you"]}, {"sent": "Im going to build it up myself .", "words": ["up", "to", "it", "build", "myself"]}, {"sent": "me by myself ?", "words": ["me", "myself", "by"]}, {"sent": "this is an I can read it all by myself book .", "words": ["can", "all", "is", "by", "this", "an", "it", "read", "myself", "book"]}, {"sent": "and I found myself lifting the nappies and going .", "words": ["the", "myself", "and"]}, {"sent": "aye just myself and my cousin .", "words": ["my", "myself", "and"]}, {"sent": "xxx keeping myself awake .", "words": ["awake", "myself"]}, {"sent": "but I just I tell myself now what I needta do is get fit .", "words": ["what", "do", "is", "fit", "get", "myself", "but"]}, {"sent": "well I cant give myself a back rub .", "words": ["give", "myself", "back", "a"]}, {"sent": "all the I mean Id be curious to myself .", "words": ["the", "all", "be", "to", "myself"]}, {"sent": "I put my right hand out I give my right hand a shake shake shake and turn myself about .", "words": ["put", "my", "hand", "shake", "and", "myself", "out", "about", "give", "a"]}, {"sent": "all by myself whispered Sam .", "words": ["all", "myself", "by"]}, {"sent": "yeah I think I Im feeling pretty optimistic about it myself .", "words": ["think", "it", "pretty", "myself", "about"]}, {"sent": "not quite sure myself really .", "words": ["not", "myself"]}, {"sent": "I enjoyed myself by buying some nursing bras online on sale .", "words": ["on", "some", "myself", "by"]}, {"sent": "now Im gonna pour some for myself .", "words": ["some", "pour", "myself", "for"]}, {"sent": "but Ill hurt myself .", "words": ["hurt", "myself", "but"]}, {"sent": "you just let me get on with it and muck myself up .", "words": ["up", "with", "you", "get", "and", "it", "myself", "on", "me"]}, {"sent": "Ill go and do something for myself then .", "words": ["do", "for", "go", "and", "myself", "then"]}, {"sent": "Ive woken myself up because I didnt think you were going to .", "words": ["up", "think", "to", "you", "myself", "were", "because"]}, {"sent": "that means that Daddy Purdie Thomas and myself can each have two .", "words": ["can", "each", "and", "have", "myself", "that"]}, {"sent": "nobody hafta just go by myself .", "words": ["by", "myself", "go"]}, {"sent": "can I um possibly buy an aeroplane that I make myself ?", "words": ["can", "an", "that", "myself", "make", "buy"]}, {"sent": "and myself an ice cream .", "words": ["ice cream", "and", "an", "myself", "ice"]}, {"sent": "I put my right foot out I give my right foot a shake shake shake and turn myself around .", "words": ["put", "my", "shake", "and", "foot", "myself", "out", "around", "give", "a"]}, {"sent": "want me to do my house by myself ?", "words": ["do", "my", "to", "by", "house", "myself", "me"]}, {"sent": "maybe I can get one for myself .", "words": ["can", "get", "myself", "for"]}, {"sent": "theres a lot of washing up to say that there was only myself you Daddy Purdie .", "words": ["up", "of", "to", "was", "a lot", "say", "you", "there", "myself", "that", "a"]}, {"sent": "you want me to be a dustbin lady and put them in the bin myself do you ?", "words": ["the", "do", "put", "to", "be", "them", "you", "and", "in", "myself", "me", "a"]}, {"sent": "Im going to take one home myself so Ill remember every day to water it and when it starts to .", "words": ["when", "to", "so", "and", "it", "myself", "every", "water", "take", "home"]}, {"sent": "thats what I said to myself thats it Im Im dying thats it .", "words": ["to", "what", "it", "myself"]}, {"sent": "I think Ill do some coffee for myself while the kettles hot .", "words": ["the", "do", "think", "for", "hot", "coffee", "myself", "some"]}, {"sent": "I wanna do it by myself .", "words": ["do", "by", "it", "myself", "wanna"]}, {"sent": "and I hafta xxx bottles of wine myself .", "words": ["of", "myself", "and"]}, {"sent": "let me do it myself Jwww .", "words": ["me", "do", "it", "myself"]}, {"sent": "listening to myself .", "words": ["to", "myself"]}, {"sent": "I eat my breakfast by myself .", "words": ["eat", "my", "myself", "by"]}, {"sent": "does that mean one and a half and I could bring myself and a child ?", "words": ["does", "bring", "and", "myself", "could", "that", "a"]}, {"sent": "I thought it was just me feeling sorry for myself .", "words": ["for", "was", "it", "myself", "me"]}, {"sent": "all by myself .", "words": ["all", "myself", "by"]}, {"sent": "I dont understand very well about God myself .", "words": ["dont", "myself", "about"]}, {"sent": "and Im Im not really sure what that oniony thing is myself .", "words": ["what", "is", "and", "myself", "that", "not"]}, {"sent": "I cant do it all by myself ?", "words": ["all", "do", "by", "it", "myself"]}, {"sent": "Id got myself that worked up .", "words": ["up", "that", "myself"]}, {"sent": "cant I cut it myself ?", "words": ["it", "cut", "myself"]}, {"sent": "I can dress myself .", "words": ["can", "dress", "myself"]}, {"sent": "you want me to brush myself .", "words": ["brush", "to", "you", "myself", "me"]}, {"sent": "just gonna get myself a drink .", "words": ["get", "myself", "drink", "a"]}, {"sent": "I said to Don look at your daughter what is she doing and then I look at myself and Im like .", "words": ["what", "look", "your", "is", "to", "like", "and", "myself", "at", "then", "she"]}, {"sent": "Im just gonna make myself a coffee .", "words": ["coffee", "make", "myself", "a"]}, {"sent": "I sorta was kicking myself for not keeping him back and I had wanted to keep him back but .", "words": ["for", "back", "was", "to", "and", "him", "myself", "not", "but"]}, {"sent": "but sometimes in the afternoon I would feel myself if I sat down .", "words": ["the", "if", "in", "myself", "would", "down", "but"]}, {"sent": "oh I stuck it on with my xxx myself .", "words": ["my", "with", "it", "stuck", "myself", "on"]}, {"sent": "Im gonna read this book all by myself .", "words": ["all", "by", "this", "read", "myself", "book"]}, {"sent": "oh Ive forgotten to pour myself a cup of tea .", "words": ["cup", "of", "to", "pour", "myself", "a"]}, {"sent": "is that what you said this myself ?", "words": ["what", "is", "you", "this", "myself", "that"]}, {"sent": "and Ill let me explain myself .", "words": ["me", "myself", "and"]}, {"sent": "I wrote xxx by myself .", "words": ["myself", "by"]}, {"sent": "I talk to myself .", "words": ["to", "myself", "talk"]}, {"sent": "I must say I like wee boys myself .", "words": ["say", "like", "myself"]}, {"sent": "Grover says ah I cant hear myself think now look at this .", "words": ["think", "hear", "this", "at", "myself", "look"]}, {"sent": "Ill pour myself a cup of tea now .", "words": ["cup", "of", "pour", "myself", "a"]}, {"sent": "I just hadta save myself from the crocodile .", "words": ["the", "myself"]}, {"sent": "I can dress myself .", "words": ["can", "dress", "myself"]}, {"sent": "want me to wind them up a little more ?", "words": ["up", "to", "them", "little", "more", "me", "a", "wind"]}, {"sent": "are you gonna wind the finger all up in that elastic ?", "words": ["the", "all", "up", "you", "finger", "in", "are", "that", "wind"]}, {"sent": "and a sudden gust of wind sprang up and blew away his beautiful bouquet .", "words": ["up", "of", "and", "away", "his", "a", "wind"]}, {"sent": "you want me to wind her .", "words": ["to", "you", "me", "her", "wind"]}, {"sent": "Mommys hairs blowing because theres lotsof wind .", "words": ["because", "wind"]}, {"sent": "how does the wind go ?", "words": ["the", "does", "how", "go", "wind"]}, {"sent": "its hard to wind .", "words": ["to", "hard", "wind"]}, {"sent": "when the wind .", "words": ["the", "when", "wind"]}, {"sent": "can you wind the lift down for me please ?", "words": ["can", "the", "for", "you", "me", "down", "wind"]}, {"sent": "they tore by like the wind made a wide circle in the grassy meadow and vanished into the woods again where they could still be heard galloping .", "words": ["the", "where", "into", "be", "like", "by", "and", "they", "in", "could", "a", "wind"]}, {"sent": "that noise was the wind and the paper .", "words": ["the", "was", "paper", "and", "that", "wind"]}, {"sent": "does it make as much wind when it xxx .", "words": ["does", "when", "much", "it", "make", "wind"]}, {"sent": "yeah see look you hafta wind him up .", "words": ["up", "you", "him", "look", "see", "wind"]}, {"sent": "can you wind that ?", "words": ["can", "that", "wind", "you"]}, {"sent": "do you want me to wind him up as well ?", "words": ["up", "do", "to", "you", "him", "me", "wind"]}, {"sent": "wind blowing .", "words": ["wind"]}, {"sent": "right are you going to wind it down now ?", "words": ["to", "you", "are", "it", "down", "wind"]}, {"sent": "wind it up .", "words": ["up", "it", "wind"]}, {"sent": "its a strong wind .", "words": ["wind", "a"]}, {"sent": "can I wind it ?", "words": ["can", "it", "wind"]}, {"sent": "then wind the lift up .", "words": ["the", "then", "up", "wind"]}, {"sent": "then the wind blows .", "words": ["the", "then", "wind"]}, {"sent": "have to wait for the wind .", "words": ["the", "wait", "for", "to", "have", "wind"]}, {"sent": "ah there were hailstones and wind .", "words": ["there", "were", "wind", "and"]}, {"sent": "oh this guys pretty hard to wind up .", "words": ["up", "to", "this", "pretty", "hard", "wind"]}, {"sent": "and the wind .", "words": ["the", "wind", "and"]}, {"sent": "here let me wind him up some more .", "words": ["up", "here", "him", "more", "me", "some", "wind"]}, {"sent": "like the wind blowing .", "words": ["the", "like", "wind"]}, {"sent": "that must be the wind .", "words": ["the", "be", "that", "wind"]}, {"sent": "and an icy wind blows across the hills .", "words": ["the", "an", "wind", "and"]}, {"sent": "hold your finger on it and then wind it up honey .", "words": ["up", "your", "hold", "and", "finger", "it", "on", "then", "wind"]}, {"sent": "when we got here yesterday a Columbia student was on his way with his notebook and the wind just took his notebook .", "words": ["the", "we", "was", "with", "wind", "and", "here", "his", "on", "when", "a"]}, {"sent": "let me wind it back up for you .", "words": ["up", "for", "back", "you", "it", "me", "wind"]}, {"sent": "but what if you wind it again ?", "words": ["what", "if", "you", "it", "but", "wind"]}, {"sent": "wind it up .", "words": ["up", "it", "wind"]}, {"sent": "Mama you want me to wind it up ?", "words": ["up", "to", "you", "it", "me", "wind"]}, {"sent": "come and wind it up .", "words": ["up", "it", "wind", "and"]}, {"sent": "oh you wind him up and he does absolutely nothing .", "words": ["up", "does", "and", "you", "him", "he", "wind"]}, {"sent": "thats a little tail that also if the wind is coming from this direction it goes prr and pushes this little flag and then it goes like that and then you know that the wind is coming from that direction .", "words": ["the", "if", "is", "flag", "like", "wind", "little", "and", "this", "you", "it", "then", "that", "a"]}, {"sent": "when when I wind it up .", "words": ["up", "it", "when", "wind"]}, {"sent": "wheres your wind ?", "words": ["your", "wind"]}, {"sent": "the wind blows the leaves through the trees .", "words": ["the", "wind"]}, {"sent": "wind does that .", "words": ["does", "that", "wind"]}, {"sent": "blowing like the wind ?", "words": ["the", "like", "wind"]}, {"sent": "heres lets wind them up again .", "words": ["up", "them", "wind"]}, {"sent": "the water runs cold between my toes apples fall off when the wind blows .", "words": ["the", "off", "my", "fall", "water", "when", "cold", "wind"]}, {"sent": "there isnt much wind at all .", "words": ["all", "there", "at", "much", "wind"]}, {"sent": "xxx can you wind it up ?", "words": ["can", "up", "you", "it", "wind"]}, {"sent": "howbout over here lets try to do it away from the wind .", "words": ["the", "do", "to", "here", "it", "away", "over", "try", "try to", "wind"]}, {"sent": "think its just the wind .", "words": ["the", "think", "wind"]}, {"sent": "when the wind blows .", "words": ["the", "when", "wind"]}, {"sent": "but why does the wind blow you up there ?", "words": ["the", "does", "up", "you", "there", "but", "blow", "why", "wind"]}, {"sent": "I can hear the wind as well Thomas .", "words": ["can", "the", "wind", "hear"]}, {"sent": "come and help me wind her .", "words": ["help", "and", "me", "her", "wind"]}, {"sent": "wanna wind it up ?", "words": ["up", "it", "wanna", "wind"]}, {"sent": "did you hear a door banging in the wind ?", "words": ["the", "wind", "hear", "you", "in", "did", "a", "door"]}, {"sent": "do you know how to wind it up and set it off ?", "words": ["up", "do", "how", "off", "to", "you", "and", "it", "wind"]}, {"sent": "I think its the wind .", "words": ["the", "think", "wind"]}, {"sent": "bubbles are very light theyre not heavy because theyre so light the wind can blow them .", "words": ["the", "can", "blow", "light", "so", "them", "wind", "are", "heavy", "not", "because", "bubbles"]}, {"sent": "and as the engines were in the engine shed listening to the wind whistling outside .", "words": ["the", "to", "outside", "and", "in", "were", "wind"]}, {"sent": "wind it wind it more .", "words": ["more", "it", "wind"]}, {"sent": "Spot ran up and down the field with his kite trying to get the wind to lift it up into the air .", "words": ["up", "the", "into", "to", "with", "and", "get", "it", "his", "down", "wind"]}, {"sent": "xxx I think Ill wind it up okay .", "words": ["up", "think", "it", "wind"]}, {"sent": "just the wind sometimes makes it creak .", "words": ["the", "it", "wind"]}, {"sent": "are you getting blown away in the wind ?", "words": ["the", "you", "in", "are", "away", "wind"]}, {"sent": "you wind it up .", "words": ["up", "it", "wind", "you"]}, {"sent": "come on wind him up .", "words": ["on", "him", "up", "wind"]}, {"sent": "wind .", "words": ["wind"]}, {"sent": "you wanna wind the watch ?", "words": ["the", "watch", "you", "wanna", "wind"]}, {"sent": "when Mummy was doing the planting I must have dropped that one and the wind has blown it down there .", "words": ["the", "was", "and", "have", "there", "that", "it", "down", "when", "wind"]}, {"sent": "well wind him up shall we ?", "words": ["up", "him", "we", "wind"]}, {"sent": "wind it up then haha .", "words": ["up", "then", "it", "wind"]}, {"sent": "like the wind tinkling through a crystal chandelier .", "words": ["the", "like", "a", "wind"]}, {"sent": "wind it up .", "words": ["up", "it", "wind"]}, {"sent": "the wind ?", "words": ["the", "wind"]}, {"sent": "not yet fox grinned I love watching the clouds move and feel the wind blow its exciting .", "words": ["the", "blow", "and", "love", "not", "wind"]}, {"sent": "its just the wind blowing the door .", "words": ["the", "door", "wind"]}, {"sent": "the wind .", "words": ["the", "wind"]}, {"sent": "wind it down .", "words": ["down", "it", "wind"]}, {"sent": "he never gets the last of his wind up .", "words": ["the", "up", "of", "last", "he", "his", "wind"]}, {"sent": "feel the wind wind is to feel .", "words": ["the", "to", "is", "wind"]}, {"sent": "did the wind blow him up there ?", "words": ["the", "up", "there", "him", "did", "blow", "wind"]}, {"sent": "were a big engine Henry Thomas said you shouldnt be afraid of a little wind .", "words": ["of", "be", "wind", "you", "little", "big", "were", "a"]}, {"sent": "the wind didnt blow this door open cause its closed see ?", "words": ["the", "open", "wind", "this", "see", "blow", "door"]}, {"sent": "early one morning the wind blew a spider across the field .", "words": ["the", "a", "wind"]}, {"sent": "but look if you wind this lookit lookit lookit lookit lookit lookit lookit .", "words": ["if", "you", "this", "look", "but", "wind"]}, {"sent": "ah yeah this is a wind up watch Daniel .", "words": ["up", "watch", "is", "this", "a", "wind"]}, {"sent": "the wind blowing outside huh ?", "words": ["the", "outside", "wind"]}, {"sent": "can you wind the chicky up ?", "words": ["can", "the", "up", "you", "wind"]}, {"sent": "early one morning the wind blew a spider across the field .", "words": ["the", "a", "wind"]}, {"sent": "wind him gently not too much ?", "words": ["him", "too", "much", "not", "wind"]}, {"sent": "let me try to wind it up okay ?", "words": ["up", "to", "it", "me", "try", "try to", "wind"]}, {"sent": "dont wind him too much dont wind him too much .", "words": ["dont", "him", "too", "much", "wind"]}, {"sent": "and the wind is saying to the sun I bet I can get that mans coat off before you can .", "words": ["the", "can", "off", "is", "to", "sun", "and", "get", "you", "that", "coat", "wind"]}, {"sent": "wind it up .", "words": ["up", "it", "wind"]}, {"sent": "in this wind .", "words": ["this", "in", "wind"]}, {"sent": "when does the wind blow you up to the sky ?", "words": ["the", "does", "up", "sky", "to", "you", "when", "blow", "wind"]}, {"sent": "you drive the cars on and wind it up .", "words": ["the", "up", "and", "drive", "you", "it", "on", "wind"]}, {"sent": "you hafta wind it up the other way .", "words": ["up", "the", "you", "other", "it", "wind"]}, {"sent": "its a mouse .", "words": ["mouse", "a"]}, {"sent": "isnt she big compared to the tiny mouse ?", "words": ["the", "tiny", "to", "big", "she", "mouse"]}, {"sent": "if theres only one its called a mouse .", "words": ["if", "mouse", "a"]}, {"sent": "thats shaped like a mouse isnt it ?", "words": ["it", "like", "mouse", "a"]}, {"sent": "the mouse ran down .", "words": ["the", "down", "mouse"]}, {"sent": "theres no opposite of mouse darling .", "words": ["of", "mouse"]}, {"sent": "mouse eat all that ?", "words": ["eat", "all", "that", "mouse"]}, {"sent": "that dont sound like Mickey mouse .", "words": ["dont", "that", "like", "mouse"]}, {"sent": "dont open your mouth because the mouse will run in .", "words": ["the", "your", "open", "mouth", "dont", "run", "mouse", "in", "will", "because"]}, {"sent": "here comes a mouse .", "words": ["here", "mouse", "a"]}, {"sent": "baby duck mouse .", "words": ["duck", "mouse"]}, {"sent": "a mouse .", "words": ["mouse", "a"]}, {"sent": "aw and look the mouse has on a dress .", "words": ["the", "and", "look", "on", "dress", "mouse", "a"]}, {"sent": "the clock struck one the mouse ran down hickory dickory dock .", "words": ["the", "down", "mouse", "clock"]}, {"sent": "house on mouse .", "words": ["on", "house", "mouse"]}, {"sent": "heres the little mouse that hes drawing a big picture of .", "words": ["the", "of", "picture", "little", "big", "that", "mouse", "a"]}, {"sent": "nitz a mouse .", "words": ["mouse", "a"]}, {"sent": "he wants to look at your Mickey mouse hat .", "words": ["look", "your", "to", "he", "at", "hat", "mouse"]}, {"sent": "mouse count one .", "words": ["mouse"]}, {"sent": "mouse ?", "words": ["mouse"]}, {"sent": "a mouse .", "words": ["mouse", "a"]}, {"sent": "when the cat chases him the mouse says argh .", "words": ["the", "cat", "him", "when", "mouse"]}, {"sent": "catching a mouse ?", "words": ["mouse", "a"]}, {"sent": "town mouse and country mouse go to bed .", "words": ["go", "to", "bed", "and", "mouse"]}, {"sent": "heres a cover for the mouse .", "words": ["the", "for", "cover", "mouse", "a"]}, {"sent": "there was a little mouse underneath the bed .", "words": ["the", "was", "bed", "little", "there", "mouse", "a"]}, {"sent": "xxx thats a mouse thats what .", "words": ["what", "mouse", "a"]}, {"sent": "grey kitten said Id like to see the house the mouse has made .", "words": ["the", "to", "like", "house", "see", "mouse"]}, {"sent": "shall we make a bed for the mouse ?", "words": ["the", "for", "we", "bed", "make", "mouse", "a"]}, {"sent": "are you a mouse ?", "words": ["are", "mouse", "a", "you"]}, {"sent": "theres a frog and a mouse isnt there ?", "words": ["frog", "and", "there", "mouse", "a"]}, {"sent": "is that a mouse ?", "words": ["mouse", "that", "a", "is"]}, {"sent": "quickly quickly very quickly runs the little mouse .", "words": ["the", "mouse", "little"]}, {"sent": "but wheat is heavy the mouse said .", "words": ["the", "is", "but", "heavy", "mouse"]}, {"sent": "town mouse says here country mouse come into my hole .", "words": ["here", "my", "into", "mouse"]}, {"sent": "grumpy mouse or something like that .", "words": ["that", "like", "mouse"]}, {"sent": "it was a mouse .", "words": ["was", "it", "mouse", "a"]}, {"sent": "whats the program that had Mickey mouse that day ?", "words": ["the", "that", "mouse"]}, {"sent": "wheres the mouse ?", "words": ["the", "mouse"]}, {"sent": "thats not a mouse .", "words": ["not", "mouse", "a"]}, {"sent": "xxx this is called the mouse .", "words": ["the", "this", "mouse", "is"]}, {"sent": "can I have that mouse please ?", "words": ["can", "have", "that", "mouse"]}, {"sent": "country mouse book .", "words": ["book", "mouse"]}, {"sent": "do you know a song about a mouse and a clock ?", "words": ["do", "clock", "you", "and", "about", "mouse", "a"]}, {"sent": "say hello to my mouse .", "words": ["to", "my", "say", "mouse"]}, {"sent": "is there a wee mouse at these biscuits ?", "words": ["these", "is", "there", "at", "mouse", "a"]}, {"sent": "twenty seven was above the mouse ?", "words": ["above", "was", "mouse", "the"]}, {"sent": "amazing mouse ?", "words": ["mouse"]}, {"sent": "a little mouse .", "words": ["mouse", "little", "a"]}, {"sent": "oh oh a mouse .", "words": ["mouse", "a"]}, {"sent": "no thats not mouse .", "words": ["not", "mouse"]}, {"sent": "chocolate chip cookie mouse ?", "words": ["cookie", "chocolate", "mouse"]}, {"sent": "a small mouse sitting on a small mushroom .", "words": ["on", "mouse", "a"]}, {"sent": "hes a mouse .", "words": ["mouse", "a"]}, {"sent": "has it got a little mouse in it ?", "words": ["little", "in", "it", "mouse", "a"]}, {"sent": "mouse .", "words": ["mouse"]}, {"sent": "did the mouse roll over into the home ?", "words": ["the", "into", "did", "over", "mouse", "home"]}, {"sent": "one on its own is a mouse .", "words": ["on", "mouse", "a", "is"]}, {"sent": "little mouse .", "words": ["mouse", "little"]}, {"sent": "the mouse ran up the .", "words": ["the", "up", "mouse"]}, {"sent": "you hafta put peas in her mouse .", "words": ["put", "peas", "you", "in", "her", "mouse"]}, {"sent": "wheres a little mouse ?", "words": ["mouse", "little", "a"]}, {"sent": "a mouse okay .", "words": ["mouse", "a"]}, {"sent": "now the mouse is tired .", "words": ["the", "tired", "mouse", "is"]}, {"sent": "is it a mouse ?", "words": ["it", "mouse", "a", "is"]}, {"sent": "thats Mickey mouse .", "words": ["mouse"]}, {"sent": "do you see the little mouse ?", "words": ["the", "do", "little", "you", "see", "mouse"]}, {"sent": "m for mouse .", "words": ["for", "mouse"]}, {"sent": "mouse ?", "words": ["mouse"]}, {"sent": "the mouse was in the bag ?", "words": ["the", "was", "in", "mouse"]}, {"sent": "a mouse after a cat .", "words": ["cat", "mouse", "a"]}, {"sent": "wheres the other mouse ?", "words": ["the", "mouse", "other"]}, {"sent": "field mouse I havent seen anything funny in a long time .", "words": ["long", "in", "mouse", "a"]}, {"sent": "is that a meesy mouse ?", "words": ["mouse", "that", "a", "is"]}, {"sent": "a mouse ?", "words": ["mouse", "a"]}, {"sent": "a little mouse came out to play .", "words": ["to", "play", "little", "out", "mouse", "a"]}, {"sent": "it says Fraser is a little mouse squeak .", "words": ["is", "little", "it", "mouse", "a"]}, {"sent": "mouse .", "words": ["mouse"]}, {"sent": "thats the mouse you take to bed with you ?", "words": ["the", "to", "with", "bed", "you", "take", "mouse"]}, {"sent": "I hope that mouse brushes its teeth before bed .", "words": ["that", "bed", "mouse"]}, {"sent": "a mouse ?", "words": ["mouse", "a"]}, {"sent": "stay here town mouse and we can have fun .", "words": ["can", "we", "and", "here", "have", "stay", "mouse"]}, {"sent": "well that thats a different kind of mouse called a dormouse .", "words": ["of", "that", "mouse", "a"]}, {"sent": "the fall mouse is next with walnuts and wheat .", "words": ["the", "is", "with", "fall", "and", "mouse"]}, {"sent": "the mouse eats a strawberry .", "words": ["the", "strawberry", "mouse", "a"]}, {"sent": "flower its a mouse .", "words": ["flower", "mouse", "a"]}, {"sent": "some horses come to see town mouse and country mouse .", "words": ["see", "to", "and", "some", "mouse"]}, {"sent": "baby mouse .", "words": ["mouse"]}, {"sent": "hello kangaroo youre the bounciest and mouse youre the smallest of us all .", "words": ["the", "all", "of", "and", "us", "mouse"]}, {"sent": "okay lets get the mouse .", "words": ["get", "the", "mouse"]}, {"sent": "whats the nursery rhyme you know where they talk about a little mouse ?", "words": ["the", "where", "you", "little", "they", "talk", "about", "mouse", "a"]}, {"sent": "put her into the flower put the mouse in the flower pot .", "words": ["the", "put", "into", "flower", "in", "her", "mouse"]}, {"sent": "he is reading a book on how to catch a mouse .", "words": ["how", "is", "to", "catch", "on", "he", "book", "mouse", "a"]}, {"sent": "a mouse again .", "words": ["mouse", "a"]}, {"sent": "there he is little mouse .", "words": ["is", "little", "there", "he", "mouse"]}, {"sent": "if you take a mouse to the movies .", "words": ["the", "if", "to", "you", "take", "mouse", "a"]}, {"sent": "is the mouse dancing ?", "words": ["the", "mouse", "is"]}, {"sent": "and youre drawing a mouse .", "words": ["mouse", "a", "and"]}, {"sent": "I wish there was more of them .", "words": ["of", "was", "them", "wish", "there", "more"]}, {"sent": "Joe youre having your wish .", "words": ["your", "wish"]}, {"sent": "I wish it was twelve oclock .", "words": ["was", "it", "wish"]}, {"sent": "I wish I did but I dont .", "words": ["dont", "did", "but", "wish"]}, {"sent": "do you wish you could be an engine driver ?", "words": ["do", "be", "wish", "you", "an", "could"]}, {"sent": "I wish you would let me know .", "words": ["me", "would", "you", "wish"]}, {"sent": "Ill tell you I wish it was .", "words": ["was", "it", "wish", "you"]}, {"sent": "I wish it were but its not .", "words": ["wish", "it", "were", "not", "but"]}, {"sent": "wish .", "words": ["wish"]}, {"sent": "I kept waking up going oh I wish someone would turn that .", "words": ["up", "that", "would", "wish"]}, {"sent": "sure wish I had a camera .", "words": ["camera", "a", "wish"]}, {"sent": "Id close my eyes and then Id wish that Id turn into a beautiful fish .", "words": ["my", "into", "then", "wish", "and", "fish", "close", "that", "a"]}, {"sent": "I wish there was somebody here .", "words": ["was", "there", "here", "wish"]}, {"sent": "I wish I could .", "words": ["could", "wish"]}, {"sent": "we wish you a merry Christmas .", "words": ["a", "wish", "you", "we"]}, {"sent": "well I wish I knew what you want .", "words": ["what", "you", "wish"]}, {"sent": "you wish you could have infinity what ?", "words": ["what", "wish", "have", "you", "could"]}, {"sent": "that very day messengers were sent throughout the land to put up posters announcing that anyone who found the spring unicorn and explained how hard the Ferians had been working would receive all the gold they could wish for .", "words": ["the", "up", "put", "how", "all", "for", "to", "would", "and", "wish", "they", "hard", "who", "could", "that", "were"]}, {"sent": "I wish youd wipe your feet before you keep going in there .", "words": ["your", "you", "wish", "there", "in", "wipe"]}, {"sent": "Im going to write your name again because Im a I wish .", "words": ["your", "to", "write", "wish", "because", "a"]}, {"sent": "wish Woo .", "words": ["wish"]}, {"sent": "a wish for Laura ?", "words": ["for", "wish", "a"]}, {"sent": "I wish I had invented the Pamper .", "words": ["the", "wish"]}, {"sent": "I wish Id just taken a piece of ordinary sliced bread .", "words": ["of", "bread", "a", "wish"]}, {"sent": "my wish came true didnt it ?", "words": ["my", "it", "wish"]}, {"sent": "he said xxx Graeme I wish uh I wish Graeme couldve stayed over xxx xxx saying xxx the man that thats doing this I wish he couldve slept over .", "words": ["the", "wish", "this", "that", "he", "over"]}, {"sent": "I wish shed sit in one spot all day and do that .", "words": ["all", "do", "and", "wish", "in", "sit", "that"]}, {"sent": "Jafar used his third wish to become a genie .", "words": ["his", "to", "a", "wish"]}, {"sent": "we wish you would .", "words": ["wish", "would", "you", "we"]}, {"sent": "I wish they really did screw they dont .", "words": ["dont", "they", "did", "wish"]}, {"sent": "are you going to wish her a happy birthday Thomas ?", "words": ["happy", "to", "wish", "you", "are", "her", "a"]}, {"sent": "wish I may wish I might have the wish I wish tonight .", "words": ["have", "the", "wish"]}, {"sent": "I wish there was someone here to do it for me .", "words": ["do", "for", "was", "to", "wish", "here", "there", "it", "me"]}, {"sent": "wish wash wish .", "words": ["wash", "wish"]}, {"sent": "I wish I had a nice chair .", "words": ["chair", "nice", "a", "wish"]}, {"sent": "is he gonna try and swipe the wish ?", "words": ["the", "is", "and", "wish", "he", "try"]}, {"sent": "wish ya were two years older .", "words": ["were", "wish"]}, {"sent": "Xavier dont you wish Mommy felt better ?", "words": ["better", "dont", "wish", "you"]}, {"sent": "I wish Naima would wear the microphone on her shirt so we would have a very good recording .", "words": ["the", "we", "shirt", "so", "wish", "have", "good", "would", "on", "her", "a"]}, {"sent": "well I wish youd make your mind up which it is .", "words": ["up", "your", "is", "which", "wish", "it", "make"]}, {"sent": "maybe we should make a wish .", "words": ["wish", "make", "a", "we"]}, {"sent": "Williams wish Wellingtons ?", "words": ["wish"]}, {"sent": "I wish I could let you play with it but I cant .", "words": ["play", "with", "you", "wish", "it", "could", "but"]}, {"sent": "I wish we had those princess .", "words": ["those", "we", "wish"]}, {"sent": "I wish I had another sock .", "words": ["another", "sock", "wish"]}, {"sent": "dont you wish you could do that ?", "words": ["do", "wish", "you", "dont", "could", "that"]}, {"sent": "you wish you had a train set ?", "words": ["train", "wish", "a", "you"]}, {"sent": "tsk oh I wish I could see Thomass lovely happy face .", "words": ["face", "happy", "wish", "see", "could"]}, {"sent": "can you see a wish ?", "words": ["can", "wish", "you", "see", "a"]}, {"sent": "I wish I could give them like raw carrots and celery and stuff like that .", "words": ["them", "like", "wish", "and", "carrots", "give", "that", "could"]}, {"sent": "were you singing we wish you a merry Christmas Ruth ?", "words": ["we", "wish", "you", "were", "a"]}, {"sent": "oh baby I wish Id seen it sooner I wish Id seen that you hit your head not your face .", "words": ["your", "face", "head", "you", "wish", "it", "hit", "that", "not"]}, {"sent": "I wish I had a friend .", "words": ["a", "wish"]}, {"sent": "I wish youd done do it quiet .", "words": ["do", "it", "quiet", "wish"]}, {"sent": "I wish I was a dragon .", "words": ["was", "a", "wish"]}, {"sent": "we wish you a merry christmas we wish you a merry christmas .", "words": ["a", "wish", "you", "we"]}, {"sent": "do more do more we wish you would .", "words": ["do", "we", "wish", "you", "more", "would"]}, {"sent": "Williams wish Wellingtons .", "words": ["wish"]}, {"sent": "I wish I was the mother .", "words": ["the", "was", "wish"]}, {"sent": "do Grovers wish book .", "words": ["book", "do", "wish"]}, {"sent": "whenever somebody has a birthday dont you wish it was your birthday ?", "words": ["your", "was", "wish", "you", "dont", "it", "a"]}, {"sent": "has he picked up the wish ?", "words": ["up", "the", "he", "wish"]}, {"sent": "I wish youd bring the balloon round .", "words": ["the", "balloon", "bring", "wish"]}, {"sent": "do you ever wish you could fly Sarah ?", "words": ["do", "could", "wish", "you"]}, {"sent": "I wish you guys knew what you did with the blue .", "words": ["the", "what", "with", "you", "wish", "blue", "did"]}, {"sent": "to wish them happy new year .", "words": ["happy", "to", "them", "wish", "new"]}, {"sent": "I wish you would more often .", "words": ["more", "would", "you", "wish"]}, {"sent": "when were cold we can say boy I wish we had some coal .", "words": ["can", "we", "say", "wish", "some", "when", "cold", "were"]}, {"sent": "you can go by stilts and you can go by fish you can go in a crock car if you wish .", "words": ["can", "if", "go", "car", "by", "and", "you", "in", "wish", "fish", "a"]}, {"sent": "I wish Fraser hadnt gone .", "words": ["wish"]}, {"sent": "wish Id never started this .", "words": ["this", "wish"]}, {"sent": "I wish I had one of they .", "words": ["of", "they", "wish"]}, {"sent": "I wish my shoes were full of money .", "words": ["money", "my", "full", "of", "wish", "were"]}, {"sent": "I wish .", "words": ["wish"]}, {"sent": "oh I wish I wish the snake would come out .", "words": ["the", "out", "would", "wish"]}, {"sent": "and I you know like I wish I knew now .", "words": ["wish", "like", "you", "and"]}, {"sent": "wish wash wish .", "words": ["wash", "wish"]}, {"sent": "would you make a wish for Laura ?", "words": ["for", "wish", "you", "would", "make", "a"]}, {"sent": "I wish it was my birthday .", "words": ["was", "my", "it", "wish"]}, {"sent": "I wish you would .", "words": ["would", "you", "wish"]}, {"sent": "wish he was fifteen .", "words": ["was", "he", "wish"]}, {"sent": "yeah I wish I wish the circus was still here right ?", "words": ["the", "was", "here", "wish"]}, {"sent": "I wish you wouldnt do that .", "words": ["do", "that", "you", "wish"]}, {"sent": "I like that garden I wish I had a garden like that .", "words": ["like", "wish", "garden", "that", "a"]}, {"sent": "I wish someone would call me xxx .", "words": ["me", "would", "wish"]}, {"sent": "I wish I stayed all day to play with Sarah .", "words": ["all", "to", "play", "with", "wish"]}, {"sent": "do more do more we wish you would .", "words": ["do", "we", "wish", "you", "more", "would"]}, {"sent": "wish shed forget that word .", "words": ["that", "wish"]}, {"sent": "wish .", "words": ["wish"]}, {"sent": "I wish you would stop tidying up my things .", "words": ["up", "my", "stop", "you", "wish", "would"]}, {"sent": "I bet you wish .", "words": ["wish", "you"]}, {"sent": "yes for a wish oh theres a w down my shirt down my shirt theres a w down my shirt down my shirt .", "words": ["my", "for", "shirt", "wish", "down", "a"]}, {"sent": "I wish xxx color xxx .", "words": ["wish"]}, {"sent": "white tip shark .", "words": ["white"]}, {"sent": "thats white too .", "words": ["white", "too"]}, {"sent": "I was going to say we could always use a little bit we could always use a little bit of that white icing there .", "words": ["we", "of", "to", "was", "say", "little", "there", "could", "white", "that", "a"]}, {"sent": "thats your white cup .", "words": ["cup", "white", "your"]}, {"sent": "youve got two white ones so put them behind eachother like that .", "words": ["put", "so", "them", "behind", "like", "white", "that"]}, {"sent": "right white and red makes pink .", "words": ["red", "white", "and"]}, {"sent": "we saw a white bust in the museum here right .", "words": ["the", "we", "here", "in", "white", "a"]}, {"sent": "therere three beds on the train now a yellow a blue and a white .", "words": ["the", "train", "blue", "and", "on", "yellow", "white", "a"]}, {"sent": "blue and white and black .", "words": ["black", "white", "and", "blue"]}, {"sent": "yeah your orange and white sports truck .", "words": ["your", "truck", "and", "orange", "white"]}, {"sent": "white this ones black .", "words": ["black", "this", "white"]}, {"sent": "yeah that babys in white with red polka dots .", "words": ["red", "with", "in", "white", "that"]}, {"sent": "cant a black daddy and a white mummy get married ?", "words": ["and", "black", "get", "white", "a"]}, {"sent": "hes a white sheep .", "words": ["white", "sheep", "a"]}, {"sent": "you put the colors on the white .", "words": ["the", "put", "you", "on", "white"]}, {"sent": "wears a white coat or something ?", "words": ["white", "coat", "a"]}, {"sent": "oh thats white milk awright .", "words": ["milk", "white"]}, {"sent": "its much much easier to cut than white bread because the white breads a lot softer and springy and more doughy isnt it ?", "words": ["the", "bread", "cut", "a lot", "to", "and", "more", "it", "white", "much", "because", "a"]}, {"sent": "can you see the little white mouse ?", "words": ["can", "the", "little", "you", "see", "white", "mouse"]}, {"sent": "a white cat .", "words": ["cat", "white", "a"]}, {"sent": "is this little white baby ?", "words": ["this", "white", "little", "is"]}, {"sent": "this is a little white kitty .", "words": ["is", "little", "this", "kitty", "white", "a"]}, {"sent": "white ?", "words": ["white"]}, {"sent": "can you see the egg with the yellow yolk and the white outside ?", "words": ["can", "the", "egg", "yellow", "outside", "with", "you", "and", "see", "white"]}, {"sent": "theyre wearing blue dungarees and a little white tshirt but it has got quite long hair .", "words": ["hair", "long", "little", "and", "blue", "it", "white", "but", "a"]}, {"sent": "its white .", "words": ["white"]}, {"sent": "its a good job we xxx white ones isnt it ?", "words": ["we", "it", "good", "white", "a"]}, {"sent": "two white ?", "words": ["white"]}, {"sent": "the green and white one isnt moving at all is it ?", "words": ["the", "all", "is", "and", "it", "green", "at", "white"]}, {"sent": "can you see the plants over there with white flowers on .", "words": ["can", "the", "with", "you", "there", "on", "see", "white", "over"]}, {"sent": "white chocolate .", "words": ["chocolate", "white"]}, {"sent": "thats a white plane .", "words": ["white", "a"]}, {"sent": "it is eight thirty on Friday night and the one hundred and five men and women dressed completely in black and white have gone to work turning the black notes on white pages into a symphony .", "words": ["the", "work", "into", "is", "to", "and", "black", "have", "in", "it", "on", "white", "a"]}, {"sent": "white .", "words": ["white"]}, {"sent": "Sues house has a white door .", "words": ["door", "white", "house", "a"]}, {"sent": "some of that white glue what is on the stick yeah ?", "words": ["the", "what", "stick", "is", "of", "on", "some", "white", "that", "glue"]}, {"sent": "and they collect them in white bags .", "words": ["them", "and", "they", "in", "white"]}, {"sent": "we could do pink and white couldnt we ?", "words": ["do", "we", "and", "could", "white"]}, {"sent": "Ive popped a little bit of white icing on it and Ive popped it in the bun tin .", "words": ["the", "of", "little", "and", "in", "it", "on", "white", "a"]}, {"sent": "why is beer white ?", "words": ["white", "why", "is"]}, {"sent": "get the white one .", "words": ["get", "the", "white"]}, {"sent": "hes got big white teeth .", "words": ["white", "big"]}, {"sent": "theres one white one .", "words": ["white"]}, {"sent": "and out of the pieces of red and white shell from the egg .", "words": ["the", "red", "egg", "of", "and", "out", "white"]}, {"sent": "but theyre nice white fluffy clouds on a blue sky arent they ?", "words": ["sky", "nice", "blue", "they", "on", "white", "but", "a"]}, {"sent": "white .", "words": ["white"]}, {"sent": "the white stuff is called snow .", "words": ["the", "white", "snow", "is"]}, {"sent": "xxx white boots .", "words": ["boots", "white"]}, {"sent": "is that Thomass blue and white painting bib ?", "words": ["is", "blue", "and", "bib", "white", "that"]}, {"sent": "you want the white glass ?", "words": ["the", "white", "glass", "you"]}, {"sent": "whats got striped on it black and white stripes ?", "words": ["and", "black", "it", "on", "white"]}, {"sent": "white ?", "words": ["white"]}, {"sent": "there was so much white .", "words": ["was", "so", "there", "white", "much"]}, {"sent": "we cant do it through the white door .", "words": ["the", "do", "we", "it", "white", "door"]}, {"sent": "it must have some white and some black do you see ?", "words": ["do", "see", "and", "have", "black", "you", "it", "some", "white"]}, {"sent": "a white on .", "words": ["on", "white", "a"]}, {"sent": "I dont think weve got white ice lollies either have we ?", "words": ["think", "we", "have", "dont", "ice", "white"]}, {"sent": "the white one .", "words": ["the", "white"]}, {"sent": "pieces of tarmac with white lines running down the middle .", "words": ["the", "of", "white", "with", "down"]}, {"sent": "what was white and had blue flashing lights ?", "words": ["what", "was", "blue", "and", "white"]}, {"sent": "its black and white .", "words": ["black", "white", "and"]}, {"sent": "it wasnt white .", "words": ["white", "it"]}, {"sent": "and then the white van comes on Tuesdays doesnt he ?", "words": ["the", "and", "he", "on", "then", "white"]}, {"sent": "and thats a white .", "words": ["white", "a", "and"]}, {"sent": "clouds should be white shouldnt they ?", "words": ["be", "white", "they"]}, {"sent": "black dogs white dogs hello .", "words": ["black", "white"]}, {"sent": "I can see the white .", "words": ["can", "see", "white", "the"]}, {"sent": "moths sleep with wings together they look like little white leaves on walls and windows and screens .", "words": ["sleep", "with", "like", "little", "and", "they", "look", "on", "white"]}, {"sent": "going to put a white one on ?", "words": ["put", "to", "on", "white", "a"]}, {"sent": "white ?", "words": ["white"]}, {"sent": "xxx some white .", "words": ["some", "white"]}, {"sent": "Mommys eating an egg white .", "words": ["an", "white", "egg"]}, {"sent": "a white envelope .", "words": ["white", "a"]}, {"sent": "theres another white one .", "words": ["another", "white"]}, {"sent": "xxx thats me white .", "words": ["me", "white"]}, {"sent": "well the the seven dwarves are angry at the witch because shes not very nice to Snow white .", "words": ["the", "to", "nice", "are", "at", "white", "not", "because"]}, {"sent": "the can is white yeah .", "words": ["the", "can", "white", "is"]}, {"sent": "the kind with white icing on it ?", "words": ["the", "with", "it", "on", "white"]}, {"sent": "I got me new shoes white .", "words": ["me", "white", "new"]}, {"sent": "you want white grape in it or some other kind ?", "words": ["you", "other", "in", "it", "some", "white"]}, {"sent": "a white dog .", "words": ["white", "dog", "a"]}, {"sent": "yeah do you Ill make a white one to be his friend .", "words": ["do", "to", "be", "you", "his", "white", "make", "a"]}, {"sent": "yes its red and white and says stop .", "words": ["red", "white", "stop", "and"]}, {"sent": "oh weve got white grape juice for you today .", "words": ["white", "juice", "for", "you"]}, {"sent": "I see a white piece .", "words": ["see", "white", "a"]}, {"sent": "white ?", "words": ["white"]}, {"sent": "you have white hair .", "words": ["have", "hair", "white", "you"]}, {"sent": "youre gonna hafta wear white socks I think .", "words": ["think", "white"]}, {"sent": "shall I go and get some white medicine for you then ?", "words": ["for", "go", "medicine", "and", "get", "you", "some", "then", "white"]}, {"sent": "a white one .", "words": ["white", "a"]}, {"sent": "do you have the same amount of water in the purple cup as you have in this white .", "words": ["the", "do", "cup", "of", "you", "have", "this", "in", "same", "water", "white"]}, {"sent": "white .", "words": ["white"]}, {"sent": "do you think this yellow butter is gonna taste different than the white butter ?", "words": ["the", "do", "think", "is", "taste", "you", "this", "butter", "yellow", "white"]}, {"sent": "a white straw with blue stripes .", "words": ["blue", "white", "with", "a"]}, {"sent": "oh grey and white .", "words": ["white", "and"]}, {"sent": "no its a white sheep .", "words": ["white", "sheep", "a"]}, {"sent": "which food items did you play with today ?", "words": ["play", "with", "you", "food", "did", "which"]}, {"sent": "does he play with pretend food with his brother ever ?", "words": ["does", "play", "with", "food", "he", "his", "pretend"]}, {"sent": "a greens a soul food fantastic .", "words": ["food", "a"]}, {"sent": "I want my food .", "words": ["my", "food"]}, {"sent": "and you shouldnt throw food anyway .", "words": ["food", "throw", "you", "and"]}, {"sent": "lorry goes on the food ?", "words": ["on", "the", "food"]}, {"sent": "feeding you real food ?", "words": ["food", "you"]}, {"sent": "and ate of the food gathered and blessed .", "words": ["of", "the", "food", "and"]}, {"sent": "food we played with last time .", "words": ["last", "food", "with", "we"]}, {"sent": "shall we get the food out then ?", "words": ["the", "we", "get", "food", "out", "then"]}, {"sent": "theres some food over here lets go feed the zebra .", "words": ["the", "zebra", "go", "here", "food", "feed", "some", "over"]}, {"sent": "does Snoopy like his food ?", "words": ["his", "does", "food", "like"]}, {"sent": "youve put your foot in the food .", "words": ["the", "put", "your", "foot", "in", "food"]}, {"sent": "should we get some food here .", "words": ["we", "get", "here", "food", "some"]}, {"sent": "he wants some food does he ?", "words": ["some", "food", "does", "he"]}, {"sent": "somewhere theres food for the dragon .", "words": ["the", "food", "for"]}, {"sent": "looks like we needta put cat food on our shopping list .", "words": ["put", "we", "like", "our", "cat", "food", "on"]}, {"sent": "um you said that she liked she went for the food last time too ?", "words": ["the", "for", "last", "you", "food", "too", "that", "she"]}, {"sent": "xxx especially with the food it was uh .", "words": ["the", "was", "with", "food", "it"]}, {"sent": "the rocks are super high the food is super high so you hafta build it up you hafta build it high for him to go up there .", "words": ["the", "up", "for", "is", "go", "so", "to", "you", "there", "food", "are", "build", "it", "him", "high"]}, {"sent": "um probably the food .", "words": ["the", "food"]}, {"sent": "Im fixing you some food Moo .", "words": ["some", "food", "you"]}, {"sent": "and then after that he went to the ah play food I think .", "words": ["the", "think", "to", "play", "and", "food", "he", "then", "that"]}, {"sent": "look at the food .", "words": ["the", "food", "at", "look"]}, {"sent": "something that looks like food .", "words": ["food", "that", "like"]}, {"sent": "full of food arent ya ?", "words": ["of", "food", "full"]}, {"sent": "oh thats what I meant thats what I thought it was called the coop thats where the food the place where you get your food the grocery place ?", "words": ["where", "what", "the", "your", "was", "you", "get", "food", "it"]}, {"sent": "like would she recognize any of those items as food that shes had at home or shes seen you guys eat ?", "words": ["those", "of", "any", "like", "you", "eat", "food", "at", "would", "that", "she", "home"]}, {"sent": "cat food .", "words": ["cat", "food"]}, {"sent": "what kind of food ?", "words": ["of", "what", "food"]}, {"sent": "oh is it chicken food ?", "words": ["chicken", "food", "it", "is"]}, {"sent": "more food .", "words": ["more", "food"]}, {"sent": "oh thats cat food .", "words": ["cat", "food"]}, {"sent": "dont talk when you have food in your mouth okay ?", "words": ["your", "mouth", "you", "have", "dont", "food", "in", "talk", "when"]}, {"sent": "food .", "words": ["food"]}, {"sent": "mostly for food now and or then also very rare .", "words": ["food", "for", "then", "and"]}, {"sent": "and what about the food and all ?", "words": ["the", "what", "all", "and", "food", "about"]}, {"sent": "um I think definitely the play food was her favorite .", "words": ["the", "think", "was", "play", "food", "her"]}, {"sent": "the food isnt gonna taste any better or anything .", "words": ["the", "better", "any", "taste", "food"]}, {"sent": "and were not gonna eat the crayon color with the crayon and we eat the food .", "words": ["the", "we", "crayon", "with", "and", "eat", "food", "were", "not"]}, {"sent": "thats dog food .", "words": ["food", "dog"]}, {"sent": "yeah with like fake food and everything .", "words": ["food", "with", "like", "and"]}, {"sent": "you wanna find more food ?", "words": ["you", "more", "food", "wanna", "find"]}, {"sent": "oh Kipper likes food .", "words": ["food"]}, {"sent": "does she have um have any like plastic food at your ha home ?", "words": ["does", "your", "any", "like", "have", "food", "at", "she", "home"]}, {"sent": "poor Pinocchio had no money no food no ticket .", "words": ["money", "food", "poor"]}, {"sent": "yeah you looked just like a dinosaur eating his food .", "words": ["like", "you", "food", "his", "a"]}, {"sent": "Mummys going to get Purdie some food and them well get some lunch .", "words": ["to", "them", "and", "get", "food", "some"]}, {"sent": "would she recognize any of the food items do you think ?", "words": ["the", "do", "think", "of", "any", "you", "food", "would", "she"]}, {"sent": "because we dont have any food at home .", "words": ["we", "any", "have", "dont", "food", "at", "because", "home"]}, {"sent": "what do they call the shop where we go and buy the food ?", "words": ["the", "what", "do", "where", "we", "go", "and", "they", "food", "buy"]}, {"sent": "it is alotof food .", "words": ["food", "it", "is"]}, {"sent": "well normally we give the pussy cats Purdies food but we havent actually got any biscuits for Purdie at the moment Thomas .", "words": ["the", "for", "we", "any", "food", "at", "give", "but"]}, {"sent": "lets just see if weve got Purdies food from yesterday .", "words": ["see", "food", "if"]}, {"sent": "like he recognizes now hes eating food more regularly .", "words": ["food", "more", "he", "like"]}, {"sent": "we have a kitchen and toy food .", "words": ["we", "and", "have", "toy", "food", "kitchen", "a"]}, {"sent": "havent had dog food before have we ?", "words": ["have", "food", "dog", "we"]}, {"sent": "and wants some food and she can smell it ?", "words": ["can", "and", "food", "it", "some", "she"]}, {"sent": "know why he told how to make the food ?", "words": ["the", "how", "to", "food", "he", "make", "why"]}, {"sent": "then you would be bigger than he but you werent so theres nothing to talk about except that youre not eating your food .", "words": ["your", "be", "so", "to", "you", "food", "he", "talk", "would", "about", "then", "that", "not", "but"]}, {"sent": "um what about the you said the food do you think she knows the names of some of the food ?", "words": ["the", "what", "do", "think", "of", "you", "food", "about", "some", "she"]}, {"sent": "and you said you didnt that she doesnt play with food at home .", "words": ["play", "with", "you", "and", "food", "at", "that", "she", "home"]}, {"sent": "have we got any food for Micahel ?", "words": ["for", "we", "any", "have", "food"]}, {"sent": "after I get my food okay .", "words": ["get", "my", "food"]}, {"sent": "so she has a grocery cart but she doesnt fill it with food she fills it with toys and .", "words": ["so", "with", "and", "food", "it", "she", "but", "a"]}, {"sent": "um the names of the food at home ?", "words": ["the", "of", "food", "at", "home"]}, {"sent": "um so uh what else besides the food was he interested in ?", "words": ["the", "what", "so", "was", "food", "in", "he"]}, {"sent": "Im getting this horrible feeling that youre watching an awful lot of tv and eating and awful lot of food on the couch with Sue .", "words": ["the", "of", "with", "couch", "and", "this", "an", "tv", "food", "on", "that"]}, {"sent": "food mommy has food .", "words": ["food"]}, {"sent": "um I dunno I mean shes everytime we come she loves the pretend food and we still dont have any pretend food but I feel like thats something that we definitely probably needta get her at some point soon .", "words": ["the", "we", "any", "like", "her", "and", "have", "dont", "food", "get", "at", "some", "pretend", "that", "she", "but"]}, {"sent": "no I dont think they eat cat food .", "words": ["think", "eat", "dont", "they", "cat", "food"]}, {"sent": "well youre not sposta eat cat food are you ?", "words": ["you", "eat", "cat", "food", "are", "not"]}, {"sent": "okay so it was really mostly the food items ?", "words": ["the", "was", "so", "food", "it"]}, {"sent": "yeah ya gonna go camping and you have food you have an egg .", "words": ["egg", "go", "you", "have", "and", "food", "an"]}, {"sent": "Im just bringing my food in now .", "words": ["in", "my", "food"]}, {"sent": "Naima are you ready to sit in your chair and have some food like some soggy oatie bites ?", "words": ["your", "to", "like", "chair", "you", "and", "have", "in", "are", "food", "some", "sit"]}, {"sent": "Dudley rushed there but the bunny lady didnt have Sampotatoes mother bunny was saying to grocer dog I have decided not to buy any food today after all .", "words": ["the", "all", "bunny", "was", "to", "not", "buy", "any", "have", "there", "food", "dog", "but"]}, {"sent": "I need the food .", "words": ["the", "food", "need"]}, {"sent": "food food .", "words": ["food"]}, {"sent": "that is your favorite food .", "words": ["your", "food", "that", "is"]}, {"sent": "lots of people eating food .", "words": ["of", "food"]}, {"sent": "you finish putting the food in and Ill put her trousers on then .", "words": ["the", "put", "finish", "you", "and", "in", "food", "on", "then", "her"]}, {"sent": "I think Ill give her some fresh food Thomas .", "words": ["think", "food", "some", "give", "her"]}, {"sent": "Jessies got all her food .", "words": ["all", "food", "her"]}, {"sent": "now weve just given her a clean plate with fresh food .", "words": ["clean", "plate", "with", "food", "her", "a"]}, {"sent": "all of the food stuff .", "words": ["of", "all", "food", "the"]}, {"sent": "she didnt want her food did she ?", "words": ["did", "food", "her", "she"]}, {"sent": "and theres food and all in it .", "words": ["all", "and", "food", "it", "in"]}, {"sent": "okay um back to the food .", "words": ["the", "to", "food", "back"]}, {"sent": "I wondered if they had goats because they are the ones that totally want all the food you have in your hand .", "words": ["the", "all", "your", "if", "hand", "you", "have", "they", "food", "are", "in", "that", "because"]}, {"sent": "some clams and mussels also get food from that same kind of bacteria .", "words": ["of", "and", "get", "food", "same", "some", "that"]}, {"sent": "she has finished her food and she has gone .", "words": ["food", "her", "she", "and"]}, {"sent": "yes if the microphone was under you it would be covered up with food and water right now .", "words": ["the", "up", "water", "if", "was", "be", "with", "you", "and", "under", "food", "it", "would"]}, {"sent": "Ill leave in ten minutes five minutes for the food to go down five minutes to eat the rest of the pie and then go out .", "words": ["the", "for", "go", "of", "to", "and", "eat", "in", "food", "out", "then", "down"]}, {"sent": "we dont have too much we just got a little bit of play food at home .", "words": ["we", "of", "play", "little", "have", "dont", "food", "at", "too", "home", "much", "a"]}, {"sent": "is that pretend food ?", "words": ["pretend", "that", "food", "is"]}, {"sent": "you want some food ?", "words": ["some", "food", "you"]}, {"sent": "are you going to put all the food back in the shopping basket ?", "words": ["the", "all", "put", "back", "to", "you", "food", "are", "in", "basket"]}, {"sent": "come on and sit on my knee .", "words": ["my", "knee", "and", "on", "sit"]}, {"sent": "do you wanna sit on mummys knee ?", "words": ["do", "knee", "you", "wanna", "on", "sit"]}, {"sent": "cmon and sit on my knee .", "words": ["my", "knee", "and", "on", "sit"]}, {"sent": "your knee ?", "words": ["knee", "your"]}, {"sent": "me knee ?", "words": ["me", "knee"]}, {"sent": "you come to sit on my knee ?", "words": ["my", "knee", "to", "you", "on", "sit"]}, {"sent": "and Im going to kiss this knee again .", "words": ["knee", "to", "and", "this", "kiss"]}, {"sent": "he did put it on your knee .", "words": ["put", "knee", "your", "it", "he", "did", "on"]}, {"sent": "right by your knee .", "words": ["knee", "your", "by"]}, {"sent": "wanna sit on my knee ?", "words": ["my", "knee", "wanna", "on", "sit"]}, {"sent": "what happened to your knee ?", "words": ["to", "what", "knee", "your"]}, {"sent": "hes going to sit on his Grandpas knee isnt he ?", "words": ["knee", "to", "his", "he", "on", "sit"]}, {"sent": "are you going to come and sit on Mummys knee ?", "words": ["knee", "to", "and", "you", "are", "on", "sit"]}, {"sent": "hes digging his claws in my knee .", "words": ["his", "in", "knee", "my"]}, {"sent": "and around her knee and back to Ethan .", "words": ["knee", "back", "to", "and", "around", "her"]}, {"sent": "on mummys knee .", "words": ["on", "knee"]}, {"sent": "on Dadas knee .", "words": ["on", "knee"]}, {"sent": "a choo choos a knee ?", "words": ["knee", "a"]}, {"sent": "do you wanna sit on my knee ?", "words": ["do", "my", "knee", "you", "wanna", "on", "sit"]}, {"sent": "wheres the other knee at ?", "words": ["the", "knee", "at", "other"]}, {"sent": "thank you for protecting my knee .", "words": ["knee", "my", "for", "you"]}, {"sent": "oh your sore knee .", "words": ["knee", "your"]}, {"sent": "on her knee ?", "words": ["on", "knee", "her"]}, {"sent": "youve a poorly knee .", "words": ["knee", "a"]}, {"sent": "are you gonna sit on my knee ?", "words": ["my", "knee", "you", "are", "on", "sit"]}, {"sent": "and when you went to Grandma and Granddads Granddad put some of his special cream on your knee didnt he ?", "words": ["put", "knee", "your", "of", "to", "you", "and", "on", "he", "his", "some", "when"]}, {"sent": "you like eating bananas on Daddys knee dont you ?", "words": ["knee", "like", "you", "dont", "on"]}, {"sent": "Mummys still got a sore knee Fraser .", "words": ["knee", "a"]}, {"sent": "thats a funny place to sit right on my crossed knee that looks like a very interesting book .", "words": ["my", "knee", "to", "like", "on", "book", "sit", "that", "a"]}, {"sent": "where lets look at Dans knee .", "words": ["where", "knee", "at", "look"]}, {"sent": "Mrs Kays knee ?", "words": ["knee"]}, {"sent": "oh you slipped off my knee .", "words": ["off", "my", "knee", "you"]}, {"sent": "good job that barriers there otherwise it wouldve bumped your knee .", "words": ["knee", "your", "there", "it", "good", "that"]}, {"sent": "when you do that I can see your knee .", "words": ["can", "do", "knee", "your", "when", "you", "see", "that"]}, {"sent": "can you put it on my knee ?", "words": ["can", "put", "my", "knee", "you", "it", "on"]}, {"sent": "whats that on your knee ?", "words": ["on", "knee", "that", "your"]}, {"sent": "is she gonna sit on my knee ?", "words": ["my", "knee", "is", "on", "sit", "she"]}, {"sent": "why dont you just come and it on my knee ?", "words": ["my", "knee", "and", "you", "dont", "it", "on", "why"]}, {"sent": "Purdies probably very happy because shes sitting on my knee and Im stroking her .", "words": ["my", "knee", "happy", "and", "on", "her", "because"]}, {"sent": "that hurt your knee ?", "words": ["hurt", "knee", "that", "your"]}, {"sent": "he bit my knee didnt he ?", "words": ["my", "knee", "he"]}, {"sent": "sit on Mamas knee .", "words": ["on", "sit", "knee"]}, {"sent": "and when we were sitting in the chair over there you were on my knee and Purdie was on your knee .", "words": ["the", "my", "knee", "your", "we", "was", "chair", "and", "you", "there", "in", "on", "over", "when", "were"]}, {"sent": "I think thats enough bashing her on the knee .", "words": ["the", "think", "knee", "on", "her"]}, {"sent": "whatre you doing with your knee ?", "words": ["your", "knee", "with", "you"]}, {"sent": "you wanna sit on my knee ?", "words": ["my", "knee", "you", "wanna", "on", "sit"]}, {"sent": "you were asleep on Mummys knee .", "words": ["knee", "you", "asleep", "on", "were"]}, {"sent": "youre gonna hafta get off my knee if you wanna do the jigsaw .", "words": ["off", "do", "my", "knee", "the", "if", "you", "get", "wanna"]}, {"sent": "youd like a drink of milk on my knee ?", "words": ["my", "knee", "drink", "of", "like", "on", "milk", "a"]}, {"sent": "knee knee knee knee .", "words": ["knee"]}, {"sent": "he doesnt wanna sit on my knee .", "words": ["my", "knee", "he", "wanna", "on", "sit"]}, {"sent": "gosh I dont think I want it on my knee .", "words": ["think", "my", "knee", "dont", "it", "on"]}, {"sent": "not on my knee just on the chair ?", "words": ["the", "my", "knee", "chair", "on", "not"]}, {"sent": "you hurt your knee .", "words": ["hurt", "knee", "your", "you"]}, {"sent": "on your knee ?", "words": ["on", "knee", "your"]}, {"sent": "the itsybitsy spider gets Phaedra on the knee .", "words": ["the", "on", "knee"]}, {"sent": "on her knee ?", "words": ["on", "knee", "her"]}, {"sent": "why are you sat on my knee sucking your thumb ?", "words": ["my", "knee", "your", "you", "are", "on", "why"]}, {"sent": "are you going to sit on my knee ?", "words": ["my", "knee", "to", "you", "are", "on", "sit"]}, {"sent": "you hit your knee ?", "words": ["knee", "hit", "your", "you"]}, {"sent": "by your knee .", "words": ["knee", "your", "by"]}, {"sent": "this is yer knee and this is yer knee .", "words": ["this", "knee", "and", "is"]}, {"sent": "xxx sit on my knee .", "words": ["on", "sit", "knee", "my"]}, {"sent": "but you know this is my bad knee .", "words": ["my", "knee", "is", "bad", "you", "this", "but"]}, {"sent": "on the knee .", "words": ["on", "the", "knee"]}, {"sent": "do you wanna come and sit on Mummys knee and talk to Mummy ?", "words": ["do", "knee", "to", "and", "you", "wanna", "talk", "on", "sit"]}, {"sent": "come and sit on my knee .", "words": ["my", "knee", "and", "on", "sit"]}, {"sent": "can you say knee ?", "words": ["can", "knee", "say", "you"]}, {"sent": "bend this knee .", "words": ["this", "knee"]}, {"sent": "sore knee Nana .", "words": ["knee"]}, {"sent": "knee knee knee knee knee knee .", "words": ["knee"]}, {"sent": "going on Snowdons knee ?", "words": ["on", "knee"]}, {"sent": "your right knee .", "words": ["knee", "your"]}, {"sent": "why dont you just sit on my knee and listen to the story .", "words": ["the", "my", "knee", "story", "listen", "to", "you", "and", "dont", "on", "sit", "why"]}, {"sent": "Ill put your hair in a bobble while while youre sitting on my knee .", "words": ["put", "my", "your", "knee", "in", "on", "hair", "a"]}, {"sent": "oh is baba coming to sit on my knee ?", "words": ["my", "knee", "is", "to", "on", "sit"]}, {"sent": "you slide off my knee ?", "words": ["off", "my", "knee", "slide", "you"]}, {"sent": "it would be Purdie on your knee .", "words": ["knee", "your", "be", "it", "would", "on"]}, {"sent": "thats your seat belt a knee belt .", "words": ["knee", "your", "belt", "a"]}, {"sent": "my hand is right here on my knee .", "words": ["my", "knee", "is", "hand", "here", "on"]}, {"sent": "well theres some by your knee arent there ?", "words": ["knee", "your", "by", "there", "some"]}, {"sent": "you make your knee caps go like this ?", "words": ["knee", "your", "go", "like", "you", "this", "make"]}, {"sent": "shes sitting on Mamas knee .", "words": ["on", "knee"]}, {"sent": "are you going to sit on Mummys knee and well read this ?", "words": ["knee", "to", "you", "and", "this", "are", "read", "on", "sit"]}, {"sent": "no her knee is right here .", "words": ["here", "knee", "her", "is"]}, {"sent": "no he just wants to sit on my knee and attack me .", "words": ["my", "knee", "to", "and", "he", "on", "me", "sit"]}, {"sent": "watch your knee .", "words": ["knee", "watch", "your"]}, {"sent": "cat sitting on her knee .", "words": ["on", "cat", "knee", "her"]}, {"sent": "by your knee .", "words": ["knee", "your", "by"]}, {"sent": "one knee .", "words": ["knee"]}, {"sent": "what do you wanna do sit on my knee ?", "words": ["what", "do", "my", "knee", "you", "wanna", "on", "sit"]}, {"sent": "are you banging your knee ?", "words": ["knee", "are", "your", "you"]}, {"sent": "your knee ?", "words": ["knee", "your"]}, {"sent": "I hate to tell you this but your knee caps are so scarred up man that theyre not a pleasant sight .", "words": ["up", "knee", "your", "to", "so", "hate", "you", "this", "are", "that", "not", "but", "a"]}, {"sent": "youve got a tiger on your knee .", "words": ["knee", "your", "tiger", "on", "a"]}, {"sent": "Scoops hurt his knee .", "words": ["hurt", "his", "knee"]}, {"sent": "come and sit on my knee .", "words": ["my", "knee", "and", "on", "sit"]}, {"sent": "on the way here I twisted my knee .", "words": ["the", "my", "knee", "here", "on"]}, {"sent": "oh my gosh it touched your foot and your knee .", "words": ["my", "knee", "your", "and", "foot", "it"]}, {"sent": "sure put it above your knee .", "words": ["put", "knee", "your", "it", "above"]}, {"sent": "its full .", "words": ["full"]}, {"sent": "getting a bit full now .", "words": ["full", "a"]}, {"sent": "no all the cups are full .", "words": ["the", "all", "are", "full"]}, {"sent": "three bags full .", "words": ["full"]}, {"sent": "xxx your mouth full .", "words": ["mouth", "your", "full"]}, {"sent": "youre full now ?", "words": ["full"]}, {"sent": "dont talk with your mouth full .", "words": ["your", "full", "mouth", "with", "dont", "talk"]}, {"sent": "two boxes full ?", "words": ["full"]}, {"sent": "you have got your hands full havent you MrsRussell ?", "words": ["have", "your", "full", "you"]}, {"sent": "hes too full now .", "words": ["full", "too"]}, {"sent": "oh its full of Cornflakes .", "words": ["of", "full"]}, {"sent": "yeah but he works full time .", "words": ["he", "full", "but"]}, {"sent": "oh is it full .", "words": ["it", "full", "is"]}, {"sent": "oh my gosh what do you have a big truck full a big truck full of stuff .", "words": ["what", "my", "do", "full", "of", "truck", "you", "have", "big", "a"]}, {"sent": "he said he aint full .", "words": ["he", "full"]}, {"sent": "is your belly full now ?", "words": ["your", "full", "is"]}, {"sent": "its a full truck .", "words": ["truck", "full", "a"]}, {"sent": "Sarah dont eat with your mouth full .", "words": ["your", "full", "mouth", "with", "eat", "dont"]}, {"sent": "you you shouldnt talk with your mouth full should you ?", "words": ["your", "full", "mouth", "with", "you", "talk"]}, {"sent": "hm whats her full name ?", "words": ["her", "full"]}, {"sent": "are you full yet ?", "words": ["are", "full", "you"]}, {"sent": "oh shes full her tummys full she doesnt want that bottle .", "words": ["full", "her", "bottle", "that", "she"]}, {"sent": "full court the basketball game of the year .", "words": ["the", "of", "game", "full"]}, {"sent": "trash can is full and .", "words": ["can", "full", "is", "trash", "and"]}, {"sent": "a pocket full of posy .", "words": ["of", "full", "a"]}, {"sent": "full of yoghurt .", "words": ["of", "full"]}, {"sent": "oh my mouth is too full .", "words": ["my", "full", "is", "mouth", "too"]}, {"sent": "that sure became full .", "words": ["that", "full"]}, {"sent": "we must eat a lot of apples because when Granddad brought that box last week it was really full wasnt it ?", "words": ["box", "full", "we", "of", "was", "last", "a lot", "eat", "that", "it", "when", "because", "a"]}, {"sent": "ohgosh the waiting rooms full now .", "words": ["the", "full"]}, {"sent": "Peter we cant understand you if you eat with your mouth full .", "words": ["your", "if", "full", "we", "mouth", "with", "you", "eat"]}, {"sent": "theyre full of seeds or maybe something like that .", "words": ["of", "that", "like", "full"]}, {"sent": "is the glass full or empty ?", "words": ["the", "full", "is", "empty", "glass"]}, {"sent": "a pocket full of posies .", "words": ["of", "full", "a"]}, {"sent": "you getting full yet ?", "words": ["full", "you"]}, {"sent": "Its not its a birds nest full of eggs .", "words": ["of", "not", "full", "a"]}, {"sent": "yes sir yes sir three pots full .", "words": ["full"]}, {"sent": "well in that case I would like another teaspoon full of flour please from the big flour sack .", "words": ["the", "full", "of", "like", "another", "in", "big", "would", "that"]}, {"sent": "and he was very very full and he had eaten so much he had a tummyache didnt he ?", "words": ["full", "was", "so", "and", "he", "much", "a"]}, {"sent": "now you got a whole car full of pizza I cant believe that that would fit in there .", "words": ["full", "of", "car", "fit", "pizza", "you", "there", "in", "would", "that", "a"]}, {"sent": "perhaps he just wanted to get a full steam ahead and go chuffing down the line .", "words": ["the", "full", "go", "to", "and", "get", "he", "down", "a"]}, {"sent": "oh I think hes getting full now .", "words": ["think", "full"]}, {"sent": "Mummy cant talk now because she has got a mouth full of toast hasnt she ?", "words": ["full", "toast", "of", "mouth", "talk", "she", "because", "a"]}, {"sent": "are you all full now ?", "words": ["all", "are", "full", "you"]}, {"sent": "sorry it was a bit full .", "words": ["was", "it", "full", "a"]}, {"sent": "you full now ?", "words": ["full", "you"]}, {"sent": "uhhuh a full one .", "words": ["full", "a"]}, {"sent": "hell be full wont he ?", "words": ["be", "he", "full"]}, {"sent": "oh its a full house when all the seats are filled and there are no more seats and people just stand at the back and watch the show .", "words": ["the", "all", "watch", "full", "back", "show", "stand", "and", "there", "more", "are", "house", "at", "when", "a"]}, {"sent": "what is your full name ?", "words": ["what", "your", "full", "is"]}, {"sent": "I think youre full .", "words": ["think", "full"]}, {"sent": "then we get full real quick .", "words": ["get", "then", "full", "we"]}, {"sent": "youre full .", "words": ["full"]}, {"sent": "two full trays .", "words": ["full"]}, {"sent": "my arms full .", "words": ["my", "full"]}, {"sent": "two full ounces of formula .", "words": ["of", "full"]}, {"sent": "do you want it full up ?", "words": ["up", "do", "full", "you", "it"]}, {"sent": "shes full of the devil all this week .", "words": ["the", "all", "full", "of", "this"]}, {"sent": "we filled it full of lollipops and brought it to open house ?", "words": ["full", "we", "of", "to", "open", "and", "it", "house"]}, {"sent": "hes full .", "words": ["full"]}, {"sent": "three bags full .", "words": ["full"]}, {"sent": "is your belly full this morning ?", "words": ["this", "your", "full", "is"]}, {"sent": "no dont talk with your mouth full .", "words": ["your", "full", "mouth", "with", "dont", "talk"]}, {"sent": "they wont fit in its full .", "words": ["they", "fit", "in", "full"]}, {"sent": "because then when the bucket is full .", "words": ["the", "full", "bucket", "is", "then", "when", "because"]}, {"sent": "it looks as if its full of gold coins .", "words": ["of", "it", "if", "full"]}, {"sent": "well youre still talking with your mouth full .", "words": ["your", "mouth", "with", "full"]}, {"sent": "the full song .", "words": ["the", "full"]}, {"sent": "when the bins are full .", "words": ["the", "are", "when", "full"]}, {"sent": "pausing here for salt he screamed heres the end of that terrible town full of Zukes who eat bread with the butter side down .", "words": ["the", "bread", "for", "full", "of", "with", "here", "eat", "salt", "he", "butter", "who", "down", "that"]}, {"sent": "is he full now ?", "words": ["he", "full", "is"]}, {"sent": "its full .", "words": ["full"]}, {"sent": "and it never really seemed full because your nappies were so small .", "words": ["your", "full", "so", "and", "it", "were", "because"]}, {"sent": "Im full up .", "words": ["up", "full"]}, {"sent": "I think Mummys hands are full now anyway .", "words": ["think", "are", "full"]}, {"sent": "ring around the rosy a pocket full of posy .", "words": ["the", "full", "of", "around", "a"]}, {"sent": "slowly the little blue engine began to move I think I can I think I can I think I can and up the mountain she went with a train full of toys .", "words": ["the", "can", "think", "up", "full", "train", "of", "to", "with", "blue", "little", "and", "she", "a"]}, {"sent": "Scoop is good at solving problems and hes full of bright ideas .", "words": ["full", "is", "of", "and", "good", "at"]}, {"sent": "got a full tummy now .", "words": ["tummy", "full", "a"]}, {"sent": "ohmygoodness you got a whole other arm full huh .", "words": ["full", "other", "you", "arm", "a"]}, {"sent": "Rachel will be full when she goes home willnt she ?", "words": ["full", "be", "will", "when", "she", "home"]}, {"sent": "youre probably full are you ?", "words": ["are", "full", "you"]}, {"sent": "its a storage container full of oatmeal .", "words": ["of", "full", "a"]}, {"sent": "is your belly full from breakfast ?", "words": ["your", "full", "is"]}, {"sent": "I think were full .", "words": ["think", "were", "full"]}, {"sent": "you are full of vim and vigor .", "words": ["full", "of", "and", "you", "are"]}, {"sent": "its very full .", "words": ["full"]}, {"sent": "Cathy put her finger in the youknow the bandage and it was full of lipstick .", "words": ["the", "put", "full", "of", "was", "and", "finger", "in", "it", "her"]}, {"sent": "Im so full .", "words": ["so", "full"]}, {"sent": "in fact we have a bag full of food for you to take .", "words": ["for", "full", "we", "of", "to", "you", "have", "in", "food", "take", "a"]}, {"sent": "well this one is full Thomas .", "words": ["this", "full", "is"]}, {"sent": "I never get a full meal nowadays .", "words": ["get", "full", "a"]}, {"sent": "Naima xxx is part of your full name .", "words": ["of", "your", "full", "is"]}, {"sent": "its not full of sweets is it ?", "words": ["full", "is", "of", "it", "not"]}, {"sent": "the taxi is full .", "words": ["the", "full", "is"]}, {"sent": "and well have a look at that and see if its full .", "words": ["if", "full", "and", "have", "at", "look", "see", "that", "a"]}, {"sent": "so that next week they can take that away full of rubbish .", "words": ["can", "full", "of", "so", "they", "that", "away", "take"]}, {"sent": "I am getting full .", "words": ["am", "full"]}, {"sent": "xxx who whats her full name ?", "words": ["her", "full", "who"]}, {"sent": "hes looking a bit full .", "words": ["full", "a"]}, {"sent": "fish sandwich ?", "words": ["sandwich", "fish"]}, {"sent": "a giraffe fish ?", "words": ["giraffe", "fish", "a"]}, {"sent": "so the whale comes in in kind of like a car wash and then these fish clean the whales off .", "words": ["the", "these", "clean", "off", "of", "so", "car", "like", "and", "in", "fish", "then", "wash", "a"]}, {"sent": "are you a fish ?", "words": ["are", "fish", "a", "you"]}, {"sent": "a fish ?", "words": ["fish", "a"]}, {"sent": "fish .", "words": ["fish"]}, {"sent": "she told you to go fish ?", "words": ["go", "to", "you", "fish", "she"]}, {"sent": "thats a fish .", "words": ["fish", "a"]}, {"sent": "because it has got fish the ladybird butterfly ducks pigs .", "words": ["the", "butterfly", "it", "fish", "because"]}, {"sent": "thats what you do when youre feeding the fish in the big ponds .", "words": ["the", "what", "do", "you", "in", "big", "fish", "when"]}, {"sent": "some fish and a fishfinger .", "words": ["some", "a", "and", "fish"]}, {"sent": "you did fishing with fish ?", "words": ["fish", "with", "did", "you"]}, {"sent": "fish swimming in there ?", "words": ["there", "in", "fish"]}, {"sent": "fish ?", "words": ["fish"]}, {"sent": "Goldie is the name of his fish .", "words": ["the", "is", "of", "fish", "his"]}, {"sent": "and a pink fish .", "words": ["fish", "a", "and"]}, {"sent": "you got fish .", "words": ["fish", "you"]}, {"sent": "oh theres a fish in the pond .", "words": ["the", "in", "fish", "a"]}, {"sent": "with some other fish .", "words": ["some", "with", "fish", "other"]}, {"sent": "do you think she might cook this fish ?", "words": ["do", "think", "you", "this", "cook", "fish", "she"]}, {"sent": "why xxx a fish here ?", "words": ["here", "fish", "why", "a"]}, {"sent": "ut a fish yes it is a fish good fish face honey .", "words": ["face", "is", "it", "good", "fish", "a"]}, {"sent": "Roger was a razor fish .", "words": ["was", "fish", "a"]}, {"sent": "is that fish cooked ?", "words": ["that", "fish", "is"]}, {"sent": "a little fish like that ?", "words": ["like", "little", "fish", "that", "a"]}, {"sent": "what were the fish doing ?", "words": ["the", "what", "were", "fish"]}, {"sent": "special type of fish .", "words": ["of", "fish"]}, {"sent": "its a funny fish isnt it ?", "words": ["it", "fish", "a"]}, {"sent": "how many fish now ?", "words": ["how", "fish"]}, {"sent": "go on make a wish burbled the fish .", "words": ["the", "go", "wish", "fish", "on", "make", "a"]}, {"sent": "do you remember that pull along fish ?", "words": ["do", "pull", "you", "fish", "that"]}, {"sent": "would you like another fish ?", "words": ["like", "would", "you", "another", "fish"]}, {"sent": "you got five fish on one line ?", "words": ["on", "fish", "you"]}, {"sent": "um one fish two fish red fish blue fish .", "words": ["red", "blue", "fish"]}, {"sent": "how many fish have you caught ?", "words": ["have", "how", "you", "fish"]}, {"sent": "you like to eat the fish ?", "words": ["the", "to", "like", "you", "eat", "fish"]}, {"sent": "where did you see fish darling ?", "words": ["where", "you", "did", "fish", "see"]}, {"sent": "well shall we throw it to the penguins because the penguins like to eat fish dont they ?", "words": ["the", "throw", "we", "to", "like", "eat", "dont", "they", "it", "fish", "because"]}, {"sent": "and whatd he do to the fish ?", "words": ["the", "do", "to", "and", "he", "fish"]}, {"sent": "got the fish to do there .", "words": ["the", "do", "to", "there", "fish"]}, {"sent": "and watch the fish .", "words": ["the", "watch", "fish", "and"]}, {"sent": "um hm look at the fish .", "words": ["the", "at", "fish", "look"]}, {"sent": "fish Max said Ruby .", "words": ["fish"]}, {"sent": "youre a fish ?", "words": ["fish", "a"]}, {"sent": "a pig out of water I mean a fish out of water .", "words": ["of", "fish", "pig", "out", "water", "a"]}, {"sent": "and what happened to the fish ?", "words": ["the", "what", "to", "and", "fish"]}, {"sent": "theres the fish mobile .", "words": ["the", "fish"]}, {"sent": "wheres the fish ?", "words": ["the", "fish"]}, {"sent": "oh sorry was that fish too big ?", "words": ["was", "big", "too", "that", "fish"]}, {"sent": "thats not a kind of fish .", "words": ["of", "fish", "not", "a"]}, {"sent": "you mean a fish net ?", "words": ["fish", "a", "you"]}, {"sent": "caught a fish .", "words": ["fish", "a"]}, {"sent": "blue fish .", "words": ["fish", "blue"]}, {"sent": "thats your fish slice .", "words": ["your", "fish"]}, {"sent": "www I wanna fish .", "words": ["wanna", "fish"]}, {"sent": "seven flipping fish .", "words": ["fish"]}, {"sent": "Doug saw a fish .", "words": ["fish", "a"]}, {"sent": "fish .", "words": ["fish"]}, {"sent": "whos looking at the fish .", "words": ["the", "fish", "at"]}, {"sent": "you dont like fish ?", "words": ["dont", "fish", "like", "you"]}, {"sent": "uhhuh so what do you call your fish again ?", "words": ["what", "do", "your", "so", "you", "fish"]}, {"sent": "even the woman at the fish store was eat inside eating lunch .", "words": ["the", "store", "was", "inside", "eat", "at", "fish"]}, {"sent": "a spotted trunk fish .", "words": ["fish", "a"]}, {"sent": "a shark golden fish ?", "words": ["fish", "a"]}, {"sent": "youre digging a fish down there .", "words": ["there", "down", "fish", "a"]}, {"sent": "to catch the fish ?", "words": ["the", "to", "catch", "fish"]}, {"sent": "what were the fish doing ?", "words": ["the", "what", "were", "fish"]}, {"sent": "thats a fish Karming very good .", "words": ["fish", "good", "a"]}, {"sent": "that you catch the fish with .", "words": ["the", "catch", "with", "you", "fish", "that"]}, {"sent": "well I could maybe xxx make them fish .", "words": ["could", "make", "fish", "them"]}, {"sent": "look what happened to this fish .", "words": ["what", "look", "to", "this", "fish"]}, {"sent": "a fish is eating the gun up ?", "words": ["the", "up", "is", "fish", "a"]}, {"sent": "sort of like a fish .", "words": ["of", "fish", "like", "a"]}, {"sent": "aw if she dont catch no fish what she gonna cook ?", "words": ["what", "if", "catch", "dont", "cook", "fish", "she"]}, {"sent": "black fish .", "words": ["black", "fish"]}, {"sent": "the fish in the sky .", "words": ["the", "in", "sky", "fish"]}, {"sent": "see how many fish you can catch .", "words": ["can", "how", "catch", "you", "fish", "see"]}, {"sent": "do you remember feeding the fish ?", "words": ["the", "do", "fish", "you"]}, {"sent": "fish no thats silly looking frogs .", "words": ["fish"]}, {"sent": "hi fish .", "words": ["fish"]}, {"sent": "say you got little fish and medium fish .", "words": ["say", "and", "little", "you", "fish"]}, {"sent": "fish faces .", "words": ["fish"]}, {"sent": "when we get dressed we can go out and bury your fish in the garden cant we ?", "words": ["can", "the", "your", "go", "we", "and", "get", "in", "garden", "out", "fish", "when"]}, {"sent": "xxx fish xxx fish .", "words": ["fish"]}, {"sent": "xxx this other fish is for mommy .", "words": ["for", "is", "other", "this", "fish"]}, {"sent": "fish dont wear clothes do they ?", "words": ["dont", "do", "they", "fish"]}, {"sent": "fish need water dont they ?", "words": ["need", "dont", "they", "water", "fish"]}, {"sent": "but Im sure all these fish and the star fish are Ariels friends very good friends .", "words": ["the", "all", "these", "star", "and", "are", "good", "fish", "but"]}, {"sent": "someone took Anne Maries fish .", "words": ["fish"]}, {"sent": "tuna fish .", "words": ["fish", "tuna"]}, {"sent": "thats tuna fish .", "words": ["fish", "tuna"]}, {"sent": "you caught a fish .", "words": ["fish", "a", "you"]}, {"sent": "fish and they like some milk .", "words": ["like", "and", "they", "fish", "milk", "some"]}, {"sent": "yes I can see three fish .", "words": ["can", "see", "fish"]}, {"sent": "more fish ?", "words": ["more", "fish"]}, {"sent": "theres a fish on it right so and it has all these beads we hadta make it you started it but you did not finish it right maybe you wanna finish it tonight and we can bring it on our trip tomorrow .", "words": ["our", "beads", "not", "can", "it", "on", "all", "these", "so", "finish", "bring", "and", "you", "make", "a", "we", "did", "wanna", "fish", "but"]}, {"sent": "where is the fish ?", "words": ["where", "the", "fish", "is"]}, {"sent": "I think hell have fish and chips uhn do you ?", "words": ["do", "think", "and", "have", "you", "fish"]}, {"sent": "are we catching some more fish ?", "words": ["we", "more", "are", "fish", "some"]}, {"sent": "pull it off your foot .", "words": ["off", "your", "pull", "foot", "it"]}, {"sent": "do you wanna pull some out ?", "words": ["do", "pull", "you", "wanna", "out", "some"]}, {"sent": "one throws it one pushes and one likes to pull .", "words": ["to", "it", "pull", "and"]}, {"sent": "oh well just hafta stand up and pull it out here a bit .", "words": ["up", "stand", "pull", "and", "here", "it", "out", "a"]}, {"sent": "pull the top up .", "words": ["the", "up", "pull"]}, {"sent": "pull it out a bit more .", "words": ["pull", "more", "it", "out", "a"]}, {"sent": "here you pull .", "words": ["here", "pull", "you"]}, {"sent": "pull pull clap .", "words": ["clap", "pull"]}, {"sent": "let me pull one off and .", "words": ["off", "me", "and", "pull"]}, {"sent": "pull .", "words": ["pull"]}, {"sent": "okay well lets have them pull the house .", "words": ["the", "them", "pull", "have", "house"]}, {"sent": "pull it down over one .", "words": ["down", "it", "over", "pull"]}, {"sent": "oh I cant get hold of the sharks tail to pull it away .", "words": ["the", "hold", "of", "to", "pull", "get", "it", "away"]}, {"sent": "did you pull it back .", "words": ["back", "pull", "you", "it", "did"]}, {"sent": "pull .", "words": ["pull"]}, {"sent": "pull very hard .", "words": ["hard", "pull"]}, {"sent": "oh you just wanna pull that dont you ?", "words": ["pull", "you", "dont", "wanna", "that"]}, {"sent": "then grab it over here and pull it on .", "words": ["and", "here", "pull", "it", "on", "then", "over"]}, {"sent": "okay now go ahead and pull .", "words": ["pull", "and", "go"]}, {"sent": "pull this down .", "words": ["this", "down", "pull"]}, {"sent": "pull that away .", "words": ["away", "that", "pull"]}, {"sent": "well pull your boots off .", "words": ["off", "boots", "your", "pull"]}, {"sent": "and then pull them up again ?", "words": ["up", "them", "pull", "and", "then"]}, {"sent": "no dont pull his eyes off .", "words": ["his", "dont", "off", "pull"]}, {"sent": "well pull his pull your jeans up .", "words": ["up", "your", "pull", "his", "jeans"]}, {"sent": "pull them up .", "words": ["up", "them", "pull"]}, {"sent": "xxx pull up mhm .", "words": ["up", "pull"]}, {"sent": "pull pull .", "words": ["pull"]}, {"sent": "and pull .", "words": ["pull", "and"]}, {"sent": "pull hard .", "words": ["hard", "pull"]}, {"sent": "pull .", "words": ["pull"]}, {"sent": "do you wanna pull it out ?", "words": ["do", "pull", "you", "it", "wanna", "out"]}, {"sent": "pull it from down here .", "words": ["here", "down", "it", "pull"]}, {"sent": "pull the strap .", "words": ["the", "pull"]}, {"sent": "pull the other ear .", "words": ["the", "other", "ear", "pull"]}, {"sent": "got ta pull them out from this side bud .", "words": ["this", "them", "out", "pull"]}, {"sent": "pull straight towards you .", "words": ["you", "pull"]}, {"sent": "dont pull it .", "words": ["dont", "it", "pull"]}, {"sent": "some engine needs to pull it .", "words": ["to", "some", "it", "pull"]}, {"sent": "pull hard .", "words": ["hard", "pull"]}, {"sent": "can you pull it apart pull it apart .", "words": ["can", "it", "pull", "you"]}, {"sent": "I think you hafta pull on this .", "words": ["think", "pull", "you", "this", "on"]}, {"sent": "you just told me to pull it apart .", "words": ["to", "pull", "you", "it", "me"]}, {"sent": "you pull this one and Ill pull that one .", "words": ["pull", "and", "this", "you", "that"]}, {"sent": "if you pull the t .", "words": ["pull", "the", "if", "you"]}, {"sent": "it doesnt pull .", "words": ["it", "pull"]}, {"sent": "can you pull it up ?", "words": ["can", "up", "pull", "you", "it"]}, {"sent": "pull it .", "words": ["it", "pull"]}, {"sent": "pull the belt .", "words": ["the", "belt", "pull"]}, {"sent": "no dont pull it .", "words": ["dont", "it", "pull"]}, {"sent": "okay now you pull .", "words": ["pull", "you"]}, {"sent": "pull .", "words": ["pull"]}, {"sent": "pull it .", "words": ["it", "pull"]}, {"sent": "why dont you pull it around that little uh rhino ?", "words": ["pull", "you", "dont", "little", "it", "around", "that", "why"]}, {"sent": "dont play with those because youre gonna pull them and break them darling .", "words": ["those", "play", "with", "them", "pull", "and", "dont", "break", "because"]}, {"sent": "here we pull them off .", "words": ["off", "we", "them", "pull", "here"]}, {"sent": "pull it oops it got all curled up you pull it okay let me let me tear it okay thats a very big piece .", "words": ["up", "all", "tear", "you", "pull", "it", "me", "big", "a"]}, {"sent": "you pull it .", "words": ["it", "pull", "you"]}, {"sent": "okay pull it up .", "words": ["up", "it", "pull"]}, {"sent": "do it you know and dont pull it off like that because uh the shavings ul fall on the floor .", "words": ["off", "do", "the", "like", "fall", "and", "you", "dont", "pull", "it", "on", "that", "because"]}, {"sent": "you cant pull the legs down on shorts .", "words": ["the", "shorts", "pull", "you", "on", "down"]}, {"sent": "you can pull the airplane around with the truck .", "words": ["can", "the", "airplane", "with", "truck", "pull", "you", "around"]}, {"sent": "you can pull all you like but they willnt come off .", "words": ["can", "all", "off", "like", "pull", "you", "they", "but"]}, {"sent": "can you pull it up ?", "words": ["can", "up", "pull", "you", "it"]}, {"sent": "here you pull it again from way back here .", "words": ["back", "pull", "here", "you", "it"]}, {"sent": "can I pull your pants leg up ?", "words": ["can", "up", "your", "pull", "pants", "leg"]}, {"sent": "and then look hes trying to pull Mowgli to say right weve got to go back to the jungle er go and get to the village .", "words": ["the", "back", "go", "to", "say", "pull", "and", "get", "look", "then"]}, {"sent": "better pull them down I think .", "words": ["better", "think", "them", "pull", "down"]}, {"sent": "you pull that up there .", "words": ["up", "pull", "you", "there", "that"]}, {"sent": "pull off the pants .", "words": ["off", "the", "pants", "pull"]}, {"sent": "what does the mousey say when you pull that ?", "words": ["the", "what", "does", "say", "you", "pull", "that", "when"]}, {"sent": "you hafta pull down on this knob .", "words": ["pull", "you", "this", "on", "down"]}, {"sent": "dont pull that off there you go .", "words": ["off", "go", "pull", "you", "dont", "there", "that"]}, {"sent": "you didnt needta pull it the way you were pulling it before did you ?", "words": ["the", "pull", "you", "it", "did", "were"]}, {"sent": "you can pull it .", "words": ["can", "it", "pull", "you"]}, {"sent": "this phone you can pull around .", "words": ["can", "pull", "you", "this", "around"]}, {"sent": "gonna pull him on the wagon ?", "words": ["on", "the", "him", "pull"]}, {"sent": "pull that little knob ?", "words": ["that", "little", "pull"]}, {"sent": "now pull that bit up like that .", "words": ["up", "that", "like", "pull"]}, {"sent": "baby what did you pull that down for ?", "words": ["what", "for", "pull", "you", "did", "down", "that"]}, {"sent": "pull it oh good job .", "words": ["it", "good", "pull"]}, {"sent": "pull my trousers up .", "words": ["up", "my", "pull"]}, {"sent": "dont you pull those down .", "words": ["those", "pull", "you", "dont", "down"]}, {"sent": "now you pull on it .", "words": ["on", "it", "pull", "you"]}, {"sent": "he could he could just pull his hair .", "words": ["pull", "he", "his", "could", "hair"]}, {"sent": "pull .", "words": ["pull"]}, {"sent": "pull pull .", "words": ["pull"]}, {"sent": "no dont pull his hair Kalie .", "words": ["his", "dont", "hair", "pull"]}, {"sent": "dont pull them out .", "words": ["dont", "them", "out", "pull"]}, {"sent": "dont be silly dont be silly because if you pull those wires out therell be trouble .", "words": ["those", "if", "be", "you", "pull", "dont", "out", "because"]}, {"sent": "oh pull tight .", "words": ["pull"]}, {"sent": "dont pull it .", "words": ["dont", "it", "pull"]}, {"sent": "pull back the battery .", "words": ["the", "back", "pull"]}, {"sent": "if you put your finger on top of it and pull it like that it will come out .", "words": ["put", "your", "if", "of", "will", "like", "you", "and", "finger", "pull", "it", "out", "on", "that"]}, {"sent": "you dont wanna pull her hair do you ?", "words": ["do", "pull", "you", "dont", "wanna", "hair", "her"]}, {"sent": "dont pull them off .", "words": ["off", "dont", "them", "pull"]}, {"sent": "pull the string .", "words": ["the", "pull"]}, {"sent": "you just needta pull it like this .", "words": ["like", "pull", "you", "this", "it"]}, {"sent": "wait a minute dont pull his head .", "words": ["wait", "head", "pull", "dont", "his", "a"]}, {"sent": "then he gets up and has a peanutbutter sandwich .", "words": ["up", "and", "he", "then", "sandwich", "a"]}, {"sent": "I dont know where she ever got this mayonnaise sandwich .", "words": ["where", "dont", "this", "she", "sandwich"]}, {"sent": "and cheese sandwich .", "words": ["cheese", "sandwich", "and"]}, {"sent": "uhhuh soup and sandwich .", "words": ["soup", "sandwich", "and"]}, {"sent": "no thats a sandwich .", "words": ["sandwich", "a"]}, {"sent": "did AuntieLwww get you that sandwich ?", "words": ["you", "get", "did", "that", "sandwich"]}, {"sent": "xxx sandwich .", "words": ["sandwich"]}, {"sent": "while Im fixing your sandwich .", "words": ["your", "sandwich"]}, {"sent": "you sure got a big mouthful of sandwich Sarah .", "words": ["of", "you", "big", "sandwich", "a"]}, {"sent": "heres a sandwich called a hero .", "words": ["sandwich", "a"]}, {"sent": "eat your sandwich .", "words": ["eat", "your", "sandwich"]}, {"sent": "you can make a sandwich .", "words": ["can", "you", "make", "sandwich", "a"]}, {"sent": "a sandwich .", "words": ["sandwich", "a"]}, {"sent": "just let Mummy make the sandwich first sweetheart .", "words": ["the", "first", "make", "sandwich"]}, {"sent": "oh thats a big sandwich .", "words": ["big", "sandwich", "a"]}, {"sent": "Im gonna have a sandwich .", "words": ["have", "sandwich", "a"]}, {"sent": "you going to make a sandwich ?", "words": ["to", "you", "make", "sandwich", "a"]}, {"sent": "I was actually thinking that I could see that cheese sandwich .", "words": ["was", "that", "see", "could", "cheese", "sandwich"]}, {"sent": "how about a dear jam sandwich ?", "words": ["sandwich", "how", "about", "a"]}, {"sent": "cheese and sausage crumb sandwich .", "words": ["cheese", "sandwich", "and"]}, {"sent": "do you want a peanutbutter sandwich ?", "words": ["do", "sandwich", "a", "you"]}, {"sent": "um and you made a sandwich right there .", "words": ["you", "and", "there", "sandwich", "a"]}, {"sent": "um yes Daddy was eating a sandwich and Mommy was eating a sandwich that day .", "words": ["was", "and", "that", "sandwich", "a"]}, {"sent": "is there a sandwich ?", "words": ["there", "sandwich", "a", "is"]}, {"sent": "sandwich for lunch ?", "words": ["for", "sandwich"]}, {"sent": "hm how about we make a peanutbutter sandwich .", "words": ["how", "we", "about", "make", "sandwich", "a"]}, {"sent": "that looks like a PlayDoh sandwich are you making a sandwich ?", "words": ["like", "you", "are", "that", "sandwich", "a"]}, {"sent": "what kind of sandwich ?", "words": ["of", "what", "sandwich"]}, {"sent": "you didnt eat the sandwich .", "words": ["eat", "the", "sandwich", "you"]}, {"sent": "or maybe these are parts of a sandwich .", "words": ["these", "of", "are", "sandwich", "a"]}, {"sent": "what else do you need for a sandwich Becky .", "words": ["what", "do", "for", "need", "you", "sandwich", "a"]}, {"sent": "you drink your juice and a sandwich and a biscuit and you might feel better .", "words": ["better", "your", "drink", "and", "you", "juice", "sandwich", "a"]}, {"sent": "wheres my sandwich ?", "words": ["my", "sandwich"]}, {"sent": "it was just a little sandwich snack but it was gorgeous .", "words": ["was", "little", "it", "but", "sandwich", "a"]}, {"sent": "xxx gotta big sandwich .", "words": ["big", "sandwich"]}, {"sent": "a little peanut butter and jelly sandwich .", "words": ["jelly", "and", "little", "butter", "peanut butter", "sandwich", "a"]}, {"sent": "youd like a sandwich ?", "words": ["like", "sandwich", "a"]}, {"sent": "is that my sandwich ?", "words": ["my", "that", "sandwich", "is"]}, {"sent": "you wanna make a sandwich ?", "words": ["you", "wanna", "make", "sandwich", "a"]}, {"sent": "I cant make you omelette sandwich .", "words": ["make", "sandwich", "you"]}, {"sent": "yeah what kind of sandwich did we make for daddy did we make him a hamburger sandwich or a ham sandwich .", "words": ["what", "for", "we", "of", "hamburger", "him", "did", "make", "sandwich", "a"]}, {"sent": "I lost my sandwich apart .", "words": ["my", "sandwich"]}, {"sent": "Ill just have a a very quick sandwich .", "words": ["have", "sandwich", "a"]}, {"sent": "did you ever really give him a bad knuckle sandwich ?", "words": ["bad", "you", "him", "did", "give", "sandwich", "a"]}, {"sent": "got a sandwich .", "words": ["sandwich", "a"]}, {"sent": "make a ham sandwich ?", "words": ["make", "sandwich", "a"]}, {"sent": "its quite a long time since youve had a cheese sandwich .", "words": ["sandwich", "cheese", "long", "a"]}, {"sent": "is it roastbeef lets make a sandwich .", "words": ["is", "it", "make", "sandwich", "a"]}, {"sent": "whats another thing you might eat in a sandwich ?", "words": ["you", "another", "eat", "in", "sandwich", "a"]}, {"sent": "lets get you dressed then we can have a sandwich .", "words": ["can", "we", "you", "get", "have", "then", "sandwich", "a"]}, {"sent": "can I have a cheese sandwich please ?", "words": ["can", "have", "cheese", "sandwich", "a"]}, {"sent": "can you put it inside the sandwich .", "words": ["can", "the", "put", "inside", "you", "it", "sandwich"]}, {"sent": "you broke my sandwich .", "words": ["my", "sandwich", "you"]}, {"sent": "want a sandwich now that you ate something good ?", "words": ["you", "good", "that", "sandwich", "a"]}, {"sent": "lets make a sandwich .", "words": ["make", "sandwich", "a"]}, {"sent": "John do you wanna make a sandwich ?", "words": ["do", "you", "wanna", "make", "sandwich", "a"]}, {"sent": "we can go out and play now if you wanna when you finish your sandwich .", "words": ["can", "your", "if", "go", "we", "play", "finish", "and", "you", "out", "wanna", "when", "sandwich"]}, {"sent": "thats your cheese sandwich .", "words": ["cheese", "your", "sandwich"]}, {"sent": "youve got your sandwich in your mouth .", "words": ["in", "mouth", "your", "sandwich"]}, {"sent": "would I let you have a sandwich at the seashore ?", "words": ["the", "you", "have", "at", "would", "sandwich", "a"]}, {"sent": "when you get to that bit of the sandwich if you wanna open it up and eat the bread separately so its not quite so thick and quite so hard .", "words": ["the", "of", "eat", "hard", "when", "not", "sandwich", "bread", "it", "that", "up", "if", "to", "so", "you", "and", "open", "get", "wanna"]}, {"sent": "you havent seen me make a sandwich yet .", "words": ["you", "me", "make", "sandwich", "a"]}, {"sent": "oh the sandwich .", "words": ["the", "sandwich"]}, {"sent": "in this sandwich .", "words": ["this", "in", "sandwich"]}, {"sent": "youve got a mouthful of cheese sandwich Thomas Lastname .", "words": ["of", "cheese", "sandwich", "a"]}, {"sent": "yeah in a minute I cant get you a sandwich when weve got to play .", "words": ["to", "play", "you", "get", "in", "when", "sandwich", "a"]}, {"sent": "I love fish sandwich .", "words": ["sandwich", "love", "fish"]}, {"sent": "does he want a peanutbutter and jelly sandwich ?", "words": ["jelly", "does", "and", "he", "sandwich", "a"]}, {"sent": "can you make a sandwich ?", "words": ["can", "you", "make", "sandwich", "a"]}, {"sent": "Im eating the sandwich that I made which is more than what youve done .", "words": ["the", "what", "is", "which", "more", "that", "sandwich"]}, {"sent": "do you want a sandwich Lara ?", "words": ["do", "sandwich", "a", "you"]}, {"sent": "in a sandwich ?", "words": ["in", "sandwich", "a"]}, {"sent": "do you think dolly might like some of your sandwich ?", "words": ["do", "think", "your", "of", "like", "you", "some", "sandwich"]}, {"sent": "turkey sandwich turkey and cheese .", "words": ["cheese", "and", "sandwich", "turkey"]}, {"sent": "thats a big sandwich .", "words": ["big", "sandwich", "a"]}, {"sent": "how about a tunafish sandwich .", "words": ["sandwich", "how", "about", "a"]}, {"sent": "fish sandwich have you ever had fish sandwich ?", "words": ["have", "you", "sandwich", "fish"]}, {"sent": "is it a jam sandwich ?", "words": ["sandwich", "it", "a", "is"]}, {"sent": "well not a cheese sandwich just cheese .", "words": ["cheese", "not", "sandwich", "a"]}, {"sent": "its the nicest sandwich Ive had for ages .", "words": ["the", "for", "sandwich"]}, {"sent": "lets make a sandwich .", "words": ["make", "sandwich", "a"]}, {"sent": "and you had Dadas sandwich .", "words": ["sandwich", "you", "and"]}, {"sent": "sandwich .", "words": ["sandwich"]}, {"sent": "the birds are eating Jwwws sandwich ?", "words": ["the", "are", "sandwich"]}, {"sent": "are we both eating Jwwws sandwich ?", "words": ["are", "sandwich", "we"]}, {"sent": "a very big sandwich .", "words": ["big", "sandwich", "a"]}, {"sent": "heres a toast sandwich .", "words": ["sandwich", "toast", "a"]}, {"sent": "peas on your sandwich .", "words": ["on", "peas", "your", "sandwich"]}, {"sent": "is that what you do when your Mum gives you a sandwich ?", "words": ["what", "do", "your", "when", "is", "you", "that", "sandwich", "a"]}, {"sent": "there now your peanutbutter sandwich .", "words": ["there", "your", "sandwich"]}, {"sent": "oh my you made a bigger sandwich too .", "words": ["my", "you", "too", "sandwich", "a"]}, {"sent": "yeah how do you make a sandwich ?", "words": ["do", "how", "you", "make", "sandwich", "a"]}, {"sent": "dyou think they would like a sandwich ?", "words": ["think", "like", "they", "would", "sandwich", "a"]}, {"sent": "and it got to be a sandwich an apple a doughnut .", "words": ["to", "be", "and", "an", "it", "apple", "sandwich", "a"]}, {"sent": "ham and plate sandwich ?", "words": ["plate", "sandwich", "and"]}, {"sent": "okay sit down Ill get you a sandwich .", "words": ["you", "get", "sit", "down", "sandwich", "a"]}, {"sent": "Im going to do myself a sandwich .", "words": ["do", "to", "myself", "sandwich", "a"]}, {"sent": "you bit a hole in your cheese sandwich too .", "words": ["your", "you", "in", "too", "cheese", "sandwich", "a"]}, {"sent": "look you got a sandwich .", "words": ["sandwich", "a", "you", "look"]}, {"sent": "you mad .", "words": ["mad", "you"]}, {"sent": "Im getting mad .", "words": ["mad"]}, {"sent": "no he gets mad .", "words": ["mad", "he"]}, {"sent": "say Im mad me .", "words": ["mad", "me", "say"]}, {"sent": "youve gone climbing mad havent you at the minute .", "words": ["the", "mad", "at", "you"]}, {"sent": "if you spill that Im gonna be mad .", "words": ["if", "be", "mad", "you", "spill", "that"]}, {"sent": "drive mad ?", "words": ["drive", "mad"]}, {"sent": "it drives you mad .", "words": ["mad", "it", "you"]}, {"sent": "they werent mad .", "words": ["mad", "they"]}, {"sent": "no now you got me mad .", "words": ["mad", "me", "you"]}, {"sent": "but we dont hafta be mad all the time right ?", "words": ["the", "all", "we", "mad", "be", "dont", "but"]}, {"sent": "are you gonna get mad at me if I put some clothes on you ?", "words": ["put", "if", "mad", "you", "get", "are", "at", "on", "me", "some"]}, {"sent": "oh you mad ?", "words": ["mad", "you"]}, {"sent": "why are you mad at him ?", "words": ["mad", "you", "him", "are", "at", "why"]}, {"sent": "or are they getting mad do you think ?", "words": ["do", "think", "mad", "you", "they", "are"]}, {"sent": "he useta bite on my clock and every thats when I got mad I hadta get rid of him .", "words": ["my", "clock", "of", "mad", "and", "get", "on", "him", "he", "every", "bite", "when"]}, {"sent": "you get mad ?", "words": ["get", "mad", "you"]}, {"sent": "because were mad at him very mad .", "words": ["mad", "him", "at", "were", "because"]}, {"sent": "yeah are you saying that hes mad ?", "words": ["mad", "are", "that", "you"]}, {"sent": "because I was so mad with him .", "words": ["mad", "was", "so", "with", "him", "because"]}, {"sent": "can I make you mad ?", "words": ["can", "mad", "make", "you"]}, {"sent": "and your mom got mad ?", "words": ["mad", "your", "and"]}, {"sent": "Jeannine has gone mad .", "words": ["mad"]}, {"sent": "was Mummy mad ?", "words": ["mad", "was"]}, {"sent": "youre going to get a mad one this time .", "words": ["to", "mad", "get", "this", "a"]}, {"sent": "yeah hes mad .", "words": ["mad"]}, {"sent": "he makes you mad by doing that ?", "words": ["mad", "by", "you", "he", "that"]}, {"sent": "probably because he was mad .", "words": ["mad", "was", "he", "because"]}, {"sent": "youre mad .", "words": ["mad"]}, {"sent": "you getting mad .", "words": ["mad", "you"]}, {"sent": "youre mad .", "words": ["mad"]}, {"sent": "hes mad at you .", "words": ["mad", "you", "at"]}, {"sent": "no I told you then I got mad at her .", "words": ["mad", "you", "at", "then", "her"]}, {"sent": "is he mad ?", "words": ["mad", "he", "is"]}, {"sent": "youre mad .", "words": ["mad"]}, {"sent": "and last night I was going mad and then I suddenly thought ah Thomass calamine so I took it out of the first aid box .", "words": ["the", "box", "of", "mad", "was", "last", "so", "first", "and", "it", "out", "then"]}, {"sent": "Ill be mad .", "words": ["mad", "be"]}, {"sent": "why were they mad at the dragon ?", "words": ["the", "mad", "they", "at", "were", "why"]}, {"sent": "are you mad about that xxx .", "words": ["mad", "you", "are", "about", "that"]}, {"sent": "oh your getting mad at me ?", "words": ["mad", "me", "your", "at"]}, {"sent": "I think shes a little bit mad .", "words": ["mad", "think", "little", "a"]}, {"sent": "I know youre gonna get mad at me but its not a toy cmon .", "words": ["mad", "get", "toy", "at", "me", "not", "but", "a"]}, {"sent": "our Mumll go bleeding mad .", "words": ["go", "mad", "our"]}, {"sent": "youre going mad in this house arent you ?", "words": ["mad", "you", "this", "in", "house"]}, {"sent": "is that why shes getting mad ?", "words": ["mad", "that", "why", "is"]}, {"sent": "Im mad .", "words": ["mad"]}, {"sent": "I got mad first and everybody else got mad .", "words": ["mad", "first", "and"]}, {"sent": "Daddy get mad ?", "words": ["get", "mad"]}, {"sent": "Melvin is mad .", "words": ["mad", "is"]}, {"sent": "no hes not mad .", "words": ["mad", "not"]}, {"sent": "I know youre mad because somebody took some away from you but she also took some from everyone else .", "words": ["mad", "you", "away", "but", "some", "she", "because"]}, {"sent": "if youre gonna eat thats fine but you cant throw things on the floor because that makes me mad that makes a mess .", "words": ["the", "if", "throw", "fine", "mad", "because", "you", "eat", "on", "me", "that", "but", "a"]}, {"sent": "mommy gets mad daddy gets mad .", "words": ["mad"]}, {"sent": "I think the teddy bears crying because shes getting mad .", "words": ["the", "mad", "think", "because"]}, {"sent": "why is Shem mad at daddy ?", "words": ["mad", "at", "why", "is"]}, {"sent": "are you mad .", "words": ["mad", "are", "you"]}, {"sent": "and Shem gets mad everybody gets mad sometimes .", "words": ["mad", "and"]}, {"sent": "someones gonna get mad at you .", "words": ["get", "mad", "you", "at"]}, {"sent": "you wanna make Mommy mad ?", "words": ["mad", "make", "wanna", "you"]}, {"sent": "that makes Daddy mad .", "words": ["mad", "that"]}, {"sent": "hes mad .", "words": ["mad"]}, {"sent": "I got mad because you ran away .", "words": ["mad", "away", "because", "you"]}, {"sent": "are you mad at him thats why youre biting are you mad at somebody ?", "words": ["mad", "you", "him", "are", "at", "why"]}, {"sent": "now stop being mad .", "words": ["mad", "stop"]}, {"sent": "you mad aint you ?", "words": ["mad", "you"]}, {"sent": "you make mommy mad and you know it .", "words": ["mad", "and", "you", "it", "make"]}, {"sent": "going mad .", "words": ["mad"]}, {"sent": "watch me get mad .", "words": ["get", "me", "mad", "watch"]}, {"sent": "uhoh Jacob is mad .", "words": ["mad", "is"]}, {"sent": "maybe hes mad at you said DW .", "words": ["mad", "you", "at"]}, {"sent": "are you mad at Maddy today ?", "words": ["mad", "are", "at", "you"]}, {"sent": "Im not mad .", "words": ["mad", "not"]}, {"sent": "hes getting mad now .", "words": ["mad"]}, {"sent": "I was too mad .", "words": ["mad", "was", "too"]}, {"sent": "that boys very mad isnt he ?", "words": ["he", "mad", "that"]}, {"sent": "why is he getting mad ?", "words": ["mad", "he", "why", "is"]}, {"sent": "dont get mad .", "words": ["get", "dont", "mad"]}, {"sent": "say mad .", "words": ["mad", "say"]}, {"sent": "I wish that some people didnt hafta sit in the car and listen to other people get mad at the bus .", "words": ["the", "listen", "car", "to", "mad", "wish", "and", "other", "in", "get", "at", "some", "sit", "that", "bus"]}, {"sent": "right sometimes they just get a little mad then they just go rrrr .", "words": ["go", "mad", "little", "get", "they", "then", "a"]}, {"sent": "youre mad .", "words": ["mad"]}, {"sent": "so I went mad Cathy .", "words": ["mad", "so"]}, {"sent": "youre mad .", "words": ["mad"]}, {"sent": "mhm youre gonna get mad huh ?", "words": ["get", "mad"]}, {"sent": "and I went mad .", "words": ["mad", "and"]}, {"sent": "whatcha mad about ?", "words": ["mad", "about"]}, {"sent": "thats that mad cow again .", "words": ["mad", "that", "cow"]}, {"sent": "hm why do you think theyre mad ?", "words": ["do", "think", "mad", "you", "why"]}, {"sent": "makes Daddy mad you know .", "words": ["mad", "you"]}, {"sent": "this ones mad ?", "words": ["mad", "this"]}, {"sent": "boy she is mad ?", "words": ["mad", "she", "is"]}, {"sent": "theyre going because shes so mad .", "words": ["mad", "so", "because"]}, {"sent": "because youre driving me mad .", "words": ["mad", "me", "because"]}, {"sent": "she is mad .", "words": ["mad", "she", "is"]}, {"sent": "did Zwww get mad this morning ?", "words": ["get", "mad", "did", "this"]}, {"sent": "make the baby mad .", "words": ["the", "mad", "make"]}, {"sent": "were they mad ?", "words": ["mad", "they", "were"]}, {"sent": "my husbands went mad .", "words": ["mad", "my"]}, {"sent": "stamp stamp kick kick kick kick kick kick kick kick kick kick kick kick .", "words": ["kick"]}, {"sent": "you like to kick the ball .", "words": ["the", "to", "like", "you", "ball", "kick"]}, {"sent": "I didnt mean you useta kick me I meant you useta kick and splash .", "words": ["and", "you", "kick", "splash", "me"]}, {"sent": "jolly by golly Barnaby said with a kick .", "words": ["kick", "with", "a", "by"]}, {"sent": "good kick .", "words": ["kick", "good"]}, {"sent": "xxx kick .", "words": ["kick"]}, {"sent": "do you kick the ball ?", "words": ["the", "do", "you", "ball", "kick"]}, {"sent": "kick and splash .", "words": ["kick", "splash", "and"]}, {"sent": "kick .", "words": ["kick"]}, {"sent": "no Daddy kick it to you .", "words": ["to", "it", "kick", "you"]}, {"sent": "you dont kick them .", "words": ["dont", "kick", "them", "you"]}, {"sent": "now Im gonna kick it .", "words": ["it", "kick"]}, {"sent": "dont kick .", "words": ["dont", "kick"]}, {"sent": "can you kick it ?", "words": ["can", "it", "kick", "you"]}, {"sent": "we kick around our building .", "words": ["around", "our", "kick", "we"]}, {"sent": "do buzzing bees swing in trees enjoy the breeze kick up their knees ?", "words": ["the", "do", "up", "their", "swing", "in", "kick"]}, {"sent": "dont you dare kick Purdie .", "words": ["dont", "kick", "you"]}, {"sent": "did you kick it ?", "words": ["it", "kick", "did", "you"]}, {"sent": "dont kick the balloon Noddy .", "words": ["the", "dont", "kick", "balloon"]}, {"sent": "dont kick .", "words": ["dont", "kick"]}, {"sent": "dont kick .", "words": ["dont", "kick"]}, {"sent": "Id like to kick .", "words": ["to", "kick", "like"]}, {"sent": "dont kick them .", "words": ["dont", "kick", "them"]}, {"sent": "can you kick the ball to granny can you ?", "words": ["can", "the", "to", "you", "ball", "kick"]}, {"sent": "kick the ball .", "words": ["the", "ball", "kick"]}, {"sent": "oh be careful you dont kick that over .", "words": ["be", "you", "dont", "kick", "careful", "over", "that"]}, {"sent": "and you mustnt kick it hard .", "words": ["you", "and", "it", "kick", "hard"]}, {"sent": "good kick ?", "words": ["kick", "good"]}, {"sent": "Id like to kick .", "words": ["to", "kick", "like"]}, {"sent": "no kick her in the backside .", "words": ["the", "in", "kick", "her"]}, {"sent": "you dont kick Mummy .", "words": ["dont", "kick", "you"]}, {"sent": "that shell throw around or kick .", "words": ["kick", "that", "throw", "around"]}, {"sent": "dont dont kick it off your tray already .", "words": ["off", "your", "dont", "kick", "it", "tray"]}, {"sent": "dont kick .", "words": ["dont", "kick"]}, {"sent": "kick kick .", "words": ["kick"]}, {"sent": "dont kick .", "words": ["dont", "kick"]}, {"sent": "little legs going kick kick kick kick kick .", "words": ["kick", "little"]}, {"sent": "dont kick that darling .", "words": ["dont", "kick", "that"]}, {"sent": "wanna kick it ?", "words": ["it", "kick", "wanna"]}, {"sent": "kick your bootie .", "words": ["kick", "your"]}, {"sent": "dont kick ?", "words": ["dont", "kick"]}, {"sent": "I know but I dont think we can kick that one in the house .", "words": ["can", "the", "think", "we", "dont", "in", "kick", "house", "that", "but"]}, {"sent": "can you kick it with your foot ?", "words": ["can", "your", "with", "you", "foot", "it", "kick"]}, {"sent": "otherwise Ill hafta kick her out in the garden .", "words": ["the", "in", "kick", "garden", "out", "her"]}, {"sent": "be careful dont kick him .", "words": ["be", "dont", "him", "kick", "careful"]}, {"sent": "kick it to me .", "words": ["to", "it", "me", "kick"]}, {"sent": "you cant kick the recorder .", "words": ["the", "kick", "you"]}, {"sent": "och Daddy no I dont wanna kick it .", "words": ["dont", "kick", "wanna", "it"]}, {"sent": "*SI1: lets kick it over here .", "words": ["here", "it", "kick", "over"]}, {"sent": "dont kick things .", "words": ["dont", "kick"]}, {"sent": "is it funny to kick Mommy in the head ?", "words": ["the", "is", "to", "head", "in", "kick", "it"]}, {"sent": "you dont kick things .", "words": ["dont", "kick", "you"]}, {"sent": "whos kick ?", "words": ["kick"]}, {"sent": "kick it frontways .", "words": ["it", "kick"]}, {"sent": "hey we dont wanna kick the toys .", "words": ["the", "we", "dont", "kick", "wanna"]}, {"sent": "dont kick her .", "words": ["dont", "kick", "her"]}, {"sent": "you didnt kick Perro .", "words": ["kick", "you"]}, {"sent": "I thought shed get a kick out of that .", "words": ["of", "get", "kick", "out", "that", "a"]}, {"sent": "then you can kick it and make it fall down .", "words": ["can", "fall", "you", "and", "it", "kick", "then", "down", "make"]}, {"sent": "dont kick .", "words": ["dont", "kick"]}, {"sent": "dont kick her .", "words": ["dont", "kick", "her"]}, {"sent": "please dont kick .", "words": ["dont", "kick"]}, {"sent": "dont kick me .", "words": ["dont", "me", "kick"]}, {"sent": "did you kick that ball ?", "words": ["you", "ball", "kick", "did", "that"]}, {"sent": "careful that you dont kick Amy .", "words": ["you", "dont", "kick", "careful", "that"]}, {"sent": "be careful you dont kick them over .", "words": ["be", "them", "you", "dont", "kick", "careful", "over"]}, {"sent": "wanna kick the ball again ?", "words": ["the", "ball", "kick", "wanna"]}, {"sent": "xxx kick me ?", "words": ["me", "kick"]}, {"sent": "dont chu kick Elmo its my Elmo .", "words": ["dont", "my", "kick"]}, {"sent": "go ahead kick that ball .", "words": ["ball", "kick", "that", "go"]}, {"sent": "soccer is when ya when ya kick xxx ball .", "words": ["ball", "kick", "when", "is"]}, {"sent": "and we go to you have a certain game that you play where you kick a ball .", "words": ["where", "game", "go", "we", "to", "play", "you", "and", "have", "ball", "kick", "that", "a"]}, {"sent": "kick it .", "words": ["it", "kick"]}, {"sent": "well that would be a kick in the pants .", "words": ["the", "be", "in", "kick", "pants", "would", "that", "a"]}, {"sent": "kick .", "words": ["kick"]}, {"sent": "do you wanna kick it ?", "words": ["do", "you", "kick", "it", "wanna"]}, {"sent": "dont kick me in the stomach please .", "words": ["the", "dont", "in", "kick", "me"]}, {"sent": "dont kick Pooh .", "words": ["dont", "kick"]}, {"sent": "if I take your shoes off and you kick the table itll really hurt your toes and thatll stop you doing it .", "words": ["off", "the", "your", "if", "table", "hurt", "stop", "and", "you", "kick", "it", "take"]}, {"sent": "dont kick .", "words": ["dont", "kick"]}, {"sent": "you wanna kick it ?", "words": ["it", "kick", "wanna", "you"]}, {"sent": "and you kick ya .", "words": ["kick", "you", "and"]}, {"sent": "kick it .", "words": ["it", "kick"]}, {"sent": "get up and kick it .", "words": ["up", "and", "get", "kick", "it"]}, {"sent": "are you gonna kick ?", "words": ["kick", "are", "you"]}, {"sent": "dont kick .", "words": ["dont", "kick"]}, {"sent": "come on dont kick that because thats Mummys sewing darling .", "words": ["dont", "kick", "on", "that", "because"]}, {"sent": "oh dont kick Mommy .", "words": ["dont", "kick"]}, {"sent": "kick his house Chris .", "words": ["his", "kick", "house"]}, {"sent": "no ya dont kick shoes ya dont kick anything .", "words": ["dont", "kick"]}, {"sent": "uh dont you kick her .", "words": ["dont", "kick", "her", "you"]}, {"sent": "dont kick .", "words": ["dont", "kick"]}, {"sent": "you kick them down .", "words": ["down", "kick", "them", "you"]}, {"sent": "kick the ball .", "words": ["the", "ball", "kick"]}, {"sent": "kick kick .", "words": ["kick"]}, {"sent": "can you go kick kick ?", "words": ["can", "kick", "go", "you"]}, {"sent": "arent you gonna kick the ball ?", "words": ["the", "ball", "kick", "you"]}, {"sent": "dont kick the table .", "words": ["the", "dont", "kick", "table"]}, {"sent": "kick it like this .", "words": ["it", "kick", "like", "this"]}, {"sent": "dont kick doggy .", "words": ["dont", "kick"]}, {"sent": "is your toe sticking outof the cover ?", "words": ["the", "your", "is", "toe", "cover"]}, {"sent": "take the cover off .", "words": ["the", "take", "off", "cover"]}, {"sent": "cover your mouth please honey .", "words": ["mouth", "your", "cover"]}, {"sent": "you need another cover ?", "words": ["another", "need", "cover", "you"]}, {"sent": "get the other box and Ill cover this one .", "words": ["the", "box", "and", "get", "other", "this", "cover"]}, {"sent": "shes going to sleep under your cover ?", "words": ["your", "sleep", "to", "under", "cover"]}, {"sent": "or a cover ?", "words": ["cover", "a"]}, {"sent": "but wheres the cover ?", "words": ["the", "but", "cover"]}, {"sent": "its a hiding cover ?", "words": ["cover", "a"]}, {"sent": "now push this hand down like that and then cover him .", "words": ["push", "hand", "like", "and", "this", "him", "cover", "then", "down", "that"]}, {"sent": "we hafta cover up everything .", "words": ["up", "cover", "we"]}, {"sent": "cover it up again .", "words": ["up", "it", "cover"]}, {"sent": "are you going to cover me ?", "words": ["to", "you", "are", "cover", "me"]}, {"sent": "he wont be able to breathe if you cover his head .", "words": ["if", "to", "be", "head", "you", "he", "cover", "his"]}, {"sent": "cover your mouth Sarah .", "words": ["mouth", "your", "cover"]}, {"sent": "real sleep with your cover ?", "words": ["sleep", "with", "cover", "your"]}, {"sent": "do you sell do you sell things to cover their mouths so they dont roar ?", "words": ["do", "their", "to", "so", "you", "they", "dont", "cover"]}, {"sent": "pap er the hard cover but its got paper inside and it has its the full length story .", "words": ["the", "full", "paper", "inside", "and", "it", "hard", "cover", "story", "but"]}, {"sent": "put the cover over .", "words": ["the", "put", "over", "cover"]}, {"sent": "Ill cover up the clowns .", "words": ["up", "the", "cover"]}, {"sent": "put your head on the pillow and Daddy will cover you .", "words": ["the", "put", "your", "head", "will", "and", "you", "cover", "on", "pillow"]}, {"sent": "back to that front cover .", "words": ["to", "that", "back", "cover"]}, {"sent": "Im all wet cause I didnt put a cover on yer diaper .", "words": ["all", "put", "wet", "diaper", "cover", "on", "a"]}, {"sent": "its the cover for the grill .", "words": ["the", "for", "cover"]}, {"sent": "there ya go very good now it looks like the picture on the cover .", "words": ["the", "go", "picture", "like", "there", "it", "good", "cover", "on"]}, {"sent": "oh well lets cover up those big muscles .", "words": ["up", "big", "those", "cover"]}, {"sent": "xxx because he was shocked and then to cover up his shock he thought he could use a clever retort .", "words": ["up", "a", "was", "to", "and", "he", "cover", "his", "then", "because", "could"]}, {"sent": "is the snake under babys cover ?", "words": ["the", "under", "cover", "is"]}, {"sent": "should I cover him up ?", "words": ["up", "him", "cover"]}, {"sent": "you should cover your mouth too .", "words": ["your", "mouth", "you", "cover", "too"]}, {"sent": "cover ya .", "words": ["cover"]}, {"sent": "here now maybe if we turn this around lets cover up the ladder wanta cover up the ladder ?", "words": ["up", "the", "if", "we", "here", "this", "cover", "around", "ladder"]}, {"sent": "cover your mouth please .", "words": ["mouth", "your", "cover"]}, {"sent": "oh heres the cover .", "words": ["the", "cover"]}, {"sent": "you want the cover or what ?", "words": ["the", "what", "cover", "you"]}, {"sent": "could you not get under the cover ?", "words": ["the", "you", "get", "under", "cover", "could", "not"]}, {"sent": "now are we going to cover ourselves with this ?", "words": ["we", "to", "with", "this", "are", "cover"]}, {"sent": "because cheese gets really hard when you dont cover it up .", "words": ["up", "when", "you", "dont", "it", "hard", "cover", "cheese", "because"]}, {"sent": "you took off the cover .", "words": ["off", "the", "cover", "you"]}, {"sent": "the cover on the wrong way .", "words": ["the", "on", "cover"]}, {"sent": "are we going to cover you both up ?", "words": ["up", "we", "to", "you", "are", "cover"]}, {"sent": "cover up the baby .", "words": ["up", "the", "cover"]}, {"sent": "you want your cover outof bed ?", "words": ["your", "bed", "cover", "you"]}, {"sent": "see under the cover under the duvet in the corner .", "words": ["the", "under", "in", "cover", "see"]}, {"sent": "did you cover yourself up ?", "words": ["up", "yourself", "you", "did", "cover"]}, {"sent": "cover the baby up .", "words": ["the", "up", "cover"]}, {"sent": "but theyre all the hard cover and I have a ton of the little hard cover books .", "words": ["the", "all", "of", "and", "have", "little", "hard", "cover", "but", "a"]}, {"sent": "I know is it the cover is it the cover for the babys pram ?", "words": ["the", "for", "is", "it", "cover"]}, {"sent": "thats the cover .", "words": ["the", "cover"]}, {"sent": "cover her up .", "words": ["up", "her", "cover"]}, {"sent": "well cover your xxx .", "words": ["your", "cover"]}, {"sent": "you put the cover on .", "words": ["the", "put", "you", "cover", "on"]}, {"sent": "cover the box with newspaper and leave it outside .", "words": ["the", "box", "outside", "with", "and", "it", "cover"]}, {"sent": "book yeah whos on the cover of the book ?", "words": ["the", "of", "on", "cover", "book"]}, {"sent": "oh that picture that Mummy has just shown you of the ginger cat and the tabby cat is on the back cover .", "words": ["the", "back", "is", "of", "picture", "you", "and", "cat", "cover", "on", "that"]}, {"sent": "Miss Kitty Cat uses the bulldozer to cover the garbage with dirt .", "words": ["the", "to", "with", "cover", "garbage"]}, {"sent": "uh you could probably cover it .", "words": ["could", "it", "cover", "you"]}, {"sent": "but I have the plastic cover .", "words": ["have", "the", "but", "cover"]}, {"sent": "cover kahvah my face for a minute .", "words": ["my", "for", "face", "cover", "a"]}, {"sent": "cover it up and well have one tomorrow .", "words": ["up", "and", "have", "it", "cover"]}, {"sent": "I think its Mummy who should cover her eyes then she says peepo .", "words": ["think", "cover", "who", "then", "her", "she"]}, {"sent": "in that cover .", "words": ["in", "that", "cover"]}, {"sent": "cover your mouth little lady when you cough .", "words": ["your", "mouth", "little", "you", "cover", "when"]}, {"sent": "does the scarf cover that ?", "words": ["the", "does", "scarf", "cover", "that"]}, {"sent": "and they to uh draw a scene from the book on the cover .", "words": ["the", "to", "draw", "and", "they", "on", "cover", "book", "a"]}, {"sent": "you dont wanna cover the words because if you cover the words you cant do what ?", "words": ["the", "do", "what", "if", "you", "dont", "cover", "wanna", "because"]}, {"sent": "cover your mouth .", "words": ["mouth", "your", "cover"]}, {"sent": "thats the cover for the the carrycot .", "words": ["the", "for", "cover"]}, {"sent": "to cover it up .", "words": ["up", "to", "it", "cover"]}, {"sent": "three cups and a spoon and a cover ?", "words": ["cover", "spoon", "a", "and"]}, {"sent": "I didnt buy a cover for you to go to sleep under .", "words": ["for", "go", "sleep", "to", "you", "under", "cover", "buy", "a"]}, {"sent": "cover your left eye .", "words": ["your", "eye", "cover"]}, {"sent": "cover your mouth .", "words": ["mouth", "your", "cover"]}, {"sent": "Megans going to get you another cover okay ?", "words": ["to", "you", "get", "another", "cover"]}, {"sent": "have you got a cover please ?", "words": ["have", "cover", "a", "you"]}, {"sent": "you wanna cover him up ?", "words": ["up", "you", "him", "cover", "wanna"]}, {"sent": "a cover .", "words": ["cover", "a"]}, {"sent": "cover your eyes .", "words": ["your", "cover"]}, {"sent": "but where is a cover ?", "words": ["where", "is", "cover", "but", "a"]}, {"sent": "and they cover up the sensor dont they ?", "words": ["up", "the", "and", "they", "dont", "cover"]}, {"sent": "and it has got a rain cover on it .", "words": ["rain", "and", "it", "cover", "on", "a"]}, {"sent": "itll cover up a little bit itll cover up one pig but it wont cover up him see you can still see him .", "words": ["up", "can", "pig", "little", "you", "him", "it", "cover", "see", "but", "a"]}, {"sent": "and listen everything in your bedroom the wallpaper the curtains and the duvet cover and the cot .", "words": ["the", "your", "listen", "and", "in", "cover", "bedroom"]}, {"sent": "what can you see on the xxx the cover of the book ?", "words": ["can", "what", "the", "of", "book", "you", "cover", "on", "see"]}, {"sent": "oh youre going to cover it up with that are you ?", "words": ["up", "to", "with", "you", "it", "are", "cover", "that"]}, {"sent": "cover your mouth cover your mouth cover your mouth .", "words": ["mouth", "your", "cover"]}, {"sent": "take the cover off for me .", "words": ["the", "off", "for", "cover", "me", "take"]}, {"sent": "lightly cover seed .", "words": ["cover"]}, {"sent": "cover up xxx piggies .", "words": ["up", "cover"]}, {"sent": "cover you up ?", "words": ["up", "cover", "you"]}, {"sent": "the cover ?", "words": ["the", "cover"]}, {"sent": "will it cover fox ?", "words": ["will", "it", "cover"]}, {"sent": "Ill put the cover on .", "words": ["the", "put", "on", "cover"]}, {"sent": "thats because you didnt cover it like Mama told ya .", "words": ["like", "you", "it", "cover", "because"]}, {"sent": "here I guess you want the cover of the picture .", "words": ["the", "of", "picture", "you", "here", "cover"]}, {"sent": "you cover your eyes and I hide it and youll find it .", "words": ["your", "hide", "and", "you", "it", "cover", "find"]}, {"sent": "have you got a cover for her we can wrap her up in ?", "words": ["can", "up", "for", "we", "you", "have", "in", "cover", "her", "a"]}, {"sent": "cover your mouth please .", "words": ["mouth", "your", "cover"]}, {"sent": "would you like to cover it up ?", "words": ["up", "to", "like", "you", "it", "cover", "would"]}, {"sent": "no that is the cover .", "words": ["the", "that", "cover", "is"]}, {"sent": "aw buddy please dont wipe it all over .", "words": ["all", "dont", "it", "wipe", "over"]}, {"sent": "let me wipe your fingers first .", "words": ["me", "wipe", "your", "first"]}, {"sent": "clean them up good wipe them good .", "words": ["clean", "up", "them", "good", "wipe"]}, {"sent": "wipe your mouth ?", "words": ["mouth", "wipe", "your"]}, {"sent": "lets wipe our hands .", "words": ["wipe", "our"]}, {"sent": "wipe your mouth .", "words": ["mouth", "wipe", "your"]}, {"sent": "come on in until I wipe your hand .", "words": ["your", "hand", "in", "on", "wipe"]}, {"sent": "Im just going to get a cloth to wipe the flower vase Thomas .", "words": ["the", "to", "flower", "get", "wipe", "a"]}, {"sent": "wipe .", "words": ["wipe"]}, {"sent": "oh dont dont wipe with that okay ?", "words": ["dont", "that", "wipe", "with"]}, {"sent": "I think we can wipe that up without too many problems .", "words": ["can", "up", "think", "we", "too", "wipe", "that"]}, {"sent": "okay lemme wipe your nose .", "words": ["wipe", "your", "nose"]}, {"sent": "sothat I can wipe off your face .", "words": ["can", "off", "your", "face", "wipe"]}, {"sent": "Im just going to wipe your face just wipe your face .", "words": ["to", "wipe", "your", "face"]}, {"sent": "wipe your face ?", "words": ["wipe", "your", "face"]}, {"sent": "wipe it on wipe it on your arm ?", "words": ["your", "it", "arm", "on", "wipe"]}, {"sent": "just give Amy a wipe and you a wash as well .", "words": ["you", "and", "give", "wipe", "wash", "a"]}, {"sent": "Eve wipe your mouth .", "words": ["mouth", "wipe", "your"]}, {"sent": "just let me wipe my top .", "words": ["me", "my", "wipe"]}, {"sent": "we just needta give that tray a quick wipe .", "words": ["we", "that", "give", "wipe", "tray", "a"]}, {"sent": "shall I wipe it off ?", "words": ["off", "it", "wipe"]}, {"sent": "well the fat controller has disappeared into the kitchen to get the dish cloth so I can wipe the table before you get down .", "words": ["the", "can", "into", "table", "to", "so", "down", "you", "get", "dish", "kitchen", "wipe"]}, {"sent": "well wipe those for now .", "words": ["wipe", "those", "for"]}, {"sent": "you want a wipe .", "words": ["wipe", "a", "you"]}, {"sent": "lets just wipe your hands and face .", "words": ["wipe", "your", "face", "and"]}, {"sent": "well I think Brian has perhaps got the towel there in case he gets hot and sweaty and then he can wipe himself .", "words": ["the", "can", "think", "and", "there", "towel", "in", "he", "hot", "then", "wipe"]}, {"sent": "and they dont hafta wipe their bottoms .", "words": ["their", "and", "dont", "they", "wipe"]}, {"sent": "wipe it off your nose Caitlin .", "words": ["off", "your", "it", "wipe", "nose"]}, {"sent": "Im gonna wipe behind you okay ?", "words": ["wipe", "behind", "you"]}, {"sent": "what did you do with that wipe Fraser ?", "words": ["what", "do", "with", "you", "did", "wipe", "that"]}, {"sent": "Ill wipe you off in just a second .", "words": ["off", "you", "in", "wipe", "a"]}, {"sent": "yeah wipe it properly .", "words": ["it", "wipe"]}, {"sent": "it doesnt matter if your fingers get sticky because we can give you a wipe cant we ?", "words": ["can", "your", "if", "we", "sticky", "you", "get", "it", "give", "wipe", "because", "a"]}, {"sent": "just let me wipe that surface over .", "words": ["over", "me", "wipe", "that"]}, {"sent": "why dont you get the rag and wipe that up .", "words": ["the", "up", "you", "get", "dont", "and", "wipe", "that", "why"]}, {"sent": "do you want me to wipe your face ?", "words": ["do", "your", "face", "to", "you", "me", "wipe"]}, {"sent": "let me wipe your chin .", "words": ["me", "wipe", "your", "chin"]}, {"sent": "gotta wipe your face .", "words": ["wipe", "your", "face"]}, {"sent": "well then lets wipe your face .", "words": ["then", "wipe", "your", "face"]}, {"sent": "when you come in youve got to wipe your feet .", "words": ["your", "to", "you", "in", "wipe", "when"]}, {"sent": "shes just getting a cloth to wipe the table .", "words": ["the", "table", "to", "wipe", "a"]}, {"sent": "okay sit here while I wipe it .", "words": ["here", "it", "sit", "wipe"]}, {"sent": "oops dont wipe it on your dress there .", "words": ["your", "dont", "there", "it", "on", "wipe", "dress"]}, {"sent": "let me wipe your face .", "words": ["me", "wipe", "your", "face"]}, {"sent": "just to wipe your nose a little bit .", "words": ["your", "nose", "to", "little", "wipe", "a"]}, {"sent": "you going to wipe her mouth ?", "words": ["to", "mouth", "you", "wipe", "her"]}, {"sent": "shall I just wipe up the mess because look your poor babys lying in the water .", "words": ["up", "the", "water", "your", "poor", "in", "look", "wipe", "because"]}, {"sent": "shall I go and wipe him ?", "words": ["him", "wipe", "and", "go"]}, {"sent": "can I just wipe your nose please Thomas ?", "words": ["can", "wipe", "your", "nose"]}, {"sent": "before we do you stand still Im going to get a tissue and wipe your nose and your hands .", "words": ["do", "your", "we", "tissue", "to", "stand", "nose", "you", "get", "and", "wipe", "a"]}, {"sent": "well hafta wipe your nose in a minute wont we ?", "words": ["your", "we", "nose", "in", "wipe", "a"]}, {"sent": "well get a cloth and wipe it out .", "words": ["and", "get", "it", "out", "wipe", "a"]}, {"sent": "if you wanna pretend that youre a unicorn why dont you get a kleenex and wipe your nose ?", "words": ["your", "if", "kleenex", "nose", "you", "get", "dont", "and", "wanna", "pretend", "wipe", "that", "why", "a"]}, {"sent": "so the jigsaw has had a wipe and so has the little book .", "words": ["the", "so", "and", "little", "book", "wipe", "a"]}, {"sent": "wipe your hands .", "words": ["wipe", "your"]}, {"sent": "wipe it up .", "words": ["up", "it", "wipe"]}, {"sent": "wipe your leg off .", "words": ["off", "wipe", "your", "leg"]}, {"sent": "can Mommy wipe your nose ?", "words": ["can", "wipe", "your", "nose"]}, {"sent": "well get down there and wipe it up .", "words": ["up", "wipe", "and", "get", "there", "it", "down"]}, {"sent": "wanna wipe your face ?", "words": ["wipe", "your", "face", "wanna"]}, {"sent": "let me wipe your nose .", "words": ["me", "wipe", "your", "nose"]}, {"sent": "and wipe up this big mess you made .", "words": ["up", "and", "you", "this", "wipe", "big"]}, {"sent": "then we saw the cat wipe the spot off the dress .", "words": ["the", "off", "we", "cat", "then", "wipe", "dress"]}, {"sent": "shall I wipe your nose ?", "words": ["wipe", "your", "nose"]}, {"sent": "trying to get some things done shes using the wipe to wipe her crayons and her face .", "words": ["the", "face", "to", "and", "get", "some", "wipe", "her"]}, {"sent": "lets just wipe the table .", "words": ["the", "wipe", "table"]}, {"sent": "you hafta wipe yourself please .", "words": ["yourself", "wipe", "you"]}, {"sent": "lets just wipe your hands .", "words": ["wipe", "your"]}, {"sent": "wipe your mouth .", "words": ["mouth", "wipe", "your"]}, {"sent": "Im just going to give them a wipe .", "words": ["to", "them", "give", "wipe", "a"]}, {"sent": "wipe your face and your hands .", "words": ["wipe", "your", "face", "and"]}, {"sent": "and Ill fetch something to wipe your hands with .", "words": ["your", "to", "with", "and", "wipe"]}, {"sent": "let Mummy wipe it .", "words": ["it", "wipe"]}, {"sent": "lets wipe your nose .", "words": ["wipe", "your", "nose"]}, {"sent": "get a tissue out the box and wipe your nose .", "words": ["the", "box", "your", "tissue", "nose", "and", "get", "out", "wipe", "a"]}, {"sent": "here let me lemme wipe that off .", "words": ["off", "here", "me", "wipe", "that"]}, {"sent": "wipe your face and hands .", "words": ["wipe", "your", "face", "and"]}, {"sent": "wipe your hands .", "words": ["wipe", "your"]}, {"sent": "okay lets wipe your hands off .", "words": ["off", "wipe", "your"]}, {"sent": "wipe your nose .", "words": ["wipe", "your", "nose"]}, {"sent": "and I dont really want you you to wipe all your giks on it .", "words": ["all", "your", "to", "you", "and", "dont", "it", "on", "wipe"]}, {"sent": "after I wipe up your bottom .", "words": ["up", "wipe", "your", "bottom"]}, {"sent": "let Daddy come in here wipe your mouth a little bit .", "words": ["your", "mouth", "little", "here", "in", "wipe", "a"]}, {"sent": "you wanna wipe your hand off ?", "words": ["off", "your", "hand", "you", "wanna", "wipe"]}, {"sent": "you wanna wipe your nose ?", "words": ["your", "you", "wanna", "wipe", "nose"]}, {"sent": "wipe your face a little bit .", "words": ["your", "face", "little", "wipe", "a"]}, {"sent": "wipe there you go .", "words": ["go", "there", "wipe", "you"]}, {"sent": "wipe .", "words": ["wipe"]}, {"sent": "okay I think youre ready to wipe arent you ?", "words": ["to", "think", "wipe", "you"]}, {"sent": "mhm now Im gonna wipe your hands okay .", "words": ["wipe", "your"]}, {"sent": "gotta wipe that up .", "words": ["up", "wipe", "that"]}, {"sent": "you will not wipe my kiss off you cheeky monkey .", "words": ["off", "my", "monkey", "you", "will", "wipe", "kiss", "not"]}, {"sent": "wipe his mouth .", "words": ["his", "wipe", "mouth"]}, {"sent": "wipe your hands and face with the napkin .", "words": ["the", "your", "face", "with", "and", "wipe", "napkin"]}, {"sent": "lets wipe your hands .", "words": ["wipe", "your"]}, {"sent": "I just needta wipe the table with a cloth .", "words": ["the", "table", "with", "wipe", "a"]}, {"sent": "have another wipe of your nose Nicole .", "words": ["your", "of", "have", "another", "wipe", "nose"]}, {"sent": "can you get the teatowel and wipe it up ?", "words": ["can", "the", "up", "and", "get", "you", "it", "wipe"]}, {"sent": "grandma will wipe it .", "words": ["will", "it", "wipe"]}, {"sent": "you need help zip it up ?", "words": ["up", "help", "need", "you", "it"]}, {"sent": "you need a pillow ?", "words": ["need", "pillow", "a", "you"]}, {"sent": "you need some more clay .", "words": ["some", "more", "need", "you"]}, {"sent": "you need help ?", "words": ["help", "need", "you"]}, {"sent": "oh you dont need the scissors .", "words": ["the", "need", "scissors", "you", "dont"]}, {"sent": "you dont need all of it .", "words": ["all", "need", "of", "you", "dont", "it"]}, {"sent": "just in case you need it .", "words": ["in", "it", "need", "you"]}, {"sent": "okay you need water ?", "words": ["water", "need", "you"]}, {"sent": "you need dem ear m you need dem things that um .", "words": ["that", "ear", "need", "you"]}, {"sent": "no I need some need some need some .", "words": ["some", "need"]}, {"sent": "lets go in here because we need to start the macaroni and cheese anyway .", "words": ["the", "need", "we", "go", "to", "need to", "and", "here", "in", "cheese", "because"]}, {"sent": "I need one more for them to sneak in .", "words": ["for", "need", "to", "them", "more", "in"]}, {"sent": "what does he need first before he goes to bed ?", "words": ["what", "does", "need", "to", "first", "bed", "he"]}, {"sent": "what other letters do you need for your name ?", "words": ["what", "do", "for", "need", "your", "you", "other"]}, {"sent": "can you wear underpants or do you need a diaper ?", "words": ["can", "do", "need", "diaper", "you", "underpants", "a"]}, {"sent": "what do you need ?", "words": ["what", "do", "need", "you"]}, {"sent": "what are all the letters you need ?", "words": ["the", "all", "what", "need", "you", "are"]}, {"sent": "now you need .", "words": ["need", "you"]}, {"sent": "I need that Laura .", "words": ["that", "need"]}, {"sent": "so we need a straight line over there .", "words": ["need", "we", "so", "there", "over", "a"]}, {"sent": "I need the bucket to put the things in dont I ?", "words": ["the", "put", "need", "bucket", "to", "dont", "in"]}, {"sent": "you dont need shoes on .", "words": ["on", "dont", "need", "you"]}, {"sent": "we need a bit more on that side dont we ?", "words": ["need", "we", "dont", "more", "on", "that", "a"]}, {"sent": "we dont need a xxx xxx .", "words": ["dont", "need", "a", "we"]}, {"sent": "you just need to put on the pink Easter bunny ears .", "words": ["the", "put", "bunny", "need", "need to", "to", "you", "on"]}, {"sent": "and then we need .", "words": ["then", "need", "we", "and"]}, {"sent": "alrighty well I dont think I need anything else .", "words": ["dont", "think", "need"]}, {"sent": "think Ill need the tongs for this .", "words": ["the", "think", "for", "need", "this"]}, {"sent": "I dont need them now .", "words": ["dont", "need", "them"]}, {"sent": "you need some paper first .", "words": ["need", "paper", "first", "you", "some"]}, {"sent": "we need another o .", "words": ["another", "need", "we"]}, {"sent": "its Tuesday today as well we need t for Tuesday theres t for Tuesday .", "words": ["for", "need", "we"]}, {"sent": "you dont need a nappie .", "words": ["dont", "need", "a", "you"]}, {"sent": "I dont think we need the I dont think we need the elevator do we ?", "words": ["the", "do", "think", "need", "we", "dont"]}, {"sent": "because youre going to need a xxx arent you ?", "words": ["need", "to", "you", "because", "a"]}, {"sent": "do you need some help with that ?", "words": ["do", "help", "need", "with", "you", "some", "that"]}, {"sent": "did you need something ?", "words": ["need", "did", "you"]}, {"sent": "I need it .", "words": ["it", "need"]}, {"sent": "yeah you need a kitchen in every one uhn dont you ?", "words": ["need", "you", "dont", "in", "every", "kitchen", "a"]}, {"sent": "and I need socks ?", "words": ["need", "and"]}, {"sent": "airplanes need a runway .", "words": ["need", "a"]}, {"sent": "say Gigi I need water Im dehydrating .", "words": ["need", "say", "water"]}, {"sent": "does it need brushing ?", "words": ["does", "it", "need"]}, {"sent": "did you need to talk to him about .", "words": ["need", "need to", "to", "you", "him", "did", "talk", "about"]}, {"sent": "what we need is a nice family picture now our new babys here .", "words": ["what", "need", "is", "we", "nice", "picture", "our", "here", "new", "a"]}, {"sent": "does she need anything else ?", "words": ["does", "need", "she"]}, {"sent": "you need another monkey ?", "words": ["another", "monkey", "need", "you"]}, {"sent": "do you need another one ?", "words": ["another", "do", "need", "you"]}, {"sent": "do they need a bowl ?", "words": ["bowl", "do", "need", "they", "a"]}, {"sent": "well we dont need it on now Thomas .", "words": ["need", "we", "dont", "it", "on"]}, {"sent": "you need help ?", "words": ["help", "need", "you"]}, {"sent": "um do I need some money ?", "words": ["money", "do", "need", "some"]}, {"sent": "you dont need a .", "words": ["dont", "need", "a", "you"]}, {"sent": "you need cinnamon on it too ?", "words": ["need", "you", "it", "too", "on"]}, {"sent": "you need a seat .", "words": ["need", "a", "you"]}, {"sent": "do you need your nose blowing ?", "words": ["do", "your", "need", "you", "nose"]}, {"sent": "I dont know that you actually need my help .", "words": ["my", "help", "need", "you", "dont", "that"]}, {"sent": "no you need another one a bigger one .", "words": ["another", "need", "a", "you"]}, {"sent": "and the other hole you need both tires .", "words": ["the", "need", "other", "you", "and"]}, {"sent": "they need gritters in Russia and snow snow removers .", "words": ["need", "snow", "and", "they", "in"]}, {"sent": "you need to .", "words": ["need to", "to", "need", "you"]}, {"sent": "you dont need these suitcases because you already have some in the back remember ?", "words": ["the", "these", "need", "back", "you", "have", "dont", "in", "some", "because"]}, {"sent": "you need your nose wiping again .", "words": ["your", "nose", "need", "you"]}, {"sent": "I do need a nose .", "words": ["nose", "do", "need", "a"]}, {"sent": "okay you need to go to the bathroom ?", "words": ["the", "need", "go", "need to", "to", "you", "bathroom"]}, {"sent": "you need a cup ?", "words": ["cup", "need", "a", "you"]}, {"sent": "then youll need the screwdriver .", "words": ["the", "then", "need"]}, {"sent": "okay you need some straws .", "words": ["some", "need", "you"]}, {"sent": "dalmatian puppies need your help .", "words": ["your", "help", "need"]}, {"sent": "oh my you need a tissue mister .", "words": ["my", "need", "tissue", "you", "a"]}, {"sent": "oh brilliant great now we need a piece for that .", "words": ["for", "need", "we", "that", "a"]}, {"sent": "well then you dont need diapers if youre Naomi .", "words": ["need", "if", "you", "dont", "then"]}, {"sent": "need help ?", "words": ["help", "need"]}, {"sent": "why do we need two fish ?", "words": ["do", "need", "we", "fish", "why"]}, {"sent": "I need I need something .", "words": ["need"]}, {"sent": "what do we need that goes with peanutbutter ?", "words": ["what", "do", "need", "we", "with", "that"]}, {"sent": "oh it broke you need a new one ?", "words": ["need", "you", "it", "new", "a"]}, {"sent": "you need the firemen uhn dont you ?", "words": ["the", "dont", "need", "you"]}, {"sent": "uh tell me what you need .", "words": ["me", "what", "need", "you"]}, {"sent": "we need some big bits uhn dont we ?", "words": ["need", "we", "dont", "some", "big"]}, {"sent": "so what does he need ?", "words": ["what", "does", "need", "so", "he"]}, {"sent": "what do you need ?", "words": ["what", "do", "need", "you"]}, {"sent": "and Im going to need some more flour in a few minutes from the flour man .", "words": ["the", "need", "to", "and", "more", "in", "some", "a"]}, {"sent": "oh we need an e too thats right .", "words": ["too", "an", "need", "we"]}, {"sent": "dont need another Daddy .", "words": ["another", "dont", "need"]}, {"sent": "no you dont need you just need to pick this you this what you do .", "words": ["what", "do", "need", "pick", "to", "need to", "you", "dont", "this"]}, {"sent": "and well dont need the scissors anymore .", "words": ["the", "need", "scissors", "and", "dont"]}, {"sent": "we need you .", "words": ["need", "you", "we"]}, {"sent": "need some more jam .", "words": ["some", "more", "need"]}, {"sent": "the fireman doesnt need the telephone today ?", "words": ["the", "need", "telephone"]}, {"sent": "it doesnt need an icecube .", "words": ["an", "it", "need"]}, {"sent": "we need a tablespoon of joy .", "words": ["of", "need", "a", "we"]}, {"sent": "I need one and you need one .", "words": ["need", "you", "and"]}, {"sent": "need somewhere to buy the drinks dont we ?", "words": ["the", "need", "we", "to", "dont", "buy"]}, {"sent": "you need a baby .", "words": ["need", "a", "you"]}, {"sent": "what do you need then ?", "words": ["what", "do", "need", "you", "then"]}, {"sent": "well if the trains going near a post office I need a stamp .", "words": ["the", "need", "if", "a"]}, {"sent": "mommy xxx said dont need my jacket .", "words": ["dont", "my", "need", "jacket"]}, {"sent": "what does she need to feel better ?", "words": ["what", "does", "better", "need", "need to", "to", "she"]}, {"sent": "ah you want your egg back .", "words": ["your", "egg", "back", "you"]}, {"sent": "you rotten egg .", "words": ["egg", "you"]}, {"sent": "the midwife had now got out her spoon and was taping the egg very gently .", "words": ["the", "egg", "was", "and", "spoon", "out", "her"]}, {"sent": "this was an egg once .", "words": ["was", "this", "an", "egg"]}, {"sent": "Ill play with the egg box then .", "words": ["the", "box", "egg", "play", "with", "then"]}, {"sent": "you want me to make you an egg ?", "words": ["egg", "to", "you", "an", "me", "make"]}, {"sent": "if Id thought I couldve saved that egg box from yesterday when we had those eggs .", "words": ["box", "egg", "if", "when", "we", "those", "that"]}, {"sent": "can you see the baby egg ?", "words": ["can", "the", "egg", "you", "see"]}, {"sent": "do you want a little egg ?", "words": ["do", "egg", "little", "you", "a"]}, {"sent": "you did it you got your egg in there .", "words": ["your", "egg", "you", "there", "in", "it", "did"]}, {"sent": "you want some egg and toast ?", "words": ["egg", "toast", "and", "you", "some"]}, {"sent": "egg .", "words": ["egg"]}, {"sent": "what kind of egg does that ?", "words": ["what", "does", "egg", "of", "that"]}, {"sent": "and it actually annoys my husband too hes like a quiet egg .", "words": ["my", "egg", "like", "and", "it", "quiet", "too", "a"]}, {"sent": "Mummys egg .", "words": ["egg"]}, {"sent": "I want you to eat some more of this egg .", "words": ["egg", "of", "to", "you", "eat", "this", "more", "some"]}, {"sent": "egg .", "words": ["egg"]}, {"sent": "but when they stop for their cup of tea or coffee they might enjoy an egg mightnt they ?", "words": ["cup", "for", "their", "egg", "of", "stop", "they", "an", "coffee", "when", "but"]}, {"sent": "thats new egg box .", "words": ["new", "box", "egg"]}, {"sent": "please can I have just one more egg please ?", "words": ["can", "have", "more", "egg"]}, {"sent": "thats like an ostrich egg .", "words": ["egg", "an", "like"]}, {"sent": "should we put the egg right here .", "words": ["the", "put", "egg", "we", "here"]}, {"sent": "shall we put that in the egg box ?", "words": ["the", "put", "box", "egg", "we", "in", "that"]}, {"sent": "and he sat all that day and he kept the egg warm and he sat all that night through a terrible storm .", "words": ["the", "all", "egg", "and", "he", "that", "a"]}, {"sent": "I was throwing it like a ball but its an egg .", "words": ["egg", "was", "like", "ball", "an", "it", "but", "a"]}, {"sent": "want some egg ?", "words": ["some", "egg"]}, {"sent": "can I eat this egg for my tea ?", "words": ["can", "my", "for", "egg", "eat", "this"]}, {"sent": "why dont you give the snake an egg ?", "words": ["the", "egg", "you", "dont", "an", "give", "why"]}, {"sent": "Nomi do you want egg ?", "words": ["do", "egg", "you"]}, {"sent": "a rooster with an egg ?", "words": ["egg", "with", "rooster", "an", "a"]}, {"sent": "then the egg will keep warm so theyll hatch .", "words": ["the", "egg", "so", "will", "then"]}, {"sent": "have we got all the orange ones in the egg ?", "words": ["the", "all", "egg", "we", "have", "in", "orange"]}, {"sent": "and when weve finished putting this into the cases were going to put the mini Easter egg .", "words": ["the", "put", "into", "egg", "to", "and", "this", "when", "were"]}, {"sent": "you got an egg ?", "words": ["an", "egg", "you"]}, {"sent": "its an egg .", "words": ["an", "egg"]}, {"sent": "did she say thats egg ?", "words": ["egg", "say", "did", "she"]}, {"sent": "and heres an egg .", "words": ["an", "egg", "and"]}, {"sent": "what egg ?", "words": ["what", "egg"]}, {"sent": "Edw is there a chicken in that egg ?", "words": ["egg", "is", "there", "in", "chicken", "that", "a"]}, {"sent": "back in the egg so your allowed to hatch .", "words": ["the", "your", "egg", "back", "so", "to", "in"]}, {"sent": "its not an egg .", "words": ["egg", "an", "not"]}, {"sent": "Horton hatches the egg .", "words": ["the", "egg"]}, {"sent": "Mummys gonna have an egg for lunch .", "words": ["have", "an", "for", "egg"]}, {"sent": "two chicks and an egg .", "words": ["an", "egg", "and"]}, {"sent": "where did you put the egg ?", "words": ["where", "the", "put", "egg", "you", "did"]}, {"sent": "open the egg .", "words": ["open", "the", "egg"]}, {"sent": "whats the egg got ?", "words": ["the", "egg"]}, {"sent": "yeah I know you can say egg .", "words": ["can", "say", "egg", "you"]}, {"sent": "make a good xxx egg .", "words": ["make", "egg", "good", "a"]}, {"sent": "okay heres the other part of your egg .", "words": ["the", "your", "egg", "of", "other"]}, {"sent": "because you eat the egg and inside is a yellow egg and you open it and theres a toy inside .", "words": ["the", "egg", "is", "open", "inside", "you", "eat", "and", "toy", "it", "yellow", "because", "a"]}, {"sent": "oh the egg is coming out .", "words": ["the", "egg", "out", "is"]}, {"sent": "thats an egg .", "words": ["an", "egg"]}, {"sent": "looks kind of like a little egg doesnt it ?", "words": ["egg", "of", "like", "little", "it", "a"]}, {"sent": "well make an egg like that .", "words": ["egg", "like", "an", "make", "that"]}, {"sent": "theres your egg .", "words": ["your", "egg"]}, {"sent": "put the eggs in the egg box .", "words": ["the", "put", "box", "egg", "in"]}, {"sent": "did we not say it looked like an ostrich egg ?", "words": ["egg", "we", "say", "like", "an", "it", "did", "not"]}, {"sent": "oh look egg Eleanor .", "words": ["egg", "look"]}, {"sent": "you dont know what an egg cup is do you ?", "words": ["cup", "what", "do", "egg", "is", "you", "dont", "an"]}, {"sent": "egg .", "words": ["egg"]}, {"sent": "we could eat your Easter egg for lunch if you want xxx .", "words": ["for", "your", "egg", "if", "we", "you", "eat", "could"]}, {"sent": "robins egg blue its the same color as a robins egg you know like a bird a robin its the same color as the egg and this ones called blue green with glitzy gold glitter glitzy gold glitter whoa .", "words": ["the", "egg", "bird", "with", "like", "blue", "you", "and", "this", "same", "green", "a"]}, {"sent": "you want your egg ?", "words": ["your", "egg", "you"]}, {"sent": "you had an egg yesterday didnt you ?", "words": ["an", "egg", "you"]}, {"sent": "and weve got I got a MiniatureHero egg and Ive ate the MiniatureHero and Ive got a Caramel egg and Ive ate the Caramel and got Caramel chocolates .", "words": ["the", "egg", "a", "and"]}, {"sent": "Im not coming in to that Nancy who cant eat her egg up .", "words": ["up", "egg", "to", "her", "eat", "in", "who", "that", "not"]}, {"sent": "theres the egg .", "words": ["the", "egg"]}, {"sent": "did you have egg today ?", "words": ["have", "did", "egg", "you"]}, {"sent": "okay I need the egg I need the egg .", "words": ["the", "need", "egg"]}, {"sent": "she does doesnt she the egg ?", "words": ["the", "does", "she", "egg"]}, {"sent": "hello little egg .", "words": ["egg", "little"]}, {"sent": "egg .", "words": ["egg"]}, {"sent": "in the egg box .", "words": ["the", "in", "box", "egg"]}, {"sent": "thats where the egg is getting broken .", "words": ["where", "the", "egg", "is", "broken"]}, {"sent": "but Max stuffed the egg into his pocket .", "words": ["the", "into", "egg", "his", "but"]}, {"sent": "I saw egg on your pants I did .", "words": ["your", "egg", "pants", "did", "on"]}, {"sent": "is it going in the egg box ?", "words": ["the", "box", "egg", "is", "in", "it"]}, {"sent": "go out and get a egg roll or something .", "words": ["egg", "go", "and", "get", "out", "a"]}, {"sent": "I can put the egg in that one .", "words": ["can", "the", "put", "egg", "in", "that"]}, {"sent": "theres an egg here .", "words": ["here", "an", "egg"]}, {"sent": "can I have an easter egg ?", "words": ["can", "have", "egg", "an"]}, {"sent": "eat your egg .", "words": ["eat", "your", "egg"]}, {"sent": "theres a baby egg there look .", "words": ["there", "look", "egg", "a"]}, {"sent": "did does Mummy have an egg cup ?", "words": ["cup", "does", "egg", "have", "an", "did"]}, {"sent": "mum has laid an egg .", "words": ["an", "egg"]}, {"sent": "I wouldnt mind a sausage with my egg .", "words": ["my", "with", "egg", "a"]}, {"sent": "well shall we have a little bit of Easter egg ?", "words": ["egg", "we", "of", "little", "have", "a"]}, {"sent": "somebody made a egg .", "words": ["egg", "a"]}, {"sent": "sighed Maisy a lazy bird hatching an egg .", "words": ["an", "bird", "egg", "a"]}, {"sent": "they come outof this little egg thing .", "words": ["they", "egg", "this", "little"]}, {"sent": "do you think you can get him without the egg getting you ?", "words": ["can", "do", "think", "the", "egg", "you", "get", "him"]}, {"sent": "thats an egg box .", "words": ["an", "box", "egg"]}, {"sent": "you want egg and toast ?", "words": ["toast", "egg", "and", "you"]}, {"sent": "a easter egg thats right this easter egg is from last year Max said Ruby into the trash bucket it goes .", "words": ["the", "egg", "into", "is", "bucket", "last", "trash", "this", "it", "a"]}, {"sent": "while Doreen and I were making egg salad sandwiches for the egg head club .", "words": ["the", "for", "egg", "head", "and", "were"]}, {"sent": "beans and egg .", "words": ["beans", "egg", "and"]}, {"sent": "Im eating egg whites with sesame seeds .", "words": ["with", "egg"]}, {"sent": "I dont want you to do that are you gonna eat some egg white and egg yolk ?", "words": ["do", "egg", "to", "you", "eat", "dont", "and", "are", "some", "white", "that"]}, {"sent": "its an ostrich egg ?", "words": ["an", "egg"]}, {"sent": "catch it .", "words": ["it", "catch"]}, {"sent": "he wants to catch a fish .", "words": ["to", "catch", "he", "fish", "a"]}, {"sent": "catch .", "words": ["catch"]}, {"sent": "here catch .", "words": ["here", "catch"]}, {"sent": "hes saying oh catch that turkey .", "words": ["catch", "that", "turkey"]}, {"sent": "catch .", "words": ["catch"]}, {"sent": "yay good catch .", "words": ["catch", "good"]}, {"sent": "catch it catch it catch it .", "words": ["it", "catch"]}, {"sent": "catch .", "words": ["catch"]}, {"sent": "you needta catch the .", "words": ["the", "catch", "you"]}, {"sent": "oh catch .", "words": ["catch"]}, {"sent": "here catch .", "words": ["here", "catch"]}, {"sent": "and hes waiting to catch the ball hes got his mitt on .", "words": ["the", "to", "catch", "and", "ball", "on", "his"]}, {"sent": "catch it .", "words": ["it", "catch"]}, {"sent": "oh can you catch it ?", "words": ["can", "it", "catch", "you"]}, {"sent": "can you play catch with Mommy ?", "words": ["can", "play", "with", "catch", "you"]}, {"sent": "and then who did he catch after hed caught the teddybear ?", "words": ["the", "catch", "teddybear", "and", "he", "did", "who", "then"]}, {"sent": "catch it catch it catch it .", "words": ["it", "catch"]}, {"sent": "you catch it .", "words": ["it", "catch", "you"]}, {"sent": "where did he catch it ?", "words": ["where", "catch", "it", "he", "did"]}, {"sent": "I cant throw it really high because she wont be able to catch it .", "words": ["throw", "be", "to", "catch", "it", "high", "she", "because"]}, {"sent": "catch it catch it .", "words": ["it", "catch"]}, {"sent": "mommy catch it .", "words": ["it", "catch"]}, {"sent": "lets catch the hoops instead .", "words": ["the", "catch"]}, {"sent": "can you catch it ?", "words": ["can", "it", "catch", "you"]}, {"sent": "I didnt catch it either .", "words": ["it", "catch"]}, {"sent": "can you catch the clown ?", "words": ["can", "the", "catch", "you"]}, {"sent": "ready catch .", "words": ["catch"]}, {"sent": "catch the ball .", "words": ["the", "ball", "catch"]}, {"sent": "will we try and catch Purdie a fish ?", "words": ["we", "catch", "and", "fish", "will", "try", "a"]}, {"sent": "thats not playing catch by the way this is playing baseball .", "words": ["the", "is", "catch", "by", "this", "not"]}, {"sent": "catch .", "words": ["catch"]}, {"sent": "can you catch it in your hat ?", "words": ["can", "your", "catch", "you", "in", "it", "hat"]}, {"sent": "shall we play catch the frog ?", "words": ["the", "we", "play", "catch", "frog"]}, {"sent": "catch the ballball .", "words": ["the", "catch"]}, {"sent": "catch it .", "words": ["it", "catch"]}, {"sent": "let mommy catch it .", "words": ["it", "catch"]}, {"sent": "Ethan catch the ball ?", "words": ["the", "ball", "catch"]}, {"sent": "catch it catch it .", "words": ["it", "catch"]}, {"sent": "catch .", "words": ["catch"]}, {"sent": "uh oh better catch him .", "words": ["better", "him", "catch"]}, {"sent": "catch a big ball .", "words": ["ball", "catch", "big", "a"]}, {"sent": "did you catch Miriam ?", "words": ["catch", "did", "you"]}, {"sent": "the whales gonna catch the shark ?", "words": ["the", "catch"]}, {"sent": "which dog shall we catch ?", "words": ["catch", "which", "dog", "we"]}, {"sent": "but he didnt catch me .", "words": ["me", "catch", "he", "but"]}, {"sent": "are you ready to catch it ?", "words": ["to", "catch", "you", "are", "it"]}, {"sent": "mummie can catch you .", "words": ["can", "catch", "you"]}, {"sent": "theres nothing to catch .", "words": ["to", "catch"]}, {"sent": "oh lets catch this jello that went on the table .", "words": ["the", "table", "catch", "this", "on", "that"]}, {"sent": "oh thats long to catch a cow with ?", "words": ["to", "catch", "with", "long", "cow", "a"]}, {"sent": "catch ?", "words": ["catch"]}, {"sent": "catch .", "words": ["catch"]}, {"sent": "they had better find that leg bone soon there is just one minute left til noon with no more time to search and look they know they will not catch that crook they failed this case is much too hard .", "words": ["more", "leg", "hard", "not", "catch", "with", "there", "much", "that", "is", "to", "and", "find", "will", "too", "better", "they", "this", "look"]}, {"sent": "*SI1: catch the bubbles ?", "words": ["the", "catch", "bubbles"]}, {"sent": "now you catch .", "words": ["catch", "you"]}, {"sent": "how many we can catch ?", "words": ["can", "how", "catch", "we"]}, {"sent": "catch .", "words": ["catch"]}, {"sent": "Ive got to literally jump in the air to catch it ?", "words": ["the", "to", "catch", "jump", "in", "it"]}, {"sent": "when they were trying to catch the pussycat ?", "words": ["the", "to", "catch", "they", "when", "were"]}, {"sent": "here you catch it .", "words": ["here", "it", "catch", "you"]}, {"sent": "Im gonna catch you .", "words": ["catch", "you"]}, {"sent": "catch .", "words": ["catch"]}, {"sent": "she swallowed the cat to catch the bird she swallowed the bird to catch ?", "words": ["the", "to", "catch", "bird", "cat", "she"]}, {"sent": "youre not trying to catch a cold are you ?", "words": ["cold", "to", "catch", "you", "are", "not", "a"]}, {"sent": "catch ready ?", "words": ["catch"]}, {"sent": "would you catch a bird please for Thomass breakfast .", "words": ["for", "bird", "catch", "you", "would", "a"]}, {"sent": "Bert can throw Ernie can catch .", "words": ["can", "catch", "throw"]}, {"sent": "you mean catch ?", "words": ["catch", "you"]}, {"sent": "can you catch him ?", "words": ["can", "him", "catch", "you"]}, {"sent": "did you catch a bee .", "words": ["bee", "catch", "you", "did", "a"]}, {"sent": "better catch it .", "words": ["better", "catch", "it"]}, {"sent": "I didnt manage to catch it .", "words": ["to", "catch", "it"]}, {"sent": "they might catch uh uh um .", "words": ["they", "catch"]}, {"sent": "try and catch it .", "words": ["it", "catch", "try", "and"]}, {"sent": "catch it .", "words": ["it", "catch"]}, {"sent": "catch it Mama .", "words": ["it", "catch"]}, {"sent": "you cant catch me Im the gingerbread man .", "words": ["the", "me", "catch", "you"]}, {"sent": "okay catch .", "words": ["catch"]}, {"sent": "good catch good catch .", "words": ["catch", "good"]}, {"sent": "catch very good .", "words": ["catch", "good"]}, {"sent": "and then we can try and then we can play a game where I throw the Jelly baby and you try and catch it in here .", "words": ["can", "where", "game", "the", "throw", "we", "play", "catch", "and", "you", "here", "in", "it", "then", "try", "a"]}, {"sent": "catch him .", "words": ["him", "catch"]}, {"sent": "you gotta catch the train and all .", "words": ["the", "all", "train", "catch", "and", "you"]}, {"sent": "well have to catch some flies .", "words": ["have", "to", "catch", "some"]}, {"sent": "can you catch a ball ?", "words": ["can", "catch", "you", "ball", "a"]}, {"sent": "and Flounder raced to catch up .", "words": ["up", "to", "catch", "and"]}, {"sent": "you wanna catch it ?", "words": ["it", "catch", "wanna", "you"]}, {"sent": "you catch here .", "words": ["here", "catch", "you"]}, {"sent": "catch it one .", "words": ["it", "catch"]}, {"sent": "you were going to catch the little bug but it went away oh well .", "words": ["the", "bug", "to", "catch", "you", "little", "it", "away", "were", "but"]}, {"sent": "you wanna play catch ?", "words": ["play", "catch", "wanna", "you"]}, {"sent": "you catch the ball .", "words": ["the", "ball", "catch", "you"]}, {"sent": "good catch .", "words": ["catch", "good"]}, {"sent": "can you catch the ball for real ?", "words": ["can", "the", "for", "catch", "you", "ball"]}, {"sent": "ah a good catch .", "words": ["catch", "good", "a"]}, {"sent": "you dont want him to catch cold .", "words": ["to", "catch", "you", "dont", "him", "cold"]}, {"sent": "catch it catch it catch it .", "words": ["it", "catch"]}, {"sent": "put your hands out to catch .", "words": ["put", "your", "to", "catch", "out"]}, {"sent": "so I catch it in my hand and then ?", "words": ["my", "hand", "so", "catch", "and", "in", "it", "then"]}, {"sent": "Im very slow .", "words": ["slow"]}, {"sent": "the taxis hafta slow down inorderto see through the smoke .", "words": ["the", "see", "down", "slow"]}, {"sent": "y you eat slow ?", "words": ["eat", "slow", "you"]}, {"sent": "slow feet .", "words": ["slow"]}, {"sent": "sorry if Im too slow .", "words": ["if", "slow", "too"]}, {"sent": "come on slow coach .", "words": ["on", "slow"]}, {"sent": "very slow ?", "words": ["slow"]}, {"sent": "right slow down .", "words": ["down", "slow"]}, {"sent": "fast and slow .", "words": ["fast", "slow", "and"]}, {"sent": "slow .", "words": ["slow"]}, {"sent": "shes a slow poke .", "words": ["slow", "a"]}, {"sent": "me slow said the bear .", "words": ["the", "me", "slow", "bear"]}, {"sent": "toe and slow .", "words": ["toe", "slow", "and"]}, {"sent": "that one going slow ?", "words": ["that", "slow"]}, {"sent": "so slow down .", "words": ["so", "down", "slow"]}, {"sent": "Im doing it slow .", "words": ["it", "slow"]}, {"sent": "fast and slow .", "words": ["fast", "slow", "and"]}, {"sent": "slow feet quick feet .", "words": ["slow"]}, {"sent": "in the winter trees hardly grow at all but slow down and rest .", "words": ["the", "all", "slow", "and", "in", "at", "down", "but"]}, {"sent": "slow .", "words": ["slow"]}, {"sent": "theyre slow .", "words": ["slow"]}, {"sent": "slow feet quick feet .", "words": ["slow"]}, {"sent": "fast and slow .", "words": ["fast", "slow", "and"]}, {"sent": "and the bus is going very slow now isnt it ?", "words": ["the", "is", "slow", "and", "it", "bus"]}, {"sent": "can you do slow ?", "words": ["can", "do", "slow", "you"]}, {"sent": "how do we go from place to place sometimes fast and sometimes slow .", "words": ["do", "how", "fast", "go", "we", "to", "slow", "and"]}, {"sent": "slow feet .", "words": ["slow"]}, {"sent": "go slow .", "words": ["slow", "go"]}, {"sent": "go slow .", "words": ["slow", "go"]}, {"sent": "slow clown ?", "words": ["slow"]}, {"sent": "I may be slow .", "words": ["be", "slow"]}, {"sent": "would he drive fast or slow ?", "words": ["fast", "slow", "drive", "he", "would"]}, {"sent": "slow down .", "words": ["down", "slow"]}, {"sent": "slow down Jwww .", "words": ["down", "slow"]}, {"sent": "slow down slow down .", "words": ["down", "slow"]}, {"sent": "you gonna go slow ?", "words": ["go", "slow", "you"]}, {"sent": "well its a slow one .", "words": ["slow", "a"]}, {"sent": "you better slow down .", "words": ["better", "down", "slow", "you"]}, {"sent": "slow .", "words": ["slow"]}, {"sent": "slow down .", "words": ["down", "slow"]}, {"sent": "slow down a bit .", "words": ["down", "slow", "a"]}, {"sent": "go slow .", "words": ["slow", "go"]}, {"sent": "slow down .", "words": ["down", "slow"]}, {"sent": "slow down .", "words": ["down", "slow"]}, {"sent": "the sign tells him to go slow .", "words": ["the", "go", "to", "slow", "him"]}, {"sent": "your slow .", "words": ["your", "slow"]}, {"sent": "lack means xxx when you race downstairs do you go fast or slow ?", "words": ["do", "fast", "go", "slow", "you", "when"]}, {"sent": "can you just slow down theyre disappearing .", "words": ["can", "down", "slow", "you"]}, {"sent": "it was sure but it was slow .", "words": ["was", "slow", "it", "but"]}, {"sent": "slow down a little bit .", "words": ["down", "little", "slow", "a"]}, {"sent": "Daddy come home slow ?", "words": ["slow", "home"]}, {"sent": "um the reason why hes been slow on solids is because we think he has food allergies so .", "words": ["the", "think", "is", "we", "so", "because", "slow", "food", "he", "on", "why"]}, {"sent": "slow down .", "words": ["down", "slow"]}, {"sent": "a slow old turtle with a wrinkly neck .", "words": ["old", "with", "slow", "turtle", "a"]}, {"sent": "hes very slow .", "words": ["slow"]}, {"sent": "youd better slow down with it .", "words": ["better", "with", "slow", "it", "down"]}, {"sent": "slow and steady get me ready .", "words": ["get", "me", "slow", "and"]}, {"sent": "and its very slow too .", "words": ["too", "slow", "and"]}, {"sent": "slow feet quick feet .", "words": ["slow"]}, {"sent": "what do you do when youre driving down the street and somebody goes too slow .", "words": ["the", "what", "do", "slow", "you", "and", "street", "too", "down", "when"]}, {"sent": "slow down .", "words": ["down", "slow"]}, {"sent": "and theres slowly which is slow .", "words": ["slow", "which", "is", "and"]}, {"sent": "well you just needta be slow .", "words": ["be", "slow", "you"]}, {"sent": "how come you walk so slow ?", "words": ["how", "so", "slow", "you", "walk"]}, {"sent": "slow .", "words": ["slow"]}, {"sent": "and youre so slow .", "words": ["so", "slow", "and"]}, {"sent": "slow .", "words": ["slow"]}, {"sent": "do it slow ?", "words": ["do", "it", "slow"]}, {"sent": "the tortoise is going slow .", "words": ["the", "slow", "is"]}, {"sent": "I think you should slow down a little bit .", "words": ["think", "slow", "you", "little", "down", "a"]}, {"sent": "kinda nice and slow .", "words": ["nice", "slow", "and"]}, {"sent": "slow a snail is slow .", "words": ["is", "slow", "a"]}, {"sent": "slow down when youre talking .", "words": ["down", "when", "slow"]}, {"sent": "slow and steady .", "words": ["slow", "and"]}, {"sent": "he moves slow .", "words": ["he", "slow"]}, {"sent": "that isnt slow .", "words": ["that", "slow"]}, {"sent": "slow down .", "words": ["down", "slow"]}, {"sent": "okay Im just gonna roll this one nice and slow .", "words": ["this", "nice", "slow", "and"]}, {"sent": "youre starting to slow down .", "words": ["to", "down", "slow"]}, {"sent": "fast and slow .", "words": ["fast", "slow", "and"]}, {"sent": "that one go slow .", "words": ["that", "slow", "go"]}, {"sent": "I think youre gonna hafta slow down a bit .", "words": ["think", "down", "slow", "a"]}, {"sent": "slow down .", "words": ["down", "slow"]}, {"sent": "yeah watch out slow down .", "words": ["slow", "down", "watch", "out"]}, {"sent": "do I go nice and slow hm ?", "words": ["do", "go", "nice", "slow", "and"]}, {"sent": "eat slow Samantha .", "words": ["eat", "slow"]}, {"sent": "slow down .", "words": ["down", "slow"]}, {"sent": "slow feet quick feet .", "words": ["slow"]}, {"sent": "I always hafta wait for Kermit because hes slow you know .", "words": ["wait", "for", "slow", "you", "because"]}, {"sent": "this one go slow .", "words": ["this", "slow", "go"]}, {"sent": "a slow sliding snail .", "words": ["slow", "a"]}, {"sent": "whats better to you fast or slow ?", "words": ["better", "fast", "to", "slow", "you"]}, {"sent": "go slow .", "words": ["slow", "go"]}, {"sent": "where are you going you slow sliming .", "words": ["where", "are", "slow", "you"]}, {"sent": "oh yeah right the slow train .", "words": ["the", "slow", "train"]}, {"sent": "go slow .", "words": ["slow", "go"]}, {"sent": "slow .", "words": ["slow"]}, {"sent": "right a vase and a flower and an umbrella .", "words": ["an", "flower", "and", "a"]}, {"sent": "and now it says next dig the weeds out of the flower beds using the fork .", "words": ["the", "of", "flower", "and", "it", "out", "fork"]}, {"sent": "Im not letting you down to play in the flower bed silly girl .", "words": ["the", "to", "flower", "play", "bed", "you", "in", "down", "not"]}, {"sent": "I put some compost in the flower pot .", "words": ["the", "put", "flower", "in", "some"]}, {"sent": "big big flower there .", "words": ["there", "flower", "big"]}, {"sent": "flower .", "words": ["flower"]}, {"sent": "Heehaw has two flowers in his flower pot .", "words": ["his", "in", "flower"]}, {"sent": "plant flower seeds indoors early in spring and they will flower in the summer .", "words": ["the", "flower", "and", "they", "in", "will", "plant"]}, {"sent": "as a bee flies from flower to flower it also gathers pollen .", "words": ["to", "flower", "bee", "it", "a"]}, {"sent": "we can use the flower .", "words": ["can", "the", "flower", "we"]}, {"sent": "it came from the flower Helen gave you .", "words": ["the", "flower", "it", "you"]}, {"sent": "there is that flower .", "words": ["there", "flower", "that", "is"]}, {"sent": "Paul can smell the flower .", "words": ["can", "the", "flower"]}, {"sent": "yes thats the flower .", "words": ["the", "flower"]}, {"sent": "can you smell the flower ?", "words": ["can", "the", "flower", "you"]}, {"sent": "a flower .", "words": ["flower", "a"]}, {"sent": "brush the flower off the chair and sit down .", "words": ["the", "off", "brush", "flower", "chair", "and", "sit", "down"]}, {"sent": "are you talking to your flower sweetie ?", "words": ["your", "to", "flower", "you", "are"]}, {"sent": "well just make the flower first .", "words": ["the", "flower", "first", "make"]}, {"sent": "do you remember that yellow flower that you picked the other day ?", "words": ["the", "do", "flower", "you", "other", "yellow", "that"]}, {"sent": "picked a flower and put it on the snowman mustnt they ?", "words": ["the", "put", "flower", "and", "they", "it", "snowman", "on", "a"]}, {"sent": "like a flower .", "words": ["flower", "like", "a"]}, {"sent": "you could say flower .", "words": ["could", "say", "flower", "you"]}, {"sent": "theres a bed and a flower .", "words": ["flower", "bed", "and", "a"]}, {"sent": "put the flower in the vase ?", "words": ["the", "in", "put", "flower"]}, {"sent": "flower .", "words": ["flower"]}, {"sent": "do you see it has a flower ?", "words": ["do", "flower", "you", "it", "see", "a"]}, {"sent": "a flower ?", "words": ["flower", "a"]}, {"sent": "Im about to feed Purdie and theres a flower in the middle of her plate .", "words": ["the", "plate", "of", "to", "flower", "and", "in", "about", "feed", "her", "a"]}, {"sent": "what about all these ones around the flower ?", "words": ["the", "all", "what", "these", "flower", "about", "around"]}, {"sent": "this big flower up here no matter how quickly it goes you can always tell the color because its just one color .", "words": ["up", "can", "how", "the", "flower", "you", "here", "this", "it", "big", "because"]}, {"sent": "up theres a tiny bit of pink flower in that piece .", "words": ["up", "tiny", "of", "flower", "in", "that", "a"]}, {"sent": "flower darling .", "words": ["flower"]}, {"sent": "do you think hes picked that flower for his Mummy ?", "words": ["do", "think", "for", "flower", "you", "his", "that"]}, {"sent": "you got the flower you got the flower .", "words": ["the", "flower", "you"]}, {"sent": "its the yellow flower .", "words": ["the", "yellow", "flower"]}, {"sent": "a flower ?", "words": ["flower", "a"]}, {"sent": "a flower .", "words": ["flower", "a"]}, {"sent": "you think you should put that the the glass with the flower in on some on top of something ?", "words": ["the", "think", "put", "of", "flower", "with", "you", "in", "glass", "on", "some", "that"]}, {"sent": "oh a flower .", "words": ["flower", "a"]}, {"sent": "is that a flower youre doing ?", "words": ["flower", "that", "a", "is"]}, {"sent": "its like a flower .", "words": ["flower", "like", "a"]}, {"sent": "I dont know why but the black birds holding a flower .", "words": ["the", "flower", "black", "dont", "why", "but", "a"]}, {"sent": "thats a lovely flower .", "words": ["flower", "a"]}, {"sent": "a flower .", "words": ["flower", "a"]}, {"sent": "shall we put that flower outside ?", "words": ["put", "we", "outside", "flower", "that"]}, {"sent": "and f is for flower .", "words": ["flower", "for", "is", "and"]}, {"sent": "and their flower the Cherokee rose .", "words": ["the", "flower", "their", "and"]}, {"sent": "is there something hiding under the flower ?", "words": ["the", "is", "flower", "there", "under"]}, {"sent": "blue flower .", "words": ["flower", "blue"]}, {"sent": "go on then you do the castle and Ill do this flower .", "words": ["the", "do", "go", "flower", "you", "and", "this", "on", "then"]}, {"sent": "flower .", "words": ["flower"]}, {"sent": "and shes got a hat with a flower in it .", "words": ["flower", "with", "and", "in", "it", "hat", "a"]}, {"sent": "a pink flower and it has a yellow centre .", "words": ["flower", "and", "it", "yellow", "a"]}, {"sent": "little flower .", "words": ["flower", "little"]}, {"sent": "flower .", "words": ["flower"]}, {"sent": "the flower ?", "words": ["the", "flower"]}, {"sent": "the flower ?", "words": ["the", "flower"]}, {"sent": "how about if we make a flower ?", "words": ["how", "if", "we", "flower", "about", "make", "a"]}, {"sent": "or stim like a flower .", "words": ["flower", "like", "a"]}, {"sent": "flower .", "words": ["flower"]}, {"sent": "flower .", "words": ["flower"]}, {"sent": "yyy you can give CaptainFeatherSword a nice big flower pot .", "words": ["can", "flower", "nice", "you", "give", "big", "a"]}, {"sent": "the little girl over here bended over to pick up a flower and the cat jumped out of the .", "words": ["the", "up", "pick", "to", "flower", "of", "little", "here", "and", "cat", "out", "over", "a"]}, {"sent": "did you have a flower ?", "words": ["flower", "you", "have", "did", "a"]}, {"sent": "the flower ?", "words": ["the", "flower"]}, {"sent": "I thought you were a flower .", "words": ["flower", "were", "a", "you"]}, {"sent": "ah thats a purple thats a purple flower is it ?", "words": ["flower", "it", "is", "a"]}, {"sent": "because its only a its a hula hoop and the little bits of blue flower .", "words": ["the", "of", "flower", "little", "and", "blue", "because", "a"]}, {"sent": "a flower right ?", "words": ["flower", "a"]}, {"sent": "smell the flower .", "words": ["the", "flower"]}, {"sent": "wow thats a beautiful flower .", "words": ["flower", "a"]}, {"sent": "lets see or we could make a a triangle flower .", "words": ["we", "flower", "see", "could", "make", "a"]}, {"sent": "its like a little flower now the way its bent .", "words": ["the", "flower", "like", "little", "a"]}, {"sent": "and I noticed when I came home from Tesco yesterday with Jo some of them are in flower arent they ?", "words": ["of", "flower", "with", "them", "and", "they", "in", "are", "some", "when", "home"]}, {"sent": "is that the flower ?", "words": ["the", "flower", "that", "is"]}, {"sent": "a flower .", "words": ["flower", "a"]}, {"sent": "the flower .", "words": ["the", "flower"]}, {"sent": "mommy is gonna draw a flower .", "words": ["flower", "draw", "a", "is"]}, {"sent": "she broke the flower .", "words": ["the", "flower", "she"]}, {"sent": "the snowy crane sleeps standing on its long leg like a flower on its stem .", "words": ["the", "flower", "like", "long", "leg", "on", "a"]}, {"sent": "and all the flowers in the flower bed are growing beautifully .", "words": ["the", "all", "flower", "bed", "and", "in", "are"]}, {"sent": "you color me flower ?", "words": ["me", "flower", "you"]}, {"sent": "what did you do with the Teletubby flower pots ?", "words": ["the", "what", "do", "flower", "with", "you", "did"]}, {"sent": "flower .", "words": ["flower"]}, {"sent": "but the flower stalks can actually go into the compost bin outside cant they ?", "words": ["the", "can", "into", "go", "outside", "flower", "they", "but"]}, {"sent": "f for flower ?", "words": ["flower", "for"]}, {"sent": "would you like a flower sandwich ?", "words": ["flower", "like", "you", "would", "sandwich", "a"]}, {"sent": "hes saying to put the flower back .", "words": ["the", "put", "back", "to", "flower"]}, {"sent": "youre not allowed to pick flower heads either are you ?", "words": ["pick", "to", "flower", "you", "are", "not"]}, {"sent": "and a red red flower yellow flower palm tree .", "words": ["red", "tree", "flower", "and", "yellow", "a"]}, {"sent": "a little flower finger puppet yeah .", "words": ["finger", "flower", "little", "a"]}, {"sent": "flower buddy .", "words": ["flower"]}, {"sent": "now the flower is smiling can you put it in right place are you gonna color it now ?", "words": ["the", "can", "put", "is", "flower", "you", "in", "it", "are"]}, {"sent": "stop bending my flower .", "words": ["my", "stop", "flower"]}, {"sent": "and what color flower has Blue drawn ?", "words": ["what", "flower", "and"]}, {"sent": "sothat flower is .", "words": ["flower", "is"]}, {"sent": "little flower bed .", "words": ["flower", "bed", "little"]}, {"sent": "how many flower ?", "words": ["how", "flower"]}, {"sent": "shower yeah .", "words": ["shower"]}, {"sent": "unless its a paint that you can just wash off in the shower .", "words": ["can", "off", "the", "paint", "shower", "wash", "you", "in", "that", "a"]}, {"sent": "Daddy put you in the shower .", "words": ["the", "put", "shower", "you", "in"]}, {"sent": "or six thirty to seven thirty in the morning when Im taking a shower and stuff .", "words": ["the", "shower", "to", "and", "in", "when", "a"]}, {"sent": "didjou take a shower with Daddy last night ?", "words": ["shower", "last", "with", "take", "a"]}, {"sent": "Daddys going in the shower .", "words": ["the", "in", "shower"]}, {"sent": "you can nearly reach the shower cant you in the club .", "words": ["can", "the", "shower", "you", "in"]}, {"sent": "now Im in the shower .", "words": ["the", "in", "shower"]}, {"sent": "who is in the shower ?", "words": ["the", "shower", "is", "in", "who"]}, {"sent": "no I didnt take a shower .", "words": ["take", "shower", "a"]}, {"sent": "do you make a mess when you take a shower ?", "words": ["do", "when", "shower", "you", "take", "make", "a"]}, {"sent": "is the shower in the garden ?", "words": ["the", "shower", "is", "garden", "in"]}, {"sent": "it was too late for him to have a bath so he hadta have a shower .", "words": ["for", "shower", "was", "to", "so", "have", "him", "it", "he", "too", "a"]}, {"sent": "do you wanna have a shower ?", "words": ["do", "shower", "you", "have", "wanna", "a"]}, {"sent": "fascinated with the shower door this morning .", "words": ["the", "shower", "with", "this", "door"]}, {"sent": "after Mommy takes a shower .", "words": ["shower", "a"]}, {"sent": "has daddy had a shower first ?", "words": ["first", "shower", "a"]}, {"sent": "did you like that shower ?", "words": ["shower", "like", "you", "did", "that"]}, {"sent": "take a shower .", "words": ["take", "shower", "a"]}, {"sent": "oh take a shower .", "words": ["take", "shower", "a"]}, {"sent": "and then one of the times there was somebody in the shower .", "words": ["the", "shower", "of", "was", "and", "there", "in", "then"]}, {"sent": "no were just going to take a shower actually Lisa .", "words": ["shower", "to", "take", "were", "a"]}, {"sent": "where she has a shower .", "words": ["where", "she", "shower", "a"]}, {"sent": "and so Renee wanted to take a shower and get clean .", "words": ["clean", "shower", "to", "so", "and", "get", "take", "a"]}, {"sent": "the watering can shower ?", "words": ["the", "can", "shower"]}, {"sent": "Smiley is taking a cold shower .", "words": ["shower", "cold", "a", "is"]}, {"sent": "Daddys gonna take a shower .", "words": ["take", "shower", "a"]}, {"sent": "you dont hafta take a shower .", "words": ["shower", "you", "dont", "take", "a"]}, {"sent": "in shower ?", "words": ["in", "shower"]}, {"sent": "a shower .", "words": ["shower", "a"]}, {"sent": "he needs a shower .", "words": ["he", "shower", "a"]}, {"sent": "and then take a shower .", "words": ["shower", "and", "then", "take", "a"]}, {"sent": "its only a mile and a half then you could shower and .", "words": ["shower", "then", "you", "and", "could", "a"]}, {"sent": "you know Gigi is in the shower .", "words": ["the", "shower", "is", "you", "in"]}, {"sent": "like a shower ?", "words": ["like", "shower", "a"]}, {"sent": "a hot shower huh ?", "words": ["hot", "shower", "a"]}, {"sent": "do you wanna go in the shower with Daddy ?", "words": ["the", "do", "go", "shower", "with", "you", "in", "wanna"]}, {"sent": "were you taking a shower ?", "words": ["shower", "were", "a", "you"]}, {"sent": "do you wanna go and have a shower ?", "words": ["do", "go", "shower", "and", "you", "have", "wanna", "a"]}, {"sent": "xxx gone in the shower ?", "words": ["the", "in", "shower"]}, {"sent": "Im in the shower .", "words": ["the", "in", "shower"]}, {"sent": "do you wanna go take a shower with me ?", "words": ["do", "go", "shower", "with", "you", "wanna", "me", "take", "a"]}, {"sent": "what are we talking about the shower for ?", "words": ["the", "what", "for", "we", "shower", "are", "about"]}, {"sent": "Daniels hiding in the shower ?", "words": ["the", "in", "shower"]}, {"sent": "do you wanna take a bath after I take a shower .", "words": ["do", "shower", "you", "wanna", "take", "a"]}, {"sent": "DoctorFoster went to Gloucester in a shower of rain .", "words": ["rain", "shower", "of", "to", "in", "a"]}, {"sent": "and whats behind the shower curtain ?", "words": ["the", "behind", "shower", "and"]}, {"sent": "they know Im down here washing so cant nobody get in the shower .", "words": ["the", "shower", "so", "here", "they", "get", "in", "down"]}, {"sent": "yeah did Pee Wee turn on the shower ?", "words": ["on", "the", "did", "shower"]}, {"sent": "this is just some diapers that are on a rinse cycle I think when theyre done when they start spinning Im gonna jump in the shower .", "words": ["the", "think", "when", "is", "shower", "this", "they", "are", "jump", "in", "on", "some", "that", "a"]}, {"sent": "be shower .", "words": ["be", "shower"]}, {"sent": "is she behind the shower curtain ?", "words": ["the", "shower", "is", "behind", "she"]}, {"sent": "because I had a shower this morning .", "words": ["shower", "this", "because", "a"]}, {"sent": "DoctorFoster went to Gloucester in a shower of rain .", "words": ["rain", "shower", "of", "to", "in", "a"]}, {"sent": "can you say shower ?", "words": ["can", "say", "shower", "you"]}, {"sent": "and you went to Aunt Berthas house for Aunt Janies shower .", "words": ["for", "shower", "to", "you", "and", "house"]}, {"sent": "the other night all over the shower curtain all over the floor .", "words": ["the", "all", "shower", "other", "over"]}, {"sent": "well it would well actually this is a special radio thats sposta be used near a shower .", "words": ["is", "radio", "be", "shower", "this", "it", "would", "a"]}, {"sent": "you cant take a shower with your hat on ?", "words": ["your", "shower", "with", "you", "on", "hat", "take", "a"]}, {"sent": "washing my hair in the shower .", "words": ["the", "my", "shower", "in", "hair"]}, {"sent": "uh you could you could come and take a shower in our barrels .", "words": ["shower", "and", "you", "our", "in", "could", "take", "a"]}, {"sent": "whos in the shower ?", "words": ["the", "in", "shower"]}, {"sent": "what are you going to do while I take a shower ?", "words": ["what", "do", "shower", "to", "you", "are", "take", "a"]}, {"sent": "it must just have been a very very quick shower because theres no rain at all .", "words": ["all", "rain", "shower", "have", "it", "at", "because", "a"]}, {"sent": "washing himself under the shower .", "words": ["the", "under", "shower"]}, {"sent": "Parrot Darrot what color is the shower curtain ?", "words": ["the", "what", "shower", "is"]}, {"sent": "but Thomas I wanna know whats behind the shower .", "words": ["the", "shower", "behind", "wanna", "but"]}, {"sent": "no shes taking a shower .", "words": ["shower", "a"]}, {"sent": "do you ever have a shower ?", "words": ["do", "shower", "you", "have", "a"]}, {"sent": "Gloucester in a shower of .", "words": ["of", "in", "shower", "a"]}, {"sent": "had a quick shower .", "words": ["shower", "a"]}, {"sent": "because our shower at home doesnt work .", "words": ["work", "shower", "our", "at", "because", "home"]}, {"sent": "having a shower isnt he ?", "words": ["he", "shower", "a"]}, {"sent": "and he has put his umbrella up when he goes in the shower so he doesnt get wet .", "words": ["up", "the", "put", "wet", "shower", "so", "and", "get", "in", "he", "his", "when"]}, {"sent": "so Mummyll go upstairs and have a shower and put some clothes on and put some make up on .", "words": ["up", "put", "go", "shower", "so", "and", "have", "on", "some", "make", "a"]}, {"sent": "didjou take a shower with Daddy at the gym ?", "words": ["the", "shower", "with", "at", "take", "a"]}, {"sent": "does he not like having a cold shower ?", "words": ["does", "cold", "shower", "like", "he", "not", "a"]}, {"sent": "is Danielle in the shower ?", "words": ["the", "in", "shower", "is"]}, {"sent": "I was up from the scrake of dawn standing in the shower .", "words": ["up", "the", "shower", "of", "was", "in"]}, {"sent": "a woman in the shower .", "words": ["the", "in", "shower", "a"]}, {"sent": "come on Raynas going to have a shower too .", "words": ["shower", "to", "have", "too", "on", "a"]}, {"sent": "what is annoying Elizabeth elephant under that nice cool shower .", "words": ["what", "is", "shower", "nice", "elephant", "under", "that"]}, {"sent": "because Mommyll need the shower next .", "words": ["the", "need", "because", "shower"]}, {"sent": "Im gonna take a shower .", "words": ["take", "shower", "a"]}, {"sent": "did you see Daddy coming out of the shower ?", "words": ["the", "shower", "of", "you", "did", "out", "see"]}, {"sent": "I think Sammy might wanna have a shower .", "words": ["think", "shower", "have", "wanna", "a"]}, {"sent": "who is in the shower ?", "words": ["the", "shower", "is", "in", "who"]}, {"sent": "you can watch a video when Mummys having a shower later .", "words": ["can", "watch", "shower", "you", "when", "a"]}, {"sent": "did you go and have a shower ?", "words": ["go", "shower", "and", "have", "you", "did", "a"]}, {"sent": "do you wanna come take a shower ?", "words": ["do", "shower", "you", "wanna", "take", "a"]}, {"sent": "shower curtain .", "words": ["shower"]}, {"sent": "I hafta take a shower .", "words": ["take", "shower", "a"]}, {"sent": "I had a shower before going to work .", "words": ["to", "work", "shower", "a"]}, {"sent": "do you have a shower or bath ?", "words": ["do", "shower", "you", "have", "a"]}, {"sent": "oh taking a shower ?", "words": ["shower", "a"]}, {"sent": "did you take a shower ?", "words": ["shower", "you", "did", "take", "a"]}, {"sent": "you were wet through when you were in the shower .", "words": ["the", "wet", "when", "shower", "you", "in", "were"]}, {"sent": "weve got a shower at home but we never really use it do we ?", "words": ["do", "a", "shower", "we", "it", "at", "but", "home"]}, {"sent": "the shower is going .", "words": ["the", "is", "shower"]}, {"sent": "we hafta get a spoon and stir it all up .", "words": ["up", "all", "we", "and", "get", "it", "spoon", "a"]}, {"sent": "let me use this spoon .", "words": ["me", "spoon", "this"]}, {"sent": "and spoon for Charlie .", "words": ["for", "spoon", "and"]}, {"sent": "lets put the spoon away now .", "words": ["the", "put", "away", "spoon"]}, {"sent": "heres your spoon .", "words": ["your", "spoon"]}, {"sent": "and she picked up her spoon and she took a taste and she said .", "words": ["up", "taste", "and", "spoon", "her", "she", "a"]}, {"sent": "oh its a spoon .", "words": ["spoon", "a"]}, {"sent": "a spoon .", "words": ["spoon", "a"]}, {"sent": "these are trees over here and thats the door to the kitchen and theres Ruby inside holding a spoon .", "words": ["the", "these", "to", "inside", "and", "here", "are", "spoon", "kitchen", "over", "a", "door"]}, {"sent": "spoon close .", "words": ["close", "spoon"]}, {"sent": "heres your spoon .", "words": ["your", "spoon"]}, {"sent": "on the spoon ?", "words": ["on", "the", "spoon"]}, {"sent": "can you get some on your spoon sweetheart ?", "words": ["can", "your", "you", "get", "spoon", "on", "some"]}, {"sent": "wheres the spoon ?", "words": ["the", "spoon"]}, {"sent": "heres the spoon .", "words": ["the", "spoon"]}, {"sent": "oh picking up the spoon ?", "words": ["up", "the", "spoon"]}, {"sent": "oh you got the spoon yeah you did .", "words": ["the", "did", "spoon", "you"]}, {"sent": "bend a spoon .", "words": ["spoon", "a"]}, {"sent": "spoon .", "words": ["spoon"]}, {"sent": "were nearly at the stage where you can lick the spoon .", "words": ["the", "where", "can", "you", "spoon", "at", "lick", "were"]}, {"sent": "a spoon and a bowl .", "words": ["bowl", "spoon", "and", "a"]}, {"sent": "another spoon ?", "words": ["another", "spoon"]}, {"sent": "is that a spoon are you gonna eat lunch ?", "words": ["is", "you", "eat", "are", "spoon", "that", "a"]}, {"sent": "spoon .", "words": ["spoon"]}, {"sent": "wheres your spoon ?", "words": ["your", "spoon"]}, {"sent": "no spoon ?", "words": ["spoon"]}, {"sent": "did you find the spoon ?", "words": ["the", "you", "spoon", "did", "find"]}, {"sent": "theres a spoon .", "words": ["spoon", "a"]}, {"sent": "oh look a purple spoon .", "words": ["spoon", "a", "look"]}, {"sent": "oh yes theres some pasta on the back of the spoon .", "words": ["the", "back", "of", "spoon", "on", "some"]}, {"sent": "youre posting your spoon underneath .", "words": ["your", "spoon"]}, {"sent": "your spoon .", "words": ["your", "spoon"]}, {"sent": "use use your ehrm spoon to eat it .", "words": ["your", "to", "eat", "it", "spoon"]}, {"sent": "his cooking spoon .", "words": ["his", "spoon"]}, {"sent": "Dadajis spoon ?", "words": ["spoon"]}, {"sent": "oh theres that spoon again .", "words": ["that", "spoon"]}, {"sent": "you have the spoon .", "words": ["have", "the", "spoon", "you"]}, {"sent": "and your spoon .", "words": ["your", "spoon", "and"]}, {"sent": "would you like this big spoon ?", "words": ["like", "you", "this", "spoon", "would", "big"]}, {"sent": "yeah spoon .", "words": ["spoon"]}, {"sent": "you dipped it you dipped the spoon .", "words": ["the", "it", "spoon", "you"]}, {"sent": "can you use the spoon to eat your icecream ?", "words": ["can", "the", "your", "to", "you", "eat", "spoon"]}, {"sent": "you and that spoon huh ?", "words": ["that", "spoon", "and", "you"]}, {"sent": "yeah theres not anything in the spoon though is there ?", "words": ["the", "is", "there", "in", "spoon", "not"]}, {"sent": "cant get anything on that spoon huh ?", "words": ["get", "on", "that", "spoon"]}, {"sent": "whats he done to the spoon and the knife and the fork ?", "words": ["the", "to", "knife", "and", "he", "spoon", "fork"]}, {"sent": "I cant find a little spoon .", "words": ["little", "spoon", "a", "find"]}, {"sent": "you can put your spoon in your hand and eat it right okay ?", "words": ["can", "put", "your", "hand", "you", "and", "eat", "in", "it", "spoon"]}, {"sent": "youd like to use a spoon .", "words": ["to", "like", "spoon", "a"]}, {"sent": "you gonna chase the spoon away ?", "words": ["the", "you", "away", "spoon", "chase"]}, {"sent": "Is that the spoon ?", "words": ["the", "that", "spoon"]}, {"sent": "put that spoon in there .", "words": ["put", "there", "in", "spoon", "that"]}, {"sent": "spoon .", "words": ["spoon"]}, {"sent": "youre gonna have a grownup spoon ?", "words": ["have", "spoon", "a"]}, {"sent": "spoon .", "words": ["spoon"]}, {"sent": "wanna get the little spoon ?", "words": ["the", "little", "get", "spoon", "wanna"]}, {"sent": "spoon .", "words": ["spoon"]}, {"sent": "use your spoon .", "words": ["your", "spoon"]}, {"sent": "wheres the spoon for feeding her ?", "words": ["the", "for", "her", "spoon"]}, {"sent": "this is a spoon .", "words": ["this", "spoon", "a", "is"]}, {"sent": "Nana and Granddad a spoon ?", "words": ["spoon", "a", "and"]}, {"sent": "oh another spoon .", "words": ["another", "spoon"]}, {"sent": "you can have the spoon .", "words": ["can", "the", "you", "have", "spoon"]}, {"sent": "and get a spoon .", "words": ["get", "spoon", "a", "and"]}, {"sent": "spoon .", "words": ["spoon"]}, {"sent": "and theres a spoon .", "words": ["spoon", "a", "and"]}, {"sent": "whos got a spoon ?", "words": ["spoon", "a"]}, {"sent": "oh it needs to be a full spoon .", "words": ["full", "to", "be", "it", "spoon", "a"]}, {"sent": "spoon .", "words": ["spoon"]}, {"sent": "your big yellow spoon .", "words": ["yellow", "big", "your", "spoon"]}, {"sent": "spoon ?", "words": ["spoon"]}, {"sent": "you can lick the spoon .", "words": ["can", "the", "you", "spoon", "lick"]}, {"sent": "thats a measuring spoon .", "words": ["spoon", "a"]}, {"sent": "lick the spoon off .", "words": ["the", "lick", "spoon", "off"]}, {"sent": "do you have a knife and a fork and a spoon ?", "words": ["do", "knife", "you", "have", "and", "spoon", "fork", "a"]}, {"sent": "with a spoon ?", "words": ["with", "spoon", "a"]}, {"sent": "spoon .", "words": ["spoon"]}, {"sent": "and heres a spoon .", "words": ["spoon", "a", "and"]}, {"sent": "spoon see .", "words": ["see", "spoon"]}, {"sent": "thats a spoon .", "words": ["spoon", "a"]}, {"sent": "youve got a special spoon today .", "words": ["spoon", "a"]}, {"sent": "for the spoon ?", "words": ["the", "for", "spoon"]}, {"sent": "and another spoon .", "words": ["another", "spoon", "and"]}, {"sent": "spoon .", "words": ["spoon"]}, {"sent": "you know how to use a spoon .", "words": ["how", "to", "you", "spoon", "a"]}, {"sent": "if you were gonna eat spaghetti and meatballs would you use a fork or a spoon ?", "words": ["if", "spaghetti", "you", "eat", "and", "spoon", "fork", "would", "were", "a"]}, {"sent": "are you stirring the tea with a spoon ?", "words": ["the", "with", "you", "are", "spoon", "a"]}, {"sent": "violent spin it around with your spoon .", "words": ["your", "with", "it", "spoon", "around"]}, {"sent": "spoon ?", "words": ["spoon"]}, {"sent": "they can have a fork and a spoon but I want two dont I ?", "words": ["can", "and", "have", "they", "dont", "spoon", "fork", "but", "a"]}, {"sent": "and this is some flour so well get a spoon and take a little bit of flour .", "words": ["is", "of", "so", "and", "get", "this", "little", "spoon", "some", "take", "a"]}, {"sent": "can I have a spoon ?", "words": ["can", "have", "spoon", "a"]}, {"sent": "wheres the spoon can you hand me that spoon over there ?", "words": ["the", "can", "hand", "you", "there", "spoon", "over", "me", "that"]}, {"sent": "I cant find your spoon Thomas .", "words": ["your", "spoon", "find"]}, {"sent": "scraping up the crumbs with a spoon .", "words": ["up", "the", "with", "spoon", "a"]}, {"sent": "spoon ?", "words": ["spoon"]}, {"sent": "then you put them in with a spoon very slowly and you boil them for five minutes .", "words": ["put", "for", "them", "with", "you", "and", "in", "spoon", "then", "a"]}, {"sent": "mm a spoon .", "words": ["spoon", "a"]}, {"sent": "spoon ?", "words": ["spoon"]}, {"sent": "is that a boat ?", "words": ["boat", "that", "a", "is"]}, {"sent": "a boat house .", "words": ["house", "boat", "a"]}, {"sent": "well youve got the boat .", "words": ["the", "boat"]}, {"sent": "a boat ?", "words": ["boat", "a"]}, {"sent": "is it sposta be a boat ?", "words": ["is", "be", "it", "boat", "a"]}, {"sent": "pushing the boat into the water .", "words": ["the", "boat", "into", "water"]}, {"sent": "Ill put the boat over here for you .", "words": ["the", "put", "for", "boat", "you", "here", "over"]}, {"sent": "wherere you going to on your boat ?", "words": ["your", "to", "you", "on", "boat"]}, {"sent": "its a boat .", "words": ["boat", "a"]}, {"sent": "boat boat boat boat .", "words": ["boat"]}, {"sent": "okay Doctor Little we need a boat .", "words": ["boat", "need", "a", "we"]}, {"sent": "it looks like a boat .", "words": ["it", "boat", "like", "a"]}, {"sent": "you wanna go on your boat ?", "words": ["your", "go", "you", "wanna", "on", "boat"]}, {"sent": "wheres the boat going ?", "words": ["the", "boat"]}, {"sent": "coat boat .", "words": ["boat", "coat"]}, {"sent": "that is a sport fishing boat .", "words": ["boat", "that", "a", "is"]}, {"sent": "boat .", "words": ["boat"]}, {"sent": "xxx boat a ship .", "words": ["boat", "a"]}, {"sent": "its not a boat .", "words": ["boat", "not", "a"]}, {"sent": "that well the man with the boat .", "words": ["the", "boat", "that", "with"]}, {"sent": "you want a boat ?", "words": ["boat", "a", "you"]}, {"sent": "so the boat doesnt float away while were in this little boat .", "words": ["the", "so", "little", "this", "in", "away", "boat", "were"]}, {"sent": "why dont you sing the boat song ?", "words": ["the", "you", "sing", "dont", "boat", "why"]}, {"sent": "och well I went over on the boat and all .", "words": ["the", "over", "all", "and", "on", "boat"]}, {"sent": "put the boat away ?", "words": ["the", "put", "away", "boat"]}, {"sent": "is panda going with you for for a ride in the boat ?", "words": ["the", "for", "is", "boat", "with", "you", "in", "ride", "a"]}, {"sent": "theres a row boat for the pirates .", "words": ["the", "for", "boat", "a"]}, {"sent": "but um there was one big boat ride she wanted on with me .", "words": ["was", "with", "ride", "there", "big", "on", "me", "boat", "she", "but"]}, {"sent": "and they could tip you outof the boat aswell .", "words": ["the", "you", "and", "they", "could", "boat"]}, {"sent": "a space boat ?", "words": ["boat", "a"]}, {"sent": "you dont like it in the boat ?", "words": ["the", "like", "you", "dont", "in", "it", "boat"]}, {"sent": "sometimes I I make alotof things there last time I make dat boat and I make a racing car dat boat and I make a racing car dat I broke it after all .", "words": ["all", "car", "last", "and", "there", "make", "it", "boat", "a"]}, {"sent": "what are you doing in your boat ?", "words": ["what", "your", "you", "in", "are", "boat"]}, {"sent": "can you find a boat ?", "words": ["can", "you", "find", "boat", "a"]}, {"sent": "yeah you can pull it by the phone receiver thats attached to the phone and the pho the boat is all tangled up on the phone right ?", "words": ["can", "the", "all", "up", "is", "to", "by", "pull", "you", "and", "it", "on", "boat"]}, {"sent": "Ill bring him over to go in the boat .", "words": ["the", "go", "to", "boat", "bring", "him", "in", "over"]}, {"sent": "or a boat .", "words": ["boat", "a"]}, {"sent": "boat ?", "words": ["boat"]}, {"sent": "boat .", "words": ["boat"]}, {"sent": "are you going to go in this boat ?", "words": ["go", "to", "you", "this", "in", "are", "boat"]}, {"sent": "I guess to help hold the uh boat up .", "words": ["the", "up", "help", "hold", "to", "boat"]}, {"sent": "yes row row row your boat gently down the .", "words": ["the", "boat", "down", "your"]}, {"sent": "did you did we go on a boat to Ireland ?", "words": ["go", "we", "to", "you", "did", "on", "boat", "a"]}, {"sent": "drive boat .", "words": ["drive", "boat"]}, {"sent": "hes on a boat .", "words": ["on", "boat", "a"]}, {"sent": "is this boat moving out away from the quay ?", "words": ["the", "is", "this", "away", "out", "boat"]}, {"sent": "whoop hit the power boat and kept on going .", "words": ["the", "and", "on", "hit", "boat"]}, {"sent": "you went camping on a boat ?", "words": ["on", "boat", "a", "you"]}, {"sent": "theyre all in the boat .", "words": ["the", "all", "in", "boat"]}, {"sent": "did you sail your boat ?", "words": ["boat", "your", "did", "you"]}, {"sent": "a boat .", "words": ["boat", "a"]}, {"sent": "row the boat .", "words": ["the", "boat"]}, {"sent": "ummhm water in the boat .", "words": ["the", "in", "boat", "water"]}, {"sent": "on the boat .", "words": ["on", "the", "boat"]}, {"sent": "xxx a boat .", "words": ["boat", "a"]}, {"sent": "probably with your boat or something right ?", "words": ["your", "boat", "with"]}, {"sent": "a boat ?", "words": ["boat", "a"]}, {"sent": "and when my son you know old enough when I can trust him on a boat .", "words": ["old", "can", "my", "boat", "you", "and", "him", "on", "when", "a"]}, {"sent": "boat .", "words": ["boat"]}, {"sent": "thats the fisherman with his boat .", "words": ["the", "his", "boat", "with"]}, {"sent": "in Daddys boat .", "words": ["in", "boat"]}, {"sent": "in the red and the blue boat ?", "words": ["the", "red", "blue", "and", "in", "boat"]}, {"sent": "youre cutting a thing for the boat ?", "words": ["the", "for", "boat", "a"]}, {"sent": "look at the wee motor boat .", "words": ["the", "at", "boat", "look"]}, {"sent": "should Daddy finish coloring this boat in ?", "words": ["this", "in", "boat", "finish"]}, {"sent": "the fisherman with the boat .", "words": ["the", "boat", "with"]}, {"sent": "this will be a nice boat this wont it ?", "words": ["be", "nice", "this", "it", "will", "boat", "a"]}, {"sent": "does Max say boat when he takes a bath ?", "words": ["does", "when", "say", "he", "boat", "a"]}, {"sent": "a house boat ?", "words": ["boat", "house", "a"]}, {"sent": "boat on the water .", "words": ["on", "the", "boat", "water"]}, {"sent": "but Max stepped into his private boat and waved goodbye .", "words": ["into", "and", "his", "boat", "but"]}, {"sent": "yup she can have that boat .", "words": ["can", "have", "that", "boat", "she"]}, {"sent": "so the people in the front of the boat get wet .", "words": ["the", "wet", "of", "so", "get", "in", "boat"]}, {"sent": "a little motor boat .", "words": ["boat", "little", "a"]}, {"sent": "these are people from the boat .", "words": ["the", "these", "are", "boat"]}, {"sent": "oh hes going to ride in the boat I see .", "words": ["the", "to", "ride", "in", "see", "boat"]}, {"sent": "do you wanna go in the ocean in your boat ?", "words": ["the", "do", "your", "go", "you", "in", "wanna", "boat"]}, {"sent": "and theres a little boat Carl .", "words": ["boat", "little", "a", "and"]}, {"sent": "is it a boat ?", "words": ["it", "boat", "a", "is"]}, {"sent": "in the boat ?", "words": ["the", "in", "boat"]}, {"sent": "look at that boat .", "words": ["boat", "at", "that", "look"]}, {"sent": "Mister Jeremy uses a lilypad for a boat he casts his line in the water and hopes to catch minnows .", "words": ["the", "for", "to", "catch", "and", "in", "he", "water", "his", "boat", "a"]}, {"sent": "digger parked with the boat .", "words": ["the", "boat", "with"]}, {"sent": "now is bunny going for a ride in the boat ?", "words": ["the", "bunny", "for", "is", "boat", "in", "ride", "a"]}, {"sent": "the boat when youre taking a swim .", "words": ["the", "when", "swim", "boat", "a"]}, {"sent": "the boat ?", "words": ["the", "boat"]}, {"sent": "it says can you find the triangle in the boat ?", "words": ["can", "the", "you", "in", "it", "find", "boat"]}, {"sent": "this guy could drive the boat couldnt he .", "words": ["the", "drive", "this", "he", "could", "boat"]}, {"sent": "and I would eat them in a boat .", "words": ["them", "and", "eat", "in", "would", "boat", "a"]}, {"sent": "you hafta have a sail in a boat dont you ?", "words": ["you", "have", "dont", "in", "boat", "a"]}, {"sent": "that is a great boat .", "words": ["boat", "that", "a", "is"]}, {"sent": "boat begins with a b .", "words": ["boat", "with", "a"]}, {"sent": "not a boat isnt it ?", "words": ["it", "boat", "not", "a"]}, {"sent": "I said its a life boat but I dont know what sort of boat it is really .", "words": ["what", "is", "of", "dont", "it", "boat", "but", "a"]}, {"sent": "thinks its a boat .", "words": ["boat", "a"]}, {"sent": "a coat and a boat .", "words": ["boat", "coat", "and", "a"]}, {"sent": "you want your boat up there do you ?", "words": ["up", "do", "your", "you", "there", "boat"]}, {"sent": "he goes row row row your boat .", "words": ["your", "boat", "he"]}, {"sent": "its a boat isnt it ?", "words": ["it", "boat", "a"]}, {"sent": "choochoo boat ?", "words": ["boat"]}, {"sent": "I need the puzzle I need the puzzle too thankyou well put the paper on the puzzle right ?", "words": ["the", "put", "need", "paper", "puzzle", "too", "on"]}, {"sent": "can you see somebody on the puzzle here where the hat is missing ?", "words": ["can", "the", "where", "is", "you", "here", "puzzle", "hat", "on", "see"]}, {"sent": "oh I thought it was a piece from the puzzle .", "words": ["the", "was", "it", "puzzle", "a"]}, {"sent": "lets do a puzzle .", "words": ["do", "puzzle", "a"]}, {"sent": "put the puzzle on it ?", "words": ["the", "put", "it", "puzzle", "on"]}, {"sent": "its its a floor puzzle .", "words": ["puzzle", "a"]}, {"sent": "oh you dont wanna do the puzzle .", "words": ["the", "do", "you", "dont", "puzzle", "wanna"]}, {"sent": "it looks like a puzzle doesnt it ?", "words": ["it", "puzzle", "like", "a"]}, {"sent": "come and do the fireengine puzzle .", "words": ["the", "do", "puzzle", "and"]}, {"sent": "I thought we were gonna put it all in and do a puzzle .", "words": ["all", "put", "do", "we", "and", "in", "it", "puzzle", "were", "a"]}, {"sent": "the puzzle edges .", "words": ["the", "puzzle"]}, {"sent": "how you gonna work in the puzzle and do your homework ?", "words": ["the", "do", "how", "work", "your", "you", "and", "in", "puzzle"]}, {"sent": "he can help us with this puzzle .", "words": ["can", "help", "with", "us", "this", "he", "puzzle"]}, {"sent": "if you dont wanna do a puzzle .", "words": ["do", "if", "you", "dont", "puzzle", "wanna", "a"]}, {"sent": "it goes into the puzzle .", "words": ["the", "it", "puzzle", "into"]}, {"sent": "lets put this puzzle together .", "words": ["this", "put", "puzzle"]}, {"sent": "it was a piece of puzzle .", "words": ["of", "was", "it", "puzzle", "a"]}, {"sent": "are you not gonna show tell me anything about the puzzle ?", "words": ["the", "show", "you", "are", "puzzle", "about", "me", "not"]}, {"sent": "look at all your puzzle pieces .", "words": ["all", "look", "your", "puzzle", "at"]}, {"sent": "thought you were helping with the jigsaw puzzle .", "words": ["the", "with", "you", "puzzle", "were"]}, {"sent": "well finish that puzzle first .", "words": ["puzzle", "first", "that", "finish"]}, {"sent": "which jigsaw puzzle ?", "words": ["puzzle", "which"]}, {"sent": "do you wanna do your puzzle now with Erin ?", "words": ["do", "your", "with", "you", "puzzle", "wanna"]}, {"sent": "where oh where could you have put that piece of puzzle ?", "words": ["where", "put", "of", "you", "have", "puzzle", "could", "that"]}, {"sent": "look theres a puzzle .", "words": ["puzzle", "a", "look"]}, {"sent": "come on lets get a puzzle .", "words": ["on", "get", "puzzle", "a"]}, {"sent": "you wanna do the puzzle here ?", "words": ["the", "do", "you", "here", "puzzle", "wanna"]}, {"sent": "gonna put the birdie in the puzzle .", "words": ["the", "in", "put", "puzzle"]}, {"sent": "that I needta do the puzzle .", "words": ["the", "do", "puzzle", "that"]}, {"sent": "look around the puzzle and see if you can find a big orange spot .", "words": ["the", "can", "look", "if", "find", "and", "you", "orange", "puzzle", "around", "see", "big", "a"]}, {"sent": "now lets see get your puzzle board .", "words": ["get", "see", "puzzle", "your"]}, {"sent": "see if you can put them where they it is like a puzzle .", "words": ["can", "where", "put", "if", "is", "them", "like", "you", "they", "it", "puzzle", "see", "a"]}, {"sent": "I can work out a puzzle .", "words": ["can", "work", "puzzle", "out", "a"]}, {"sent": "lets bring the puzzle back .", "words": ["the", "puzzle", "bring", "back"]}, {"sent": "does BigBird wanna do the puzzle too ?", "words": ["the", "does", "do", "puzzle", "wanna", "too"]}, {"sent": "youve seen a hippo in your puzzle ?", "words": ["puzzle", "in", "your", "a"]}, {"sent": "puzzle .", "words": ["puzzle"]}, {"sent": "you wanna come with me to get the puzzle ?", "words": ["the", "to", "with", "you", "get", "puzzle", "wanna", "me"]}, {"sent": "these look like little puzzle pieces .", "words": ["these", "like", "little", "puzzle", "look"]}, {"sent": "half a puzzle .", "words": ["puzzle", "a"]}, {"sent": "although we have like cl I mean hes yeah I mean we dont have like the clock puzzle out or anything for him .", "words": ["the", "for", "we", "clock", "like", "have", "dont", "him", "puzzle", "out"]}, {"sent": "well you build up the puzzle for me Aran .", "words": ["up", "the", "for", "you", "puzzle", "build", "me"]}, {"sent": "do ya wanna get your puzzle ?", "words": ["do", "your", "get", "puzzle", "wanna"]}, {"sent": "can you maybe you can get another puzzle for us to do eh ?", "words": ["can", "do", "for", "to", "you", "get", "another", "us", "puzzle"]}, {"sent": "you want a puzzle ?", "words": ["puzzle", "a", "you"]}, {"sent": "cant you get the puzzle ?", "words": ["get", "the", "puzzle", "you"]}, {"sent": "lets get our puzzle .", "words": ["get", "puzzle", "our"]}, {"sent": "what did what xxx the puzzle has a name ?", "words": ["the", "what", "puzzle", "did", "a"]}, {"sent": "she has more problems with this puzzle than she does any of the other ones that she has .", "words": ["the", "does", "of", "with", "any", "other", "this", "more", "puzzle", "that", "she"]}, {"sent": "a puzzle ?", "words": ["puzzle", "a"]}, {"sent": "look look at the puzzle come back down here and look .", "words": ["the", "look", "back", "and", "here", "puzzle", "at", "down"]}, {"sent": "no no puzzle .", "words": ["puzzle"]}, {"sent": "wheres that puzzle ?", "words": ["puzzle", "that"]}, {"sent": "are you going to come and look at this puzzle ?", "words": ["to", "and", "you", "this", "are", "at", "puzzle", "look"]}, {"sent": "you wanna do this puzzle ?", "words": ["do", "you", "this", "puzzle", "wanna"]}, {"sent": "what would you like would you like to play with this puzzle ?", "words": ["what", "to", "play", "with", "like", "you", "this", "puzzle", "would"]}, {"sent": "and would you maybe like a jigsaw puzzle too ?", "words": ["like", "you", "and", "puzzle", "would", "too", "a"]}, {"sent": "I wanna do a puzzle do you wanna do a puzzle ?", "words": ["do", "you", "puzzle", "wanna", "a"]}, {"sent": "because the jigsaw puzzle was a picture of a tree with leaves wasnt it ?", "words": ["the", "tree", "of", "was", "picture", "with", "it", "puzzle", "because", "a"]}, {"sent": "well put everything away and you can show me how you do the puzzle one more time before you go .", "words": ["can", "do", "put", "how", "the", "go", "show", "you", "and", "more", "away", "puzzle", "me"]}, {"sent": "why dont you put the puzzle over here so Lois can see ?", "words": ["the", "can", "put", "so", "you", "here", "dont", "puzzle", "see", "over", "why"]}, {"sent": "whos missing from the puzzle now ?", "words": ["the", "puzzle"]}, {"sent": "lets do this puzzle darling .", "words": ["do", "puzzle", "this"]}, {"sent": "and look theres a jigsaw puzzle here .", "words": ["and", "here", "puzzle", "look", "a"]}, {"sent": "do you wanna do a puzzle with Mommy ?", "words": ["do", "with", "you", "puzzle", "wanna", "a"]}, {"sent": "like a jigsaw puzzle .", "words": ["puzzle", "like", "a"]}, {"sent": "is it your puzzle making a noise ?", "words": ["your", "is", "it", "puzzle", "a"]}, {"sent": "do a puzzle yeah do a puzzle what else did we do ?", "words": ["do", "what", "we", "puzzle", "did", "a"]}, {"sent": "he would be crying okay lets finish the puzzle and then we can play with some of your cars on it or something .", "words": ["the", "can", "your", "we", "of", "be", "play", "with", "finish", "and", "it", "he", "puzzle", "would", "on", "some", "then"]}, {"sent": "wheres the rest of your puzzle pieces honey ?", "words": ["the", "of", "puzzle", "your"]}, {"sent": "a puzzle .", "words": ["puzzle", "a"]}, {"sent": "what about your puzzle ?", "words": ["what", "puzzle", "your", "about"]}, {"sent": "and we bought Caitlin a jigsaw puzzle .", "words": ["puzzle", "we", "a", "and"]}, {"sent": "this is a very complicated jigsaw puzzle .", "words": ["this", "puzzle", "a", "is"]}, {"sent": "but you bring your jigs put the jigsaw puzzle in there because youll lose it darling .", "words": ["the", "put", "your", "bring", "because", "you", "there", "in", "it", "puzzle", "but"]}, {"sent": "mm guess thats not really a puzzle .", "words": ["puzzle", "not", "a"]}, {"sent": "well do the puzzle over here where theres more space .", "words": ["the", "do", "where", "here", "more", "puzzle", "over"]}, {"sent": "theres another puzzle over there if you wanna do another one .", "words": ["do", "if", "you", "another", "there", "puzzle", "wanna", "over"]}, {"sent": "a puzzle ?", "words": ["puzzle", "a"]}, {"sent": "were wanna put this puzzle together dont we ?", "words": ["put", "we", "this", "dont", "puzzle", "wanna", "were"]}, {"sent": "theres only a pig in the puzzle and some birds .", "words": ["the", "pig", "and", "in", "puzzle", "some", "a"]}, {"sent": "can you put Big Bird back in the puzzle ?", "words": ["can", "the", "put", "back", "you", "in", "puzzle"]}, {"sent": "Mama I want that other puzzle Mama where this go ?", "words": ["where", "go", "other", "this", "puzzle", "that"]}, {"sent": "did you finish the puzzle ?", "words": ["the", "finish", "you", "puzzle", "did"]}, {"sent": "you know what though its a puzzle of Donald Duck .", "words": ["what", "of", "you", "puzzle", "a"]}, {"sent": "here is a puzzle .", "words": ["here", "puzzle", "a", "is"]}, {"sent": "you wanna do the other puzzle ?", "words": ["the", "do", "other", "you", "puzzle", "wanna"]}, {"sent": "mm this looks like kindof a hard puzzle .", "words": ["like", "this", "puzzle", "hard", "a"]}, {"sent": "well I hope we werent missing a piece when it was a new puzzle .", "words": ["we", "was", "it", "puzzle", "new", "when", "a"]}, {"sent": "that is a big puzzle .", "words": ["is", "that", "puzzle", "big", "a"]}, {"sent": "put that puzzle back in there and then well play with those puzzles .", "words": ["put", "those", "back", "play", "with", "and", "there", "in", "puzzle", "then", "that"]}, {"sent": "thats that puzzle we made remember ?", "words": ["puzzle", "that", "we"]}, {"sent": "it was a big puzzle .", "words": ["was", "it", "puzzle", "big", "a"]}, {"sent": "oh a puzzle cool .", "words": ["puzzle", "a"]}, {"sent": "is it like a puzzle to get it match up ?", "words": ["up", "is", "to", "like", "get", "it", "puzzle", "a"]}, {"sent": "put the pot in the puzzle okay ?", "words": ["the", "in", "put", "puzzle"]}, {"sent": "its a puzzle .", "words": ["puzzle", "a"]}, {"sent": "this is a nice puzzle .", "words": ["is", "nice", "this", "puzzle", "a"]}, {"sent": "well find out ?", "words": ["out", "find"]}, {"sent": "seems to me you need to find some more .", "words": ["need", "need to", "to", "you", "more", "find", "me", "some"]}, {"sent": "since I cant find yours .", "words": ["find"]}, {"sent": "and we cant find him .", "words": ["him", "we", "find", "and"]}, {"sent": "find this orange for me please .", "words": ["for", "this", "orange", "find", "me"]}, {"sent": "we need to find um some square things .", "words": ["need", "we", "need to", "to", "find", "some"]}, {"sent": "well find out about .", "words": ["about", "out", "find"]}, {"sent": "if we find the sticker we can see what .", "words": ["the", "can", "what", "if", "we", "find", "see"]}, {"sent": "what did they find ?", "words": ["what", "did", "they", "find"]}, {"sent": "see if you can find something to play with .", "words": ["can", "if", "to", "play", "with", "you", "find", "see"]}, {"sent": "youll find it will you ?", "words": ["will", "it", "you", "find"]}, {"sent": "find that first .", "words": ["first", "that", "find"]}, {"sent": "can you find blocks to put in the bucket .", "words": ["can", "the", "put", "bucket", "to", "you", "in", "find"]}, {"sent": "look at this find their heads where does this go ?", "words": ["where", "does", "look", "their", "go", "find", "this", "at"]}, {"sent": "find out whats wrong with her .", "words": ["her", "with", "out", "find"]}, {"sent": "should we try to find her other shoe ?", "words": ["try", "we", "to", "other", "shoe", "find", "her", "try to"]}, {"sent": "find the little dot put it right in the hole .", "words": ["the", "put", "little", "in", "it", "find"]}, {"sent": "can you find it ?", "words": ["can", "it", "find", "you"]}, {"sent": "you find a jigsaw piece with Jess on it then .", "words": ["with", "you", "it", "find", "on", "then", "a"]}, {"sent": "can you find the other one xxx ?", "words": ["can", "the", "other", "you", "find"]}, {"sent": "you willnt find a sweetie up here .", "words": ["up", "you", "here", "find", "a"]}, {"sent": "yes Joseph what did you find ?", "words": ["what", "did", "find", "you"]}, {"sent": "I thought you were trying to hide him Fraser so we couldnt find him .", "words": ["hide", "we", "to", "so", "you", "him", "find", "were"]}, {"sent": "find the circle .", "words": ["the", "find"]}, {"sent": "go and find it then .", "words": ["go", "and", "it", "find", "then"]}, {"sent": "thats Et screaming loudly can you find Et sadly saying goodbye ?", "words": ["can", "find", "you"]}, {"sent": "can you find any animals ?", "words": ["can", "any", "find", "you"]}, {"sent": "did you find the lid ?", "words": ["the", "did", "find", "you"]}, {"sent": "can you find the bug in this picture ?", "words": ["can", "the", "bug", "picture", "you", "this", "in", "find"]}, {"sent": "but that was because I couldnt find my purse .", "words": ["my", "purse", "was", "but", "find", "that", "because"]}, {"sent": "can you find the circle ?", "words": ["can", "the", "find", "you"]}, {"sent": "shall I find this for you ?", "words": ["this", "for", "you", "find"]}, {"sent": "daddy cant find it .", "words": ["it", "find"]}, {"sent": "you try to find where it goes .", "words": ["where", "to", "you", "it", "find", "try", "try to"]}, {"sent": "if shes managed to find a pair of scissors .", "words": ["if", "of", "to", "scissors", "find", "a"]}, {"sent": "let me see if I can find out a good one to read .", "words": ["can", "if", "to", "me", "read", "out", "find", "good", "see", "a"]}, {"sent": "so if you want them back now youll find theyve got milk .", "words": ["if", "back", "so", "them", "you", "find", "milk"]}, {"sent": "because how long does she expect uh people to be members of the class in which they presently find themselves ?", "words": ["the", "does", "how", "of", "to", "be", "long", "they", "in", "find", "which", "she", "because"]}, {"sent": "just tryina find out what children hafta say at different times of the day .", "words": ["the", "what", "of", "say", "find", "out", "at"]}, {"sent": "where did you find these ?", "words": ["where", "these", "you", "did", "find"]}, {"sent": "Father will hafta investigate this and find out where the Siamese cats come and everything else .", "words": ["where", "the", "and", "this", "out", "find", "will"]}, {"sent": "shall shall we see if we can find a picture of Poppy and Sam in the middle ?", "words": ["can", "the", "if", "we", "of", "picture", "and", "in", "find", "see", "a"]}, {"sent": "see cant find .", "words": ["see", "find"]}, {"sent": "can you find the little ball ?", "words": ["can", "the", "little", "you", "ball", "find"]}, {"sent": "can you find the page where you play peek a boo ?", "words": ["can", "the", "where", "play", "you", "find", "a"]}, {"sent": "where did you find those ?", "words": ["where", "those", "you", "did", "find"]}, {"sent": "I think I can find her .", "words": ["can", "think", "her", "find"]}, {"sent": "can you find .", "words": ["can", "find", "you"]}, {"sent": "I cant find a top for you to wear Fraser .", "words": ["for", "to", "you", "find", "a"]}, {"sent": "can you find a lambie ?", "words": ["can", "a", "find", "you"]}, {"sent": "cant find the end .", "words": ["the", "find"]}, {"sent": "I want you to try and find uncle snake again .", "words": ["to", "and", "you", "find", "try"]}, {"sent": "and Ill wash my hands and find another little pot for us .", "words": ["my", "for", "and", "another", "little", "us", "find", "wash"]}, {"sent": "lets see if we can find two green .", "words": ["can", "if", "we", "green", "find", "see"]}, {"sent": "Ill go and find that story in a minute .", "words": ["go", "and", "in", "find", "story", "that", "a"]}, {"sent": "well I cant find the thing .", "words": ["the", "find"]}, {"sent": "you find number two .", "words": ["find", "you"]}, {"sent": "we gotta find look for the mailtruck .", "words": ["the", "for", "we", "find", "look"]}, {"sent": "shall we find this one next ?", "words": ["this", "find", "we"]}, {"sent": "see if you can find it .", "words": ["can", "if", "you", "it", "find", "see"]}, {"sent": "we we needta find you some footwear .", "words": ["you", "some", "find", "we"]}, {"sent": "can you find the one with no tail ?", "words": ["can", "the", "with", "you", "find"]}, {"sent": "okay we hafta find a purple circle .", "words": ["a", "find", "we"]}, {"sent": "well well hafta go and find it later .", "words": ["find", "it", "and", "go"]}, {"sent": "now find the picture .", "words": ["the", "picture", "find"]}, {"sent": "put all the pieces down and find his ears .", "words": ["the", "all", "put", "and", "find", "his", "down"]}, {"sent": "and telling me which you find .", "words": ["you", "and", "find", "me", "which"]}, {"sent": "I cant find it darling .", "words": ["it", "find"]}, {"sent": "can you find the little picture of the telephone then ?", "words": ["can", "the", "of", "picture", "little", "you", "find", "then", "telephone"]}, {"sent": "well find an I .", "words": ["an", "find"]}, {"sent": "oh did you find another can ?", "words": ["can", "you", "another", "did", "find"]}, {"sent": "who can we find to play with you ?", "words": ["can", "we", "to", "play", "with", "you", "find", "who"]}, {"sent": "should we put it aside and do a different one since we cant find that one ?", "words": ["do", "put", "we", "and", "it", "find", "that", "a"]}, {"sent": "you find the nest .", "words": ["the", "find", "you"]}, {"sent": "shall we find somebody to go in the broken down car ?", "words": ["the", "go", "we", "to", "car", "broken", "in", "find", "down"]}, {"sent": "shes helping Flossie find a book to read .", "words": ["to", "read", "find", "book", "a"]}, {"sent": "you find it ?", "words": ["it", "find", "you"]}, {"sent": "and lets group here to the alphabet and find the letter l .", "words": ["the", "to", "and", "here", "find"]}, {"sent": "lets see if I can find a longer stick .", "words": ["can", "stick", "if", "find", "see", "a"]}, {"sent": "but we cant find Humptydumpty now can we ?", "words": ["can", "find", "but", "we"]}, {"sent": "you never know what youre gonna find .", "words": ["what", "find", "you"]}, {"sent": "I did find them Becky .", "words": ["them", "did", "find"]}, {"sent": "everybody find a partner .", "words": ["a", "find"]}, {"sent": "can you find me the deer in there ?", "words": ["can", "the", "deer", "you", "there", "in", "find", "me"]}, {"sent": "lets see what other things you can find .", "words": ["can", "what", "you", "other", "find", "see"]}, {"sent": "look in the bag and see what you find to write on .", "words": ["the", "what", "to", "find", "and", "you", "write", "in", "look", "on", "see"]}, {"sent": "can you find number eleven ?", "words": ["can", "find", "you"]}, {"sent": "now find now find someone wearing blue .", "words": ["blue", "find"]}, {"sent": "what cant you find ?", "words": ["what", "find", "you"]}, {"sent": "and then Ill find that one see if I can find it anywhere .", "words": ["can", "if", "and", "it", "find", "see", "then", "that"]}, {"sent": "youll find a place for them .", "words": ["for", "them", "a", "find"]}, {"sent": "lets see if I can find kite .", "words": ["can", "see", "if", "find"]}, {"sent": "lets go find Perro .", "words": ["find", "go"]}, {"sent": "see if you can find em .", "words": ["can", "if", "you", "find", "see"]}, {"sent": "when I leave the house whats another thing I look for and I sometimes cant find do you know ?", "words": ["the", "do", "for", "find", "and", "another", "you", "house", "look", "when"]}, {"sent": "lots see if we can find some more animals in there .", "words": ["can", "if", "we", "there", "more", "in", "find", "see", "some"]}, {"sent": "here maybe you can find the piece that goes to the foot .", "words": ["the", "can", "to", "you", "here", "foot", "find", "that"]}, {"sent": "mike you know what I want you to find now ?", "words": ["to", "what", "find", "you"]}, {"sent": "can you find the end of his string ?", "words": ["can", "the", "of", "you", "find", "his"]}, {"sent": "mm strawberry ice cream .", "words": ["ice", "ice cream", "strawberry"]}, {"sent": "could be a blueberry or a strawberry tree ?", "words": ["tree", "strawberry", "be", "could", "a"]}, {"sent": "your strawberry milk is ready .", "words": ["milk", "your", "is", "strawberry"]}, {"sent": "thats raspberry or apricot or another apricot or peach and strawberry another peach and strawberry .", "words": ["another", "strawberry", "and"]}, {"sent": "it was very good strawberry .", "words": ["was", "it", "good", "strawberry"]}, {"sent": "and strawberry .", "words": ["strawberry", "and"]}, {"sent": "*OP1: you got a big strawberry .", "words": ["strawberry", "big", "a", "you"]}, {"sent": "a strawberry .", "words": ["strawberry", "a"]}, {"sent": "he says he wants strawberry ice cream with whipped cream on top .", "words": ["ice cream", "strawberry", "with", "on", "he", "ice"]}, {"sent": "heres a strawberry too .", "words": ["too", "strawberry", "a"]}, {"sent": "would you like some strawberry ?", "words": ["strawberry", "like", "you", "would", "some"]}, {"sent": "would you like chocolate milk or strawberry milk Honey ?", "words": ["strawberry", "chocolate", "like", "you", "would", "milk"]}, {"sent": "a chocolate cake or a strawberry cake ?", "words": ["chocolate", "strawberry", "cake", "a"]}, {"sent": "how bout a blue strawberry ?", "words": ["how", "blue", "strawberry", "a"]}, {"sent": "the bit of strawberry has gone ?", "words": ["the", "of", "strawberry"]}, {"sent": "do you like strawberry ?", "words": ["do", "strawberry", "like", "you"]}, {"sent": "strawberry juice .", "words": ["juice", "strawberry"]}, {"sent": "a strawberry on the pizza ?", "words": ["the", "strawberry", "pizza", "on", "a"]}, {"sent": "he has a strawberry hat and a red nose .", "words": ["red", "strawberry", "nose", "and", "he", "hat", "a"]}, {"sent": "thats strawberry right ?", "words": ["strawberry"]}, {"sent": "how did you know that was strawberry milk ?", "words": ["how", "strawberry", "was", "you", "did", "milk", "that"]}, {"sent": "now how about this strawberry ?", "words": ["this", "how", "about", "strawberry"]}, {"sent": "yeah they have the strawberry syrup .", "words": ["have", "they", "strawberry", "the"]}, {"sent": "its the same size as the strawberry .", "words": ["the", "same", "strawberry"]}, {"sent": "theres some nice big pieces of strawberry in that strawberry jam arent there ?", "words": ["strawberry", "of", "nice", "there", "in", "that", "some", "big"]}, {"sent": "he hes like uh I think hell be like a strawberry .", "words": ["think", "strawberry", "be", "like", "he", "a"]}, {"sent": "strawberry shortcake is ?", "words": ["is", "strawberry"]}, {"sent": "you had a red one but it wasnt strawberry was it ?", "words": ["red", "strawberry", "was", "you", "it", "but", "a"]}, {"sent": "theres a strawberry in here look .", "words": ["strawberry", "here", "in", "look", "a"]}, {"sent": "strawberry mm .", "words": ["strawberry"]}, {"sent": "theres three eggs and a strawberry actually .", "words": ["strawberry", "a", "and"]}, {"sent": "oh you want strawberry milk ?", "words": ["milk", "strawberry", "you"]}, {"sent": "hello strawberry .", "words": ["strawberry"]}, {"sent": "its the way Mummy has cut the strawberry .", "words": ["the", "cut", "strawberry"]}, {"sent": "its a plastic strawberry .", "words": ["strawberry", "a"]}, {"sent": "what was that about the strawberry ?", "words": ["the", "what", "strawberry", "was", "about", "that"]}, {"sent": "one strawberry .", "words": ["strawberry"]}, {"sent": "strawberry tarts ?", "words": ["strawberry"]}, {"sent": "have you dropped a strawberry ?", "words": ["have", "strawberry", "a", "you"]}, {"sent": "its more strawberry .", "words": ["more", "strawberry"]}, {"sent": "did you have your strawberry tart then ?", "words": ["your", "strawberry", "you", "have", "did", "then"]}, {"sent": "going to buy him a strawberry ?", "words": ["strawberry", "to", "him", "buy", "a"]}, {"sent": "hows your strawberry ?", "words": ["your", "strawberry"]}, {"sent": "looks like a strawberry its an apple .", "words": ["strawberry", "like", "an", "apple", "a"]}, {"sent": "and theres the tom the strawberry .", "words": ["the", "strawberry", "and"]}, {"sent": "a rectangle shaped strawberry cake .", "words": ["strawberry", "cake", "a"]}, {"sent": "and you put a strawberry on top .", "words": ["put", "strawberry", "you", "and", "on", "a"]}, {"sent": "well hafta tell Grandma that they make a strawberry kind and maybe shell get it when you got to visit .", "words": ["when", "strawberry", "to", "and", "get", "they", "that", "it", "you", "make", "a"]}, {"sent": "how bout a pink strawberry ?", "words": ["how", "strawberry", "a"]}, {"sent": "shall we go strawberry picking this summer ?", "words": ["go", "this", "strawberry", "we"]}, {"sent": "Im sure Daddy could manage the odd strawberry dont you ?", "words": ["the", "strawberry", "you", "dont", "could"]}, {"sent": "oh is strawberry a fruit or a vegetable ?", "words": ["strawberry", "a", "is"]}, {"sent": "do you have homemade strawberry jam ?", "words": ["have", "do", "strawberry", "you"]}, {"sent": "so a bit of cardboard has gone over the picture of the strawberry .", "words": ["the", "strawberry", "of", "so", "picture", "over", "a"]}, {"sent": "whos a strawberry ?", "words": ["strawberry", "a"]}, {"sent": "hows your strawberry ?", "words": ["your", "strawberry"]}, {"sent": "no all the strawberry is gone .", "words": ["the", "all", "is", "strawberry"]}, {"sent": "the xxx strawberry and the mouse are wearing glasses too that is so silly .", "words": ["the", "strawberry", "is", "so", "and", "glasses", "are", "too", "that", "mouse"]}, {"sent": "its strawberry yoghurt .", "words": ["strawberry"]}, {"sent": "which one do you like strawberry okay .", "words": ["do", "strawberry", "like", "you", "which"]}, {"sent": "is that for your strawberry shortcake ?", "words": ["for", "your", "strawberry", "is", "that"]}, {"sent": "Naima hold still theres a strawberry down here .", "words": ["hold", "strawberry", "here", "down", "a"]}, {"sent": "find the strawberry .", "words": ["the", "strawberry", "find"]}, {"sent": "I dont know but well hafta wait and see heres a strawberry wagon .", "words": ["wait", "strawberry", "and", "dont", "see", "but", "a"]}, {"sent": "strawberry tarts .", "words": ["strawberry"]}, {"sent": "strawberry ?", "words": ["strawberry"]}, {"sent": "can you find me a strawberry ?", "words": ["can", "strawberry", "you", "find", "me", "a"]}, {"sent": "a strawberry .", "words": ["strawberry", "a"]}, {"sent": "oh strawberry .", "words": ["strawberry"]}, {"sent": "wow blueberry and strawberry .", "words": ["strawberry", "and"]}, {"sent": "its darker red than the strawberry jam we had last week .", "words": ["red", "the", "we", "strawberry", "last"]}, {"sent": "its hard to play when you have strawberry in your mouth .", "words": ["your", "strawberry", "to", "play", "mouth", "you", "have", "in", "hard", "when"]}, {"sent": "oh she is enjoying this strawberry icecream .", "words": ["this", "strawberry", "she", "is"]}, {"sent": "is it strawberry jam ?", "words": ["it", "strawberry", "is"]}, {"sent": "Mummy and Daddy had a nice meal and for dessert we had strawberry meringue raspberry meringue .", "words": ["for", "we", "strawberry", "nice", "and", "a"]}, {"sent": "and a strawberry .", "words": ["strawberry", "a", "and"]}, {"sent": "strawberry jam ?", "words": ["strawberry"]}, {"sent": "a red strawberry .", "words": ["red", "strawberry", "a"]}, {"sent": "strawberry ?", "words": ["strawberry"]}, {"sent": "I can see another strawberry in that bowl .", "words": ["can", "bowl", "strawberry", "another", "in", "see", "that"]}, {"sent": "we had strawberry pancakes in Pennsylvania didnt we ?", "words": ["in", "strawberry", "we"]}, {"sent": "strawberry .", "words": ["strawberry"]}, {"sent": "theres a strawberry .", "words": ["strawberry", "a"]}, {"sent": "is that a strawberry ?", "words": ["strawberry", "that", "a", "is"]}, {"sent": "I think thats a strawberry icecream .", "words": ["think", "strawberry", "a"]}, {"sent": "you put in such a big piece of strawberry .", "words": ["put", "strawberry", "of", "you", "in", "big", "a"]}, {"sent": "here have some strawberry .", "words": ["here", "some", "have", "strawberry"]}, {"sent": "strawberry ice cream in the middle .", "words": ["the", "ice cream", "strawberry", "in", "ice"]}, {"sent": "you had chips and strawberry .", "words": ["strawberry", "and", "you"]}, {"sent": "do you remember going strawberry picking yesterday ?", "words": ["do", "strawberry", "you"]}, {"sent": "the strawberry one until teatime .", "words": ["the", "strawberry"]}, {"sent": "a strawberry .", "words": ["strawberry", "a"]}, {"sent": "strawberry in a box ?", "words": ["in", "box", "a", "strawberry"]}, {"sent": "oh theres one strawberry on its own there .", "words": ["on", "there", "strawberry"]}, {"sent": "uh huh huh huh oh my god were being recordered or strawberry .", "words": ["my", "were", "strawberry"]}, {"sent": "its just like it only its strawberry instead .", "words": ["it", "like", "strawberry"]}, {"sent": "Naimas eating a strawberry .", "words": ["strawberry", "a"]}, {"sent": "I think shes got her hand stuck .", "words": ["think", "stuck", "her", "hand"]}, {"sent": "you getting stuck ?", "words": ["stuck", "you"]}, {"sent": "jingle bells and when Santa got stuck up the chimney didnt you ?", "words": ["up", "the", "and", "you", "stuck", "when"]}, {"sent": "I think hes stuck he wants to get out but hes stuck .", "words": ["think", "to", "get", "he", "stuck", "out", "but"]}, {"sent": "is the big car stuck ?", "words": ["the", "is", "car", "stuck", "big"]}, {"sent": "it looks almost as if the frogs are saying watch out dont get stuck .", "words": ["the", "watch", "if", "get", "dont", "it", "are", "stuck", "out"]}, {"sent": "has she got a fishbone stuck ?", "words": ["stuck", "she", "a"]}, {"sent": "my hand is stuck in the car .", "words": ["the", "my", "is", "hand", "car", "in", "stuck"]}, {"sent": "is he stuck ?", "words": ["stuck", "he", "is"]}, {"sent": "then she heard a bug cry Im stuck in a spout and she climbed to the top and hoisted him out .", "words": ["bug", "the", "to", "cry", "and", "in", "him", "stuck", "out", "then", "she", "a"]}, {"sent": "and Bella was stuck in the lift .", "words": ["the", "was", "and", "in", "stuck"]}, {"sent": "I think its more likely to be stuck up a fire uh stuck up a tree and needing the services of the fireman .", "words": ["up", "the", "think", "tree", "of", "to", "be", "and", "more", "stuck", "a"]}, {"sent": "I have all of them stuck on .", "words": ["all", "of", "them", "have", "stuck", "on"]}, {"sent": "hell get stuck .", "words": ["get", "stuck"]}, {"sent": "is it stuck ?", "words": ["it", "stuck", "is"]}, {"sent": "or else you might get stuck .", "words": ["get", "stuck", "you"]}, {"sent": "youre laughing because Purdies stuck ?", "words": ["stuck", "because"]}, {"sent": "is there a fire no hes stuck up a tree so they need their ladders uhn dont they ?", "words": ["up", "tree", "need", "their", "is", "so", "there", "they", "dont", "stuck", "a"]}, {"sent": "has the ambulance got stuck in the puzzle ?", "words": ["the", "puzzle", "in", "stuck"]}, {"sent": "okay its stuck then .", "words": ["then", "stuck"]}, {"sent": "stuck together ?", "words": ["stuck"]}, {"sent": "your fingers stuck in there .", "words": ["there", "in", "stuck", "your"]}, {"sent": "are your feet stuck in the mud ?", "words": ["the", "your", "in", "are", "stuck"]}, {"sent": "just then Gordon returned and he frowned when he saw Thomas stuck in the middle of the bridge .", "words": ["the", "of", "and", "in", "stuck", "he", "then", "when"]}, {"sent": "theyre not stuck together .", "words": ["stuck", "not"]}, {"sent": "stuck on there ?", "words": ["on", "there", "stuck"]}, {"sent": "are you a bit stuck ?", "words": ["are", "stuck", "a", "you"]}, {"sent": "its stuck .", "words": ["stuck"]}, {"sent": "bit stuck actually .", "words": ["stuck"]}, {"sent": "are you stuck ?", "words": ["are", "stuck", "you"]}, {"sent": "whats this one of my traps stuck in a knights helmet .", "words": ["my", "of", "this", "in", "stuck", "a"]}, {"sent": "is it stuck ?", "words": ["it", "stuck", "is"]}, {"sent": "youre stuck in there .", "words": ["there", "in", "stuck"]}, {"sent": "its stuck on the road .", "words": ["on", "the", "stuck"]}, {"sent": "stuck where ?", "words": ["where", "stuck"]}, {"sent": "well hes stuck stuck stuck in there .", "words": ["there", "in", "stuck"]}, {"sent": "youre stuck in the tree .", "words": ["the", "tree", "in", "stuck"]}, {"sent": "hes going to get stuck .", "words": ["get", "to", "stuck"]}, {"sent": "more stuck ?", "words": ["more", "stuck"]}, {"sent": "the bits of bread got stuck in the toaster I cant get them out .", "words": ["the", "bread", "of", "them", "get", "in", "stuck", "out"]}, {"sent": "does it go when Santa got stuck up the chimney .", "words": ["up", "does", "the", "go", "it", "stuck", "when"]}, {"sent": "and pretend that that now Im stuck in here .", "words": ["and", "here", "in", "stuck", "pretend", "that"]}, {"sent": "and who was stuck ?", "words": ["was", "who", "stuck", "and"]}, {"sent": "stuck to your finger ohp .", "words": ["your", "to", "finger", "stuck"]}, {"sent": "its stuck is it ?", "words": ["it", "stuck", "is"]}, {"sent": "hes stuck under a bridge .", "words": ["under", "stuck", "a"]}, {"sent": "it stuck on your hand ?", "words": ["your", "hand", "it", "stuck", "on"]}, {"sent": "if you get stuck on the rocks you can fly straight back out into the ocean cant you ?", "words": ["the", "can", "if", "back", "into", "you", "get", "stuck", "out", "on"]}, {"sent": "its stuck .", "words": ["stuck"]}, {"sent": "oh youve got some stuck to your socks .", "words": ["to", "some", "stuck", "your"]}, {"sent": "Purdie has whiskers on her face because if shes going somewhere small and her whiskers get caught she knows that shes going to get stuck as well .", "words": ["face", "if", "to", "and", "get", "that", "stuck", "on", "her", "she", "because"]}, {"sent": "youve got your finger stuck in your sleeve .", "words": ["finger", "in", "stuck", "your"]}, {"sent": "are you stuck ?", "words": ["are", "stuck", "you"]}, {"sent": "I think hes scared because hes stuck in that tree Fraser .", "words": ["think", "tree", "scared", "in", "stuck", "that", "because"]}, {"sent": "the pages are stuck .", "words": ["the", "are", "stuck"]}, {"sent": "three here stuck together .", "words": ["here", "stuck"]}, {"sent": "hes got his tongue stuck out .", "words": ["his", "stuck", "tongue", "out"]}, {"sent": "youre stuck .", "words": ["stuck"]}, {"sent": "this is gonna make it stuck .", "words": ["is", "this", "it", "stuck", "make"]}, {"sent": "xxx you really were stuck .", "words": ["stuck", "were", "you"]}, {"sent": "remember when you stuck marshmallows in that ?", "words": ["when", "you", "in", "stuck", "that"]}, {"sent": "is it stuck ?", "words": ["it", "stuck", "is"]}, {"sent": "the duck stays stuck deep in the muck .", "words": ["the", "in", "stuck", "duck"]}, {"sent": "and the driver and the bus is stuck in the traffic arent they ?", "words": ["the", "is", "and", "they", "in", "stuck", "bus"]}, {"sent": "stuck in my chair at noon hes having lunch its lunchtime .", "words": ["my", "chair", "in", "stuck", "at"]}, {"sent": "just a minute it has stuck round here as well .", "words": ["here", "it", "stuck", "a"]}, {"sent": "you got your arm stuck oh no your arms are so stuck .", "words": ["your", "so", "you", "are", "stuck", "arm"]}, {"sent": "stuck stuck stuck .", "words": ["stuck"]}, {"sent": "you youve got that yoghurt pot stuck on your ear .", "words": ["your", "you", "stuck", "ear", "on", "that"]}, {"sent": "oh is your hand stuck ?", "words": ["stuck", "your", "hand", "is"]}, {"sent": "stuck .", "words": ["stuck"]}, {"sent": "are they stuck ?", "words": ["they", "are", "stuck"]}, {"sent": "shes stuck there .", "words": ["there", "stuck"]}, {"sent": "youre stuck now arent you ?", "words": ["stuck", "you"]}, {"sent": "his hip was stuck .", "words": ["his", "was", "stuck"]}, {"sent": "stuck .", "words": ["stuck"]}, {"sent": "its stuck .", "words": ["stuck"]}, {"sent": "two stuck the rope ?", "words": ["the", "stuck"]}, {"sent": "stuck is it ?", "words": ["it", "stuck", "is"]}, {"sent": "are you stuck ?", "words": ["are", "stuck", "you"]}, {"sent": "thats stuck .", "words": ["stuck"]}, {"sent": "uh oh I think its stuck .", "words": ["think", "stuck"]}, {"sent": "does that mean you stuck it on each one ?", "words": ["does", "each", "you", "it", "stuck", "on", "that"]}, {"sent": "because if he goes up to the top hes gonna get stuck right ?", "words": ["up", "the", "if", "to", "get", "he", "stuck", "because"]}, {"sent": "its stuck .", "words": ["stuck"]}, {"sent": "who got stuck in the tunnel ?", "words": ["the", "in", "stuck", "who"]}, {"sent": "I know youre stuck in your dress sweetie .", "words": ["your", "dress", "in", "stuck"]}, {"sent": "he wasnt stuck on very well .", "words": ["on", "stuck", "he"]}, {"sent": "youre going to get stuck in the box are you ?", "words": ["the", "box", "to", "you", "get", "in", "are", "stuck"]}, {"sent": "stuck .", "words": ["stuck"]}, {"sent": "Annes fireengines stuck .", "words": ["stuck"]}, {"sent": "you stuck it together did you ?", "words": ["it", "stuck", "did", "you"]}, {"sent": "is she stuck ?", "words": ["stuck", "she", "is"]}, {"sent": "you are stuck .", "words": ["are", "stuck", "you"]}, {"sent": "its stuck to your hand ?", "words": ["your", "to", "stuck", "hand"]}, {"sent": "I cant get it off its stuck pull pull hard uh .", "words": ["off", "pull", "get", "it", "stuck", "hard"]}, {"sent": "you have a piece of blue clay stuck right here .", "words": ["of", "blue", "have", "you", "here", "stuck", "a"]}, {"sent": "youre getting crabby when its stuck down .", "words": ["down", "stuck", "when"]}, {"sent": "are you stuck ?", "words": ["are", "stuck", "you"]}, {"sent": "duck .", "words": ["duck"]}, {"sent": "youre saying quackquacks but thats not because of the toy duck in the bath .", "words": ["the", "of", "toy", "in", "but", "duck", "not", "because"]}, {"sent": "a duck ?", "words": ["duck", "a"]}, {"sent": "duck duck duck .", "words": ["duck"]}, {"sent": "just round the corner and up the hill and theres a nice duck pond .", "words": ["the", "up", "nice", "and", "duck", "a"]}, {"sent": "a duck .", "words": ["duck", "a"]}, {"sent": "and theres the baby duck .", "words": ["the", "duck", "and"]}, {"sent": "duck duck duck duck duck duck duck duck .", "words": ["duck"]}, {"sent": "lets put the duck back too .", "words": ["the", "put", "back", "too", "duck"]}, {"sent": "is that duck ?", "words": ["duck", "that", "is"]}, {"sent": "in every page youll hafta find a little duck .", "words": ["little", "in", "every", "find", "duck", "a"]}, {"sent": "do you think thats a duck aswell Matthew ?", "words": ["do", "think", "you", "duck", "a"]}, {"sent": "the duck clears his house .", "words": ["the", "his", "house", "duck"]}, {"sent": "Donald duck whos at ?", "words": ["duck", "at"]}, {"sent": "and this duck has gone to the left hand side .", "words": ["the", "hand", "to", "and", "this", "duck"]}, {"sent": "no chairs for that duck ?", "words": ["duck", "for", "that"]}, {"sent": "wheres the duck ?", "words": ["the", "duck"]}, {"sent": "what does a duck say Morgan ?", "words": ["what", "does", "say", "duck", "a"]}, {"sent": "if you hafta make poo you can do it in duck pot .", "words": ["can", "do", "if", "you", "in", "it", "duck", "make"]}, {"sent": "oh youve got a duck ?", "words": ["duck", "a"]}, {"sent": "what does the duck say ?", "words": ["the", "what", "does", "say", "duck"]}, {"sent": "Mother duck swims with her ducklings .", "words": ["duck", "her", "with"]}, {"sent": "duck puzzle .", "words": ["duck", "puzzle"]}, {"sent": "duck .", "words": ["duck"]}, {"sent": "I think its a duck .", "words": ["duck", "think", "a"]}, {"sent": "duck needs your help to seek them out .", "words": ["help", "your", "to", "them", "out", "duck"]}, {"sent": "wheres your duck ?", "words": ["duck", "your"]}, {"sent": "quack quack no thats a duck .", "words": ["duck", "a"]}, {"sent": "JJ wheres the duck go ?", "words": ["the", "duck", "go"]}, {"sent": "a duck ?", "words": ["duck", "a"]}, {"sent": "thats a special duck .", "words": ["duck", "a"]}, {"sent": "a duck .", "words": ["duck", "a"]}, {"sent": "wanna touch the duck the fuzzy duck ?", "words": ["the", "duck", "touch", "wanna"]}, {"sent": "wheres the duck ?", "words": ["the", "duck"]}, {"sent": "duck for Donie .", "words": ["duck", "for"]}, {"sent": "see the duck .", "words": ["the", "see", "duck"]}, {"sent": "I dont think thats a duck .", "words": ["duck", "dont", "think", "a"]}, {"sent": "look duck .", "words": ["duck", "look"]}, {"sent": "a duck .", "words": ["duck", "a"]}, {"sent": "a duck begins with d .", "words": ["duck", "with", "a"]}, {"sent": "whats the bunny saying to the duck ?", "words": ["the", "to", "bunny", "duck"]}, {"sent": "following their mommy duck .", "words": ["duck", "their"]}, {"sent": "yes the duck goes in the water .", "words": ["the", "in", "duck", "water"]}, {"sent": "duck says quack .", "words": ["duck"]}, {"sent": "is that a little duck ?", "words": ["is", "little", "duck", "that", "a"]}, {"sent": "duck .", "words": ["duck"]}, {"sent": "duck .", "words": ["duck"]}, {"sent": "well I see the duck .", "words": ["the", "see", "duck"]}, {"sent": "duck .", "words": ["duck"]}, {"sent": "how about the duck .", "words": ["the", "how", "duck", "about"]}, {"sent": "a duck ?", "words": ["duck", "a"]}, {"sent": "what does the duck say ?", "words": ["the", "what", "does", "say", "duck"]}, {"sent": "be like a duck ?", "words": ["duck", "be", "like", "a"]}, {"sent": "a yellow duck .", "words": ["duck", "yellow", "a"]}, {"sent": "duck .", "words": ["duck"]}, {"sent": "this is Mummy duck feeding baby duck .", "words": ["duck", "this", "is"]}, {"sent": "what did mother duck say ?", "words": ["duck", "what", "say", "did"]}, {"sent": "wheres your duck ?", "words": ["duck", "your"]}, {"sent": "the yellow duck button .", "words": ["the", "yellow", "button", "duck"]}, {"sent": "can you see a little duck in any of these pictures ?", "words": ["can", "these", "of", "any", "little", "you", "in", "duck", "see", "a"]}, {"sent": "Ive got one about a duck .", "words": ["duck", "about", "a"]}, {"sent": "is that a duck ?", "words": ["duck", "that", "a", "is"]}, {"sent": "what happened to duck the other day ?", "words": ["the", "what", "to", "other", "duck"]}, {"sent": "a silly duck .", "words": ["duck", "a"]}, {"sent": "Im gonna put the duck back .", "words": ["the", "put", "back", "duck"]}, {"sent": "how about the duck .", "words": ["the", "how", "duck", "about"]}, {"sent": "because suppose it looks a bit like a duck to you but he is actually called a stork .", "words": ["is", "to", "like", "you", "it", "he", "but", "duck", "because", "a"]}, {"sent": "what does a duck say ?", "words": ["what", "does", "say", "duck", "a"]}, {"sent": "wheres where does the duck live ?", "words": ["where", "does", "duck", "the"]}, {"sent": "one day I saw a downy duck with feathers on her back .", "words": ["back", "with", "on", "duck", "her", "a"]}, {"sent": "whats that sort of TweetiePie duck thing called ?", "words": ["of", "that", "duck"]}, {"sent": "find mummy a duck .", "words": ["duck", "a", "find"]}, {"sent": "are you gonna give the the chicken one or is it a duck ?", "words": ["the", "is", "you", "are", "it", "duck", "chicken", "give", "a"]}, {"sent": "ah shall we have a look at this duck ?", "words": ["we", "have", "this", "at", "look", "duck", "a"]}, {"sent": "uhhuh it is a duck .", "words": ["duck", "it", "a", "is"]}, {"sent": "following their morning duck splish their mommy duck .", "words": ["duck", "their"]}, {"sent": "heres the duck .", "words": ["the", "duck"]}, {"sent": "so now we gotta look through these and see if we can find any tha of the duck ones or the cup ones .", "words": ["can", "these", "the", "cup", "if", "we", "of", "so", "any", "find", "and", "look", "duck", "see"]}, {"sent": "a duck .", "words": ["duck", "a"]}, {"sent": "duck ?", "words": ["duck"]}, {"sent": "well hear the duck .", "words": ["the", "duck", "hear"]}, {"sent": "oh whats the duck doing ?", "words": ["the", "duck"]}, {"sent": "duck right .", "words": ["duck"]}, {"sent": "duck .", "words": ["duck"]}, {"sent": "but doctor duck did not ask Felixs mama to leave the room doctor duck let Felixs mama stay with him the whole time .", "words": ["the", "to", "with", "him", "room", "did", "duck", "stay", "not", "but"]}, {"sent": "duck looking at me .", "words": ["duck", "me", "at"]}, {"sent": "so DippyDuck begins duck begins with DippyDuck .", "words": ["duck", "so", "with"]}, {"sent": "we made it to the duck pond .", "words": ["the", "we", "to", "it", "duck"]}, {"sent": "I see a yellow duck looking at me .", "words": ["yellow", "me", "at", "duck", "see", "a"]}, {"sent": "look theres a duck .", "words": ["duck", "a", "look"]}, {"sent": "Im not a duck xxx that costume .", "words": ["duck", "that", "not", "a"]}, {"sent": "not sure that might be a duck .", "words": ["be", "duck", "that", "not", "a"]}, {"sent": "a duck .", "words": ["duck", "a"]}, {"sent": "no thats the duck .", "words": ["the", "duck"]}, {"sent": "is it a duck ?", "words": ["duck", "it", "a", "is"]}, {"sent": "is that a duck ?", "words": ["duck", "that", "a", "is"]}, {"sent": "whatd mother duck say ?", "words": ["duck", "say"]}, {"sent": "a fixit store ?", "words": ["store", "a"]}, {"sent": "are you gonna work at the store ?", "words": ["the", "work", "store", "you", "are", "at"]}, {"sent": "you going go the store ?", "words": ["go", "the", "store", "you"]}, {"sent": "and heres wheres the lady that would sell at the store ?", "words": ["the", "store", "would", "and", "at", "that"]}, {"sent": "okay youre going to the store .", "words": ["the", "to", "store"]}, {"sent": "see theres mommy went to the store to get some food .", "words": ["the", "to", "store", "get", "food", "see", "some"]}, {"sent": "hes gonna get inside the store stuck inside the store .", "words": ["the", "store", "inside", "get", "stuck"]}, {"sent": "this is a store right ?", "words": ["this", "store", "a", "is"]}, {"sent": "xxx xxx shop in the store .", "words": ["the", "store", "in"]}, {"sent": "this store will be closing in thirty minutes .", "words": ["store", "be", "this", "in", "will"]}, {"sent": "she te she was g she liked the doll at first and she does seem to like babydolls when we see them in the store and what not .", "words": ["the", "does", "what", "we", "was", "to", "first", "them", "like", "doll", "and", "store", "not", "in", "at", "see", "when", "she"]}, {"sent": "why dont you wait just a minute until we look at the store at the at behind here okay ?", "words": ["the", "wait", "we", "store", "behind", "you", "here", "dont", "at", "look", "why", "a"]}, {"sent": "okay youd better get your store back in order .", "words": ["better", "your", "back", "store", "get", "in"]}, {"sent": "do you wanna go shopping in the store ?", "words": ["the", "do", "go", "store", "you", "in", "wanna"]}, {"sent": "one store was even having its picture taken .", "words": ["was", "store", "picture"]}, {"sent": "he doesnt go to the store to get his food .", "words": ["the", "go", "to", "store", "get", "food", "he", "his"]}, {"sent": "lets start like were going to the grocery store .", "words": ["the", "to", "store", "like", "were"]}, {"sent": "I think this is a store under here .", "words": ["think", "is", "store", "here", "this", "under", "a"]}, {"sent": "shall we take dolly to the store ?", "words": ["the", "we", "to", "store", "take"]}, {"sent": "Ive never seen it in a store .", "words": ["store", "in", "it", "a"]}, {"sent": "the Kim and carrots store has toys cookies and fruit for sale .", "words": ["the", "for", "store", "and", "carrots"]}, {"sent": "do you buy some chips at the food store ?", "words": ["the", "do", "store", "you", "food", "at", "some", "buy"]}, {"sent": "the store .", "words": ["the", "store"]}, {"sent": "you be the man who runs the store .", "words": ["the", "be", "store", "you", "who"]}, {"sent": "weve got some boxes of cornflakes and one box of RiceKrispies and we havent a lot of room to store them .", "words": ["box", "we", "of", "to", "store", "a lot", "them", "and", "room", "some", "a"]}, {"sent": "you cant buy babies at the store ?", "words": ["the", "store", "you", "at", "buy"]}, {"sent": "on Friday Max and Ruby went to the store to buy a birthday present for Grandma .", "words": ["the", "for", "present", "to", "store", "and", "on", "buy", "a"]}, {"sent": "did you go to the store last night ?", "words": ["the", "go", "to", "store", "last", "you", "did"]}, {"sent": "you get it from the store ?", "words": ["the", "store", "you", "get", "it"]}, {"sent": "store ?", "words": ["store"]}, {"sent": "well go the store in a minute .", "words": ["the", "go", "store", "in", "a"]}, {"sent": "xxx and he followed her in the store .", "words": ["the", "store", "and", "in", "he", "her"]}, {"sent": "right whats the name of the coffee store ?", "words": ["the", "of", "store", "coffee"]}, {"sent": "one in the grocery store .", "words": ["the", "store", "in"]}, {"sent": "and go in the store and say hi .", "words": ["the", "go", "store", "say", "and", "in"]}, {"sent": "see if you can find the store while Dad puts this together for you .", "words": ["can", "the", "for", "if", "store", "you", "this", "find", "see"]}, {"sent": "youre sposta store the Mister Potatohead pieces in there but you can put Bert and Ernie in there for now .", "words": ["the", "can", "put", "for", "store", "you", "and", "there", "in", "but"]}, {"sent": "I was at the store and oh .", "words": ["the", "was", "store", "and", "at"]}, {"sent": "so hes going inside the store .", "words": ["the", "so", "inside", "store"]}, {"sent": "this little store .", "words": ["this", "store", "little"]}, {"sent": "hurry we hafta go to the store pretty soon .", "words": ["the", "hurry", "go", "we", "to", "store", "pretty"]}, {"sent": "its a little intercom so if somebodys in another part of the store like lets say so if somebody brought you something that didnt have a price tag on it you could say okay we need a price check for spaghetti .", "words": ["the", "of", "like", "could", "store", "another", "have", "it", "on", "that", "for", "if", "spaghetti", "so", "little", "you", "a", "need", "we", "say", "in"]}, {"sent": "oh you took it off the box already in the store ?", "words": ["off", "the", "box", "store", "you", "in", "it"]}, {"sent": "thats him he has a grocery store see ?", "words": ["store", "him", "he", "see", "a"]}, {"sent": "we went up the escalator at the store just like Corduroy .", "words": ["up", "the", "we", "store", "like", "at"]}, {"sent": "youre gonna be the man who owns the store huh ?", "words": ["the", "be", "store", "who"]}, {"sent": "did you have fun at the store ?", "words": ["the", "store", "you", "have", "did", "at"]}, {"sent": "they watch the new neighbor come out of the store .", "words": ["the", "watch", "of", "store", "they", "out", "new"]}, {"sent": "why shouldnt I go in the store ?", "words": ["the", "go", "store", "in", "why"]}, {"sent": "a dozen eggs from the grocery store .", "words": ["the", "store", "a"]}, {"sent": "in the store ?", "words": ["the", "store", "in"]}, {"sent": "a radido at the store .", "words": ["the", "store", "at", "a"]}, {"sent": "do you wanna look at a book first or play with the toys first or play store ?", "words": ["the", "do", "store", "first", "play", "with", "you", "at", "wanna", "look", "book", "a"]}, {"sent": "mhm mama came back from the store .", "words": ["the", "store", "back"]}, {"sent": "the German store .", "words": ["the", "store"]}, {"sent": "remember the name of the store ?", "words": ["the", "of", "store"]}, {"sent": "grandpa and grandma went to the store .", "words": ["the", "to", "store", "and"]}, {"sent": "at the grocery store .", "words": ["the", "store", "at"]}, {"sent": "did you tell Daddy what store we went to ?", "words": ["what", "we", "store", "to", "you", "did"]}, {"sent": "what else would the dog buy at the store anything else ?", "words": ["the", "what", "dog", "store", "at", "would", "buy"]}, {"sent": "except the convenient store .", "words": ["the", "store"]}, {"sent": "and at the store I work at theres babies a lot too .", "words": ["the", "work", "a lot", "store", "and", "at", "too", "a"]}, {"sent": "what other stuff do you have in your store .", "words": ["what", "do", "your", "store", "you", "have", "other", "in"]}, {"sent": "did you see it in the store ?", "words": ["the", "store", "you", "in", "it", "did", "see"]}, {"sent": "here heres the store .", "words": ["here", "the", "store"]}, {"sent": "grocery store ?", "words": ["store"]}, {"sent": "okay let me go back to the store .", "words": ["the", "back", "go", "to", "store", "me"]}, {"sent": "this store will close in five minutes .", "words": ["store", "this", "in", "will", "close"]}, {"sent": "were you a good girl at the store ?", "words": ["the", "store", "you", "good", "at", "were", "a"]}, {"sent": "mom they have these things at the xxx store ?", "words": ["the", "these", "store", "have", "they", "at"]}, {"sent": "okay now what do you wanna store in this bag ?", "words": ["what", "do", "store", "you", "this", "in", "wanna"]}, {"sent": "where are you gonna store them ?", "words": ["where", "store", "them", "you", "are"]}, {"sent": "come back from the store .", "words": ["the", "store", "back"]}, {"sent": "take the car apart and play store .", "words": ["the", "car", "store", "play", "and", "take"]}, {"sent": "look at our store at home .", "words": ["look", "store", "our", "at", "home"]}, {"sent": "well the ones that had price stickers were bought at the store xxx store .", "words": ["the", "store", "at", "that", "were"]}, {"sent": "is Patches at the store ?", "words": ["the", "store", "at", "is"]}, {"sent": "you go the the big store every day ?", "words": ["the", "go", "store", "you", "every", "big"]}, {"sent": "you look for the pictures on the on the store fronts and tell me which one is the barber shop .", "words": ["the", "for", "is", "store", "you", "and", "look", "on", "me", "which"]}, {"sent": "who once lived in the toy department of a big store .", "words": ["the", "of", "store", "toy", "in", "who", "big", "a"]}, {"sent": "and then you went to see the football shoes in the store .", "words": ["the", "to", "store", "you", "and", "in", "see", "then"]}, {"sent": "or do you think MissisDyllisPrice wants some at the store to sell ?", "words": ["the", "do", "think", "store", "to", "you", "at", "some"]}, {"sent": "Kim is pretending to be the woman behind the counter that we buy things from at the store .", "words": ["the", "is", "we", "to", "be", "store", "behind", "at", "that", "buy"]}, {"sent": "a store ?", "words": ["store", "a"]}, {"sent": "are you back from the store ?", "words": ["the", "back", "store", "you", "are"]}, {"sent": "does it look like a store ?", "words": ["does", "store", "like", "it", "look", "a"]}, {"sent": "and its where they store things in .", "words": ["where", "store", "and", "they", "in"]}, {"sent": "you remember that store ?", "words": ["store", "that", "you"]}, {"sent": "I think its gonna be time to go out to the store soon .", "words": ["the", "think", "go", "to", "be", "store", "out"]}, {"sent": "I went to the store and I got you a little present .", "words": ["the", "present", "to", "store", "you", "and", "little", "a"]}, {"sent": "we passed two nuns in the grocery store and Adam told his daddy .", "words": ["the", "we", "store", "and", "in", "his"]}, {"sent": "I hafta go shopping and youre very bad to handle in the sh store ?", "words": ["the", "go", "to", "store", "bad", "and", "in"]}, {"sent": "you wanna get them at the toy store ?", "words": ["the", "store", "them", "you", "get", "toy", "wanna", "at"]}, {"sent": "xxx we went to the pet store but we stopped by another by the bank xxx we looked in and I told you about a pichur maybe you didnt .", "words": ["the", "we", "to", "store", "by", "another", "and", "in", "you", "about", "but", "a"]}, {"sent": "well get to the store sometime today .", "words": ["get", "to", "store", "the"]}, {"sent": "I got it home I bought it I bought the shade in a different store but I bought the base at pottery barn kids .", "words": ["the", "store", "in", "it", "at", "home", "but", "a"]}, {"sent": "thats fish soup .", "words": ["soup", "fish"]}, {"sent": "soup .", "words": ["soup"]}, {"sent": "want some of that soup ?", "words": ["of", "some", "soup", "that"]}, {"sent": "you had your chance to have soup and you said you didnt want any more .", "words": ["your", "to", "any", "you", "have", "and", "more", "soup"]}, {"sent": "its soup .", "words": ["soup"]}, {"sent": "and carrot soup .", "words": ["soup", "and"]}, {"sent": "what kind of soup is it today ?", "words": ["what", "is", "of", "it", "soup"]}, {"sent": "youve got two tins of soup .", "words": ["of", "soup"]}, {"sent": "do you wanna make some soup ?", "words": ["do", "you", "wanna", "soup", "some", "make"]}, {"sent": "soup .", "words": ["soup"]}, {"sent": "we just wanta have a little soup because were not very hungry so well just drink it outof these .", "words": ["these", "drink", "we", "so", "hungry", "little", "have", "it", "soup", "were", "not", "because", "a"]}, {"sent": "lets eat a little bit more soup and then Ill get you some juice .", "words": ["little", "eat", "and", "more", "get", "you", "soup", "some", "then", "juice", "a"]}, {"sent": "get over here and eat your soup and then ya can have your popsicle .", "words": ["can", "your", "and", "get", "here", "eat", "have", "soup", "then", "popsicle", "over"]}, {"sent": "heres a tin of tomato soup .", "words": ["of", "soup", "a"]}, {"sent": "do you think she wants to eat some soup ?", "words": ["do", "think", "to", "you", "eat", "soup", "some", "she"]}, {"sent": "making lentil soup .", "words": ["soup"]}, {"sent": "you hafta cook the soup .", "words": ["the", "cook", "soup", "you"]}, {"sent": "theres sugar and tea in my soup ?", "words": ["soup", "in", "my", "and"]}, {"sent": "well thats a little piece of styrofoam from the day you and Jwww made lentil soup .", "words": ["the", "of", "you", "little", "and", "soup", "a"]}, {"sent": "oh we have potato leek soup .", "words": ["have", "soup", "potato", "we"]}, {"sent": "you want more soup ?", "words": ["soup", "more", "you"]}, {"sent": "a can of soup .", "words": ["can", "of", "soup", "a"]}, {"sent": "Louie brought them a bowl of banana soup .", "words": ["bowl", "of", "them", "banana", "soup", "a"]}, {"sent": "soup okay .", "words": ["soup"]}, {"sent": "some vegetable soup ?", "words": ["soup", "some"]}, {"sent": "appetizer before your soup ?", "words": ["soup", "your"]}, {"sent": "you gonna eat the soup thats in the dish ?", "words": ["the", "you", "eat", "dish", "in", "soup"]}, {"sent": "I wonder if hes gonna put it in his soup .", "words": ["put", "if", "soup", "in", "it", "his"]}, {"sent": "chocolate soup ?", "words": ["soup", "chocolate"]}, {"sent": "the soup ?", "words": ["the", "soup"]}, {"sent": "TyrannasaurusRex likes Frasers puzzle soup .", "words": ["soup", "puzzle"]}, {"sent": "want some carrot soup ?", "words": ["soup", "some"]}, {"sent": "I dont want any soup .", "words": ["soup", "dont", "any"]}, {"sent": "that is soup Dominic .", "words": ["soup", "that", "is"]}, {"sent": "theyre cooking soup ?", "words": ["soup"]}, {"sent": "its the color is orange but its actually soup .", "words": ["the", "is", "orange", "soup", "but"]}, {"sent": "soup .", "words": ["soup"]}, {"sent": "are you gonna have some alphabet soup too ?", "words": ["you", "have", "are", "too", "soup", "some"]}, {"sent": "duck sauce soup .", "words": ["duck", "soup", "sauce"]}, {"sent": "thats soup .", "words": ["soup"]}, {"sent": "lets go have soup .", "words": ["have", "soup", "go"]}, {"sent": "Ive never had carrot soup before .", "words": ["soup"]}, {"sent": "should we pour some soup in here and these can be each of our little bowls ?", "words": ["can", "these", "each", "we", "of", "be", "pour", "and", "here", "our", "in", "little", "soup", "some"]}, {"sent": "soup in the pan .", "words": ["soup", "the", "in"]}, {"sent": "monkey pour duh soup in dieir .", "words": ["soup", "in", "monkey", "pour"]}, {"sent": "vegetable soup .", "words": ["soup"]}, {"sent": "thats veggie soup .", "words": ["soup"]}, {"sent": "you dont put soup xxx .", "words": ["soup", "dont", "put", "you"]}, {"sent": "are we ready for soup soup is on .", "words": ["for", "is", "we", "on", "are", "soup"]}, {"sent": "youre thinking about chicken eggs or chicken soup today ?", "words": ["soup", "chicken", "about"]}, {"sent": "does monkey want some soup ?", "words": ["soup", "does", "monkey", "some"]}, {"sent": "theres some soup for you .", "words": ["soup", "some", "for", "you"]}, {"sent": "bean and bacon soup ?", "words": ["soup", "and"]}, {"sent": "okay how would you like some soup ?", "words": ["how", "like", "you", "would", "soup", "some"]}, {"sent": "Nathaniels making soup .", "words": ["soup"]}, {"sent": "well have some soup and bread .", "words": ["bread", "and", "have", "soup", "some"]}, {"sent": "these stones will make excellent soup said Siew .", "words": ["will", "these", "soup", "make"]}, {"sent": "soup .", "words": ["soup"]}, {"sent": "thats soup .", "words": ["soup"]}, {"sent": "make some soup .", "words": ["soup", "some", "make"]}, {"sent": "can you pour the soup into the pan ?", "words": ["can", "the", "into", "pour", "you", "soup"]}, {"sent": "theres a waffle on your soup can .", "words": ["can", "your", "soup", "on", "a"]}, {"sent": "you want the peas to go in the soup ?", "words": ["the", "go", "to", "peas", "you", "in", "soup"]}, {"sent": "soup ?", "words": ["soup"]}, {"sent": "pea soup .", "words": ["soup"]}, {"sent": "that is soup .", "words": ["soup", "that", "is"]}, {"sent": "what do you eat soup with ?", "words": ["what", "do", "with", "you", "eat", "soup"]}, {"sent": "excuse me how about baloney and soup ?", "words": ["how", "and", "about", "soup", "me"]}, {"sent": "soup .", "words": ["soup"]}, {"sent": "thats soup .", "words": ["soup"]}, {"sent": "carrot soup .", "words": ["soup"]}, {"sent": "so youre having soup at your feast ?", "words": ["soup", "so", "your", "at"]}, {"sent": "yes Mommys going to have some soup .", "words": ["have", "to", "soup", "some"]}, {"sent": "did you eat soup at Maggies house ?", "words": ["you", "eat", "house", "did", "at", "soup"]}, {"sent": "heres some tomato soup instead .", "words": ["soup", "some"]}, {"sent": "oh you put some tuna in the soup ?", "words": ["the", "put", "tuna", "you", "in", "soup", "some"]}, {"sent": "icecream and cupcake soup ?", "words": ["soup", "and"]}, {"sent": "could I have a tin of tomato soup please .", "words": ["of", "have", "soup", "could", "a"]}, {"sent": "okay you have carrot soup .", "words": ["have", "soup", "you"]}, {"sent": "did you make some soup ?", "words": ["you", "did", "soup", "some", "make"]}, {"sent": "vegetable soup .", "words": ["soup"]}, {"sent": "is that bread for the soup ?", "words": ["the", "bread", "for", "is", "soup", "that"]}, {"sent": "what do you eat soup with ?", "words": ["what", "do", "with", "you", "eat", "soup"]}, {"sent": "would you make some soup for mommy ?", "words": ["for", "you", "would", "soup", "some", "make"]}, {"sent": "cracker soup ?", "words": ["soup", "cracker"]}, {"sent": "oh we have soup .", "words": ["have", "soup", "we"]}, {"sent": "to go in the soup ?", "words": ["the", "go", "to", "in", "soup"]}, {"sent": "sandwich soup ?", "words": ["soup", "sandwich"]}, {"sent": "soup ?", "words": ["soup"]}, {"sent": "eat your soup .", "words": ["eat", "soup", "your"]}, {"sent": "are you making kiwi soup ?", "words": ["soup", "are", "you"]}, {"sent": "good pouring that soup .", "words": ["soup", "that", "good"]}, {"sent": "what kind of things do you wanna put in your soup ?", "words": ["what", "do", "put", "your", "of", "you", "in", "wanna", "soup"]}, {"sent": "you gonna make some soup ?", "words": ["soup", "some", "make", "you"]}, {"sent": "about a whole village and they all make a big pot of soup together with a xxx magic stone .", "words": ["all", "of", "with", "and", "they", "big", "about", "stone", "soup", "make", "a"]}, {"sent": "soup first then dessert .", "words": ["soup", "then", "first"]}, {"sent": "tomato soup .", "words": ["soup"]}, {"sent": "do you like soup ?", "words": ["soup", "do", "like", "you"]}, {"sent": "some asparagus soup .", "words": ["soup", "some"]}, {"sent": "yummy soup .", "words": ["soup"]}, {"sent": "you must stay beside Mummy .", "words": ["stay", "beside", "you"]}, {"sent": "but he loved to be beside the seaside .", "words": ["the", "to", "beside", "be", "he", "but"]}, {"sent": "and sat down beside her .", "words": ["beside", "down", "her", "and"]}, {"sent": "underneath beside or above the witch ?", "words": ["above", "the", "beside"]}, {"sent": "put them why dont you put them beside eachother ?", "words": ["put", "beside", "them", "you", "dont", "why"]}, {"sent": "are you beside Mama ?", "words": ["beside", "are", "you"]}, {"sent": "whats beside side you ?", "words": ["beside", "you"]}, {"sent": "beside and below .", "words": ["beside", "and"]}, {"sent": "oh beside the cupcake .", "words": ["the", "beside"]}, {"sent": "thats a little child sitting on the seat beside his daddy .", "words": ["the", "beside", "little", "his", "on", "a"]}, {"sent": "but Im sitting right beside you .", "words": ["beside", "but", "you"]}, {"sent": "he knelt beside the princess and kissed her gently .", "words": ["the", "beside", "and", "he", "her"]}, {"sent": "now pat her beside the Mommy .", "words": ["the", "beside", "her"]}, {"sent": "stand it up beside the chair so it cant fall over .", "words": ["up", "the", "stand", "beside", "so", "fall", "chair", "it", "over"]}, {"sent": "you must never go beside the cooker because youll hurt yourself .", "words": ["the", "yourself", "go", "hurt", "beside", "you", "because"]}, {"sent": "there it is right beside your foot .", "words": ["your", "is", "beside", "there", "foot", "it"]}, {"sent": "you must stay beside Mummy .", "words": ["stay", "beside", "you"]}, {"sent": "so I brought him in beside me .", "words": ["so", "beside", "him", "in", "me"]}, {"sent": "you sit beside your Mummy and you can show her the puzzle .", "words": ["can", "the", "your", "show", "beside", "and", "you", "puzzle", "sit", "her"]}, {"sent": "and you take it out and you put it beside the other ones .", "words": ["the", "put", "beside", "you", "and", "other", "it", "out", "take"]}, {"sent": "these go beside the church .", "words": ["the", "these", "go", "church", "beside"]}, {"sent": "beside her .", "words": ["beside", "her"]}, {"sent": "I thought I was sitting down beside you .", "words": ["was", "beside", "down", "you"]}, {"sent": "beside little elephant .", "words": ["beside", "elephant", "little"]}, {"sent": "beside that it tastes good .", "words": ["beside", "it", "that", "good"]}, {"sent": "let me get a piece of paper over there beside that .", "words": ["of", "paper", "beside", "get", "there", "that", "me", "over", "a"]}, {"sent": "thatll hafta go beside the this xxx .", "words": ["the", "this", "beside", "go"]}, {"sent": "just beside that .", "words": ["beside", "that"]}, {"sent": "theres a plate beside you there .", "words": ["plate", "beside", "you", "there", "a"]}, {"sent": "do you hafta sit beside the same person every day when youre getting your lunch ?", "words": ["the", "do", "your", "beside", "you", "same", "every", "sit", "when"]}, {"sent": "beside beside the zoo theres a farm .", "words": ["the", "zoo", "beside", "a"]}, {"sent": "Teddy loved to be beside the seaside .", "words": ["the", "to", "beside", "be"]}, {"sent": "its over there beside the lamp .", "words": ["the", "beside", "there", "lamp", "over"]}, {"sent": "i you do the l beside the a .", "words": ["the", "do", "i", "beside", "you", "a"]}, {"sent": "Jonsey can get in beside you .", "words": ["can", "beside", "you", "get", "in"]}, {"sent": "and I ended up bringing him in beside me .", "words": ["up", "beside", "and", "in", "him", "me"]}, {"sent": "ooh look Mickeys interested now look hes sit beside you .", "words": ["beside", "sit", "you", "look"]}, {"sent": "who did you two sit beside today ?", "words": ["beside", "you", "did", "who", "sit"]}, {"sent": "come on come on up beside me .", "words": ["on", "me", "beside", "up"]}, {"sent": "and sat down beside her .", "words": ["beside", "down", "her", "and"]}, {"sent": "beside the doll .", "words": ["the", "beside", "doll"]}, {"sent": "and theres a fish there swimming beside him .", "words": ["beside", "and", "there", "him", "fish", "a"]}, {"sent": "let me give you some friends first to lie beside you and keep you comforted .", "words": ["to", "beside", "first", "you", "and", "me", "give", "some"]}, {"sent": "you didnt sit beside anybody ?", "words": ["beside", "sit", "you"]}, {"sent": "whats that beside the table ?", "words": ["the", "beside", "that", "table"]}, {"sent": "she wants to sit beside me .", "words": ["to", "beside", "me", "sit", "she"]}, {"sent": "well first of all come and sit beside me .", "words": ["all", "of", "beside", "first", "and", "me", "sit"]}, {"sent": "wanna try something else beside the pizza ?", "words": ["the", "beside", "pizza", "wanna", "try"]}, {"sent": "could he sit beside him ?", "words": ["beside", "him", "he", "sit", "could"]}, {"sent": "who did you sleep beside ?", "words": ["sleep", "beside", "you", "did", "who"]}, {"sent": "whats that thing there beside the rat ?", "words": ["the", "there", "beside", "that"]}, {"sent": "uhhuh youre beside the car .", "words": ["the", "car", "beside"]}, {"sent": "Bear loved to be beside the sea .", "words": ["the", "to", "beside", "be"]}, {"sent": "and standing beside the fish ?", "words": ["the", "beside", "fish", "and"]}, {"sent": "and there is a piece of metal right there is a metal pole right beside it .", "words": ["is", "of", "beside", "and", "there", "it", "a"]}, {"sent": "just sitting beside me .", "words": ["me", "beside"]}, {"sent": "right you sit down on the floor beside me okay ?", "words": ["the", "beside", "you", "on", "me", "sit", "down"]}, {"sent": "you can sit beside her .", "words": ["can", "beside", "you", "sit", "her"]}, {"sent": "and when you woke up was Jwww not in bed there beside you ?", "words": ["up", "was", "beside", "bed", "you", "and", "there", "in", "when", "not"]}, {"sent": "it is right beside your chair .", "words": ["your", "is", "beside", "chair", "it"]}, {"sent": "well if we put her beside the counter like that then weve got some room for some letters and parcels havent we ?", "words": ["the", "put", "for", "if", "we", "beside", "like", "and", "that", "room", "some", "then", "her"]}, {"sent": "you wanna sit beside this monkey and read ?", "words": ["beside", "monkey", "you", "and", "this", "read", "wanna", "sit"]}, {"sent": "walking beside the worm .", "words": ["the", "beside"]}, {"sent": "brought him up put him down in the bed put his dodie in lay beside me in the bed .", "words": ["up", "the", "put", "beside", "bed", "in", "him", "his", "me", "down"]}, {"sent": "Bob jogged along the side beside Muck .", "words": ["the", "beside"]}, {"sent": "then settled down beside it .", "words": ["beside", "then", "down", "it"]}, {"sent": "why dont you sit beside Mr Cromer and write what he writes .", "words": ["what", "beside", "write", "you", "and", "dont", "he", "sit", "why"]}, {"sent": "beside you .", "words": ["beside", "you"]}, {"sent": "are they beside the eyes ?", "words": ["the", "they", "beside", "are"]}, {"sent": "what do you do at school beside have naps ?", "words": ["what", "do", "school", "beside", "you", "have", "at"]}, {"sent": "will we move it over here and put the box beside it ?", "words": ["the", "put", "box", "we", "beside", "and", "here", "it", "will", "over"]}, {"sent": "beside .", "words": ["beside"]}, {"sent": "and did you sit beside him ?", "words": ["beside", "you", "and", "him", "did", "sit"]}, {"sent": "cant sit beside you .", "words": ["beside", "sit", "you"]}, {"sent": "well just hafta put them beside the bunny .", "words": ["the", "put", "bunny", "beside", "them"]}, {"sent": "why dont you sit down beside Margot ?", "words": ["beside", "you", "dont", "sit", "down", "why"]}, {"sent": "theyre running beside it arent they ?", "words": ["they", "beside", "it"]}, {"sent": "do you have any beside you ?", "words": ["do", "beside", "any", "you", "have"]}, {"sent": "youll hafta sit beside me and give me a cuddle .", "words": ["give", "beside", "and", "me", "sit", "a"]}, {"sent": "you come into you come into bed beside Mummy and Daddy in the morning dont you ?", "words": ["the", "into", "beside", "bed", "and", "you", "dont", "in"]}, {"sent": "and you put it in beside it .", "words": ["put", "beside", "you", "and", "in", "it"]}, {"sent": "come and stand beside it .", "words": ["stand", "beside", "it", "and"]}, {"sent": "shall I put it beside hippo Harryhippo ?", "words": ["beside", "put", "it"]}, {"sent": "put her beside the daddy .", "words": ["the", "beside", "put", "her"]}, {"sent": "his glasses beside his bed .", "words": ["his", "beside", "bed", "glasses"]}, {"sent": "are you going to sit beside her .", "words": ["to", "beside", "you", "are", "sit", "her"]}, {"sent": "its uh beside the rug .", "words": ["the", "beside"]}, {"sent": "what either beside the .", "words": ["the", "what", "beside"]}, {"sent": "it is right beside you I think .", "words": ["think", "is", "beside", "you", "it"]}, {"sent": "let me get a piece of paper over there beside that .", "words": ["of", "paper", "beside", "get", "there", "that", "me", "over", "a"]}, {"sent": "theres a little dog running beside the tractor isnt there ?", "words": ["the", "beside", "little", "there", "tractor", "dog", "a"]}, {"sent": "our car was parked beside him .", "words": ["was", "car", "beside", "our", "him"]}, {"sent": "well its a little table beside the couch just like those tables over there .", "words": ["the", "those", "table", "beside", "couch", "like", "little", "there", "over", "a"]}, {"sent": "right beside this .", "words": ["this", "beside"]}, {"sent": "you can sit down beside my bottom .", "words": ["can", "my", "bottom", "beside", "you", "sit", "down"]}, {"sent": "why dont you put her put her beside the daddy .", "words": ["the", "put", "beside", "you", "dont", "her", "why"]}, {"sent": "and shes sitting beside xxx on Christmas day .", "words": ["on", "beside", "and"]}, {"sent": "did you put it beside the cup ?", "words": ["the", "cup", "put", "beside", "you", "it", "did"]}, {"sent": "what do you usually do at this library beside get books ?", "words": ["what", "do", "beside", "you", "get", "this", "at"]}, {"sent": "down came the rain and .", "words": ["the", "down", "rain", "and"]}, {"sent": "oh James in the rain .", "words": ["the", "in", "rain"]}, {"sent": "theres uh that door theres rain coming in through the door .", "words": ["the", "rain", "in", "that", "door"]}, {"sent": "that dried up all the rain .", "words": ["up", "all", "the", "rain", "that"]}, {"sent": "oh look at the rain now .", "words": ["the", "at", "rain", "look"]}, {"sent": "out came the sun and dried up all the rain .", "words": ["the", "up", "all", "rain", "sun", "and", "out"]}, {"sent": "thered be lots and lotsof rain wouldnt there ?", "words": ["be", "there", "rain", "and"]}, {"sent": "I think it is xxx to the rain ?", "words": ["the", "think", "rain", "is", "to", "it"]}, {"sent": "it started to rain .", "words": ["to", "it", "rain"]}, {"sent": "do you like the rain ?", "words": ["the", "do", "rain", "like", "you"]}, {"sent": "theres no rain is there ?", "words": ["there", "rain", "is"]}, {"sent": "precipatation is like rain .", "words": ["like", "rain", "is"]}, {"sent": "dont rain on us cloud .", "words": ["rain", "dont", "us", "cloud", "on"]}, {"sent": "I thought it was just rain .", "words": ["was", "it", "rain"]}, {"sent": "oh Im glad the rain has stopped .", "words": ["the", "rain"]}, {"sent": "rain rain go away come back another day .", "words": ["back", "rain", "go", "another", "away"]}, {"sent": "it looks like rain .", "words": ["it", "like", "rain"]}, {"sent": "oh right theres the rain .", "words": ["the", "rain"]}, {"sent": "a very sharp hat this is the top of the table gosh this will keep the rain out wont it ?", "words": ["the", "table", "rain", "is", "of", "will", "this", "it", "out", "hat", "a"]}, {"sent": "its pouring with rain Anne .", "words": ["with", "rain"]}, {"sent": "its about rain .", "words": ["rain", "about"]}, {"sent": "what else have you got for the rain ?", "words": ["the", "what", "for", "rain", "you", "have"]}, {"sent": "I had got the rain cap on me just in time a little while after I closed duh umbrella .", "words": ["the", "rain", "little", "in", "on", "me", "a"]}, {"sent": "summer breezes feel like hands touching you gently like this you see leafy branches of trees dance in the wind wind may bring rain clouds when the rain is over wind blows the clouds away and the sun comes out then the air smells clean and fresh .", "words": ["the", "of", "like", "when", "clean", "dance", "away", "then", "is", "sun", "bring", "you", "and", "see", "wind", "rain", "this", "in", "out", "over"]}, {"sent": "oh it has just started to spot with rain outside .", "words": ["rain", "outside", "to", "with", "it"]}, {"sent": "green rain ?", "words": ["rain", "green"]}, {"sent": "it is not gonna rain ?", "words": ["it", "not", "rain", "is"]}, {"sent": "rain and storms .", "words": ["rain", "and"]}, {"sent": "then the rain washes him out .", "words": ["the", "rain", "him", "out", "then"]}, {"sent": "you dont like rain do you ?", "words": ["do", "rain", "like", "you", "dont"]}, {"sent": "he can sound like the rain .", "words": ["can", "the", "rain", "like", "he"]}, {"sent": "what goes up when the rain comes down ?", "words": ["up", "what", "the", "rain", "down", "when"]}, {"sent": "oh hes under the hes sheltering is he outof the rain ?", "words": ["the", "rain", "is", "under", "he"]}, {"sent": "think how much rain would be inside our house .", "words": ["think", "how", "rain", "be", "inside", "our", "house", "would", "much"]}, {"sent": "you can not go out in the rain .", "words": ["can", "the", "rain", "go", "you", "in", "out", "not"]}, {"sent": "what happens when the rain comes down ?", "words": ["the", "what", "rain", "down", "when"]}, {"sent": "rain .", "words": ["rain"]}, {"sent": "dried up all the rain .", "words": ["up", "all", "rain", "the"]}, {"sent": "it looks like it might rain later .", "words": ["it", "like", "rain"]}, {"sent": "is weve had a lot of rain and Mummy didnt put holes in the bottom of the Teletubbie pots .", "words": ["the", "put", "bottom", "rain", "is", "of", "a lot", "and", "in", "a"]}, {"sent": "theres a little girl in the rain .", "words": ["the", "rain", "little", "in", "a"]}, {"sent": "youre going to bang the rain and the clouds away and pretend its it just sunshining .", "words": ["the", "rain", "to", "and", "away", "it", "pretend"]}, {"sent": "he can sound like the rain .", "words": ["can", "the", "rain", "like", "he"]}, {"sent": "the cars in the rain yeah .", "words": ["the", "in", "rain"]}, {"sent": "yeah thats rain .", "words": ["rain"]}, {"sent": "like theres some people riding bicycles who were stuck in the rain .", "words": ["the", "rain", "like", "in", "stuck", "who", "some", "were"]}, {"sent": "did it rain when we were on holiday ?", "words": ["rain", "we", "it", "did", "on", "when", "were"]}, {"sent": "and then the rain fell down and a lot of the water collected in the compartments and we saw Purdie leaning into it and drinking some water didnt we ?", "words": ["the", "into", "rain", "we", "of", "a lot", "and", "in", "it", "water", "some", "then", "down", "a"]}, {"sent": "and the rain hood up .", "words": ["the", "up", "rain", "and"]}, {"sent": "wanna go outside and look at the rain ?", "words": ["the", "rain", "go", "outside", "and", "at", "wanna", "look"]}, {"sent": "mhm its rain Ill play with you rain .", "words": ["play", "with", "rain", "you"]}, {"sent": "you see all that rain coming down ?", "words": ["all", "rain", "you", "see", "down", "that"]}, {"sent": "xxx the rain with the sun xxx .", "words": ["the", "sun", "with", "rain"]}, {"sent": "do you hear the rain outside on the leaves .", "words": ["the", "do", "rain", "outside", "hear", "you", "on"]}, {"sent": "Im singing in the rain .", "words": ["the", "in", "rain"]}, {"sent": "it sounds like the rain outside doesnt it ?", "words": ["the", "rain", "outside", "like", "it"]}, {"sent": "down like the rain .", "words": ["the", "down", "like", "rain"]}, {"sent": "I think well leave her out a little bit longer because it might rain this afternoon and at least now the suns out .", "words": ["the", "think", "rain", "little", "and", "this", "it", "out", "at", "her", "because", "a"]}, {"sent": "uh its really bad outside and Id hate tuh have tuh walk two or three blocks in the rain .", "words": ["the", "rain", "outside", "hate", "bad", "and", "have", "in", "walk"]}, {"sent": "what do we do in the rain ?", "words": ["the", "what", "do", "rain", "we", "in"]}, {"sent": "well put you in shorts but I think its going to rain later .", "words": ["put", "shorts", "think", "rain", "to", "you", "in", "but"]}, {"sent": "in the rain singing .", "words": ["the", "in", "rain"]}, {"sent": "then the rain stopped .", "words": ["the", "then", "rain"]}, {"sent": "rain .", "words": ["rain"]}, {"sent": "gosh it looks like it has got loads of rain in it .", "words": ["rain", "of", "like", "in", "it"]}, {"sent": "wasnt there a lot of rain today ?", "words": ["rain", "of", "a lot", "there", "a"]}, {"sent": "can you hear the rain ?", "words": ["can", "the", "rain", "hear", "you"]}, {"sent": "its probably going to rain later on .", "words": ["on", "to", "rain"]}, {"sent": "rain comes out of clouds .", "words": ["of", "out", "rain"]}, {"sent": "and when it started to rain theres a lady outside with her dogs and she hadta start running .", "words": ["rain", "to", "outside", "with", "her", "and", "it", "when", "she", "a"]}, {"sent": "go away rain .", "words": ["away", "rain", "go"]}, {"sent": "going to rain Amy .", "words": ["to", "rain"]}, {"sent": "what about the rain ?", "words": ["the", "what", "rain", "about"]}, {"sent": "it wouldnt be much fun in the rain would it ?", "words": ["the", "rain", "be", "in", "it", "would", "much"]}, {"sent": "and dried all the rain .", "words": ["the", "all", "rain", "and"]}, {"sent": "I hope it doesnt rain tonight .", "words": ["it", "rain"]}, {"sent": "all that rain water has gone away now hasnt it ?", "words": ["all", "rain", "away", "it", "water", "that"]}, {"sent": "it is gonna rain on us .", "words": ["rain", "is", "us", "it", "on"]}, {"sent": "all of a sudden it started to rain .", "words": ["all", "rain", "of", "to", "it", "a"]}, {"sent": "its gonna rain .", "words": ["rain"]}, {"sent": "DoctorFoster went to Gloucester in a shower of rain .", "words": ["rain", "shower", "of", "to", "in", "a"]}, {"sent": "rain and storm .", "words": ["rain", "and"]}, {"sent": "Pete you think its gonna rain today ?", "words": ["think", "rain", "you"]}, {"sent": "is that rain ?", "words": ["that", "rain", "is"]}, {"sent": "down came the rain and washed the spider out .", "words": ["the", "rain", "and", "out", "down"]}, {"sent": "joey hops to a pool filled with fresh rain water .", "words": ["rain", "to", "pool", "with", "water", "a"]}, {"sent": "so we walked round to xxx and got him the Beano summer special and back again in the pouring rain .", "words": ["the", "back", "we", "rain", "to", "so", "and", "him", "in"]}, {"sent": "lotsof rain .", "words": ["rain"]}, {"sent": "ya left it out in the rain Sarah .", "words": ["the", "rain", "in", "it", "out"]}, {"sent": "whats the rain made ?", "words": ["the", "rain"]}, {"sent": "do you know why he likes rain ?", "words": ["do", "rain", "you", "he", "why"]}, {"sent": "we went to that ladys house the other day didnt we in the rain ?", "words": ["the", "rain", "we", "to", "other", "in", "house", "that"]}, {"sent": "it has just started to rain .", "words": ["to", "it", "rain"]}, {"sent": "fishes like rain .", "words": ["like", "rain"]}, {"sent": "were looking at the rain arent we ?", "words": ["the", "rain", "we", "at", "were"]}, {"sent": "it looks like rain .", "words": ["it", "like", "rain"]}, {"sent": "is it goinu rain today Graeme ?", "words": ["it", "rain", "is"]}, {"sent": "by the time they arrived rain was falling hard .", "words": ["the", "rain", "was", "by", "they", "hard"]}, {"sent": "what would happen to me if it didnt rain ?", "words": ["what", "if", "rain", "to", "it", "would", "me"]}, {"sent": "thats a hard one isnt it ?", "words": ["it", "hard", "a"]}, {"sent": "throw it down hard .", "words": ["down", "it", "throw", "hard"]}, {"sent": "youve got to push hard .", "words": ["to", "push", "hard"]}, {"sent": "oh that was hard .", "words": ["was", "that", "hard"]}, {"sent": "its kind of hard .", "words": ["of", "hard"]}, {"sent": "is it hard ?", "words": ["it", "hard", "is"]}, {"sent": "after a hard days work Thomas and his friends .", "words": ["work", "and", "hard", "his", "a"]}, {"sent": "youre making it very hard to people to see huh ?", "words": ["see", "to", "it", "hard"]}, {"sent": "dont bite too hard because its made outof plastic and it might break right ?", "words": ["and", "dont", "it", "hard", "too", "bite", "break", "because"]}, {"sent": "hey not so hard Becca .", "words": ["so", "not", "hard"]}, {"sent": "you rub that hard .", "words": ["that", "hard", "you"]}, {"sent": "hey not so hard please .", "words": ["so", "not", "hard"]}, {"sent": "its too hard to throw its not something to throw its not something to throw .", "words": ["throw", "to", "hard", "too", "not"]}, {"sent": "this is hard .", "words": ["this", "hard", "is"]}, {"sent": "ohmygoodness its really hard .", "words": ["hard"]}, {"sent": "er we hafta pull hard .", "words": ["pull", "hard", "we"]}, {"sent": "hard .", "words": ["hard"]}, {"sent": "its very hard .", "words": ["hard"]}, {"sent": "hard ?", "words": ["hard"]}, {"sent": "oh its so hard to get up there .", "words": ["up", "to", "so", "get", "there", "hard"]}, {"sent": "hard .", "words": ["hard"]}, {"sent": "here is a hard one .", "words": ["here", "hard", "a", "is"]}, {"sent": "no you know its kind of hard because we have that tape on .", "words": ["we", "of", "tape", "you", "have", "hard", "on", "that", "because"]}, {"sent": "its very hard .", "words": ["hard"]}, {"sent": "but theyre just a bit hard .", "words": ["hard", "but", "a"]}, {"sent": "pull hard .", "words": ["hard", "pull"]}, {"sent": "I dont think you have to push quite that hard .", "words": ["think", "push", "to", "you", "have", "dont", "hard", "that"]}, {"sent": "push it look really hard .", "words": ["it", "push", "hard", "look"]}, {"sent": "now this is going to be hard .", "words": ["is", "to", "be", "this", "hard"]}, {"sent": "are you do you think you need your drink because weve been doing such a lot of hard work today havent we ?", "words": ["do", "think", "need", "your", "drink", "work", "of", "we", "a lot", "you", "are", "hard", "because", "a"]}, {"sent": "dont do it hard though .", "words": ["dont", "do", "it", "hard"]}, {"sent": "dont dont do everything so hard Rachael .", "words": ["dont", "do", "so", "hard"]}, {"sent": "pull hard .", "words": ["hard", "pull"]}, {"sent": "lemme see before you press it on too hard .", "words": ["you", "it", "hard", "too", "on", "see"]}, {"sent": "okay now let me ask you a hard question okay ?", "words": ["me", "hard", "a", "you"]}, {"sent": "well I thought Id worked very hard on that .", "words": ["on", "that", "hard"]}, {"sent": "or hell smack you hard .", "words": ["hard", "you"]}, {"sent": "is it hard work ?", "words": ["it", "work", "hard", "is"]}, {"sent": "this is a hard one but I can do it .", "words": ["can", "do", "is", "this", "it", "hard", "but", "a"]}, {"sent": "thats right push hard .", "words": ["push", "hard"]}, {"sent": "this is too hard .", "words": ["this", "too", "hard", "is"]}, {"sent": "you reaching so hard .", "words": ["so", "hard", "you"]}, {"sent": "I guess hard to resist .", "words": ["to", "hard"]}, {"sent": "it is hard to open the cream top yes .", "words": ["the", "is", "open", "to", "it", "hard"]}, {"sent": "its hard to do it again .", "words": ["to", "do", "it", "hard"]}, {"sent": "its hard to remember .", "words": ["to", "hard"]}, {"sent": "hard to tell with these shoes .", "words": ["to", "with", "hard", "these"]}, {"sent": "its hard to be older ?", "words": ["to", "be", "hard"]}, {"sent": "theyre hard to do .", "words": ["to", "do", "hard"]}, {"sent": "but Honey you cant really eat it because its dangerous for a little girl to eat hard candy .", "words": ["for", "to", "you", "eat", "little", "it", "but", "hard", "candy", "because", "a"]}, {"sent": "okay its big and hard and you cant eat it .", "words": ["you", "eat", "and", "it", "hard", "big"]}, {"sent": "because seeds are hard right ?", "words": ["hard", "are", "because"]}, {"sent": "push hard .", "words": ["push", "hard"]}, {"sent": "its too hard for you ?", "words": ["you", "for", "hard", "too"]}, {"sent": "hes hard to close up .", "words": ["up", "to", "close", "hard"]}, {"sent": "and the carrots was hard and all .", "words": ["the", "all", "was", "and", "carrots", "hard"]}, {"sent": "thats hard that .", "words": ["that", "hard"]}, {"sent": "it is hard .", "words": ["it", "hard", "is"]}, {"sent": "and its very very hard to make a noise like that .", "words": ["to", "like", "and", "that", "hard", "make", "a"]}, {"sent": "there its hard to do isnt it ?", "words": ["do", "to", "there", "it", "hard"]}, {"sent": "oh you make hard work .", "words": ["work", "make", "hard", "you"]}, {"sent": "have you been running and working hard today ?", "words": ["have", "hard", "and", "you"]}, {"sent": "um I just wanna find something thats not so hard .", "words": ["so", "find", "hard", "wanna", "not"]}, {"sent": "its its not real hard .", "words": ["not", "hard"]}, {"sent": "that was hard in the house .", "words": ["the", "was", "in", "house", "hard", "that"]}, {"sent": "thats a small hard ball I like the soft ones in the house .", "words": ["the", "soft", "like", "ball", "in", "house", "hard", "a"]}, {"sent": "they need the hard work hard .", "words": ["the", "work", "need", "they", "hard"]}, {"sent": "push hard .", "words": ["push", "hard"]}, {"sent": "dont throw it so hard .", "words": ["throw", "so", "dont", "it", "hard"]}, {"sent": "Cinderellas stepmother stepmother makes Cinderella work very hard .", "words": ["work", "hard"]}, {"sent": "dont work too hard .", "words": ["dont", "work", "hard", "too"]}, {"sent": "I think that ones a bit hard Fraser for you .", "words": ["think", "for", "you", "hard", "that", "a"]}, {"sent": "have you bit your thumb too hard ?", "words": ["your", "you", "have", "hard", "too"]}, {"sent": "maybe you need the hard hat on .", "words": ["the", "need", "you", "on", "hard", "hat"]}, {"sent": "dont even have a good hard floor to bang your feet on do you .", "words": ["do", "your", "to", "you", "have", "dont", "hard", "good", "on", "a"]}, {"sent": "push it real hard .", "words": ["it", "push", "hard"]}, {"sent": "real hard .", "words": ["hard"]}, {"sent": "its a bit hard .", "words": ["hard", "a"]}, {"sent": "oh its hard isnt it ?", "words": ["it", "hard"]}, {"sent": "there is so much food on the table it is hard to find Peter .", "words": ["the", "table", "is", "so", "to", "there", "food", "it", "hard", "find", "on", "much"]}, {"sent": "that was hard work Peter .", "words": ["work", "was", "that", "hard"]}, {"sent": "sometimes these are hard for me too .", "words": ["these", "for", "are", "hard", "too", "me"]}, {"sent": "was it hard or easy ?", "words": ["was", "it", "hard"]}, {"sent": "you bump too hard sometime .", "words": ["bump", "too", "hard", "you"]}, {"sent": "oh so hard work .", "words": ["so", "work", "hard"]}, {"sent": "itll be hard for her though too .", "words": ["for", "be", "hard", "too", "her"]}, {"sent": "push him hard .", "words": ["him", "push", "hard"]}, {"sent": "its hard to snap .", "words": ["to", "hard"]}, {"sent": "yeah catch up with Ramsey its hard to catch up with Ramsey .", "words": ["up", "to", "catch", "with", "hard"]}, {"sent": "oh yeah you can hear the tv if you listen hard enough you can hear the water tank .", "words": ["can", "the", "if", "listen", "hear", "you", "tv", "hard", "water"]}, {"sent": "I think if you push them down really hard that theyll go all the way through .", "words": ["the", "all", "think", "push", "if", "go", "them", "you", "hard", "down", "that"]}, {"sent": "blow hard hard hard .", "words": ["blow", "hard"]}, {"sent": "I bet its hard work to be a Mommy .", "words": ["work", "to", "be", "hard", "a"]}, {"sent": "he working hard .", "words": ["he", "hard"]}, {"sent": "xxx hard .", "words": ["hard"]}, {"sent": "it is hard .", "words": ["it", "hard", "is"]}, {"sent": "those are hard to do that .", "words": ["do", "those", "to", "are", "hard", "that"]}, {"sent": "hard to hold ?", "words": ["to", "hold", "hard"]}, {"sent": "its not too hard .", "words": ["not", "hard", "too"]}, {"sent": "hard balls ?", "words": ["hard"]}, {"sent": "the wind blew Pappa bears hat into the street .", "words": ["the", "into", "hat", "street", "wind"]}, {"sent": "ohgoodness his nose his hat his mouth and his pipe have come off .", "words": ["off", "mouth", "and", "have", "hat", "his", "nose"]}, {"sent": "thats a feather see the he took the feather outof her hat see the feathers in her hat the feathers in her hat then he took the feather thats why shes mad .", "words": ["the", "mad", "in", "he", "hat", "see", "then", "her", "why", "a"]}, {"sent": "look we can put another hat on .", "words": ["can", "put", "we", "another", "on", "look", "hat"]}, {"sent": "his hat on right .", "words": ["his", "on", "hat"]}, {"sent": "heres his hat .", "words": ["his", "hat"]}, {"sent": "and a hat too .", "words": ["hat", "too", "a", "and"]}, {"sent": "whos under the hat ?", "words": ["the", "under", "hat"]}, {"sent": "it looks like the lamppost is wearing a hat doesnt it ?", "words": ["the", "is", "like", "it", "hat", "a"]}, {"sent": "looks like Johnny Applesed with a pot on a wearing a pot for a hat .", "words": ["for", "with", "like", "hat", "on", "a"]}, {"sent": "it looks like a hat .", "words": ["hat", "it", "like", "a"]}, {"sent": "hat do you want ?", "words": ["hat", "do", "you"]}, {"sent": "what colors his hat ?", "words": ["his", "what", "hat"]}, {"sent": "flat hat .", "words": ["hat"]}, {"sent": "hat hat .", "words": ["hat"]}, {"sent": "have you seen Jasons cowboy hat Stan ?", "words": ["have", "hat", "you"]}, {"sent": "Cwwws got an umbrella and Ewwws got a hat .", "words": ["hat", "an", "a", "and"]}, {"sent": "xxx Im wearing a funny hat on my head .", "words": ["my", "head", "on", "hat", "a"]}, {"sent": "you think it could be a hat ?", "words": ["think", "be", "you", "it", "hat", "could", "a"]}, {"sent": "do you think his hat at the top might be good ?", "words": ["the", "do", "think", "be", "you", "hat", "at", "his", "good"]}, {"sent": "well I can see I can see somebody here wearing a hat .", "words": ["can", "here", "hat", "see", "a"]}, {"sent": "hiho hiho Ethans got a hat .", "words": ["hat", "a"]}, {"sent": "it looks like a hat .", "words": ["hat", "it", "like", "a"]}, {"sent": "whats hat doing Nomi ?", "words": ["hat"]}, {"sent": "your hat its your hat isnt it ?", "words": ["hat", "it", "your"]}, {"sent": "what color is the Eskimos hat ?", "words": ["the", "what", "hat", "is"]}, {"sent": "thats Robin Hoods hat uhn isnt it ?", "words": ["hat", "it"]}, {"sent": "maybe it would make a good hat for the baby .", "words": ["the", "for", "it", "good", "would", "hat", "make", "a"]}, {"sent": "who has a hat ?", "words": ["hat", "a", "who"]}, {"sent": "put it on with your hat .", "words": ["put", "your", "with", "it", "hat", "on"]}, {"sent": "you have a hat .", "words": ["have", "hat", "a", "you"]}, {"sent": "can you make a hat ?", "words": ["can", "you", "hat", "make", "a"]}, {"sent": "take off your hat so youre not so hot darling .", "words": ["off", "your", "so", "hot", "hat", "take", "not"]}, {"sent": "no its not a hat .", "words": ["hat", "not", "a"]}, {"sent": "did you drop the hat ?", "words": ["the", "you", "drop", "did", "hat"]}, {"sent": "thats a nice hat I might keep it for myself .", "words": ["for", "nice", "it", "myself", "hat", "a"]}, {"sent": "grandma made you a newspaper hat last night didnt she ?", "words": ["last", "you", "hat", "she", "a"]}, {"sent": "can you imagine if you went to a party wearing a sieve instead of a party hat ?", "words": ["can", "if", "of", "to", "you", "party", "hat", "a"]}, {"sent": "he needs a hat .", "words": ["hat", "he", "a"]}, {"sent": "a daddy hat .", "words": ["hat", "a"]}, {"sent": "put your cowboy hat on .", "words": ["hat", "put", "your", "on"]}, {"sent": "a big hat for a big boss .", "words": ["hat", "for", "big", "a"]}, {"sent": "hat .", "words": ["hat"]}, {"sent": "you wanna make a black hat on your snowman ?", "words": ["your", "you", "black", "on", "snowman", "wanna", "hat", "make", "a"]}, {"sent": "is there another hat ?", "words": ["another", "there", "hat", "is"]}, {"sent": "oh putting a hat on him ?", "words": ["hat", "on", "him", "a"]}, {"sent": "you wanna put the hat on the lamb ?", "words": ["the", "put", "you", "on", "wanna", "hat", "lamb"]}, {"sent": "there now she has a hat .", "words": ["hat", "there", "she", "a"]}, {"sent": "thats a hat .", "words": ["hat", "a"]}, {"sent": "well before you put it on you needta put your tabard on so I mean you cant get it over your head if your wearing a hat so just carry the hat down please .", "words": ["the", "put", "your", "if", "so", "head", "you", "get", "carry", "it", "hat", "on", "down", "over", "a"]}, {"sent": "the cats wearing a hat and the birds on the girls hair .", "words": ["the", "and", "hat", "on", "hair", "a"]}, {"sent": "its got a spooky hat on it .", "words": ["hat", "on", "it", "a"]}, {"sent": "oh to get Mummys hat .", "words": ["get", "to", "hat"]}, {"sent": "ya missing a hat .", "words": ["hat", "a"]}, {"sent": "with an UncleSam hat on .", "words": ["hat", "an", "with", "on"]}, {"sent": "has she got her hat on ?", "words": ["hat", "on", "her", "she"]}, {"sent": "get the hat off .", "words": ["get", "the", "off", "hat"]}, {"sent": "go get his hat June .", "words": ["get", "his", "hat", "go"]}, {"sent": "xxx got my hat on .", "words": ["hat", "my", "on"]}, {"sent": "and youre wearing that beautiful hat that you made at nursery arent you ?", "words": ["you", "and", "at", "hat", "that"]}, {"sent": "have no fear said the cat in the hat .", "words": ["the", "have", "cat", "in", "hat"]}, {"sent": "yes put his hat on .", "words": ["his", "put", "on", "hat"]}, {"sent": "a top hat .", "words": ["hat", "a"]}, {"sent": "silly hat .", "words": ["hat"]}, {"sent": "green hat .", "words": ["hat", "green"]}, {"sent": "this hat .", "words": ["hat", "this"]}, {"sent": "bear needs a hat will daddys yellow hat fit ?", "words": ["fit", "will", "bear", "hat", "yellow", "a"]}, {"sent": "wheres your hat ?", "words": ["hat", "your"]}, {"sent": "your butterfly hat ?", "words": ["butterfly", "your", "hat"]}, {"sent": "girls hat .", "words": ["hat"]}, {"sent": "new hat new hat new hat new hat .", "words": ["hat", "new"]}, {"sent": "you putting your foot on the clowns hat ?", "words": ["the", "your", "you", "foot", "hat", "on"]}, {"sent": "you wanna put on my hat ?", "words": ["put", "my", "you", "wanna", "hat", "on"]}, {"sent": "well well put the hat on assoonas we put the jacket on awright ?", "words": ["the", "put", "we", "hat", "on", "jacket"]}, {"sent": "hat .", "words": ["hat"]}, {"sent": "red hat .", "words": ["red", "hat"]}, {"sent": "whos under the hat ?", "words": ["the", "under", "hat"]}, {"sent": "thats a pointed hat .", "words": ["hat", "a"]}, {"sent": "a hat ?", "words": ["hat", "a"]}, {"sent": "and that hat she had yeah uhn didnt she ?", "words": ["hat", "that", "she", "and"]}, {"sent": "her hat keeps falling .", "words": ["hat", "her"]}, {"sent": "and she wears a pointed hat on her head .", "words": ["head", "and", "on", "hat", "her", "she", "a"]}, {"sent": "I havent seen the green hat for a long time .", "words": ["the", "for", "long", "green", "hat", "a"]}, {"sent": "get mittens and hat .", "words": ["get", "mittens", "hat", "and"]}, {"sent": "with a funny hat .", "words": ["hat", "with", "a"]}, {"sent": "a big fat lady sat upon my hat .", "words": ["hat", "my", "big", "a"]}, {"sent": "look at the dolls hat .", "words": ["the", "at", "hat", "look"]}, {"sent": "my hat fell off ?", "words": ["hat", "my", "off"]}, {"sent": "everybodys got a hat but you and me .", "words": ["you", "and", "hat", "me", "but", "a"]}, {"sent": "thats a cute hat .", "words": ["cute", "hat", "a"]}, {"sent": "you got the hat ?", "words": ["the", "hat", "you"]}, {"sent": "does the mans hat come off aswell ?", "words": ["the", "does", "off", "hat"]}, {"sent": "well you gotta take the hat off I think .", "words": ["the", "off", "think", "you", "hat", "take"]}, {"sent": "oh youre putting your BobtheBuilder hat on now .", "words": ["hat", "on", "your"]}, {"sent": "sun hat and veil xxx then he put the space cadet into .", "words": ["the", "put", "into", "sun", "and", "he", "hat", "then"]}, {"sent": "hat .", "words": ["hat"]}, {"sent": "Mummys hat ?", "words": ["hat"]}, {"sent": "lets find your hat and then we can go outside .", "words": ["can", "your", "we", "go", "outside", "and", "find", "hat", "then"]}, {"sent": "ah hor horse right .", "words": ["horse"]}, {"sent": "give horse a ride now cant you ?", "words": ["you", "horse", "give", "ride", "a"]}, {"sent": "do you think the horse is perhaps a police horse and the police mans going to ride it through the streets of Manchester ?", "words": ["the", "do", "think", "is", "of", "to", "you", "and", "it", "horse", "ride", "a"]}, {"sent": "is it a horse ?", "words": ["horse", "it", "a", "is"]}, {"sent": "we put your horse away .", "words": ["put", "your", "we", "away", "horse"]}, {"sent": "the horse .", "words": ["the", "horse"]}, {"sent": "mm thats a horse .", "words": ["horse", "a"]}, {"sent": "heres a horse .", "words": ["horse", "a"]}, {"sent": "you ride a horse ?", "words": ["horse", "ride", "a", "you"]}, {"sent": "and the Mummy horse is running after it .", "words": ["the", "is", "and", "it", "horse"]}, {"sent": "the horse .", "words": ["the", "horse"]}, {"sent": "what the horse eating the bear ?", "words": ["the", "what", "horse", "bear"]}, {"sent": "neigh neigh said the horse .", "words": ["the", "horse"]}, {"sent": "Im a horse .", "words": ["horse", "a"]}, {"sent": "dont think Ive ever sat at the breakfast table with a horse before .", "words": ["the", "think", "table", "with", "dont", "at", "horse", "a"]}, {"sent": "did that horse go mieow ?", "words": ["horse", "that", "did", "go"]}, {"sent": "another horse ?", "words": ["another", "horse"]}, {"sent": "this is a horse .", "words": ["horse", "this", "a", "is"]}, {"sent": "if youd got a horse we could have the horse jumping over those couldnt you ?", "words": ["the", "those", "if", "we", "you", "have", "horse", "could", "over", "a"]}, {"sent": "eats a horse does he ?", "words": ["horse", "does", "he", "a"]}, {"sent": "whats the horse doing ?", "words": ["the", "horse"]}, {"sent": "a big horse .", "words": ["horse", "big", "a"]}, {"sent": "poor little horse .", "words": ["horse", "little", "poor"]}, {"sent": "the horse likes its horse food .", "words": ["the", "horse", "food"]}, {"sent": "but uh it was an ambulance and it said horse ambulance .", "words": ["was", "and", "an", "it", "horse", "but"]}, {"sent": "and the horse .", "words": ["the", "horse", "and"]}, {"sent": "okay what is the horse doing ?", "words": ["the", "what", "horse", "is"]}, {"sent": "the horse dose dont have a horse .", "words": ["the", "have", "dont", "horse", "a"]}, {"sent": "have you ever seen a real horse ?", "words": ["have", "horse", "a", "you"]}, {"sent": "horse .", "words": ["horse"]}, {"sent": "what would you do with a horse ?", "words": ["what", "do", "with", "you", "would", "horse", "a"]}, {"sent": "because Anna has this horse I think aswell .", "words": ["horse", "this", "think", "because"]}, {"sent": "I think the horse piece might go there .", "words": ["the", "think", "go", "there", "horse"]}, {"sent": "you want the other horse ?", "words": ["the", "horse", "other", "you"]}, {"sent": "the horse is going to drive the train is he ?", "words": ["the", "is", "train", "to", "drive", "he", "horse"]}, {"sent": "theres the rocking horse but whered the baby go ?", "words": ["the", "horse", "but", "go"]}, {"sent": "well if that was a real horse itd be fairly squashed .", "words": ["if", "was", "be", "horse", "that", "a"]}, {"sent": "that ones a baby horse ?", "words": ["horse", "that", "a"]}, {"sent": "ah is that a horse ?", "words": ["horse", "that", "a", "is"]}, {"sent": "what do you think the horse says ?", "words": ["the", "what", "do", "think", "you", "horse"]}, {"sent": "can the horse reach now ?", "words": ["can", "the", "horse"]}, {"sent": "you get you get a bit lost when you get to one horse open sleigh dont you petal ?", "words": ["open", "to", "you", "get", "dont", "horse", "when", "a"]}, {"sent": "wheres your horse ?", "words": ["horse", "your"]}, {"sent": "if horse says tiger likes school then tiger says doesnt he ?", "words": ["if", "school", "he", "tiger", "horse", "then"]}, {"sent": "blue horse blue horse what do you see ?", "words": ["what", "do", "you", "blue", "horse", "see"]}, {"sent": "whats the horse doing up there ?", "words": ["the", "horse", "up", "there"]}, {"sent": "at the finish of his bare back act he leapt gracefully from the horse .", "words": ["the", "back", "of", "finish", "horse", "he", "at", "his"]}, {"sent": "which horse can I hafta play with ?", "words": ["can", "play", "with", "horse", "which"]}, {"sent": "thank you horse .", "words": ["horse", "you"]}, {"sent": "you getting off the horse ?", "words": ["off", "the", "horse", "you"]}, {"sent": "horse .", "words": ["horse"]}, {"sent": "that was a horse wasnt it ?", "words": ["was", "it", "horse", "that", "a"]}, {"sent": "its a big horse ?", "words": ["horse", "big", "a"]}, {"sent": "well I think actually the horse is on the choo choo .", "words": ["the", "think", "is", "on", "horse"]}, {"sent": "is it a horse ?", "words": ["horse", "it", "a", "is"]}, {"sent": "Joseph can Joseph touch the horsie Thomas can you bring your horse here please ?", "words": ["can", "the", "your", "bring", "touch", "you", "here", "horse"]}, {"sent": "why whyve you brought horse and cat in .", "words": ["and", "you", "cat", "in", "horse", "why"]}, {"sent": "what are you doing to the horse ?", "words": ["the", "what", "to", "you", "are", "horse"]}, {"sent": "can you say horse ?", "words": ["can", "horse", "say", "you"]}, {"sent": "oh a horse .", "words": ["horse", "a"]}, {"sent": "or or have you got to take the horse to the doctor ?", "words": ["the", "to", "you", "have", "horse", "take"]}, {"sent": "a naughty horse .", "words": ["horse", "naughty", "a"]}, {"sent": "Ill choose this horse .", "words": ["horse", "this"]}, {"sent": "is that your horse ?", "words": ["your", "horse", "that", "is"]}, {"sent": "hes on his horse .", "words": ["on", "his", "horse"]}, {"sent": "yep its a horse the bouncy horse .", "words": ["horse", "the", "a"]}, {"sent": "oh howd the horse get in there horse doesnt go in there .", "words": ["the", "go", "get", "there", "in", "horse"]}, {"sent": "oh you wanna put the horse up there ?", "words": ["the", "up", "put", "you", "there", "wanna", "horse"]}, {"sent": "oh I see you mean horse .", "words": ["horse", "see", "you"]}, {"sent": "well its like a horse but its not a horse .", "words": ["like", "horse", "not", "but", "a"]}, {"sent": "and therell be horse jumping .", "words": ["horse", "be", "and"]}, {"sent": "give the apple to the horse to eat .", "words": ["the", "to", "eat", "horse", "apple", "give"]}, {"sent": "thats a horse yeah .", "words": ["horse", "a"]}, {"sent": "mummie and baby horse .", "words": ["horse", "and"]}, {"sent": "have you ever seen a horse sit down like that ?", "words": ["like", "you", "have", "horse", "sit", "down", "that", "a"]}, {"sent": "to stay strong and healthy the brown horse eats all day .", "words": ["the", "all", "brown", "to", "and", "horse", "stay"]}, {"sent": "horse .", "words": ["horse"]}, {"sent": "horse .", "words": ["horse"]}, {"sent": "a horse ?", "words": ["horse", "a"]}, {"sent": "to his horse ?", "words": ["his", "to", "horse"]}, {"sent": "did you get that from school that horse ?", "words": ["school", "you", "get", "did", "horse", "that"]}, {"sent": "can I have the horse ?", "words": ["can", "the", "have", "horse"]}, {"sent": "a blue horse .", "words": ["horse", "blue", "a"]}, {"sent": "what does the horse say ?", "words": ["the", "what", "does", "say", "horse"]}, {"sent": "a horse gun ?", "words": ["horse", "a"]}, {"sent": "thats the horse .", "words": ["the", "horse"]}, {"sent": "whats that horse doing ?", "words": ["horse", "that"]}, {"sent": "okay you give that to the horse then .", "words": ["the", "to", "then", "you", "horse", "give", "that"]}, {"sent": "hungry horse .", "words": ["horse", "hungry"]}, {"sent": "I dont think its really a horse .", "words": ["horse", "dont", "think", "a"]}, {"sent": "horse and all the kings .", "words": ["horse", "all", "the", "and"]}, {"sent": "thats the horse .", "words": ["the", "horse"]}, {"sent": "is the horse eating some hay ?", "words": ["the", "horse", "some", "is"]}, {"sent": "whats a rocking horse do ?", "words": ["horse", "do", "a"]}, {"sent": "and thats a horse .", "words": ["horse", "a", "and"]}, {"sent": "I think the horse goes under here .", "words": ["the", "think", "here", "under", "horse"]}, {"sent": "you were playing with a horse werent you ?", "words": ["with", "you", "horse", "were", "a"]}, {"sent": "this is a nice horse .", "words": ["is", "nice", "this", "horse", "a"]}, {"sent": "do you know whit the horse says ?", "words": ["the", "do", "horse", "you"]}, {"sent": "and this is a horse shoe .", "words": ["is", "and", "shoe", "this", "horse", "a"]}, {"sent": "let me just get my toast .", "words": ["get", "me", "my", "toast"]}, {"sent": "and Fraser Jamie had um a slice of toast in between didnt he ?", "words": ["toast", "of", "and", "in", "he", "a"]}, {"sent": "butter toast .", "words": ["butter", "toast"]}, {"sent": "or toast ?", "words": ["toast"]}, {"sent": "now eat some more toast please .", "words": ["eat", "some", "more", "toast"]}, {"sent": "the horsie would have toast aswell ?", "words": ["the", "have", "toast", "would"]}, {"sent": "would you like some toast ?", "words": ["toast", "like", "you", "would", "some"]}, {"sent": "and some toast with jam on it .", "words": ["toast", "with", "and", "it", "on", "some"]}, {"sent": "and then you will be able to have your toast .", "words": ["your", "toast", "be", "to", "you", "and", "have", "will", "then"]}, {"sent": "were just gonna toast it a little bit .", "words": ["toast", "little", "it", "were", "a"]}, {"sent": "did you eat that toast would you like more toast ?", "words": ["toast", "like", "you", "eat", "more", "did", "would", "that"]}, {"sent": "you hold my toast .", "words": ["my", "hold", "toast", "you"]}, {"sent": "you had Marmite on that piece of toast .", "words": ["toast", "of", "you", "on", "that"]}, {"sent": "do you like your toast ?", "words": ["do", "your", "toast", "like", "you"]}, {"sent": "the toast has come up .", "words": ["the", "up", "toast"]}, {"sent": "toast on Tigger ?", "words": ["on", "toast"]}, {"sent": "do you want toast Stan ?", "words": ["do", "toast", "you"]}, {"sent": "can I put your fruit on the toast plate ?", "words": ["can", "the", "put", "your", "plate", "toast", "on"]}, {"sent": "Im having a job to talk because Ive got a mouthful of toast and cheese .", "words": ["toast", "of", "to", "and", "talk", "cheese", "because", "a"]}, {"sent": "mummys gonna bring you some toast .", "words": ["some", "bring", "toast", "you"]}, {"sent": "and heres another toast .", "words": ["another", "toast", "and"]}, {"sent": "you ate up the toast .", "words": ["up", "the", "toast", "you"]}, {"sent": "you wanna have toast and Jwww and Jwww have a bagel .", "words": ["toast", "and", "have", "you", "wanna", "a"]}, {"sent": "what would you like bread or toast ?", "words": ["what", "bread", "toast", "like", "you", "would"]}, {"sent": "Ill bet he burned the toast .", "words": ["the", "he", "toast"]}, {"sent": "and what do you like on your toast ?", "words": ["what", "do", "your", "toast", "like", "you", "and", "on"]}, {"sent": "as soon as shes made her toast and gone pee pee .", "words": ["her", "toast", "and"]}, {"sent": "theres some toast for you .", "words": ["some", "for", "toast", "you"]}, {"sent": "you like toast do you ?", "words": ["do", "like", "toast", "you"]}, {"sent": "I pushed your toast towards you and it nearly went on the floor didnt it ?", "words": ["the", "your", "toast", "and", "you", "it", "on"]}, {"sent": "and then you can have some Cornflakes and some toast .", "words": ["can", "toast", "you", "have", "and", "some", "then"]}, {"sent": "why dont you give oh is Pilchard having some toast ?", "words": ["toast", "is", "you", "dont", "some", "give", "why"]}, {"sent": "I think Ill have some jam on toast .", "words": ["think", "toast", "have", "on", "some"]}, {"sent": "well weve got some toast in the toaster .", "words": ["the", "some", "in", "toast"]}, {"sent": "you want two cheese on the toast ?", "words": ["the", "toast", "you", "on", "cheese"]}, {"sent": "you can always have a piece of bread or a piece of toast instead .", "words": ["can", "bread", "toast", "of", "you", "have", "a"]}, {"sent": "we want some more toast .", "words": ["some", "more", "toast", "we"]}, {"sent": "are you eating your toast as well ?", "words": ["are", "your", "toast", "you"]}, {"sent": "but youre talking now about the toast being outside because when it caught fire I threw it out of the patio doors .", "words": ["the", "toast", "of", "outside", "because", "it", "about", "out", "when", "but"]}, {"sent": "Ive just put put a bit of tea and toast on .", "words": ["put", "toast", "of", "and", "on", "a"]}, {"sent": "would you like a slice of toast ?", "words": ["toast", "of", "like", "you", "would", "a"]}, {"sent": "you dont want the toast .", "words": ["the", "dont", "toast", "you"]}, {"sent": "you had toast what did you have on top of your toast ?", "words": ["what", "your", "toast", "of", "you", "have", "did", "on"]}, {"sent": "well Ill eat mine when I finish my toast .", "words": ["my", "toast", "finish", "eat", "mine", "when"]}, {"sent": "who had toast ?", "words": ["toast", "who"]}, {"sent": "what are you gonna want on your toast Nathaniel ?", "words": ["what", "your", "toast", "you", "are", "on"]}, {"sent": "just hold on until Ive finished my toast .", "words": ["on", "my", "hold", "toast"]}, {"sent": "they were talking about making french toast with eggnog .", "words": ["toast", "with", "they", "about", "were"]}, {"sent": "would you like toast with jam ?", "words": ["toast", "with", "like", "you", "would"]}, {"sent": "butter toast .", "words": ["butter", "toast"]}, {"sent": "you dont hate toast .", "words": ["dont", "hate", "toast", "you"]}, {"sent": "could I have some toast please ?", "words": ["have", "some", "could", "toast"]}, {"sent": "Ill have some toast please .", "words": ["have", "some", "toast"]}, {"sent": "thats right when you move the toast you have pictures of Teletubbies under the plate on the plate dont you ?", "words": ["the", "plate", "toast", "of", "you", "have", "under", "dont", "on", "when"]}, {"sent": "xxx another piece of toast in the toaster .", "words": ["the", "toast", "of", "another", "in"]}, {"sent": "um he hasnt liked solid food at all except for just toast and crackers .", "words": ["all", "for", "toast", "and", "food", "he", "at"]}, {"sent": "the toast has gone pop .", "words": ["the", "pop", "toast"]}, {"sent": "cheese on toast with jam ?", "words": ["on", "with", "cheese", "toast"]}, {"sent": "Daddy doesnt want any toast but you can have some toast .", "words": ["can", "toast", "any", "you", "have", "some", "but"]}, {"sent": "just toast and spaghetti ?", "words": ["spaghetti", "toast", "and"]}, {"sent": "toast and rice crispies .", "words": ["toast", "and"]}, {"sent": "do you have something on your toast ?", "words": ["do", "your", "toast", "you", "have", "on"]}, {"sent": "you said you wanted some toast with marmalade .", "words": ["some", "with", "toast", "you"]}, {"sent": "what happens to the toast ?", "words": ["to", "what", "toast", "the"]}, {"sent": "a nice piece of warm toast .", "words": ["of", "nice", "toast", "a"]}, {"sent": "a piece of toast for Mummy .", "words": ["of", "for", "toast", "a"]}, {"sent": "you want toast ?", "words": ["toast", "you"]}, {"sent": "Ill toast you some little bagel .", "words": ["some", "little", "toast", "you"]}, {"sent": "you have toast in front of you dear .", "words": ["toast", "of", "you", "have", "in"]}, {"sent": "can I put it on my toast then ?", "words": ["can", "put", "my", "toast", "it", "on", "then"]}, {"sent": "cinnamon toast .", "words": ["toast"]}, {"sent": "thats Mummys toast ready now .", "words": ["toast"]}, {"sent": "you going to have some toast with some Golly on ?", "words": ["toast", "to", "with", "you", "have", "on", "some"]}, {"sent": "I said by the time Ive done my toast you will have eaten .", "words": ["the", "my", "toast", "you", "by", "have", "will"]}, {"sent": "toast you some bagel ?", "words": ["some", "toast", "you"]}, {"sent": "do you think the fireman and the policeman are ready for a piece of toast now ?", "words": ["the", "do", "think", "for", "toast", "of", "and", "you", "are", "a"]}, {"sent": "you can eat your toast .", "words": ["can", "your", "toast", "you", "eat"]}, {"sent": "xxx I havent got the toast ready yet .", "words": ["the", "toast"]}, {"sent": "I just had a little piece of cheese when you had your cheese toast .", "words": ["your", "when", "toast", "of", "little", "you", "cheese", "a"]}, {"sent": "are you counting your toast ?", "words": ["are", "your", "toast", "you"]}, {"sent": "can you put your er toast in the bin if youve finished it ?", "words": ["can", "the", "put", "your", "if", "toast", "you", "in", "it"]}, {"sent": "she just gets a snack they do like juice and toast or juice and an apple or something like that .", "words": ["do", "toast", "like", "and", "they", "an", "apple", "juice", "that", "she", "a"]}, {"sent": "I havent got any real toast .", "words": ["any", "toast"]}, {"sent": "Weener that piece of toast looks like an old piece of leather .", "words": ["old", "toast", "of", "like", "an", "that"]}, {"sent": "Mummy better butter and put some marmalade on her piece of toast .", "words": ["better", "put", "toast", "of", "and", "butter", "on", "some", "her"]}, {"sent": "oh you know what Matthew I think the dog wants your cream cheese toast Ill be back okay ?", "words": ["the", "what", "think", "your", "toast", "back", "be", "you", "cheese", "dog"]}, {"sent": "he fixed the toast tongs ?", "words": ["the", "he", "toast"]}, {"sent": "when the toast is ready it will say pop wont it ?", "words": ["the", "pop", "toast", "is", "say", "it", "will", "when"]}, {"sent": "you want your toast or should I eat it ?", "words": ["your", "toast", "you", "eat", "it"]}, {"sent": "shall I put some toast give you some toast and you can put the jam on ?", "words": ["can", "the", "put", "toast", "you", "and", "on", "some", "give"]}, {"sent": "have some more toast .", "words": ["have", "some", "more", "toast"]}, {"sent": "then we can put toast on .", "words": ["can", "put", "toast", "we", "on", "then"]}, {"sent": "Ive put some toast in the machine as well .", "words": ["the", "put", "toast", "in", "some"]}, {"sent": "I thought you wanted to eat that french toast .", "words": ["toast", "to", "you", "eat", "that"]}, {"sent": "youre eating banana Im eating toast .", "words": ["banana", "toast"]}, {"sent": "no matter which toast .", "words": ["which", "toast"]}, {"sent": "and well have some toast .", "words": ["have", "some", "toast", "and"]}, {"sent": "dyou eat french toast sometimes ?", "words": ["eat", "toast"]}, {"sent": "she sent them out to play whilst she made some I think it was hot buttered toast wasnt it ?", "words": ["think", "toast", "to", "was", "play", "them", "hot", "it", "out", "some", "she"]}, {"sent": "cluck chicken .", "words": ["chicken"]}, {"sent": "which would be like a like a little chicken xxx no its a whats a little chicken called like when theyre like little yellow yeah chick there ya go .", "words": ["when", "go", "be", "yellow", "like", "little", "there", "would", "chicken", "which", "a"]}, {"sent": "well hes a chicken who likes the water huh ?", "words": ["the", "water", "who", "chicken", "a"]}, {"sent": "Megan give her the chicken and her animals .", "words": ["the", "and", "chicken", "give", "her"]}, {"sent": "or chicken soup ?", "words": ["soup", "chicken"]}, {"sent": "its chicken .", "words": ["chicken"]}, {"sent": "chicken .", "words": ["chicken"]}, {"sent": "oh we could have the chicken and the bee play together .", "words": ["the", "we", "bee", "play", "and", "have", "chicken", "could"]}, {"sent": "and an orange and a tomato and some chicken .", "words": ["chicken", "and", "an", "orange", "some", "a"]}, {"sent": "can you not find the chicken ?", "words": ["can", "the", "you", "find", "chicken", "not"]}, {"sent": "you like chicken ?", "words": ["chicken", "like", "you"]}, {"sent": "youve found the chicken didnt you ?", "words": ["the", "chicken", "you"]}, {"sent": "is that a chicken ?", "words": ["chicken", "that", "a", "is"]}, {"sent": "here is some chicken for you here is some chicken eat the chicken .", "words": ["the", "for", "chicken", "is", "you", "here", "eat", "some"]}, {"sent": "well this is meant to be a chicken leg I think .", "words": ["think", "is", "to", "be", "this", "leg", "chicken", "a"]}, {"sent": "chicken ?", "words": ["chicken"]}, {"sent": "oh the horse is hugging the chicken isnt he ?", "words": ["the", "is", "he", "horse", "chicken"]}, {"sent": "I got I got two chicken nuggets .", "words": ["chicken"]}, {"sent": "one chicken .", "words": ["chicken"]}, {"sent": "would you like me to cook you some chicken wings ?", "words": ["chicken", "to", "like", "you", "cook", "would", "me", "some"]}, {"sent": "the chicken do you put him in them ?", "words": ["the", "do", "put", "them", "you", "him", "in", "chicken"]}, {"sent": "chicken chicken chicken chicken chicken chicken chicken chicken .", "words": ["chicken"]}, {"sent": "whats the chicken say ?", "words": ["the", "chicken", "say"]}, {"sent": "or no a chicken ?", "words": ["chicken", "a"]}, {"sent": "Mama wheres wheres two chicken nuggets ?", "words": ["chicken"]}, {"sent": "was it nice chicken as well ?", "words": ["nice", "was", "it", "chicken"]}, {"sent": "oh is she having some chicken ?", "words": ["some", "chicken", "she", "is"]}, {"sent": "the chicken leg .", "words": ["the", "chicken", "leg"]}, {"sent": "is it chicken or fish ?", "words": ["chicken", "it", "fish", "is"]}, {"sent": "chicken ?", "words": ["chicken"]}, {"sent": "theres a chicken .", "words": ["chicken", "a"]}, {"sent": "a chicken .", "words": ["chicken", "a"]}, {"sent": "theres another chicken .", "words": ["another", "chicken"]}, {"sent": "you said chicken .", "words": ["chicken", "you"]}, {"sent": "a chi lotsof chicken .", "words": ["chicken", "a"]}, {"sent": "some chicken .", "words": ["some", "chicken"]}, {"sent": "do you like being a lady better or a chicken ?", "words": ["better", "do", "like", "you", "chicken", "a"]}, {"sent": "here Megan theres a rooster a chicken .", "words": ["here", "rooster", "chicken", "a"]}, {"sent": "apples and chicken .", "words": ["chicken", "and"]}, {"sent": "so Id like some some cooked chicken please .", "words": ["so", "chicken", "like", "some"]}, {"sent": "whats a chicken say ?", "words": ["chicken", "say", "a"]}, {"sent": "give me chicken .", "words": ["me", "give", "chicken"]}, {"sent": "see this chicken ?", "words": ["see", "this", "chicken"]}, {"sent": "another chicken yeah .", "words": ["another", "chicken"]}, {"sent": "is the chicken hot ?", "words": ["the", "chicken", "hot", "is"]}, {"sent": "roti with chicken .", "words": ["chicken", "with"]}, {"sent": "what about chicken and chips ?", "words": ["what", "and", "about", "chicken"]}, {"sent": "I wonder why this chicken usually cooks too fast in here on on four fifty how can xxx chicken in there for an hour and its not done yet I the oven must not be working right or something Ive never seen it cook so slowly .", "words": ["the", "an", "cook", "oven", "not", "can", "fast", "be", "there", "it", "on", "chicken", "why", "how", "for", "so", "and", "here", "too", "this", "in"]}, {"sent": "beautiful chicken curry .", "words": ["chicken"]}, {"sent": "how many eggs do you think the chicken has laid ?", "words": ["the", "do", "how", "think", "you", "chicken"]}, {"sent": "thats chicken .", "words": ["chicken"]}, {"sent": "yeah you see the chicken ?", "words": ["the", "see", "chicken", "you"]}, {"sent": "chicken .", "words": ["chicken"]}, {"sent": "chick chick chick chick chicken .", "words": ["chicken"]}, {"sent": "so dog food soup and chicken .", "words": ["so", "and", "food", "soup", "chicken", "dog"]}, {"sent": "PostmanPats catching the chicken is he ?", "words": ["the", "chicken", "he", "is"]}, {"sent": "that is the chicken pick it up .", "words": ["the", "up", "is", "pick", "it", "chicken", "that"]}, {"sent": "we bought some chicken and er we bought some vegetables didnt we ?", "words": ["some", "chicken", "and", "we"]}, {"sent": "it was a chicken .", "words": ["was", "it", "chicken", "a"]}, {"sent": "there is a chicken and a rooster .", "words": ["is", "and", "there", "rooster", "chicken", "a"]}, {"sent": "I thought it was why does the chicken cross the street .", "words": ["the", "does", "was", "it", "chicken", "street", "why"]}, {"sent": "Eleanor would you like some mushrooms and chicken ?", "words": ["chicken", "like", "and", "you", "would", "some"]}, {"sent": "Eleanor heres your chicken and mushroom .", "words": ["chicken", "your", "and"]}, {"sent": "no we said chicken .", "words": ["chicken", "we"]}, {"sent": "theyre putting the prices of pork and chicken up .", "words": ["the", "up", "of", "and", "chicken"]}, {"sent": "do you like chicken ?", "words": ["do", "chicken", "like", "you"]}, {"sent": "we have some chicken tetrazzini .", "words": ["have", "some", "chicken", "we"]}, {"sent": "where does the chicken go ?", "words": ["where", "does", "the", "go", "chicken"]}, {"sent": "well they come from a chicken and theyre white .", "words": ["and", "they", "chicken", "white", "a"]}, {"sent": "I didnt give you any chicken did I ?", "words": ["any", "you", "did", "chicken", "give"]}, {"sent": "I like to eat chicken .", "words": ["eat", "to", "like", "chicken"]}, {"sent": "chicken .", "words": ["chicken"]}, {"sent": "its chicken .", "words": ["chicken"]}, {"sent": "now Ive got the most delicious chicken just cooked .", "words": ["the", "chicken"]}, {"sent": "wheres my other chicken ?", "words": ["chicken", "my", "other"]}, {"sent": "the corn and the chicken .", "words": ["the", "chicken", "corn", "and"]}, {"sent": "if weve got some chicken or some salmon .", "words": ["some", "if", "chicken"]}, {"sent": "we better keep the chicken away from them .", "words": ["the", "better", "we", "them", "away", "chicken"]}, {"sent": "chicken .", "words": ["chicken"]}, {"sent": "I know you like chicken .", "words": ["chicken", "like", "you"]}, {"sent": "gimme chicken leg .", "words": ["chicken", "leg"]}, {"sent": "its a chicken leg .", "words": ["chicken", "leg", "a"]}, {"sent": "do you think shed like chicken ?", "words": ["do", "think", "like", "you", "chicken"]}, {"sent": "Mummy putting the chicken off the bone for you .", "words": ["the", "off", "for", "you", "chicken"]}, {"sent": "okay you can go out in a little while have some salad too and some chicken just sit down .", "words": ["can", "go", "chicken", "you", "little", "have", "in", "and", "out", "too", "some", "sit", "down", "a"]}, {"sent": "a chicken .", "words": ["chicken", "a"]}, {"sent": "oh a chicken .", "words": ["chicken", "a"]}, {"sent": "chicken .", "words": ["chicken"]}, {"sent": "please please eat chichahaen chicken haoha please .", "words": ["eat", "chicken"]}, {"sent": "chicken says bak bak bak .", "words": ["chicken"]}, {"sent": "I think shes a chicken isnt she ?", "words": ["chicken", "think", "she", "a"]}, {"sent": "is she a little chicken ?", "words": ["is", "little", "chicken", "she", "a"]}, {"sent": "you had chicken and chips .", "words": ["chicken", "and", "you"]}, {"sent": "can I have some chicken ?", "words": ["can", "some", "have", "chicken"]}, {"sent": "um the way you prepared the the chicken ?", "words": ["the", "chicken", "you"]}, {"sent": "and then what about the chicken ?", "words": ["the", "what", "and", "about", "chicken", "then"]}, {"sent": "chicken ?", "words": ["chicken"]}, {"sent": "chicken .", "words": ["chicken"]}, {"sent": "and the chicken .", "words": ["the", "chicken", "and"]}, {"sent": "you could use some dry jeans .", "words": ["dry", "you", "some", "could", "jeans"]}, {"sent": "pull those jeans up .", "words": ["jeans", "up", "those", "pull"]}, {"sent": "one in your jeans ?", "words": ["in", "your", "jeans"]}, {"sent": "is what these jeans are .", "words": ["what", "these", "is", "are", "jeans"]}, {"sent": "how do you like my blue jeans ?", "words": ["do", "how", "my", "like", "you", "blue", "jeans"]}, {"sent": "here put his jeans on .", "words": ["put", "here", "on", "his", "jeans"]}, {"sent": "see if theres all different types of jeans dont bother .", "words": ["all", "if", "of", "dont", "see", "jeans"]}, {"sent": "Eleanor get your jeans .", "words": ["get", "your", "jeans"]}, {"sent": "cover your jeans up Eleanor .", "words": ["up", "your", "cover", "jeans"]}, {"sent": "do you wanna give daddy some jeans ?", "words": ["do", "you", "wanna", "some", "give", "jeans"]}, {"sent": "should she wear jeans too ?", "words": ["too", "she", "jeans"]}, {"sent": "blue jeans .", "words": ["jeans", "blue"]}, {"sent": "jeans but they mean .", "words": ["they", "but", "jeans"]}, {"sent": "you know the denim jeans ?", "words": ["the", "jeans", "you"]}, {"sent": "so I had got a pair of black jeans cut all up like this here for him .", "words": ["up", "all", "for", "cut", "of", "so", "like", "black", "this", "here", "him", "a", "jeans"]}, {"sent": "youve not put your jeans on .", "words": ["put", "your", "on", "not", "jeans"]}, {"sent": "jeans ?", "words": ["jeans"]}, {"sent": "theres a hole in these jeans .", "words": ["these", "in", "jeans", "a"]}, {"sent": "where are the jeans ?", "words": ["where", "the", "are", "jeans"]}, {"sent": "see a pair of shoes something like that with higher heels to wear with those jeans .", "words": ["those", "of", "to", "with", "like", "jeans", "see", "that", "a"]}, {"sent": "we get to put on your blue jeans now .", "words": ["put", "your", "we", "to", "blue", "get", "on", "jeans"]}, {"sent": "hey I like your I like your shirt and your jeans .", "words": ["your", "shirt", "like", "and", "jeans"]}, {"sent": "a hole in his jeans ?", "words": ["his", "in", "jeans", "a"]}, {"sent": "xxx jeans Mamas gonna put her jeans on now .", "words": ["on", "put", "her", "jeans"]}, {"sent": "shes wearing jeans .", "words": ["jeans"]}, {"sent": "jeans on .", "words": ["on", "jeans"]}, {"sent": "you can give give daddy some jeans .", "words": ["can", "you", "some", "give", "jeans"]}, {"sent": "you could wear some new jeans .", "words": ["you", "some", "could", "new", "jeans"]}, {"sent": "I have my oldest and dirtiest jeans on .", "words": ["my", "and", "have", "on", "jeans"]}, {"sent": "theyre jeans .", "words": ["jeans"]}, {"sent": "there is your jeans .", "words": ["jeans", "there", "your", "is"]}, {"sent": "its like jeans .", "words": ["like", "jeans"]}, {"sent": "a pair of jeans or something .", "words": ["of", "jeans", "a"]}, {"sent": "a pair of jeans .", "words": ["of", "jeans", "a"]}, {"sent": "oh yes you could put it on like this on your jeans couldnt you ?", "words": ["put", "your", "like", "you", "this", "it", "on", "could", "jeans"]}, {"sent": "Mama bought your jeans .", "words": ["your", "jeans"]}, {"sent": "do you want your jeans on ?", "words": ["do", "your", "you", "on", "jeans"]}, {"sent": "let Daddy get his jeans on ?", "words": ["get", "his", "on", "jeans"]}, {"sent": "put the jeans .", "words": ["the", "put", "jeans"]}, {"sent": "can you say jeans ?", "words": ["can", "jeans", "say", "you"]}, {"sent": "like Id prefer to run about pair of jeans and a tshirt .", "words": ["of", "to", "like", "and", "run", "about", "a", "jeans"]}, {"sent": "got blue jeans on .", "words": ["on", "jeans", "blue"]}, {"sent": "can you see any jeans ?", "words": ["can", "any", "you", "see", "jeans"]}, {"sent": "she has some cute jeans on .", "words": ["cute", "on", "some", "she", "jeans"]}, {"sent": "do you want your jeans on ?", "words": ["do", "your", "you", "on", "jeans"]}, {"sent": "Ive got my jeans on .", "words": ["on", "my", "jeans"]}, {"sent": "put jeans on him .", "words": ["on", "put", "him", "jeans"]}, {"sent": "and your jeans .", "words": ["jeans", "your", "and"]}, {"sent": "yes those jeans are very wet .", "words": ["wet", "are", "those", "jeans"]}, {"sent": "no jeans allowed .", "words": ["jeans"]}, {"sent": "did you get some new jeans ?", "words": ["you", "get", "did", "some", "new", "jeans"]}, {"sent": "shes got jeans on .", "words": ["on", "jeans"]}, {"sent": "thats not as bad as blue jeans .", "words": ["jeans", "not", "bad", "blue"]}, {"sent": "money is in my jeans .", "words": ["money", "my", "is", "in", "jeans"]}, {"sent": "let us put some jeans on him .", "words": ["put", "us", "him", "on", "some", "jeans"]}, {"sent": "when you pull your jeans up you pull them from the top .", "words": ["up", "the", "your", "them", "pull", "you", "when", "jeans"]}, {"sent": "put your jeans on .", "words": ["on", "put", "your", "jeans"]}, {"sent": "Thomas jeans Thomas jeans .", "words": ["jeans"]}, {"sent": "why dont we put these blue jeans on ?", "words": ["these", "put", "we", "blue", "dont", "on", "why", "jeans"]}, {"sent": "I cant even get into my jeans .", "words": ["get", "my", "into", "jeans"]}, {"sent": "shes already wearing jeans .", "words": ["jeans"]}, {"sent": "and your new jeans .", "words": ["jeans", "new", "your", "and"]}, {"sent": "shop keepers wear jeans uhn dont they ?", "words": ["dont", "they", "jeans"]}, {"sent": "youve got to put your jeans back on .", "words": ["put", "your", "back", "to", "on", "jeans"]}, {"sent": "should we put the other doll in jeans ?", "words": ["the", "put", "we", "doll", "other", "in", "jeans"]}, {"sent": "Im only putting jeans on you .", "words": ["on", "you", "jeans"]}, {"sent": "and maybe a cute little pair of jeans huh ?", "words": ["cute", "of", "little", "and", "jeans", "a"]}, {"sent": "wheres your jeans come here .", "words": ["here", "your", "jeans"]}, {"sent": "he can get his blue jeans dirty .", "words": ["can", "blue", "get", "he", "dirty", "his", "jeans"]}, {"sent": "so its great getting them out there jeans and all .", "words": ["all", "so", "them", "and", "there", "out", "jeans"]}, {"sent": "here are some blue jeans and some shirts and some sheets .", "words": ["blue", "here", "and", "are", "some", "jeans"]}, {"sent": "listen I think Ill put jeans on you .", "words": ["think", "put", "listen", "you", "on", "jeans"]}, {"sent": "and then they bought him jeans aswell or something .", "words": ["and", "they", "him", "then", "jeans"]}, {"sent": "jeans .", "words": ["jeans"]}, {"sent": "and you wear jeans sometimes you still play with bell bottoms .", "words": ["play", "with", "you", "and", "jeans"]}, {"sent": "you have signs that say no jeans quite often Thomas .", "words": ["say", "you", "have", "that", "jeans"]}, {"sent": "everything they do sweater they do jeans .", "words": ["they", "do", "sweater", "jeans"]}, {"sent": "what about are you going to show your new jeans as well ?", "words": ["what", "your", "show", "to", "you", "are", "about", "new", "jeans"]}, {"sent": "Matt has jeans on doesnt he ?", "words": ["on", "he", "jeans"]}, {"sent": "do you want your jeans on ?", "words": ["do", "your", "you", "on", "jeans"]}, {"sent": "put jeans on ?", "words": ["on", "put", "jeans"]}, {"sent": "you like your jeans dont you ?", "words": ["your", "like", "you", "dont", "jeans"]}, {"sent": "and you cant go in casual clothes or jeans .", "words": ["go", "you", "and", "in", "jeans"]}, {"sent": "let us put his blue jeans on him .", "words": ["put", "blue", "us", "on", "him", "his", "jeans"]}, {"sent": "jeans and trainers .", "words": ["and", "jeans"]}, {"sent": "yep wash your jeans .", "words": ["your", "wash", "jeans"]}, {"sent": "if theyre blue trousers I presume theyll be jeans willnt they ?", "words": ["if", "be", "blue", "they", "jeans"]}, {"sent": "she has jeans on .", "words": ["on", "she", "jeans"]}, {"sent": "your green Guess jeans .", "words": ["your", "green", "jeans"]}, {"sent": "jeans ?", "words": ["jeans"]}, {"sent": "and they said oh you cant come in because theres no jeans allowed .", "words": ["you", "and", "they", "in", "because", "jeans"]}, {"sent": "you dont have jeans on .", "words": ["you", "have", "dont", "on", "jeans"]}, {"sent": "your jeans are wet I think thats because you spilled the water there right ?", "words": ["the", "wet", "think", "your", "you", "there", "are", "water", "because", "jeans"]}, {"sent": "hes a black tee shirt and black jeans .", "words": ["shirt", "and", "black", "a", "jeans"]}, {"sent": "nice jeans on .", "words": ["on", "nice", "jeans"]}, {"sent": "Daddys jeans .", "words": ["jeans"]}, {"sent": "xxx Daddy went to work in his jeans as well didnt he ?", "words": ["work", "to", "in", "he", "his", "jeans"]}, {"sent": "well write Carls name .", "words": ["write"]}, {"sent": "are you sposta write write on that ?", "words": ["write", "you", "are", "on", "that"]}, {"sent": "now how do you write that down ?", "words": ["do", "how", "write", "you", "down", "that"]}, {"sent": "do wanna write do you wanna write your name ?", "words": ["do", "your", "write", "you", "wanna"]}, {"sent": "let me write let me write c r s .", "words": ["me", "write"]}, {"sent": "you write that there .", "words": ["there", "that", "write", "you"]}, {"sent": "okay Ill write mine too .", "words": ["mine", "write", "too"]}, {"sent": "then Im gonna write a .", "words": ["then", "write", "a"]}, {"sent": "to write on .", "words": ["on", "to", "write"]}, {"sent": "let us mummy write your name on this so I know which one did this .", "words": ["your", "so", "write", "us", "this", "did", "on", "which"]}, {"sent": "so Im gonna put the ribbon there and Im gonna write for Griz .", "words": ["the", "put", "for", "so", "write", "and", "there"]}, {"sent": "dont write on the walls .", "words": ["on", "dont", "write", "the"]}, {"sent": "youll hafta write a letter and tell him to bring our gifts here instead of to Texas .", "words": ["of", "to", "bring", "write", "and", "our", "here", "him", "a"]}, {"sent": "look let us write red .", "words": ["red", "us", "write", "look"]}, {"sent": "you get a postcard and you write on those .", "words": ["those", "write", "and", "get", "you", "on", "a"]}, {"sent": "because Im trying to write .", "words": ["to", "write", "because"]}, {"sent": "okay thats my pen Peter and my paper so we better Im the only one who can write on it .", "words": ["the", "better", "my", "can", "we", "so", "paper", "write", "and", "it", "pen", "on", "who"]}, {"sent": "they write .", "words": ["they", "write"]}, {"sent": "let us write Stes name again .", "words": ["us", "write"]}, {"sent": "sometimes you can write your name in them .", "words": ["can", "your", "them", "write", "you", "in"]}, {"sent": "so oh I was gonna write welcome to right ?", "words": ["was", "so", "to", "write"]}, {"sent": "can you write my paper for me for school honey ?", "words": ["can", "my", "for", "school", "paper", "write", "you", "me"]}, {"sent": "then AuntieMabel hadta write a letter thanking Dora for a nice time and saying but Pippin has forgotten her suitcase .", "words": ["for", "nice", "write", "and", "then", "her", "but", "a"]}, {"sent": "you write your name .", "words": ["your", "write", "you"]}, {"sent": "write cat .", "words": ["cat", "write"]}, {"sent": "do you think we should also write it down maybe ?", "words": ["do", "think", "we", "write", "you", "it", "down"]}, {"sent": "just look at all these long thin things you can write with .", "words": ["can", "all", "these", "look", "with", "long", "you", "write", "at"]}, {"sent": "okay Im going to write now Peter there thats your name .", "words": ["to", "your", "write", "there"]}, {"sent": "no you dont write on buttons .", "words": ["on", "dont", "write", "you"]}, {"sent": "of all the things that weve been given so that we can write to the right people to thank them for the right present .", "words": ["the", "all", "can", "for", "present", "we", "of", "so", "to", "them", "write", "that"]}, {"sent": "dw oh to write to write ?", "words": ["to", "write"]}, {"sent": "I dont know if theyre gonna write on those things either .", "words": ["those", "if", "write", "dont", "on"]}, {"sent": "wanna write a Lois a letter ?", "words": ["write", "wanna", "a"]}, {"sent": "why cant you write ?", "words": ["write", "why", "you"]}, {"sent": "you wanna write on the paper ?", "words": ["the", "paper", "write", "you", "wanna", "on"]}, {"sent": "Ill write a lady .", "words": ["write", "a"]}, {"sent": "he is gonna write .", "words": ["he", "write", "is"]}, {"sent": "write circle ?", "words": ["write"]}, {"sent": "let me see you write your name .", "words": ["your", "me", "write", "you", "see"]}, {"sent": "Chrissie could write uh words .", "words": ["could", "write"]}, {"sent": "thats not how you write it .", "words": ["how", "write", "you", "it", "not"]}, {"sent": "then he tried to write his name on the steamy glass with his paw .", "words": ["the", "to", "with", "write", "on", "he", "glass", "his", "then"]}, {"sent": "you write your name .", "words": ["your", "write", "you"]}, {"sent": "I just hafta write down the words .", "words": ["the", "down", "write"]}, {"sent": "we could make one now and you can write your name as well on that paper .", "words": ["can", "your", "we", "paper", "write", "you", "and", "that", "on", "could", "make"]}, {"sent": "he hasta write with that .", "words": ["that", "with", "he", "write"]}, {"sent": "so Ill write one of these and then weve made a start havent we ?", "words": ["these", "we", "of", "so", "write", "and", "then", "a"]}, {"sent": "how can I write ?", "words": ["can", "how", "write"]}, {"sent": "first were gonna write Gla ?", "words": ["first", "were", "write"]}, {"sent": "now you have a pen but whatre you gonna write with write on ?", "words": ["with", "write", "you", "have", "pen", "on", "but", "a"]}, {"sent": "lets see xxx what can we let you write on ?", "words": ["can", "what", "we", "write", "you", "on", "see"]}, {"sent": "you can write with it .", "words": ["can", "with", "write", "you", "it"]}, {"sent": "right Im not I havent written anymore cards yet though so we hafta wait to write more cards .", "words": ["wait", "we", "to", "so", "write", "more", "not"]}, {"sent": "sometimes adults just write notes but it must be fun to put little stickers on special days huh ?", "words": ["put", "to", "be", "write", "little", "it", "on", "but"]}, {"sent": "I need something to write with .", "words": ["to", "with", "need", "write"]}, {"sent": "you no write on table .", "words": ["on", "table", "write", "you"]}, {"sent": "okay write Michael okay this is what were gonna do to that .", "words": ["what", "do", "is", "to", "write", "this", "that", "were"]}, {"sent": "yes shall we write um Raynas name now ?", "words": ["write", "we"]}, {"sent": "I can write .", "words": ["can", "write"]}, {"sent": "you write your name then .", "words": ["then", "your", "write", "you"]}, {"sent": "well youll needta write on paper if you write with that one I think you want ta write on some paper ?", "words": ["think", "if", "paper", "with", "write", "you", "on", "some", "that"]}, {"sent": "write ?", "words": ["write"]}, {"sent": "dont write in the book .", "words": ["the", "write", "dont", "in", "book"]}, {"sent": "how about we write your name .", "words": ["how", "your", "we", "write", "about"]}, {"sent": "you you write Ill hold it .", "words": ["it", "hold", "write", "you"]}, {"sent": "you know how to write your name .", "words": ["how", "your", "to", "write", "you"]}, {"sent": "could write your name with it couldnt you ?", "words": ["your", "with", "write", "you", "it", "could"]}, {"sent": "should I write your name now ?", "words": ["your", "write"]}, {"sent": "you you go and find Sneezy and Dopey and write it down .", "words": ["go", "write", "and", "you", "it", "find", "down"]}, {"sent": "write write .", "words": ["write"]}, {"sent": "Im gonna write them all down when youve been naughty .", "words": ["all", "them", "write", "down", "when", "naughty"]}, {"sent": "Im gonna let you write .", "words": ["write", "you"]}, {"sent": "and now we will write this story this week .", "words": ["we", "write", "and", "this", "will", "story"]}, {"sent": "write Wix .", "words": ["write"]}, {"sent": "what else do you wanna write ?", "words": ["what", "do", "write", "you", "wanna"]}, {"sent": "you gonna write with one at a time .", "words": ["with", "write", "you", "at", "a"]}, {"sent": "your mommy write on your hand ?", "words": ["on", "your", "write", "hand"]}, {"sent": "shall I write again ?", "words": ["write"]}, {"sent": "so shall we write .", "words": ["so", "write", "we"]}, {"sent": "what should I write ?", "words": ["what", "write"]}, {"sent": "and were going to write this to .", "words": ["to", "write", "and", "this", "were"]}, {"sent": "you like to write too dont you Peter ?", "words": ["to", "like", "write", "you", "dont", "too"]}, {"sent": "but weve also got to write eggs on our shopping list because theres only one egg left now .", "words": ["egg", "to", "because", "write", "our", "on", "but"]}, {"sent": "must write it down before I forget .", "words": ["down", "it", "write"]}, {"sent": "well itll be a long time now before we can write to FatherChristmas again wont it ?", "words": ["can", "we", "be", "to", "long", "write", "it", "a"]}, {"sent": "write on here .", "words": ["on", "here", "write"]}, {"sent": "lets write .", "words": ["write"]}, {"sent": "what are you going to write ?", "words": ["what", "to", "write", "you", "are"]}, {"sent": "are you gonna write on the paper ?", "words": ["the", "paper", "write", "you", "are", "on"]}, {"sent": "I didnt write the letter .", "words": ["the", "write"]}, {"sent": "youre a good storyteller Honey you should write a book .", "words": ["write", "you", "good", "book", "a"]}, {"sent": "I think Ill write that down .", "words": ["think", "down", "that", "write"]}, {"sent": "want me to write for Rwww .", "words": ["to", "me", "for", "write"]}, {"sent": "what would you like to write ?", "words": ["what", "to", "like", "write", "you", "would"]}, {"sent": "I have to write Auntie Marie a letter .", "words": ["have", "to", "write", "a"]}, {"sent": "gonna write his name .", "words": ["his", "write"]}, {"sent": "we hafta write Hwwws card as well dont we ?", "words": ["dont", "write", "we"]}, {"sent": "dyou wanna write the eight ?", "words": ["the", "write", "wanna"]}, {"sent": "now see if you can write it down here .", "words": ["can", "if", "write", "you", "here", "it", "see", "down"]}, {"sent": "right well well get your telephone and you can talk to SarahLastname and write some notes down .", "words": ["can", "your", "to", "write", "you", "get", "and", "talk", "some", "down", "telephone"]}, {"sent": "and how many is the one with the big bear ?", "words": ["the", "how", "is", "with", "and", "bear", "big"]}, {"sent": "the bear .", "words": ["the", "bear"]}, {"sent": "bear bear .", "words": ["bear"]}, {"sent": "you got this teddy bear here .", "words": ["bear", "this", "here", "you"]}, {"sent": "teddy bear .", "words": ["bear"]}, {"sent": "is that your bear ?", "words": ["your", "bear", "that", "is"]}, {"sent": "why doesnt the bear like the bear food ?", "words": ["the", "like", "bear", "food", "why"]}, {"sent": "its a big bear isnt it ?", "words": ["bear", "it", "big", "a"]}, {"sent": "point to the bear .", "words": ["the", "to", "bear"]}, {"sent": "he has all those like brown bear books and .", "words": ["all", "those", "brown", "like", "bear", "and", "he"]}, {"sent": "thats the big bear .", "words": ["the", "big", "bear"]}, {"sent": "blueberry pie said the bear I love it .", "words": ["the", "it", "love", "bear"]}, {"sent": "and a bear .", "words": ["bear", "a", "and"]}, {"sent": "you sitting with your bear in the chair .", "words": ["the", "your", "with", "chair", "bear", "you", "in"]}, {"sent": "in the bear ?", "words": ["the", "in", "bear"]}, {"sent": "its a bear a big bear daddy bear .", "words": ["bear", "big", "a"]}, {"sent": "koala bear and a baby koala bear .", "words": ["a", "and", "bear"]}, {"sent": "bear gets a piggyback ride .", "words": ["ride", "a", "bear"]}, {"sent": "he finds a little bear wheres the bears mommy ?", "words": ["the", "bear", "little", "he", "a"]}, {"sent": "are there any bear cubs in the river ?", "words": ["the", "any", "bear", "there", "in", "are"]}, {"sent": "and wheres dancing bear going to go ?", "words": ["bear", "to", "go", "and"]}, {"sent": "baby koala bear rides on Mothers back .", "words": ["on", "back", "bear"]}, {"sent": "oh its another a different bear ?", "words": ["another", "bear", "a"]}, {"sent": "did you give bear hugs ?", "words": ["bear", "give", "did", "you"]}, {"sent": "do you think theres room for rabbit and little bear in that plane ?", "words": ["do", "think", "for", "you", "and", "little", "bear", "room", "in", "that"]}, {"sent": "whos behind the teddy bear ?", "words": ["the", "behind", "bear"]}, {"sent": "isnt it funny how a bear likes honey ?", "words": ["bear", "how", "it", "a"]}, {"sent": "little bear what ?", "words": ["bear", "what", "little"]}, {"sent": "and heres your other teddy bear .", "words": ["bear", "other", "your", "and"]}, {"sent": "baby bear ran into the kitchen .", "words": ["the", "into", "kitchen", "bear"]}, {"sent": "bear in the box .", "words": ["the", "in", "box", "bear"]}, {"sent": "whose behind the teddy bear ?", "words": ["the", "behind", "bear"]}, {"sent": "orange bear ?", "words": ["orange", "bear"]}, {"sent": "but a big birthday bear hug is a nice present .", "words": ["present", "is", "nice", "bear", "hug", "big", "but", "a"]}, {"sent": "see the bear ?", "words": ["the", "see", "bear"]}, {"sent": "my little bunny its time now for bed a bear on your pillow a star overhead .", "words": ["my", "bunny", "for", "your", "pillow", "star", "bed", "little", "bear", "on", "a"]}, {"sent": "wheres the bear ?", "words": ["the", "bear"]}, {"sent": "bear yeah good .", "words": ["good", "bear"]}, {"sent": "is the bear funny ?", "words": ["the", "bear", "is"]}, {"sent": "hes a lucky bear .", "words": ["bear", "a"]}, {"sent": "weve just got big bear .", "words": ["big", "bear"]}, {"sent": "theres a bear .", "words": ["bear", "a"]}, {"sent": "what kind of bear is that ?", "words": ["what", "is", "of", "bear", "that"]}, {"sent": "Sleepys a big bear .", "words": ["bear", "big", "a"]}, {"sent": "you want your bear ?", "words": ["bear", "your", "you"]}, {"sent": "polar bear polar bear .", "words": ["bear"]}, {"sent": "oh is Pooh bear a lil dirty ?", "words": ["bear", "dirty", "a", "is"]}, {"sent": "nightnight bear .", "words": ["bear"]}, {"sent": "oh the bear said Im a bear .", "words": ["the", "a", "bear"]}, {"sent": "mama bear .", "words": ["bear"]}, {"sent": "sweet dreams Pooh bear and goodnight to you little one .", "words": ["to", "little", "you", "and", "bear"]}, {"sent": "I think so I think its a bear .", "words": ["bear", "so", "think", "a"]}, {"sent": "huh oh my gosh a koala bear .", "words": ["bear", "my", "a"]}, {"sent": "uhhuh bear .", "words": ["bear"]}, {"sent": "its a bear .", "words": ["bear", "a"]}, {"sent": "oh Pooh bear .", "words": ["bear"]}, {"sent": "put it around the bear .", "words": ["the", "put", "bear", "it", "around"]}, {"sent": "and theres the brown bear again .", "words": ["the", "bear", "brown", "and"]}, {"sent": "yeah thats a baby koala bear .", "words": ["bear", "a"]}, {"sent": "want Mommy to hug the bear ?", "words": ["hug", "to", "bear", "the"]}, {"sent": "so those are the two main ones we dont we dont do a whole lot oh a panda bear too .", "words": ["the", "do", "those", "we", "so", "bear", "dont", "are", "too", "a"]}, {"sent": "so obviously the rabbit must go back down the plane and just leave Little bear to go into the attic .", "words": ["the", "into", "back", "go", "so", "to", "and", "bear", "down"]}, {"sent": "bear is in the box .", "words": ["the", "box", "is", "bear", "in"]}, {"sent": "say bear .", "words": ["say", "bear"]}, {"sent": "Ive brought you the moon little bear said big bear .", "words": ["the", "moon", "bear", "little", "you", "big"]}, {"sent": "a teddy bear ?", "words": ["bear", "a"]}, {"sent": "a bear said Mrs Brown .", "words": ["bear", "a"]}, {"sent": "that is right bear .", "words": ["bear", "that", "is"]}, {"sent": "is he a big scary bear ?", "words": ["is", "bear", "he", "big", "a"]}, {"sent": "and a panda bear .", "words": ["bear", "a", "and"]}, {"sent": "lets let him see teddy bear .", "words": ["see", "him", "bear"]}, {"sent": "the I know the bear walk one where we took your shoes off ?", "words": ["the", "where", "off", "your", "we", "bear", "walk"]}, {"sent": "a big bear in the jungle .", "words": ["the", "bear", "in", "big", "a"]}, {"sent": "big bear ?", "words": ["big", "bear"]}, {"sent": "I bet he he does wish he could reach the table because hes only a tiny bear isnt he ?", "words": ["the", "does", "tiny", "table", "wish", "bear", "he", "could", "because", "a"]}, {"sent": "bear .", "words": ["bear"]}, {"sent": "thats where Daddy bear sits isnt it ?", "words": ["where", "it", "bear"]}, {"sent": "did you have a nice sleep bear ?", "words": ["sleep", "nice", "you", "have", "bear", "did", "a"]}, {"sent": "that bear .", "words": ["that", "bear"]}, {"sent": "um so the papa bear looked at his bowl he didnt even hafta take a bite .", "words": ["the", "bowl", "bite", "so", "bear", "he", "at", "his", "take", "a"]}, {"sent": "this bear belongs to that .", "words": ["to", "this", "that", "bear"]}, {"sent": "shall we give the bear some juice ?", "words": ["the", "we", "bear", "some", "give", "juice"]}, {"sent": "look at the bear .", "words": ["the", "bear", "at", "look"]}, {"sent": "so generally any type of lion or bear .", "words": ["of", "so", "any", "bear", "lion"]}, {"sent": "Pooh bear .", "words": ["bear"]}, {"sent": "oh wait or do I gotta try to catch you little bear ?", "words": ["do", "wait", "to", "catch", "you", "little", "bear", "try", "try to"]}, {"sent": "a teddy bear and a blankie .", "words": ["bear", "and", "a"]}, {"sent": "Big Bears the big bear and Little Bears the little bear .", "words": ["the", "little", "and", "bear", "big"]}, {"sent": "ah nice bear .", "words": ["nice", "bear"]}, {"sent": "theres a teddy bear .", "words": ["bear", "a"]}, {"sent": "teddy bear teddy bear say your prayers .", "words": ["your", "say", "bear"]}, {"sent": "polar bear .", "words": ["bear"]}, {"sent": "is that a little bear or a kittycat ?", "words": ["is", "bear", "little", "that", "a"]}, {"sent": "and theres here teddy bear .", "words": ["here", "bear", "and"]}, {"sent": "meanwhile downstairs the bears had come home there was a daddy bear a mama bear .", "words": ["the", "was", "bear", "there", "home", "a"]}, {"sent": "bear .", "words": ["bear"]}, {"sent": "a bear bear bear .", "words": ["bear", "a"]}, {"sent": "Pooh bear .", "words": ["bear"]}, {"sent": "and the bear goes .", "words": ["the", "bear", "and"]}, {"sent": "zoo z z z z .", "words": ["zoo"]}, {"sent": "that says zoo look there .", "words": ["zoo", "there", "that", "look"]}, {"sent": "remember when we went to the zoo ?", "words": ["the", "zoo", "we", "to", "when"]}, {"sent": "just like at the zoo .", "words": ["the", "zoo", "like", "at"]}, {"sent": "are just the gates to get inside to the zoo and then this is the zoo inside of here .", "words": ["the", "zoo", "is", "of", "to", "inside", "and", "get", "this", "here", "are", "then"]}, {"sent": "put me in the zoo .", "words": ["the", "zoo", "put", "in", "me"]}, {"sent": "whos he going to see at the zoo ?", "words": ["the", "zoo", "to", "he", "at", "see"]}, {"sent": "how about when we went to the zoo ?", "words": ["the", "zoo", "how", "we", "to", "about", "when"]}, {"sent": "lets go see the animals of the zoo .", "words": ["the", "zoo", "go", "of", "see"]}, {"sent": "not a very good zoo is it ?", "words": ["zoo", "is", "it", "good", "not", "a"]}, {"sent": "oh they would put me in the zoo if they could see what I could do .", "words": ["the", "zoo", "put", "what", "do", "if", "see", "they", "in", "would", "me", "could"]}, {"sent": "youve broken the zoo .", "words": ["the", "zoo", "broken"]}, {"sent": "should we turn the page and see what they see at the zoo ?", "words": ["the", "what", "zoo", "we", "and", "they", "at", "see"]}, {"sent": "whos in the zoo ?", "words": ["the", "zoo", "in"]}, {"sent": "the zoo ?", "words": ["the", "zoo"]}, {"sent": "xxx go byebye to zoo .", "words": ["zoo", "to", "go"]}, {"sent": "why have you sat on the zoo ?", "words": ["the", "zoo", "you", "have", "on", "why"]}, {"sent": "I wanna make the pet park zoo .", "words": ["the", "zoo", "park", "wanna", "make"]}, {"sent": "who would like to go to the zoo today ?", "words": ["the", "zoo", "go", "to", "like", "would", "who"]}, {"sent": "blue orange green and violet too oh they would put me in the zoo if they could see what I can do .", "words": ["the", "zoo", "put", "what", "can", "if", "do", "see", "would", "and", "blue", "they", "in", "orange", "green", "too", "me", "could"]}, {"sent": "what animals would you wanna see the most at the zoo ?", "words": ["the", "what", "zoo", "you", "at", "wanna", "would", "see"]}, {"sent": "well at the zoo they have remember they have elephant washers at the zoo ?", "words": ["the", "zoo", "elephant", "have", "they", "at"]}, {"sent": "what happened when she got to the zoo ?", "words": ["the", "what", "zoo", "to", "when", "she"]}, {"sent": "did you see penguins at the zoo ?", "words": ["the", "zoo", "you", "did", "at", "see"]}, {"sent": "go to the zoo ?", "words": ["the", "to", "zoo", "go"]}, {"sent": "you dont why dont you want to go to the zoo ?", "words": ["the", "zoo", "go", "to", "you", "dont", "want to", "why"]}, {"sent": "what did the seals do at the zoo when we went .", "words": ["the", "what", "do", "zoo", "we", "did", "at", "when"]}, {"sent": "wheres the zoo keeper ?", "words": ["the", "zoo"]}, {"sent": "did Jwww go to the zoo ?", "words": ["the", "zoo", "go", "to", "did"]}, {"sent": "is he go to the zoo ?", "words": ["the", "zoo", "go", "is", "to", "he"]}, {"sent": "oh thats xxx my new bear from the zoo .", "words": ["the", "zoo", "my", "bear", "new"]}, {"sent": "the zoo ?", "words": ["the", "zoo"]}, {"sent": "she saw one at the circus and you saw them at the zoo .", "words": ["the", "zoo", "them", "you", "and", "at", "she"]}, {"sent": "is that is this train going to your zoo park ?", "words": ["zoo", "your", "train", "is", "to", "this", "park", "that"]}, {"sent": "you not wanna go to the zoo then ?", "words": ["the", "zoo", "go", "to", "you", "wanna", "then", "not"]}, {"sent": "and who else went to the zoo with Nina ?", "words": ["the", "zoo", "to", "with", "and", "who"]}, {"sent": "theyre going to the zoo .", "words": ["the", "to", "zoo"]}, {"sent": "I dont think the train is gonna fit in the zoo .", "words": ["the", "zoo", "think", "is", "train", "fit", "dont", "in"]}, {"sent": "sometimes they hide when you go to the zoo .", "words": ["the", "zoo", "hide", "go", "to", "you", "they", "when"]}, {"sent": "to Blackpool zoo .", "words": ["zoo", "to"]}, {"sent": "oh thats suppose theyre going to visit the zoo .", "words": ["the", "to", "zoo"]}, {"sent": "thats a zoo .", "words": ["zoo", "a"]}, {"sent": "weve been to the zoo a handful of times its been since the beginning of summer .", "words": ["the", "zoo", "of", "to", "a"]}, {"sent": "I hafta pick out a fun zoo outfit for you .", "words": ["zoo", "for", "pick", "you", "out", "a"]}, {"sent": "put me in the zoo .", "words": ["the", "zoo", "put", "in", "me"]}, {"sent": "you went to the San Diego zoo ?", "words": ["the", "to", "zoo", "you"]}, {"sent": "zoo .", "words": ["zoo"]}, {"sent": "the zoo fell down ?", "words": ["the", "zoo", "down"]}, {"sent": "did we see a giraffe at the zoo ?", "words": ["the", "zoo", "giraffe", "we", "did", "at", "see", "a"]}, {"sent": "heres the zoo .", "words": ["the", "zoo"]}, {"sent": "because I hafta stay in a cage all day and Im not allowed to come out oops here comes my zoo keeper .", "words": ["zoo", "all", "my", "to", "and", "here", "in", "out", "stay", "not", "because", "a"]}, {"sent": "we have some zoo stuff .", "words": ["have", "some", "zoo", "we"]}, {"sent": "do you want um do you want a hippo in your in your zoo ?", "words": ["zoo", "do", "your", "you", "in", "a"]}, {"sent": "because then um if you go on tuesday then well go to the zoo on monday .", "words": ["the", "zoo", "if", "go", "to", "you", "on", "then", "because"]}, {"sent": "and Rachel went to the zoo yesterday .", "words": ["the", "to", "zoo", "and"]}, {"sent": "shall we build a zoo to put them in John ?", "words": ["zoo", "put", "we", "to", "them", "in", "build", "a"]}, {"sent": "oh they would put me in a zoo if they could see what I could do .", "words": ["zoo", "what", "put", "do", "if", "see", "they", "in", "would", "me", "could", "a"]}, {"sent": "we could make a zoo couldnt we ?", "words": ["zoo", "we", "could", "make", "a"]}, {"sent": "but we have no taken him to a zoo yet .", "words": ["zoo", "we", "to", "have", "him", "but", "a"]}, {"sent": "would ya like to play with your new zoo ?", "words": ["zoo", "your", "to", "play", "with", "like", "would", "new"]}, {"sent": "may favorite at the zoo is the bears .", "words": ["the", "zoo", "is", "at"]}, {"sent": "what were the penguins doing when you saw them at the zoo ?", "words": ["the", "what", "zoo", "when", "them", "you", "at", "were"]}, {"sent": "she doesnt know why the giraffe isnt in the zoo .", "words": ["the", "zoo", "giraffe", "in", "she", "why"]}, {"sent": "put me in the zoo .", "words": ["the", "zoo", "put", "in", "me"]}, {"sent": "zoo .", "words": ["zoo"]}, {"sent": "sweetie pie what animals are we gonna see at the zoo ?", "words": ["the", "what", "zoo", "we", "are", "at", "see"]}, {"sent": "it says if I ran the zoo .", "words": ["the", "zoo", "it", "if"]}, {"sent": "put your zoo back ?", "words": ["zoo", "put", "your", "back"]}, {"sent": "Im making a zoo .", "words": ["zoo", "a"]}, {"sent": "what were the moon bears doing when we saw them at the zoo ?", "words": ["the", "what", "zoo", "moon", "when", "we", "them", "at", "were"]}, {"sent": "have you made a track to the zoo yet ?", "words": ["the", "zoo", "to", "you", "have", "a"]}, {"sent": "she sees at the little petting zoo farm .", "words": ["the", "zoo", "little", "at", "she"]}, {"sent": "we only can see in this part because what is a zoo a zoo is for people .", "words": ["can", "what", "zoo", "for", "we", "is", "this", "in", "see", "because", "a"]}, {"sent": "did we go to the zoo on Saturday ?", "words": ["the", "zoo", "go", "we", "to", "did", "on"]}, {"sent": "youve been to the zoo with Abi .", "words": ["the", "to", "zoo", "with"]}, {"sent": "thats in the zoo .", "words": ["the", "zoo", "in"]}, {"sent": "its almost like a zoo kinda right ?", "words": ["zoo", "like", "a"]}, {"sent": "I will go into the zoo .", "words": ["the", "zoo", "into", "go", "will"]}, {"sent": "youre setting that up a nice zoo .", "words": ["up", "zoo", "nice", "that", "a"]}, {"sent": "put them on the zoo then .", "words": ["the", "zoo", "put", "them", "on", "then"]}, {"sent": "wanna make a zoo with Mommy ?", "words": ["zoo", "with", "wanna", "make", "a"]}, {"sent": "thats a zoo .", "words": ["zoo", "a"]}, {"sent": "Mr and Mrs mouse took their kids to the zoo .", "words": ["the", "zoo", "their", "to", "and", "mouse"]}, {"sent": "lets play with the animal car with the zoo car .", "words": ["the", "zoo", "car", "play", "with", "animal"]}, {"sent": "put me in the zoo .", "words": ["the", "zoo", "put", "in", "me"]}, {"sent": "what did you see at the zoo ?", "words": ["the", "what", "zoo", "you", "did", "at", "see"]}, {"sent": "youre at the zoo .", "words": ["the", "zoo", "at"]}, {"sent": "we do not want you in the zoo out you go out out with you .", "words": ["the", "do", "zoo", "we", "go", "with", "you", "in", "out", "not"]}, {"sent": "youve been all the way to the zoo .", "words": ["the", "all", "zoo", "to"]}, {"sent": "oh hes taking them back to the zoo .", "words": ["the", "zoo", "back", "to", "them"]}, {"sent": "oh youre going to make them go around in a zoo ?", "words": ["zoo", "go", "to", "them", "in", "around", "make", "a"]}, {"sent": "the none of the zoo animals ?", "words": ["the", "none", "zoo", "of"]}, {"sent": "this is a book all about the zoo .", "words": ["the", "all", "zoo", "is", "this", "about", "book", "a"]}, {"sent": "these look like some animals from the zoo .", "words": ["the", "these", "zoo", "like", "look", "some"]}, {"sent": "and then at the national zoo they have the farm at the zoo .", "words": ["the", "zoo", "and", "have", "they", "at", "then"]}, {"sent": "mm zoo .", "words": ["zoo"]}, {"sent": "we do not want you to go in the zoo out youd go out with you .", "words": ["the", "do", "zoo", "we", "go", "to", "with", "you", "in", "out", "not"]}, {"sent": "have you guys been to the zoo or something ?", "words": ["the", "zoo", "to", "you", "have"]}, {"sent": "what did you see at the zoo ?", "words": ["the", "what", "zoo", "you", "did", "at", "see"]}, {"sent": "we do not want you in the zoo out you go out out with you .", "words": ["the", "do", "zoo", "we", "go", "with", "you", "in", "out", "not"]}, {"sent": "but is this tape sticky ?", "words": ["is", "sticky", "tape", "this", "but"]}, {"sent": "getting kind of sticky .", "words": ["of", "sticky"]}, {"sent": "hm look at that sticky hand .", "words": ["look", "hand", "sticky", "at", "that"]}, {"sent": "theyre sticky .", "words": ["sticky"]}, {"sent": "sticky on mommys chin .", "words": ["sticky", "on", "chin"]}, {"sent": "this also has that sticky kind of stuff .", "words": ["sticky", "this", "that", "of"]}, {"sent": "its very sticky .", "words": ["sticky"]}, {"sent": "the sticky the sticky part needs to go face down like this .", "words": ["the", "face", "go", "sticky", "to", "like", "this", "down"]}, {"sent": "theyre sticky from the jello .", "words": ["sticky", "the"]}, {"sent": "are you all sticky Abe ?", "words": ["sticky", "all", "are", "you"]}, {"sent": "sticky .", "words": ["sticky"]}, {"sent": "look at all these sticky finger marks .", "words": ["all", "these", "look", "sticky", "finger", "at"]}, {"sent": "and sticky .", "words": ["sticky", "and"]}, {"sent": "dont you go wiping your sticky fingers on everything .", "words": ["your", "go", "sticky", "you", "dont", "on"]}, {"sent": "because youll get sticky fingers .", "words": ["get", "sticky", "because"]}, {"sent": "sticky out hair .", "words": ["sticky", "hair", "out"]}, {"sent": "a sticky one .", "words": ["sticky", "a"]}, {"sent": "washing because its all sticky was the melon .", "words": ["the", "all", "sticky", "was", "melon", "because"]}, {"sent": "because shell get all sticky .", "words": ["get", "all", "sticky", "because"]}, {"sent": "can you see the sticky up bits there ?", "words": ["can", "the", "up", "sticky", "you", "there", "see"]}, {"sent": "is the lollipop sticky ?", "words": ["the", "sticky", "lollipop", "is"]}, {"sent": "sticky and ugh .", "words": ["sticky", "and"]}, {"sent": "sticky .", "words": ["sticky"]}, {"sent": "Amy gets sticky hand sticky nose and sticky face when she eats banana dont you Amy ?", "words": ["face", "hand", "sticky", "and", "you", "banana", "dont", "when", "she", "nose"]}, {"sent": "what sticky white thing ?", "words": ["sticky", "what", "white"]}, {"sent": "its pretty sticky isnt it ?", "words": ["sticky", "it", "pretty"]}, {"sent": "I dont think he wants to be sticky .", "words": ["think", "sticky", "to", "be", "dont", "he"]}, {"sent": "youre sticky .", "words": ["sticky"]}, {"sent": "because youre bound to be a little bit sticky .", "words": ["sticky", "to", "be", "little", "because", "a"]}, {"sent": "its pretty sticky chicken isnt it ?", "words": ["sticky", "chicken", "it", "pretty"]}, {"sent": "youre a sticky stick .", "words": ["sticky", "stick", "a"]}, {"sent": "little still sticky here .", "words": ["sticky", "here", "little"]}, {"sent": "just then Ruby stepped on something sticky .", "words": ["on", "then", "sticky"]}, {"sent": "no youve got sticky hands so dont touch them okay ?", "words": ["sticky", "so", "them", "touch", "dont"]}, {"sent": "Im worried a little bit that your hands are sticky and youre gonna get everything super sticky .", "words": ["your", "sticky", "little", "and", "get", "are", "that", "a"]}, {"sent": "so theyve got a little bit sticky havent they ?", "words": ["sticky", "so", "little", "they", "a"]}, {"sent": "itll be all sticky if I hold it .", "words": ["all", "hold", "if", "sticky", "be", "it"]}, {"sent": "youve got sticky hands ?", "words": ["sticky"]}, {"sent": "a sticky .", "words": ["sticky", "a"]}, {"sent": "sticky sticky sticky you want another piece ?", "words": ["sticky", "another", "you"]}, {"sent": "sticky and drawing and playing ?", "words": ["sticky", "and"]}, {"sent": "maple syrup is sticky .", "words": ["sticky", "is"]}, {"sent": "oh you put the sticky on the box .", "words": ["the", "put", "box", "sticky", "you", "on"]}, {"sent": "xxx still sticky .", "words": ["sticky"]}, {"sent": "what do you call sticky trains ?", "words": ["sticky", "what", "do", "you"]}, {"sent": "so I dont want sticky finger prints .", "words": ["dont", "so", "finger", "sticky"]}, {"sent": "Im not ripping the car Im ripping the sticky stuff off .", "words": ["the", "off", "sticky", "car", "not"]}, {"sent": "sticky sticky sticky and let me clean your sticky mouth too .", "words": ["clean", "your", "sticky", "mouth", "and", "too", "me"]}, {"sent": "its sticky .", "words": ["sticky"]}, {"sent": "its sticky .", "words": ["sticky"]}, {"sent": "sticky sticky hands .", "words": ["sticky"]}, {"sent": "is it sticky ?", "words": ["sticky", "it", "is"]}, {"sent": "somethings a bit sticky in there .", "words": ["sticky", "there", "in", "a"]}, {"sent": "I thought we were going to do a sticky ?", "words": ["do", "we", "sticky", "to", "were", "a"]}, {"sent": "be careful because youve got sticky hands .", "words": ["sticky", "be", "because", "careful"]}, {"sent": "you dont want a sticky bun today .", "words": ["sticky", "dont", "a", "you"]}, {"sent": "does he have sticky stuff on the back ?", "words": ["the", "does", "back", "sticky", "have", "he", "on"]}, {"sent": "this glue is very very sticky .", "words": ["sticky", "this", "is", "glue"]}, {"sent": "okay can I clean your sticky sticky hand .", "words": ["can", "clean", "your", "hand", "sticky"]}, {"sent": "youre not sticky .", "words": ["sticky", "not"]}, {"sent": "youre sticky sleeve .", "words": ["sticky"]}, {"sent": "Im a tree frog I have sticky pads on my fingers .", "words": ["tree", "my", "sticky", "frog", "have", "on", "a"]}, {"sent": "you dont care if you put your sticky fingers on your cheek .", "words": ["put", "your", "if", "sticky", "cheek", "you", "dont", "on"]}, {"sent": "sticky .", "words": ["sticky"]}, {"sent": "ah because shes all sticky .", "words": ["sticky", "all", "because"]}, {"sent": "and then we have um we have a book thats called TouchandFeelWildAnimals that has a frog in it and then the little frog has because its touch and feel because its the one with sticky fingers on it .", "words": ["the", "we", "sticky", "with", "touch", "and", "have", "frog", "in", "it", "little", "on", "book", "then", "that", "because", "a"]}, {"sent": "is it sticky ?", "words": ["sticky", "it", "is"]}, {"sent": "not sticky .", "words": ["sticky", "not"]}, {"sent": "on the sticky part .", "words": ["on", "the", "sticky"]}, {"sent": "more sticky fingers oh my gosh you are still so sticky .", "words": ["my", "sticky", "so", "you", "more", "are"]}, {"sent": "did you your hands are a lil sticky ?", "words": ["your", "sticky", "you", "are", "did", "a"]}, {"sent": "its sticky stuff look .", "words": ["sticky", "look"]}, {"sent": "what do you need sticky tape for Awww ?", "words": ["what", "do", "for", "need", "sticky", "tape", "you"]}, {"sent": "oh youve got a sticky label stuck to your shorts Thomas .", "words": ["shorts", "your", "sticky", "to", "stuck", "a"]}, {"sent": "its sticky isnt it ?", "words": ["sticky", "it"]}, {"sent": "sticky .", "words": ["sticky"]}, {"sent": "do you like sticky dog kisses ?", "words": ["do", "sticky", "like", "you", "dog"]}, {"sent": "oh you have sticky hands .", "words": ["have", "sticky", "you"]}, {"sent": "its got sticky fibers from the rug on it .", "words": ["sticky", "the", "on", "it"]}, {"sent": "thats like when when you drink Motrin or Tylenol and it spills and it gets very sticky .", "words": ["drink", "sticky", "like", "and", "you", "it", "when"]}, {"sent": "sticky sticky sticky .", "words": ["sticky"]}, {"sent": "your nose still sticky ?", "words": ["sticky", "your", "nose"]}, {"sent": "sugar cake sugar cake slice some coconut sweet and sticky brown and gooey .", "words": ["cake", "brown", "sticky", "and", "some"]}, {"sent": "the asphalt mixer made hot sticky asphalt .", "words": ["the", "hot", "sticky"]}, {"sent": "let me see the web is sticky .", "words": ["the", "is", "sticky", "me", "see"]}, {"sent": "its sticky .", "words": ["sticky"]}, {"sent": "what sort of sticky bun would you like bus driver ?", "words": ["what", "of", "sticky", "like", "you", "would", "bus"]}, {"sent": "see how sticky it is ?", "words": ["how", "is", "sticky", "it", "see"]}, {"sent": "put them sticky side down .", "words": ["sticky", "put", "down", "them"]}, {"sent": "are your hands sticky from that cookie ?", "words": ["cookie", "your", "sticky", "are", "that"]}, {"sent": "that may be the sticky side .", "words": ["the", "be", "sticky", "that"]}, {"sent": "think theyve lost their sticky back .", "words": ["sticky", "think", "their", "back"]}, {"sent": "sticky hands .", "words": ["sticky"]}, {"sent": "thats kind of sticky huh ?", "words": ["of", "sticky"]}, {"sent": "yeah thats pretty sticky stuff isnt it ?", "words": ["sticky", "it", "pretty"]}, {"sent": "its sticky .", "words": ["sticky"]}, {"sent": "get your hands kinda sticky when you that dont you ?", "words": ["your", "sticky", "you", "get", "dont", "that", "when"]}, {"sent": "the sticky letter .", "words": ["the", "sticky"]}, {"sent": "sticky sticky sticky .", "words": ["sticky"]}, {"sent": "do you wanna finish off the wild animals ?", "words": ["off", "do", "the", "finish", "you", "wanna"]}, {"sent": "okay lets finish this xxx .", "words": ["this", "finish"]}, {"sent": "finish your juice .", "words": ["juice", "your", "finish"]}, {"sent": "after we finish this then well go .", "words": ["go", "we", "finish", "this", "then"]}, {"sent": "let let me finish this .", "words": ["me", "finish", "this"]}, {"sent": "do you wanna finish this game ?", "words": ["do", "game", "finish", "you", "this", "wanna"]}, {"sent": "you can have some yogurt after you finish your toast .", "words": ["can", "your", "toast", "yogurt", "finish", "you", "have", "some"]}, {"sent": "going to the ferryboat soon so they can finish their dinner ?", "words": ["the", "can", "their", "to", "so", "finish", "they"]}, {"sent": "would you like to finish it ?", "words": ["to", "finish", "like", "you", "it", "would"]}, {"sent": "finish the rest of your juice .", "words": ["the", "your", "of", "finish", "juice"]}, {"sent": "okay Billy finish it .", "words": ["it", "finish"]}, {"sent": "oh well I might aswell finish building my aquarium mightnt I ?", "words": ["my", "finish"]}, {"sent": "Samantha would you please finish your juice ?", "words": ["your", "finish", "you", "would", "juice"]}, {"sent": "now Ill just finish this and then Ill see too Purdie .", "words": ["finish", "and", "this", "too", "see", "then"]}, {"sent": "train you wanna finish the book ?", "words": ["the", "train", "finish", "you", "wanna", "book"]}, {"sent": "finish eating your cookie .", "words": ["cookie", "your", "finish"]}, {"sent": "now you finish the rest of the story .", "words": ["the", "of", "finish", "you", "story"]}, {"sent": "we try to get her and my son to learn to pick up um what they finish with cause I get tired of picking up toys all all the time .", "words": ["up", "what", "my", "try", "all", "the", "we", "pick", "to", "of", "with", "finish", "and", "get", "they", "tired", "her", "try to"]}, {"sent": "finish this first you want some juice ?", "words": ["first", "finish", "you", "this", "some", "juice"]}, {"sent": "did you finish drawing ?", "words": ["finish", "did", "you"]}, {"sent": "come on lets just finish this and well look at them and you can tell me whats in them okay ?", "words": ["can", "them", "finish", "and", "you", "this", "in", "at", "look", "on", "me"]}, {"sent": "shall we finish it off ?", "words": ["off", "it", "finish", "we"]}, {"sent": "no darling you just sit there and finish it all .", "words": ["all", "finish", "and", "you", "there", "it", "sit"]}, {"sent": "whyncha finish the smoothie thats in the glass .", "words": ["the", "in", "glass", "finish"]}, {"sent": "Im gonna finish my chores .", "words": ["my", "finish"]}, {"sent": "do you just wanna go back up there while we finish getting you dressed ?", "words": ["up", "do", "back", "go", "we", "finish", "you", "there", "wanna"]}, {"sent": "xxx finish your food .", "words": ["food", "your", "finish"]}, {"sent": "were gonna go out and were gonna garden if we ever finish this book yes .", "words": ["if", "go", "we", "finish", "and", "this", "garden", "out", "book", "were"]}, {"sent": "I was going to play with the new jig jigsaw with you when finish .", "words": ["the", "to", "was", "play", "with", "finish", "you", "new", "when"]}, {"sent": "you finish .", "words": ["finish", "you"]}, {"sent": "lets finish doing this nappie first .", "words": ["this", "first", "finish"]}, {"sent": "uh could you finish that as fast as possible please ?", "words": ["fast", "finish", "you", "could", "that"]}, {"sent": "Im gonna we are gonna finish all this cereal this morning .", "words": ["all", "we", "finish", "this", "are", "cereal"]}, {"sent": "well are you going to cut a little piece for me to finish wrapping this present for Rory .", "words": ["for", "cut", "present", "to", "finish", "you", "little", "this", "are", "me", "a"]}, {"sent": "Mummy just finish her lunch .", "words": ["her", "finish"]}, {"sent": "shall we finish the village ?", "words": ["the", "finish", "we"]}, {"sent": "do you wanna finish it Stan ?", "words": ["do", "finish", "you", "it", "wanna"]}, {"sent": "put those back and I want you to sit down and finish whats on your plate .", "words": ["put", "those", "your", "back", "plate", "to", "finish", "you", "and", "on", "sit", "down"]}, {"sent": "you didnt wanna make them finish cooking have them finish cooking ?", "words": ["them", "finish", "you", "have", "wanna", "make"]}, {"sent": "now lets go back and get in your chair and finish your pizza .", "words": ["your", "back", "go", "finish", "chair", "and", "get", "pizza", "in"]}, {"sent": "I think you better finish up her bath dont you ?", "words": ["up", "better", "think", "finish", "you", "dont", "her"]}, {"sent": "then finish whats on your plate .", "words": ["plate", "your", "finish", "on", "then"]}, {"sent": "xxx did you finish all of those ?", "words": ["all", "those", "of", "finish", "you", "did"]}, {"sent": "can you finish up the report from there ?", "words": ["can", "the", "up", "finish", "you", "there"]}, {"sent": "are you gonna finish that one ?", "words": ["are", "that", "finish", "you"]}, {"sent": "did you not put them back in there when you finish ?", "words": ["put", "back", "them", "finish", "you", "there", "in", "did", "when", "not"]}, {"sent": "would you like to have finish these plates ?", "words": ["these", "to", "like", "finish", "you", "have", "would"]}, {"sent": "can you help me finish ?", "words": ["can", "help", "finish", "you", "me"]}, {"sent": "when we finish .", "words": ["when", "finish", "we"]}, {"sent": "arent we going to finish looking at it ?", "words": ["we", "to", "finish", "it", "at"]}, {"sent": "you wanna finish that first do you ?", "words": ["do", "first", "finish", "you", "wanna", "that"]}, {"sent": "well finish what you got there first .", "words": ["what", "first", "finish", "you", "there"]}, {"sent": "so you finish you finish with this story ?", "words": ["so", "with", "finish", "you", "this", "story"]}, {"sent": "I dont want I dont want Fraser finish .", "words": ["dont", "finish"]}, {"sent": "and then later on when Dada finish his breakfast .", "words": ["finish", "and", "his", "on", "then", "when"]}, {"sent": "we didnt finish .", "words": ["finish", "we"]}, {"sent": "now you wanna finish this book ?", "words": ["finish", "you", "this", "wanna", "book"]}, {"sent": "Ill finish this off then .", "words": ["off", "this", "then", "finish"]}, {"sent": "lets finish there .", "words": ["there", "finish"]}, {"sent": "you wanna finish the juice ?", "words": ["the", "finish", "you", "wanna", "juice"]}, {"sent": "you need the other piece to finish it dont you ?", "words": ["the", "need", "to", "finish", "other", "you", "dont", "it"]}, {"sent": "you finish that first okay ?", "words": ["first", "that", "finish", "you"]}, {"sent": "assoonas I finish my soup .", "words": ["soup", "my", "finish"]}, {"sent": "are you going to finish this milk .", "words": ["to", "finish", "you", "this", "are", "milk"]}, {"sent": "you finish this milk and Ill give you some more right away .", "words": ["finish", "and", "you", "this", "more", "away", "milk", "some", "give"]}, {"sent": "shall we finish building a bigger house so hat you can sit in it ?", "words": ["can", "we", "so", "finish", "you", "in", "it", "house", "hat", "sit", "a"]}, {"sent": "but first lets finish your dinner okay ?", "words": ["first", "your", "finish", "but"]}, {"sent": "you put your ThomasTheTankEngine hat on while I finish cleaning you up .", "words": ["up", "put", "your", "finish", "you", "on", "hat"]}, {"sent": "let me finish the corners off .", "words": ["the", "me", "finish", "off"]}, {"sent": "you finish .", "words": ["finish", "you"]}, {"sent": "lets finish .", "words": ["finish"]}, {"sent": "when Ive done most of it youll finish it off and get the credit wont you ?", "words": ["off", "the", "of", "finish", "and", "get", "you", "it", "when"]}, {"sent": "well lets finish our breakfast first shall we ?", "words": ["we", "first", "finish", "our"]}, {"sent": "lets just leave it there while you finish .", "words": ["there", "it", "finish", "you"]}, {"sent": "no you can do when you finish your breakfast well go and have a play with it .", "words": ["can", "do", "your", "go", "play", "with", "finish", "you", "and", "have", "it", "when", "a"]}, {"sent": "youd hafta do it with little ones if you wanna finish it .", "words": ["do", "if", "with", "finish", "little", "you", "it", "wanna"]}, {"sent": "Ill finish .", "words": ["finish"]}, {"sent": "you finish the story .", "words": ["the", "story", "finish", "you"]}, {"sent": "why dont you finish your dinner .", "words": ["your", "finish", "you", "dont", "why"]}, {"sent": "can you finish finish it up and xxx .", "words": ["can", "up", "finish", "you", "and", "it"]}, {"sent": "I think Ill just finish it off sweetheart .", "words": ["off", "think", "it", "finish"]}, {"sent": "okay now the prince and princess and dragon finish decorating the Christmas tree .", "words": ["the", "tree", "finish", "and"]}, {"sent": "well finish this off quickly then and then you can see the animals .", "words": ["off", "can", "the", "finish", "and", "you", "this", "see", "then"]}, {"sent": "do you want daddy to finish it ?", "words": ["do", "to", "finish", "you", "it"]}, {"sent": "no you didnt finish them .", "words": ["them", "finish", "you"]}, {"sent": "youre gonna finish in no time at all Im gonna finish the story ?", "words": ["the", "all", "finish", "in", "at", "story"]}, {"sent": "daddy and I wanna talk for a few minutes so please finish eating and then itll be time for your bath .", "words": ["for", "your", "so", "be", "finish", "and", "talk", "wanna", "then", "a"]}, {"sent": "think youll finish that tonight ?", "words": ["think", "that", "finish"]}, {"sent": "finish unscrewing that ?", "words": ["that", "finish"]}, {"sent": "I think we did finish the puzzle wanna go see it ?", "words": ["the", "think", "we", "go", "finish", "it", "puzzle", "did", "wanna", "see"]}, {"sent": "please finish your work aunt Jen Jen otherwise you cant come to the Microspace .", "words": ["the", "work", "your", "to", "finish", "you"]}, {"sent": "Ill just finish doing this .", "words": ["this", "finish"]}, {"sent": "let just let Mummy finish mixing it .", "words": ["it", "finish"]}, {"sent": "oh after you finish eating gramma wants you to have some of this vitamin c alright ?", "words": ["of", "to", "finish", "you", "have", "this", "some"]}, {"sent": "yep were doing it its on you gotta finish getting dressed .", "words": ["finish", "you", "it", "on", "were"]}, {"sent": "lemme get yer knife .", "words": ["get", "knife"]}, {"sent": "yeah a wee tiny knife .", "words": ["tiny", "knife", "a"]}, {"sent": "be careful with the knife .", "words": ["the", "be", "knife", "with", "careful"]}, {"sent": "that was the noise of me cutting the toast with a knife .", "words": ["the", "toast", "of", "was", "knife", "with", "me", "that", "a"]}, {"sent": "cut cut with a knife .", "words": ["knife", "with", "cut", "a"]}, {"sent": "well I hafta get a knife heres a knife .", "words": ["get", "knife", "a"]}, {"sent": "put the knife here Julia .", "words": ["the", "put", "here", "knife"]}, {"sent": "give me the knife .", "words": ["the", "me", "give", "knife"]}, {"sent": "what are you doing with the knife buddy ?", "words": ["the", "what", "knife", "with", "you", "are"]}, {"sent": "let Mummy use the knife .", "words": ["the", "knife"]}, {"sent": "knife and fork for shorty .", "words": ["knife", "for", "fork", "and"]}, {"sent": "lets see if itll climb over that knife .", "words": ["if", "knife", "over", "see", "climb", "that"]}, {"sent": "youre going to use the little knife .", "words": ["the", "to", "knife", "little"]}, {"sent": "hey look RobinHood has a knife and a bow and arrow .", "words": ["knife", "and", "a", "look"]}, {"sent": "and I just caught my arm on the knife block .", "words": ["the", "my", "block", "knife", "and", "arm", "on"]}, {"sent": "a knife for cutting isnt it ?", "words": ["it", "knife", "for", "a"]}, {"sent": "tell him Im gonna put it on with a knife .", "words": ["put", "knife", "with", "him", "it", "on", "a"]}, {"sent": "Ill cut it with a knife .", "words": ["cut", "knife", "with", "it", "a"]}, {"sent": "youd like the cookie cutters and the knife ?", "words": ["the", "cookie", "knife", "like", "and"]}, {"sent": "the knife goes in there .", "words": ["the", "in", "knife", "there"]}, {"sent": "thats his knife and fork and his plate .", "words": ["plate", "knife", "and", "fork", "his"]}, {"sent": "do you see this special knife ?", "words": ["do", "knife", "you", "this", "see"]}, {"sent": "the knife is just there .", "words": ["the", "there", "knife", "is"]}, {"sent": "this knife is not good .", "words": ["is", "knife", "this", "good", "not"]}, {"sent": "he knows everything but a knife .", "words": ["knife", "he", "but", "a"]}, {"sent": "mommy can I have a knife ?", "words": ["can", "have", "knife", "a"]}, {"sent": "you can have the little knife if you want Amy .", "words": ["can", "the", "if", "knife", "little", "have", "you"]}, {"sent": "right now with this knife were going to put a little slit .", "words": ["put", "to", "knife", "with", "little", "this", "were", "a"]}, {"sent": "this is the electric bread knife .", "words": ["the", "bread", "is", "knife", "this"]}, {"sent": "hes got its called a knife darling .", "words": ["knife", "a"]}, {"sent": "we are going to use a different knife now Thomas .", "words": ["we", "to", "knife", "are", "a"]}, {"sent": "take your knife .", "words": ["your", "knife", "take"]}, {"sent": "whoa the knife fell in my lap .", "words": ["the", "in", "knife", "my"]}, {"sent": "with a knife and fork .", "words": ["knife", "with", "and", "fork", "a"]}, {"sent": "now we stir that knife in a xxx water .", "words": ["we", "knife", "in", "water", "that", "a"]}, {"sent": "youre not having the sharp knife .", "words": ["the", "knife", "not"]}, {"sent": "with a knife then Im gonna give it to you to eat .", "words": ["to", "knife", "with", "then", "you", "eat", "it", "give", "a"]}, {"sent": "a real knife yeah .", "words": ["knife", "a"]}, {"sent": "okay Ill get a knife .", "words": ["get", "knife", "a"]}, {"sent": "get a knife and a fork please .", "words": ["knife", "and", "get", "fork", "a"]}, {"sent": "go and get a knife .", "words": ["go", "knife", "and", "get", "a"]}, {"sent": "a knife ya .", "words": ["knife", "a"]}, {"sent": "you got a big knife ?", "words": ["knife", "big", "a", "you"]}, {"sent": "do you wanna knife ?", "words": ["do", "knife", "wanna", "you"]}, {"sent": "knife .", "words": ["knife"]}, {"sent": "do you think your gun is a scissor or a knife ?", "words": ["do", "think", "your", "is", "knife", "you", "a"]}, {"sent": "Ill see if theres a knife over here .", "words": ["if", "knife", "here", "see", "over", "a"]}, {"sent": "wheres my knife ?", "words": ["knife", "my"]}, {"sent": "he let you cut it with a knife yes .", "words": ["cut", "knife", "with", "you", "it", "he", "a"]}, {"sent": "do you want this knife ?", "words": ["do", "knife", "this", "you"]}, {"sent": "the electric bread knife .", "words": ["the", "bread", "knife"]}, {"sent": "pass pass me the knife .", "words": ["the", "me", "knife"]}, {"sent": "see I think this looks more like a knife .", "words": ["think", "knife", "like", "this", "more", "see", "a"]}, {"sent": "hes putting caterpillar icing on the cake with his knife .", "words": ["the", "cake", "knife", "with", "his", "on"]}, {"sent": "yeah but its not alright to put butter on a knife and wave it around in the air and get it all over the floor .", "words": ["the", "all", "put", "to", "knife", "and", "get", "in", "it", "butter", "around", "on", "over", "not", "but", "a"]}, {"sent": "oh theres your knife .", "words": ["knife", "your"]}, {"sent": "wheres that knife Alex ?", "words": ["knife", "that"]}, {"sent": "were putting those pictures on top now lets see if we can find heres a knife fork and spoon can you see another knife fork and a spoon ?", "words": ["can", "those", "if", "we", "knife", "and", "you", "another", "spoon", "fork", "find", "on", "see", "were", "a"]}, {"sent": "a knife to cut wood .", "words": ["to", "knife", "cut", "a"]}, {"sent": "thats a very sharp knife .", "words": ["knife", "a"]}, {"sent": "I dont know whose knife it is .", "words": ["dont", "knife", "it", "is"]}, {"sent": "wheres your plastic knife ?", "words": ["knife", "your"]}, {"sent": "the other knife ?", "words": ["the", "knife", "other"]}, {"sent": "wheres the knife ?", "words": ["the", "knife"]}, {"sent": "we need a sharper knife .", "words": ["knife", "need", "a", "we"]}, {"sent": "why dont you get the big sharp knife ?", "words": ["the", "knife", "you", "get", "dont", "big", "why"]}, {"sent": "Ill get the other knife .", "words": ["get", "the", "knife", "other"]}, {"sent": "youll break the knife .", "words": ["the", "knife", "break"]}, {"sent": "I should use the letter knife really to open this .", "words": ["the", "open", "to", "knife", "this"]}, {"sent": "you want a knife and a fork .", "words": ["knife", "and", "you", "fork", "a"]}, {"sent": "lets get a knife and cut this loaf of bread .", "words": ["bread", "cut", "of", "knife", "and", "get", "this", "a"]}, {"sent": "Hunka Munka stood up on her chair and chopped at the ham with another lead knife .", "words": ["up", "the", "knife", "with", "chair", "and", "another", "at", "on", "her"]}, {"sent": "right mix that little bit up there with the knife .", "words": ["up", "the", "knife", "with", "little", "there", "that"]}, {"sent": "you should cut with your knife .", "words": ["your", "cut", "knife", "with", "you"]}, {"sent": "his knife he lost it .", "words": ["his", "knife", "it", "he"]}, {"sent": "get an orange and a knife xxx .", "words": ["knife", "and", "get", "an", "orange", "a"]}, {"sent": "theres that knife .", "words": ["knife", "that"]}, {"sent": "well you just feel like I guess the physics of it because like the heat rises and whatever that its so simple that unless like you know some flying knife comes out and hits your balloon like that theres just not that much that can go assuming youre not because they do weather reporting so youre not gonna get hopefully caught in a thunderstorm or whatever .", "words": ["the", "go", "of", "like", "balloon", "not", "can", "knife", "it", "some", "much", "that", "so", "you", "and", "because", "a", "do", "your", "get", "they", "in", "out"]}, {"sent": "a knife .", "words": ["knife", "a"]}, {"sent": "theres your knife here .", "words": ["here", "knife", "your"]}, {"sent": "no I dont need a small knife .", "words": ["dont", "knife", "need", "a"]}, {"sent": "theres a knife in your back .", "words": ["your", "back", "knife", "in", "a"]}, {"sent": "have you made marks with your knife ?", "words": ["your", "knife", "with", "you", "have"]}, {"sent": "would you like a knife ?", "words": ["knife", "like", "you", "would", "a"]}, {"sent": "with your fingers sometimes you do that sometimes you can do that you wanna knife or do you wanna do it with your fingers .", "words": ["can", "do", "your", "knife", "with", "you", "it", "wanna", "that"]}, {"sent": "Ill just go and get the bread knife .", "words": ["the", "bread", "go", "knife", "and", "get"]}, {"sent": "knife and fork .", "words": ["knife", "fork", "and"]}, {"sent": "a knife .", "words": ["knife", "a"]}, {"sent": "knife .", "words": ["knife"]}, {"sent": "Mummys got a carving knife .", "words": ["knife", "a"]}, {"sent": "youre gonna get it out with a knife ?", "words": ["knife", "with", "get", "it", "out", "a"]}, {"sent": "Daddys got a pen knife .", "words": ["knife", "pen", "a"]}, {"sent": "right you mix it up with the knife .", "words": ["up", "the", "knife", "with", "you", "it"]}, {"sent": "oh a knife .", "words": ["knife", "a"]}, {"sent": "watch your fingers with that sharp knife .", "words": ["watch", "your", "knife", "with", "that"]}, {"sent": "well hafta get a knife or something .", "words": ["get", "knife", "a"]}, {"sent": "yes I was cutting the Kamut with a knife .", "words": ["the", "was", "knife", "with", "a"]}, {"sent": "and anything like that on the table Granddad said would be carved with a sharp knife .", "words": ["the", "table", "be", "knife", "with", "like", "and", "would", "on", "that", "a"]}, {"sent": "Ill get you a knife .", "words": ["get", "knife", "a", "you"]}, {"sent": "you didnt get you finger in the way did you ?", "words": ["the", "you", "get", "finger", "in", "did"]}, {"sent": "whats a finger toe ?", "words": ["toe", "finger", "a"]}, {"sent": "does that finger have a name ?", "words": ["does", "have", "finger", "that", "a"]}, {"sent": "no dont stick your finger in there .", "words": ["stick", "your", "dont", "finger", "in", "there"]}, {"sent": "you see they go on your finger .", "words": ["your", "go", "you", "they", "finger", "on", "see"]}, {"sent": "paste you have paste on your finger .", "words": ["your", "you", "have", "finger", "on"]}, {"sent": "that is your finger part this is the thumb part of things .", "words": ["the", "your", "is", "of", "finger", "this", "that"]}, {"sent": "you got your finger in there .", "words": ["your", "you", "finger", "in", "there"]}, {"sent": "put your finger on the two on the two put your finger on the two on the two put your finger on the two not on the shoe .", "words": ["the", "put", "your", "shoe", "finger", "on", "not"]}, {"sent": "Ive hurt my finger .", "words": ["hurt", "finger", "my"]}, {"sent": "no pop it with one finger .", "words": ["finger", "it", "pop", "with"]}, {"sent": "whats wrong with your finger ?", "words": ["your", "finger", "with"]}, {"sent": "its finger licking good Michael .", "words": ["finger", "good"]}, {"sent": "Id hafta sit with my finger on the button .", "words": ["the", "my", "with", "finger", "on", "sit", "button"]}, {"sent": "oo you bit my finger it ate my finger all up .", "words": ["up", "all", "my", "you", "finger", "it"]}, {"sent": "one little finger clap clap clap .", "words": ["finger", "clap", "little"]}, {"sent": "use your finger .", "words": ["finger", "your"]}, {"sent": "what a green finger youve got now .", "words": ["what", "finger", "green", "a"]}, {"sent": "if you get if you get your finger in my tea youll get burned again .", "words": ["my", "your", "if", "you", "get", "finger", "in"]}, {"sent": "tickle tickle your finger .", "words": ["finger", "your", "tickle"]}, {"sent": "you hit your finger on something and the thing bit you ?", "words": ["the", "your", "you", "and", "finger", "on", "hit"]}, {"sent": "got your finger stuck .", "words": ["finger", "stuck", "your"]}, {"sent": "dont put your finger in Henry no .", "words": ["put", "your", "dont", "finger", "in"]}, {"sent": "can you take it off of your finger ?", "words": ["can", "off", "your", "of", "you", "finger", "it", "take"]}, {"sent": "before the sun sets on her sixteenth birthday she shall prick her finger on the spindle of a spinning wheel .", "words": ["the", "of", "sun", "finger", "on", "her", "she", "a"]}, {"sent": "Cadburys finger biscuits .", "words": ["finger"]}, {"sent": "look you had your finger on it .", "words": ["your", "you", "finger", "it", "look", "on"]}, {"sent": "oh its stuck to your finger .", "words": ["your", "to", "finger", "stuck"]}, {"sent": "keep your finger on it .", "words": ["on", "finger", "it", "your"]}, {"sent": "stick your finger in there and pull it along like that .", "words": ["stick", "your", "like", "and", "pull", "finger", "in", "there", "it", "that"]}, {"sent": "the next finger going next finger is MisterPointer .", "words": ["the", "finger", "is"]}, {"sent": "follow Daddys finger .", "words": ["finger"]}, {"sent": "its on my finger .", "words": ["on", "finger", "my"]}, {"sent": "how did you hurt your finger ?", "words": ["how", "your", "hurt", "you", "finger", "did"]}, {"sent": "shes got me other finger .", "words": ["me", "finger", "other"]}, {"sent": "this little finger on my right .", "words": ["my", "little", "this", "finger", "on"]}, {"sent": "you hurt your finger ?", "words": ["hurt", "finger", "your", "you"]}, {"sent": "with your finger darling .", "words": ["your", "finger", "with"]}, {"sent": "did you hurt your finger ?", "words": ["your", "hurt", "you", "finger", "did"]}, {"sent": "and I think its a story book where you hafta keep putting your finger through the page to feel something .", "words": ["where", "the", "think", "your", "to", "and", "you", "finger", "book", "story", "a"]}, {"sent": "youll hafta wear it on your finger for the rest of your life then .", "words": ["the", "for", "your", "of", "finger", "it", "on", "then"]}, {"sent": "Thomas what is this going to do to my finger ?", "words": ["what", "do", "my", "is", "to", "this", "finger"]}, {"sent": "hes just kinda started to eat them as finger food .", "words": ["to", "them", "eat", "finger", "food"]}, {"sent": "you got my finger .", "words": ["finger", "my", "you"]}, {"sent": "what were you saying about your finger ?", "words": ["what", "your", "you", "finger", "about", "were"]}, {"sent": "your finger is stuck ?", "words": ["finger", "stuck", "your", "is"]}, {"sent": "did you hurt your finger ?", "words": ["your", "hurt", "you", "finger", "did"]}, {"sent": "where your big finger ?", "words": ["where", "finger", "big", "your"]}, {"sent": "hows your finger ?", "words": ["finger", "your"]}, {"sent": "point with your finger so I can see better .", "words": ["can", "better", "your", "so", "with", "finger", "see"]}, {"sent": "if you move your finger .", "words": ["finger", "your", "if", "you"]}, {"sent": "get your finger outof your ear .", "words": ["get", "finger", "your", "ear"]}, {"sent": "look at that little finger .", "words": ["look", "little", "finger", "at", "that"]}, {"sent": "you wanna piece to do some finger painting on you can have a piece .", "words": ["can", "do", "to", "you", "have", "finger", "wanna", "on", "some", "a"]}, {"sent": "no this is index finger .", "words": ["this", "finger", "is"]}, {"sent": "put your finger in the hole ?", "words": ["the", "put", "your", "finger", "in"]}, {"sent": "April Im not going to put my finger in the end of your nose no .", "words": ["the", "put", "my", "your", "of", "to", "finger", "in", "not", "nose"]}, {"sent": "index finger .", "words": ["finger"]}, {"sent": "the little finger .", "words": ["the", "finger", "little"]}, {"sent": "are you gonna bite Annas finger .", "words": ["bite", "finger", "are", "you"]}, {"sent": "do you remember what it was she hadta prick her finger on ?", "words": ["do", "what", "was", "you", "finger", "it", "on", "her", "she"]}, {"sent": "put your finger on your nose .", "words": ["put", "your", "finger", "on", "nose"]}, {"sent": "you gonna tie it around your finger ?", "words": ["your", "you", "finger", "it", "around"]}, {"sent": "get your finger out your mouth then .", "words": ["your", "mouth", "get", "finger", "out", "then"]}, {"sent": "would you like a little bit of sticky tape for your finger ?", "words": ["for", "your", "of", "sticky", "like", "tape", "little", "you", "finger", "would", "a"]}, {"sent": "put your finger here .", "words": ["here", "finger", "put", "your"]}, {"sent": "and take your finger out .", "words": ["your", "and", "finger", "out", "take"]}, {"sent": "point your finger like this ?", "words": ["finger", "your", "like", "this"]}, {"sent": "its a finger .", "words": ["finger", "a"]}, {"sent": "I have got poorly finger .", "words": ["have", "finger"]}, {"sent": "put your finger in Mey .", "words": ["finger", "put", "your", "in"]}, {"sent": "your finger print ?", "words": ["finger", "your"]}, {"sent": "he sucked my finger .", "words": ["finger", "my", "he"]}, {"sent": "well if mummie puts her finger underneath like that then it wont collapse will it ?", "words": ["if", "her", "like", "finger", "it", "will", "then", "that"]}, {"sent": "you dont wanna use up all the paint on your finger ?", "words": ["up", "all", "the", "paint", "your", "you", "dont", "finger", "wanna", "on"]}, {"sent": "whose finger ?", "words": ["finger"]}, {"sent": "on my finger .", "words": ["on", "finger", "my"]}, {"sent": "you said Daddy was poorly when he came home with a plaster on his finger .", "words": ["was", "with", "you", "his", "finger", "he", "on", "when", "a", "home"]}, {"sent": "now you wanna do it with your hand with your finger .", "words": ["do", "your", "hand", "with", "you", "finger", "it", "wanna"]}, {"sent": "oops a shoelace got your finger .", "words": ["finger", "your", "a"]}, {"sent": "you have some blue on your finger ?", "words": ["your", "blue", "have", "you", "finger", "on", "some"]}, {"sent": "oh now youve have cut your mouth on my finger nail havent you ?", "words": ["my", "your", "cut", "mouth", "nail", "you", "have", "finger", "on"]}, {"sent": "my finger .", "words": ["finger", "my"]}, {"sent": "you had a piece of wood stuck in your finger and Mama was holding you like this and Dada .", "words": ["your", "of", "was", "like", "you", "and", "finger", "in", "this", "stuck", "a"]}, {"sent": "drool on the babys finger Sarah .", "words": ["on", "the", "finger"]}, {"sent": "dont poke it with your finger .", "words": ["your", "with", "dont", "finger", "it"]}, {"sent": "it looks like a finger .", "words": ["finger", "it", "like", "a"]}, {"sent": "oh Eleanor look hes biting my finger .", "words": ["finger", "my", "look"]}, {"sent": "actually you know what I can put them on my nails my finger nails .", "words": ["can", "what", "put", "my", "them", "you", "finger", "on"]}, {"sent": "take your finger outof your mouth and let me ask you something .", "words": ["your", "mouth", "and", "you", "finger", "me", "take"]}, {"sent": "you always leave out the middle finger .", "words": ["the", "finger", "out", "you"]}, {"sent": "stick your finger in the window here .", "words": ["the", "stick", "your", "here", "finger", "in", "window"]}, {"sent": "do you have a spot on your little finger too ?", "words": ["do", "your", "you", "have", "little", "finger", "too", "on", "a"]}, {"sent": "finger puppets .", "words": ["finger"]}, {"sent": "your finger .", "words": ["finger", "your"]}, {"sent": "its nice to do some finger paints as well .", "words": ["do", "to", "nice", "finger", "some"]}, {"sent": "thats a finger .", "words": ["finger", "a"]}, {"sent": "get your finger out your mouth Moo .", "words": ["your", "mouth", "get", "finger", "out"]}, {"sent": "its allgone on your finger has it ?", "words": ["on", "finger", "it", "your"]}, {"sent": "too cold to bite ?", "words": ["bite", "to", "cold", "too"]}, {"sent": "bite ya bite ya .", "words": ["bite"]}, {"sent": "with one bite of this poisoned apple Snow Whites eyes will close forever she said the only cure for the sleeping spell was loves first kiss .", "words": ["the", "for", "of", "was", "will", "with", "first", "this", "bite", "apple", "close", "kiss", "she"]}, {"sent": "now Im gonna bite you .", "words": ["bite", "you"]}, {"sent": "do you want a bite of this ?", "words": ["do", "of", "you", "this", "bite", "a"]}, {"sent": "oo does mommy get a little bite ?", "words": ["does", "little", "get", "bite", "a"]}, {"sent": "no you didnt bite a hole in it .", "words": ["you", "in", "it", "bite", "a"]}, {"sent": "big bite .", "words": ["bite", "big"]}, {"sent": "well have a bite each shall we ?", "words": ["each", "we", "have", "bite", "a"]}, {"sent": "oh look at that big bite .", "words": ["look", "that", "at", "bite", "big"]}, {"sent": "no biting is naughty bite .", "words": ["bite", "naughty", "is"]}, {"sent": "they bite my nose ?", "words": ["bite", "they", "my", "nose"]}, {"sent": "here take a good bite now .", "words": ["here", "good", "bite", "take", "a"]}, {"sent": "you wanna bite my finger .", "words": ["my", "you", "finger", "wanna", "bite"]}, {"sent": "have a bite .", "words": ["have", "bite", "a"]}, {"sent": "can I take your spoon and feed you a bite of of rice and spaghetti and stuff ?", "words": ["can", "your", "spaghetti", "of", "and", "you", "spoon", "feed", "bite", "take", "a"]}, {"sent": "a bite ?", "words": ["bite", "a"]}, {"sent": "is good bite of oatmeal ?", "words": ["bite", "of", "good", "is"]}, {"sent": "shell take a bite of one and then just go on to another one and another one .", "words": ["go", "of", "to", "and", "another", "on", "bite", "then", "take", "a"]}, {"sent": "I really dont wanna bite .", "words": ["bite", "dont", "wanna"]}, {"sent": "grr Im gonna bite them .", "words": ["bite", "them"]}, {"sent": "can I have a bite ?", "words": ["can", "bite", "have", "a"]}, {"sent": "did you just say bite all by yourself ?", "words": ["all", "yourself", "say", "you", "by", "did", "bite"]}, {"sent": "trying to bite me ?", "words": ["bite", "to", "me"]}, {"sent": "no don dont bite that one .", "words": ["bite", "dont", "that"]}, {"sent": "I was just thinking at least our hands are clean because we washed them but you dont then go and bite it .", "words": ["clean", "we", "go", "was", "them", "our", "you", "dont", "and", "are", "it", "but", "at", "bite", "then", "because"]}, {"sent": "no dont bite it .", "words": ["bite", "dont", "it"]}, {"sent": "you got to have a big mouth to eat all that in one bite .", "words": ["all", "to", "mouth", "you", "have", "eat", "that", "in", "bite", "big", "a"]}, {"sent": "do you have a bug bite Sweetie ?", "words": ["bug", "do", "you", "have", "bite", "a"]}, {"sent": "another bite for you .", "words": ["another", "bite", "for", "you"]}, {"sent": "take a good bite .", "words": ["bite", "take", "good", "a"]}, {"sent": "hes trying to bite you and Im gonna try and get him .", "words": ["to", "and", "you", "get", "him", "bite", "try"]}, {"sent": "just take a little bite .", "words": ["bite", "take", "little", "a"]}, {"sent": "and took another bite .", "words": ["another", "bite", "and"]}, {"sent": "no Matthew isnt going to bite you .", "words": ["bite", "to", "you"]}, {"sent": "you dont bite that please .", "words": ["bite", "dont", "that", "you"]}, {"sent": "just bite a tiny bit off .", "words": ["bite", "tiny", "off", "a"]}, {"sent": "could I have a bite of your cookie ?", "words": ["cookie", "your", "of", "have", "bite", "could", "a"]}, {"sent": "Ill put a little bite in a bowl .", "words": ["bowl", "put", "little", "in", "bite", "a"]}, {"sent": "you have a bite .", "words": ["have", "bite", "a", "you"]}, {"sent": "I think we got a ant bite or a mosquito bite down in the park .", "words": ["the", "think", "we", "in", "park", "ant", "bite", "down", "a"]}, {"sent": "its very naughty to bite anyone .", "words": ["bite", "to", "naughty"]}, {"sent": "I wanna bite .", "words": ["bite", "wanna"]}, {"sent": "do you want your last bite of bread and butter ?", "words": ["do", "bread", "your", "of", "last", "you", "and", "butter", "bite"]}, {"sent": "ha did you get a really big bite ?", "words": ["you", "get", "did", "bite", "big", "a"]}, {"sent": "no bite no bite .", "words": ["bite"]}, {"sent": "he could bite it right there and it would pop .", "words": ["pop", "and", "there", "it", "he", "would", "bite", "could"]}, {"sent": "okay ready for a bite ?", "words": ["bite", "for", "a"]}, {"sent": "and Ill have a little bite of yours .", "words": ["of", "little", "have", "and", "bite", "a"]}, {"sent": "yes whered the bee bite you ?", "words": ["the", "bite", "bee", "you"]}, {"sent": "will he bite ?", "words": ["will", "bite", "he"]}, {"sent": "and bite .", "words": ["bite", "and"]}, {"sent": "Ill bite your nose .", "words": ["bite", "your", "nose"]}, {"sent": "you seem to love to bite on them .", "words": ["to", "them", "you", "on", "love", "bite"]}, {"sent": "can I bite this ?", "words": ["can", "bite", "this"]}, {"sent": "dont bite me .", "words": ["bite", "dont", "me"]}, {"sent": "xxx xxx wanna bite ?", "words": ["bite", "wanna"]}, {"sent": "taste a lil bite of that .", "words": ["of", "taste", "bite", "that", "a"]}, {"sent": "going to bite ?", "words": ["bite", "to"]}, {"sent": "did he ever bite anybody ?", "words": ["bite", "he", "did"]}, {"sent": "bite Annes toes ?", "words": ["bite"]}, {"sent": "bite into them theyre really juicy .", "words": ["bite", "them", "into"]}, {"sent": "will he bite us ?", "words": ["will", "bite", "he", "us"]}, {"sent": "dont really bite it pretend .", "words": ["bite", "dont", "pretend", "it"]}, {"sent": "here is a bite .", "words": ["here", "bite", "a", "is"]}, {"sent": "no dont bite Megan .", "words": ["bite", "dont"]}, {"sent": "dont let the bedbugs bite .", "words": ["the", "dont", "bite"]}, {"sent": "take another bite .", "words": ["another", "bite", "take"]}, {"sent": "oh dont actually bite it please .", "words": ["bite", "dont", "it"]}, {"sent": "you wanna bite his head ?", "words": ["head", "you", "his", "wanna", "bite"]}, {"sent": "I think youre going to bite me .", "words": ["bite", "to", "think", "me"]}, {"sent": "bite it ?", "words": ["bite", "it"]}, {"sent": "have a bite of oatmeal .", "words": ["have", "bite", "of", "a"]}, {"sent": "Mommys bite .", "words": ["bite"]}, {"sent": "Im gonna get you and bite your butt .", "words": ["your", "and", "get", "you", "bite"]}, {"sent": "dont tell me horses bite .", "words": ["bite", "dont", "me"]}, {"sent": "I wonder does Diffy Daffy bite your nose ?", "words": ["bite", "does", "your", "nose"]}, {"sent": "dont bite Mommy .", "words": ["bite", "dont"]}, {"sent": "do you wanna bite ?", "words": ["bite", "do", "wanna", "you"]}, {"sent": "do you want a bite of your ?", "words": ["do", "your", "of", "you", "bite", "a"]}, {"sent": "do you want me to bite your finger ?", "words": ["do", "your", "to", "you", "finger", "bite", "me"]}, {"sent": "have bite .", "words": ["have", "bite"]}, {"sent": "give me a bite .", "words": ["bite", "me", "give", "a"]}, {"sent": "bite .", "words": ["bite"]}, {"sent": "bite it .", "words": ["bite", "it"]}, {"sent": "oh Im gonna bite them .", "words": ["bite", "them"]}, {"sent": "a bite ?", "words": ["bite", "a"]}, {"sent": "well if you go in the tunnel Ill make sure that he doesnt bite you .", "words": ["the", "if", "go", "you", "in", "that", "he", "bite", "make"]}, {"sent": "oh Im gonna bite them .", "words": ["bite", "them"]}, {"sent": "you bite the table ?", "words": ["bite", "the", "table", "you"]}, {"sent": "Ill hafta wear the ones that Andy took the bite outof .", "words": ["the", "bite", "that"]}, {"sent": "before I bite it .", "words": ["bite", "it"]}, {"sent": "you take a bite .", "words": ["bite", "take", "a", "you"]}, {"sent": "they dont bite .", "words": ["dont", "they", "bite"]}, {"sent": "bite it .", "words": ["bite", "it"]}, {"sent": "they didnt bite you .", "words": ["bite", "they", "you"]}, {"sent": "you gave everybody a bite ?", "words": ["bite", "a", "you"]}, {"sent": "ya gotta bite hard .", "words": ["bite", "hard"]}, {"sent": "there Im gonna hide myself so that you cannot bite me .", "words": ["hide", "so", "you", "there", "myself", "bite", "me", "that"]}, {"sent": "oh that was your bite .", "words": ["your", "was", "bite", "that"]}, {"sent": "have a look in the box .", "words": ["the", "box", "have", "in", "look", "a"]}, {"sent": "shall we look in the other box ?", "words": ["the", "box", "we", "other", "in", "look"]}, {"sent": "a letter box .", "words": ["box", "a"]}, {"sent": "what else is in the box ?", "words": ["the", "what", "box", "is", "in"]}, {"sent": "that was your box Nathan ?", "words": ["your", "was", "box", "that"]}, {"sent": "yes thats a box .", "words": ["box", "a"]}, {"sent": "how bout the drink box ?", "words": ["the", "how", "box", "drink"]}, {"sent": "this box .", "words": ["this", "box"]}, {"sent": "well when youve finished the carrots in the box will you please come and help me ?", "words": ["the", "help", "box", "you", "and", "in", "carrots", "will", "me", "when"]}, {"sent": "box .", "words": ["box"]}, {"sent": "actually Ive gotta give you the last box now .", "words": ["the", "box", "last", "you", "give"]}, {"sent": "yeah thats a box .", "words": ["box", "a"]}, {"sent": "thats the box that the tapes in the wheels .", "words": ["the", "in", "that", "box"]}, {"sent": "theres no pictures on the box that shows what you can build with them .", "words": ["the", "what", "can", "box", "with", "them", "you", "build", "on", "that"]}, {"sent": "close the box and close the drawer .", "words": ["the", "box", "and", "drawer", "close"]}, {"sent": "whats in the box ?", "words": ["the", "in", "box"]}, {"sent": "are you putting it in the box ?", "words": ["the", "box", "you", "in", "it", "are"]}, {"sent": "let us see how much you remember about this box .", "words": ["how", "box", "you", "us", "this", "about", "see", "much"]}, {"sent": "now if we put the box down there .", "words": ["the", "put", "box", "if", "we", "there", "down"]}, {"sent": "go get another box .", "words": ["get", "box", "another", "go"]}, {"sent": "there must be something else in the other box .", "words": ["the", "box", "be", "other", "there", "in"]}, {"sent": "I cant I cant fit in the box unless I do .", "words": ["the", "do", "box", "fit", "in"]}, {"sent": "that box .", "words": ["box", "that"]}, {"sent": "so you never opened that box up .", "words": ["up", "box", "so", "you", "that"]}, {"sent": "youre not sposta climb in the box .", "words": ["the", "box", "in", "climb", "not"]}, {"sent": "no no no hes not in the box .", "words": ["the", "in", "box", "not"]}, {"sent": "there was a wonderful post box today at playgroup .", "words": ["box", "was", "there", "at", "a"]}, {"sent": "heres your box when you get through with the book put it back in the box .", "words": ["the", "put", "box", "your", "back", "with", "you", "get", "in", "it", "book", "when"]}, {"sent": "theyre all in the box together .", "words": ["the", "all", "in", "box"]}, {"sent": "shall Mummy put Spot away in the other box ?", "words": ["the", "put", "box", "other", "in", "away"]}, {"sent": "is there a football on the box ?", "words": ["the", "box", "is", "there", "on", "a"]}, {"sent": "I dont want you to break the crayon box .", "words": ["the", "box", "crayon", "to", "you", "dont", "break"]}, {"sent": "it was a blue box .", "words": ["box", "was", "blue", "it", "a"]}, {"sent": "you can hold the box .", "words": ["can", "the", "box", "hold", "you"]}, {"sent": "be oh in that box ?", "words": ["be", "in", "box", "that"]}, {"sent": "how do we open this box ?", "words": ["do", "how", "box", "we", "open", "this"]}, {"sent": "whats in the box ?", "words": ["the", "in", "box"]}, {"sent": "is this a big box or a little box ?", "words": ["box", "is", "little", "this", "big", "a"]}, {"sent": "youre weighing the empty Smartie box now ?", "words": ["the", "box", "empty"]}, {"sent": "what color is that box ?", "words": ["what", "box", "that", "is"]}, {"sent": "well youre lying on the video box Thomas .", "words": ["on", "the", "box"]}, {"sent": "of the box ?", "words": ["of", "the", "box"]}, {"sent": "lets open the box .", "words": ["open", "the", "box"]}, {"sent": "thats right into a different post box .", "words": ["box", "into", "a"]}, {"sent": "ah where is the box ?", "words": ["where", "the", "box", "is"]}, {"sent": "anything else in the box now ?", "words": ["the", "in", "box"]}, {"sent": "Ill pop it in this little red box then you can help yourself when youre ready .", "words": ["red", "can", "help", "pop", "box", "yourself", "little", "you", "this", "in", "it", "then", "when"]}, {"sent": "unless you take this blue box off my head .", "words": ["off", "my", "box", "head", "blue", "you", "this", "take"]}, {"sent": "put just put it in this box .", "words": ["put", "box", "this", "in", "it"]}, {"sent": "by the red box .", "words": ["the", "red", "box", "by"]}, {"sent": "well shall we put the crayons back in the box then ?", "words": ["the", "put", "box", "back", "we", "in", "then"]}, {"sent": "and the lid back on the box .", "words": ["the", "box", "back", "and", "on"]}, {"sent": "whats in that box then ?", "words": ["in", "box", "that", "then"]}, {"sent": "theyre in the box arent they ?", "words": ["the", "they", "in", "box"]}, {"sent": "can we put that back in the box ?", "words": ["can", "the", "put", "box", "back", "we", "in", "that"]}, {"sent": "oh the little mans in the box .", "words": ["the", "in", "box", "little"]}, {"sent": "thats the sm very very tiny single malt whisky box .", "words": ["the", "tiny", "box"]}, {"sent": "well put the cotton wool in the box as well please .", "words": ["the", "in", "put", "box"]}, {"sent": "well you get me the box with all the nuts and bolts in .", "words": ["the", "all", "box", "with", "you", "get", "and", "in", "me"]}, {"sent": "are you going to put your hat in your magic box ?", "words": ["put", "your", "box", "to", "you", "in", "are", "hat"]}, {"sent": "do you think hes in the box ?", "words": ["the", "do", "think", "box", "you", "in"]}, {"sent": "he threw that box in didnt he ?", "words": ["in", "that", "box", "he"]}, {"sent": "youre breaking the box .", "words": ["the", "box"]}, {"sent": "now leave the paper on the box .", "words": ["the", "paper", "box", "on"]}, {"sent": "what else is in the box ?", "words": ["the", "what", "box", "is", "in"]}, {"sent": "theres a surprise in the box for you .", "words": ["the", "for", "box", "you", "in", "a"]}, {"sent": "in that box huh ?", "words": ["in", "box", "that"]}, {"sent": "which one do you think is the second box ?", "words": ["the", "do", "think", "box", "is", "you", "which"]}, {"sent": "I think what we needta do in a moment is put the waterwheel in the big white box .", "words": ["the", "what", "think", "do", "put", "box", "we", "is", "in", "white", "big", "a"]}, {"sent": "move that box outof the way .", "words": ["the", "box", "that"]}, {"sent": "I fell on the box .", "words": ["on", "the", "box"]}, {"sent": "um I dont know where the box for those books are .", "words": ["where", "the", "for", "box", "those", "dont", "are"]}, {"sent": "what else did she bring you in the toy box ?", "words": ["the", "what", "box", "bring", "you", "in", "toy", "did", "she"]}, {"sent": "I dont think youll find the egg box pet .", "words": ["the", "think", "box", "egg", "dont", "find"]}, {"sent": "wheres your money box John ?", "words": ["money", "box", "your"]}, {"sent": "your tissues are in the box with the Teletubbies on them .", "words": ["the", "your", "box", "with", "them", "in", "are", "on"]}, {"sent": "the music box ?", "words": ["the", "box"]}, {"sent": "its not worth putting them back in the box is it ?", "words": ["the", "box", "back", "is", "them", "in", "it", "not"]}, {"sent": "theres not really enough room in that box is there ?", "words": ["box", "is", "there", "in", "room", "that", "not"]}, {"sent": "see put him in the water box .", "words": ["the", "put", "box", "in", "him", "water", "see"]}, {"sent": "knocks in box .", "words": ["in", "box"]}, {"sent": "the box is ripped .", "words": ["the", "box", "is"]}, {"sent": "I dont think the red box eh the green box has a lid does it ?", "words": ["the", "red", "think", "does", "box", "dont", "it", "green", "a"]}, {"sent": "who is on the blue box ?", "words": ["the", "box", "is", "blue", "who", "on"]}, {"sent": "you see all the pictures on the box ?", "words": ["the", "all", "box", "you", "on", "see"]}, {"sent": "toy box ?", "words": ["toy", "box"]}, {"sent": "another one okay watch out for your little box okay .", "words": ["watch", "your", "for", "box", "little", "another", "out"]}, {"sent": "if I took the Eggo box out she would recognize what it is .", "words": ["the", "what", "box", "if", "is", "it", "out", "would", "she"]}, {"sent": "well why dont we use the ones in the box ?", "words": ["the", "box", "we", "dont", "in", "why"]}, {"sent": "what do you think happened the box that the puzzle was in ?", "words": ["the", "what", "do", "think", "box", "was", "you", "in", "puzzle", "that"]}, {"sent": "look this boy has got a brown cardboard box and hes rushing for the bus .", "words": ["the", "for", "box", "brown", "and", "this", "look", "bus", "a"]}, {"sent": "wheres the chocolate on the box ?", "words": ["the", "on", "chocolate", "box"]}, {"sent": "theres nothing in the box .", "words": ["the", "in", "box"]}, {"sent": "your lunch box okay come on .", "words": ["on", "box", "your"]}, {"sent": "you have a look in the box and see .", "words": ["the", "box", "you", "have", "and", "in", "look", "see", "a"]}, {"sent": "well go outside in a little bit theres your white box .", "words": ["your", "box", "go", "outside", "little", "in", "white", "a"]}, {"sent": "but thats the box that we stand the other box in before it goes into the loft .", "words": ["the", "box", "into", "we", "stand", "other", "in", "it", "that", "but"]}, {"sent": "theyre in a little box arent they ?", "words": ["box", "little", "they", "in", "a"]}, {"sent": "all tucked in the box now .", "words": ["the", "all", "in", "box"]}, {"sent": "well I dont think we can fix it well hafta look for a new one .", "words": ["can", "think", "for", "we", "fix", "dont", "it", "look", "new", "a"]}, {"sent": "you think you can fix my car next time it breaks ?", "words": ["can", "think", "my", "fix", "car", "you", "it"]}, {"sent": "and what you gonna fix now ?", "words": ["what", "fix", "you", "and"]}, {"sent": "not fix it .", "words": ["fix", "it", "not"]}, {"sent": "did you did you fix the uh television ?", "words": ["the", "fix", "did", "you"]}, {"sent": "can daddy fix it ?", "words": ["can", "fix", "it"]}, {"sent": "xxx fix it .", "words": ["fix", "it"]}, {"sent": "I wish there was somebody there to fix something I I tell ya .", "words": ["was", "to", "fix", "wish", "there"]}, {"sent": "Im gonna go fix us some food alright ?", "words": ["go", "fix", "us", "food", "some"]}, {"sent": "lets see if I can fix it .", "words": ["can", "if", "fix", "it", "see"]}, {"sent": "fix it ?", "words": ["fix", "it"]}, {"sent": "shall we take it apart and fix it ?", "words": ["we", "fix", "and", "it", "take"]}, {"sent": "okay want me ta fix it ?", "words": ["me", "it", "fix"]}, {"sent": "okay fix your hands .", "words": ["fix", "your"]}, {"sent": "now lemme fix it right okay ?", "words": ["fix", "it"]}, {"sent": "can you fix it ?", "words": ["can", "fix", "it", "you"]}, {"sent": "now Ill fix her arms .", "words": ["fix", "her"]}, {"sent": "can you fix it in the back ?", "words": ["can", "the", "back", "fix", "you", "in", "it"]}, {"sent": "whilst you do that Im going to fix these stairs .", "words": ["do", "these", "to", "fix", "you", "stairs", "that"]}, {"sent": "Mommy hasta fix it since you broke it though .", "words": ["fix", "it", "you"]}, {"sent": "when you want me to fix it Im not gonna fix it .", "words": ["to", "fix", "you", "it", "me", "when", "not"]}, {"sent": "can I fix your sleeves ?", "words": ["can", "fix", "your"]}, {"sent": "oh itll work in a minute I just gotta fix the batterys .", "words": ["the", "work", "fix", "in", "a"]}, {"sent": "somebody to fix the rail tracks quick .", "words": ["the", "to", "fix"]}, {"sent": "grandma fix it .", "words": ["fix", "it"]}, {"sent": "you want me to fix this to this ?", "words": ["to", "fix", "you", "this", "me"]}, {"sent": "it fix ?", "words": ["fix", "it"]}, {"sent": "if I cant fix it who are we gonna have fix it ?", "words": ["if", "we", "fix", "have", "it", "are", "who"]}, {"sent": "you hafta fix it ?", "words": ["fix", "it", "you"]}, {"sent": "what did Daddy fix your cart ?", "words": ["what", "your", "did", "fix"]}, {"sent": "fix that up lets see what it picked up wanna hear what it picked up .", "words": ["up", "what", "fix", "hear", "it", "wanna", "see", "that"]}, {"sent": "Ill see if I can fix that pool .", "words": ["can", "if", "fix", "pool", "see", "that"]}, {"sent": "well wait a minute Im trying to fix it for you .", "words": ["wait", "for", "to", "fix", "you", "it", "a"]}, {"sent": "let me fix this please .", "words": ["me", "fix", "this"]}, {"sent": "fix your lettuce .", "words": ["fix", "your"]}, {"sent": "Ill fix it so it doesnt hurt your arm does it hurt your arm ?", "words": ["does", "your", "hurt", "fix", "so", "it", "arm"]}, {"sent": "oh come here and Ill fix it .", "words": ["here", "fix", "it", "and"]}, {"sent": "he can fix things .", "words": ["can", "fix", "he"]}, {"sent": "Im just trying to remember how to fix it .", "words": ["to", "how", "it", "fix"]}, {"sent": "Mama fix that for you .", "words": ["fix", "for", "that", "you"]}, {"sent": "can you fix the screwdriver ?", "words": ["can", "fix", "the", "you"]}, {"sent": "fix what ?", "words": ["fix", "what"]}, {"sent": "well well fix them .", "words": ["fix", "them"]}, {"sent": "what do you needta fix it ?", "words": ["what", "do", "fix", "you", "it"]}, {"sent": "let me see if I cant fix this xxx .", "words": ["if", "fix", "me", "this", "see"]}, {"sent": "if you try and fix that side too then I think your fire engines going to be okay .", "words": ["think", "your", "if", "fix", "to", "be", "and", "you", "that", "too", "then", "try"]}, {"sent": "gonna fix it .", "words": ["fix", "it"]}, {"sent": "mommys gonna fix this airconditioner so it .", "words": ["fix", "it", "so", "this"]}, {"sent": "waimit let me fix it for you and then Ill give it to you .", "words": ["give", "for", "fix", "to", "you", "and", "it", "me", "then"]}, {"sent": "fix your what ?", "words": ["fix", "what", "your"]}, {"sent": "you can fix it .", "words": ["can", "fix", "it", "you"]}, {"sent": "fix it all up .", "words": ["all", "fix", "it", "up"]}, {"sent": "yeah Im gonna try to fix a little snack because hes grumpy .", "words": ["to", "fix", "little", "try", "try to", "because", "a"]}, {"sent": "Ill fix it so it doesnt so it feels better .", "words": ["better", "fix", "it", "so"]}, {"sent": "Papa will fix it when he comes home .", "words": ["fix", "it", "he", "will", "when", "home"]}, {"sent": "youre gonna hafta fix up the pieces .", "words": ["up", "fix", "the"]}, {"sent": "do you want me to fix it ?", "words": ["do", "to", "fix", "you", "it", "me"]}, {"sent": "are you gonna fix it again ?", "words": ["fix", "are", "it", "you"]}, {"sent": "here Im going to fix you something to drink .", "words": ["drink", "to", "fix", "you", "here"]}, {"sent": "here Ill fix it .", "words": ["here", "fix", "it"]}, {"sent": "Ill fix it for you dont worry .", "words": ["for", "fix", "you", "dont", "it"]}, {"sent": "thats alright just fix it .", "words": ["fix", "it"]}, {"sent": "you can fix it put it down put it down on the floor theres one two put that one on the end three .", "words": ["can", "the", "put", "fix", "you", "it", "on", "down", "that"]}, {"sent": "maybe daddy can fix it .", "words": ["can", "fix", "it"]}, {"sent": "we just hafta fix it thats all .", "words": ["all", "fix", "it", "we"]}, {"sent": "you fix my tv ?", "words": ["fix", "my", "tv", "you"]}, {"sent": "and Mummy said dont worry Ill fix it .", "words": ["dont", "it", "fix", "and"]}, {"sent": "can we fix it ?", "words": ["can", "fix", "it", "we"]}, {"sent": "want Mommy fix it ?", "words": ["fix", "it"]}, {"sent": "fix him .", "words": ["fix", "him"]}, {"sent": "you fix it back .", "words": ["fix", "it", "back", "you"]}, {"sent": "yeah just leave that there and then well fix it later .", "words": ["fix", "and", "there", "it", "then", "that"]}, {"sent": "Ill try to fix it after I take a shower .", "words": ["try", "shower", "to", "fix", "it", "take", "try to", "a"]}, {"sent": "and hes going to fix them with the .", "words": ["the", "to", "fix", "with", "them", "and"]}, {"sent": "Ill fix it .", "words": ["fix", "it"]}, {"sent": "you gonna fix up the house ?", "words": ["up", "the", "fix", "you", "house"]}, {"sent": "fix the truck .", "words": ["the", "fix", "truck"]}, {"sent": "fix it what ?", "words": ["fix", "what", "it"]}, {"sent": "pliers yeah why dont you fix the train with the pliers ?", "words": ["the", "train", "fix", "with", "you", "dont", "why"]}, {"sent": "do you think we could fix one ?", "words": ["do", "think", "we", "fix", "you", "could"]}, {"sent": "you gonna fix it ?", "words": ["fix", "it", "you"]}, {"sent": "you can fix it .", "words": ["can", "fix", "it", "you"]}, {"sent": "get Bob to fix it .", "words": ["get", "to", "it", "fix"]}, {"sent": "they fix you .", "words": ["they", "fix", "you"]}, {"sent": "I cant fix it its broke .", "words": ["fix", "it"]}, {"sent": "and maybe well fix it later tomorrow .", "words": ["fix", "it", "and"]}, {"sent": "is he gonna fix something ?", "words": ["fix", "he", "is"]}, {"sent": "fix the head .", "words": ["the", "fix", "head"]}, {"sent": "that mans going to fix that mans going to fix it isnt he ?", "words": ["to", "fix", "it", "he", "that"]}, {"sent": "are you going to fix it all by yourself ?", "words": ["all", "yourself", "to", "fix", "you", "by", "are", "it"]}, {"sent": "I make I fix your plate next .", "words": ["your", "fix", "make", "plate"]}, {"sent": "Ill fix it .", "words": ["fix", "it"]}, {"sent": "Mummy fix it .", "words": ["fix", "it"]}, {"sent": "who you gonna fix me ?", "words": ["fix", "me", "you", "who"]}, {"sent": "whos gonna fix the battery whos gonna fix the car sweetheart .", "words": ["the", "fix", "car"]}, {"sent": "but Ill Ill fix it to help you move it up .", "words": ["up", "help", "to", "fix", "you", "it", "but"]}, {"sent": "are you gonna fix it Peter ?", "words": ["fix", "are", "it", "you"]}, {"sent": "oh fix your sleeves for ya .", "words": ["fix", "for", "your"]}, {"sent": "Mommy fix the picnic table .", "words": ["the", "fix", "table"]}, {"sent": "can Carl not fix it ?", "words": ["can", "fix", "it", "not"]}, {"sent": "theres the bottle .", "words": ["the", "bottle"]}, {"sent": "where is that spray bottle ?", "words": ["where", "bottle", "that", "is"]}, {"sent": "the spoon the peanutbutter the pizzas the waffles the bread the bottle .", "words": ["the", "bread", "bottle", "spoon"]}, {"sent": "putting the shampoo bottle in your mouth ?", "words": ["the", "your", "mouth", "in", "bottle"]}, {"sent": "bottle empty .", "words": ["bottle", "empty"]}, {"sent": "you better give dolly some bottle and make her feel better .", "words": ["better", "her", "you", "and", "bottle", "some", "give", "make"]}, {"sent": "right we dont normally shes maybe had well she actually had a bottle the other day .", "words": ["the", "we", "other", "dont", "bottle", "she", "a"]}, {"sent": "and you put you like turn it upside down in the bottle .", "words": ["the", "put", "like", "you", "and", "in", "it", "bottle", "down"]}, {"sent": "bottle .", "words": ["bottle"]}, {"sent": "in the bottle ?", "words": ["the", "in", "bottle"]}, {"sent": "soda bottle .", "words": ["bottle", "soda"]}, {"sent": "a bottle .", "words": ["bottle", "a"]}, {"sent": "gonna give her her bottle ?", "words": ["bottle", "give", "her"]}, {"sent": "its your bottle top .", "words": ["bottle", "your"]}, {"sent": "thats a bottle of milk .", "words": ["of", "milk", "bottle", "a"]}, {"sent": "Im going to get Amys bottle .", "words": ["get", "to", "bottle"]}, {"sent": "you ready to nightnight with your bottle ?", "words": ["your", "to", "with", "you", "bottle"]}, {"sent": "she may recognize thats a bottle .", "words": ["bottle", "she", "a"]}, {"sent": "everybody in bed and hes roaming looking for a bottle .", "words": ["for", "bed", "and", "in", "bottle", "a"]}, {"sent": "yeah you can drink it outof the bottle sometimes sometimes you can drink it outof a glass or a cup .", "words": ["can", "the", "cup", "drink", "you", "it", "glass", "bottle", "a"]}, {"sent": "it looked like she was trying to coordinate her hands to to do whatever she wanted to do with the baby and the bottle .", "words": ["the", "do", "to", "was", "with", "like", "and", "it", "bottle", "her", "she"]}, {"sent": "she liked the fake bottle .", "words": ["the", "bottle", "she"]}, {"sent": "mm yes it is a babys bottle .", "words": ["bottle", "it", "a", "is"]}, {"sent": "and uh hes got an empty bottle here .", "words": ["empty", "here", "and", "an", "bottle"]}, {"sent": "look at that little baby cow having a bottle .", "words": ["look", "little", "at", "bottle", "that", "cow", "a"]}, {"sent": "a bottle of what ?", "words": ["of", "what", "bottle", "a"]}, {"sent": "for the bottle .", "words": ["the", "bottle", "for"]}, {"sent": "is that your bottle ?", "words": ["your", "bottle", "that", "is"]}, {"sent": "that one or does this bottle belong to him ?", "words": ["does", "to", "this", "him", "bottle", "that"]}, {"sent": "do you have apple juice in your bottle ?", "words": ["do", "your", "you", "have", "in", "apple", "bottle", "juice"]}, {"sent": "its a bottle huh .", "words": ["bottle", "a"]}, {"sent": "bottle ?", "words": ["bottle"]}, {"sent": "its the same Lubriderm that came out of that bottle .", "words": ["the", "of", "same", "out", "bottle", "that"]}, {"sent": "well you dont drink from the bottle anymore .", "words": ["the", "drink", "you", "dont", "bottle"]}, {"sent": "bottle .", "words": ["bottle"]}, {"sent": "do you want the bottle ?", "words": ["the", "do", "bottle", "you"]}, {"sent": "look what the puppets gonna drink from the bottle Stef .", "words": ["the", "what", "drink", "look", "bottle"]}, {"sent": "thats a baby bottle .", "words": ["bottle", "a"]}, {"sent": "you wanna feed the bottle to the baby ?", "words": ["the", "to", "you", "wanna", "feed", "bottle"]}, {"sent": "fits the bottle .", "words": ["the", "bottle"]}, {"sent": "bottle ?", "words": ["bottle"]}, {"sent": "but what happened to the bottle ?", "words": ["the", "what", "to", "bottle", "but"]}, {"sent": "the bottle and these measuring spoons .", "words": ["the", "these", "bottle", "and"]}, {"sent": "bottle .", "words": ["bottle"]}, {"sent": "and give her a bottle just like you get a bottle .", "words": ["give", "like", "and", "you", "get", "bottle", "her", "a"]}, {"sent": "oh you found a bottle ?", "words": ["bottle", "a", "you"]}, {"sent": "you get off and play with your Playmobil while I give Amy her bottle and then well have a story .", "words": ["off", "story", "your", "play", "with", "then", "and", "get", "you", "have", "bottle", "give", "her", "a"]}, {"sent": "thats a bottle for the baby .", "words": ["the", "bottle", "for", "a"]}, {"sent": "wheres the babys bottle ?", "words": ["the", "bottle"]}, {"sent": "you really want that bottle because it looks like yours ?", "words": ["like", "you", "it", "bottle", "that", "because"]}, {"sent": "thats not your bottle Im sorry .", "words": ["bottle", "your", "not"]}, {"sent": "thats a bottle .", "words": ["bottle", "a"]}, {"sent": "heres your bottle .", "words": ["bottle", "your"]}, {"sent": "you wanna eat the bottle .", "words": ["the", "you", "eat", "wanna", "bottle"]}, {"sent": "bottle .", "words": ["bottle"]}, {"sent": "she never took to the bottle unfortunately so .", "words": ["the", "to", "so", "bottle", "she"]}, {"sent": "theres the bottle .", "words": ["the", "bottle"]}, {"sent": "bottle ?", "words": ["bottle"]}, {"sent": "bottle sit .", "words": ["sit", "bottle"]}, {"sent": "its bottle .", "words": ["bottle"]}, {"sent": "a bottle .", "words": ["bottle", "a"]}, {"sent": "is that your bottle ?", "words": ["your", "bottle", "that", "is"]}, {"sent": "we have another bottle .", "words": ["have", "bottle", "another", "we"]}, {"sent": "you going to feed the baby her bottle ?", "words": ["the", "to", "you", "feed", "bottle", "her"]}, {"sent": "do you want a bottle ?", "words": ["do", "bottle", "a", "you"]}, {"sent": "yeah heres the bottle .", "words": ["the", "bottle"]}, {"sent": "so she did that a lot which she doesnt have a she has babies at home but she doesnt have a bottle so that was interesting .", "words": ["a lot", "so", "was", "which", "have", "did", "at", "home", "bottle", "that", "she", "but", "a"]}, {"sent": "who brought you a bottle ?", "words": ["bottle", "a", "you", "who"]}, {"sent": "thats a pretend bottle baby bottle .", "words": ["pretend", "bottle", "a"]}, {"sent": "and he had I think one of the I think the bottle is what he had first .", "words": ["the", "what", "think", "is", "of", "first", "and", "he", "bottle"]}, {"sent": "a bottle full of um .", "words": ["of", "bottle", "full", "a"]}, {"sent": "Mommas putting on the bottle fer ya .", "words": ["on", "the", "bottle"]}, {"sent": "you want the bottle ?", "words": ["the", "bottle", "you"]}, {"sent": "are you pressing the top of the juice bottle ?", "words": ["the", "of", "you", "are", "bottle", "juice"]}, {"sent": "when you have little baby lambs sometimes you hafta give them milk from a bottle .", "words": ["them", "little", "have", "you", "bottle", "milk", "give", "when", "a"]}, {"sent": "heehee feed the clown the bottle .", "words": ["the", "bottle", "feed"]}, {"sent": "oh okay I was gonna wash off the bottle cause it had a lot of food on the outside of it .", "words": ["off", "the", "of", "was", "outside", "a lot", "food", "it", "on", "bottle", "wash", "a"]}, {"sent": "its a bottle .", "words": ["bottle", "a"]}, {"sent": "oh we got a bottle with orange juice .", "words": ["we", "with", "orange", "bottle", "juice", "a"]}, {"sent": "oh we put away Aislinns bottle .", "words": ["put", "away", "bottle", "we"]}, {"sent": "xxx give her a bottle ?", "words": ["bottle", "give", "her", "a"]}, {"sent": "have you got the bottle ?", "words": ["have", "the", "bottle", "you"]}, {"sent": "out of mommys bottle ?", "words": ["of", "bottle", "out"]}, {"sent": "is that a bottle ?", "words": ["bottle", "that", "a", "is"]}, {"sent": "and he threw the bottle as far as he could .", "words": ["the", "and", "he", "bottle", "could"]}, {"sent": "I was impressed he kn knew what to do with the bottle cause hes never had a bottle .", "words": ["the", "what", "do", "to", "was", "with", "he", "bottle", "a"]}, {"sent": "its milk in the bottle yes .", "words": ["milk", "the", "in", "bottle"]}, {"sent": "I can see your babys bottle down there look .", "words": ["can", "your", "there", "look", "see", "bottle", "down"]}, {"sent": "give the baby a ba no give the baby the bottle .", "words": ["the", "give", "bottle", "a"]}, {"sent": "not just one bottle .", "words": ["bottle", "not"]}, {"sent": "no no dont drink bottle .", "words": ["dont", "bottle", "drink"]}, {"sent": "I think she needs a bottle really .", "words": ["think", "bottle", "she", "a"]}, {"sent": "its a pretend bottle ?", "words": ["pretend", "bottle", "a"]}, {"sent": "bottle the bottle ?", "words": ["the", "bottle"]}, {"sent": "bottle bottle .", "words": ["bottle"]}, {"sent": "youre going to get your bottle whoops Im sorry .", "words": ["get", "to", "bottle", "your"]}, {"sent": "the bottle ?", "words": ["the", "bottle"]}, {"sent": "this is a bottle right ?", "words": ["this", "bottle", "a", "is"]}, {"sent": "what comes inside the bottle ?", "words": ["the", "what", "inside", "bottle"]}, {"sent": "Mommy close the door .", "words": ["the", "close", "door"]}, {"sent": "close the door .", "words": ["the", "close", "door"]}, {"sent": "would you close the dishwasher for me now ?", "words": ["the", "for", "you", "would", "me", "close"]}, {"sent": "close your eyes and count to ten and Ill hide again .", "words": ["your", "hide", "to", "and", "close"]}, {"sent": "nono dont close that door .", "words": ["dont", "close", "that", "door"]}, {"sent": "close it thats fine .", "words": ["close", "it", "fine"]}, {"sent": "thats close by .", "words": ["close", "by"]}, {"sent": "five little fingers snuggle up close .", "words": ["up", "close", "little"]}, {"sent": "mm well it wont close without another one .", "words": ["another", "close", "it"]}, {"sent": "you cant close it .", "words": ["close", "it", "you"]}, {"sent": "close them tight .", "words": ["close", "them"]}, {"sent": "you got to close your drawer first .", "words": ["your", "to", "first", "you", "drawer", "close"]}, {"sent": "well close to the clouds but not past the clouds .", "words": ["the", "to", "close", "not", "but"]}, {"sent": "then we close it .", "words": ["close", "then", "it", "we"]}, {"sent": "what was it close to ?", "words": ["what", "was", "to", "it", "close"]}, {"sent": "close it and try again .", "words": ["close", "it", "try", "and"]}, {"sent": "and we are gonna close it up .", "words": ["up", "we", "and", "it", "are", "close"]}, {"sent": "shoo that was a close one huh ?", "words": ["was", "close", "that", "a"]}, {"sent": "close may I please have some Chewies ?", "words": ["have", "some", "close"]}, {"sent": "you dont needta get close .", "words": ["get", "dont", "close", "you"]}, {"sent": "and Im close to the camera now so Im gonna walk outside and if you can hear me then .", "words": ["the", "can", "camera", "if", "to", "so", "outside", "then", "and", "you", "hear", "me", "close", "walk"]}, {"sent": "so maybe whilst were setting up well close the shop .", "words": ["up", "the", "so", "close", "were"]}, {"sent": "now you wanna close it ?", "words": ["close", "it", "wanna", "you"]}, {"sent": "close the door .", "words": ["the", "close", "door"]}, {"sent": "close .", "words": ["close"]}, {"sent": "close the door .", "words": ["the", "close", "door"]}, {"sent": "close your eyes .", "words": ["close", "your"]}, {"sent": "she always stayed very close to Mama Duck .", "words": ["to", "close", "she"]}, {"sent": "can you close the door ?", "words": ["can", "the", "you", "close", "door"]}, {"sent": "now its all close up .", "words": ["up", "all", "close"]}, {"sent": "close the door .", "words": ["the", "close", "door"]}, {"sent": "close your eyes .", "words": ["close", "your"]}, {"sent": "close the back door .", "words": ["the", "close", "back", "door"]}, {"sent": "close it up ?", "words": ["up", "close", "it"]}, {"sent": "close the box of the furniture ?", "words": ["the", "of", "close", "box"]}, {"sent": "and now close goes this way .", "words": ["this", "close", "and"]}, {"sent": "now you can close it .", "words": ["can", "close", "it", "you"]}, {"sent": "but she had gone into the house so quickly she had forgotten to close the gate .", "words": ["the", "into", "so", "to", "house", "close", "she", "but"]}, {"sent": "close .", "words": ["close"]}, {"sent": "or pretty close .", "words": ["close", "pretty"]}, {"sent": "and close them .", "words": ["close", "them", "and"]}, {"sent": "and sometimes hell decide halfway through the book that hes done and hell grab the back of the book and close it and say byebye .", "words": ["the", "back", "of", "say", "and", "it", "book", "close", "that"]}, {"sent": "mm close it .", "words": ["close", "it"]}, {"sent": "okay but Im gonna close this door a little its cold .", "words": ["little", "this", "close", "door", "cold", "but", "a"]}, {"sent": "close the other side .", "words": ["the", "close", "other"]}, {"sent": "close .", "words": ["close"]}, {"sent": "Im gonna close that window because its a bit cold isnt it ?", "words": ["close", "it", "window", "that", "cold", "because", "a"]}, {"sent": "yeah go go close you gotta close your eye .", "words": ["your", "go", "eye", "you", "close"]}, {"sent": "can you close it ?", "words": ["can", "close", "it", "you"]}, {"sent": "close it up .", "words": ["up", "close", "it"]}, {"sent": "can you close it ?", "words": ["can", "close", "it", "you"]}, {"sent": "now close it .", "words": ["close", "it"]}, {"sent": "and they dont wanna get up so close do they ?", "words": ["up", "do", "so", "and", "get", "dont", "they", "wanna", "close"]}, {"sent": "open and close open and close .", "words": ["open", "close", "and"]}, {"sent": "close it up .", "words": ["up", "close", "it"]}, {"sent": "okay Ill close it .", "words": ["close", "it"]}, {"sent": "and then you close the drawer .", "words": ["the", "then", "you", "and", "drawer", "close"]}, {"sent": "can you close the door ?", "words": ["can", "the", "you", "close", "door"]}, {"sent": "close it and put it on ?", "words": ["put", "and", "it", "on", "close"]}, {"sent": "close your eyes .", "words": ["close", "your"]}, {"sent": "did you close it ?", "words": ["close", "it", "did", "you"]}, {"sent": "can ya close it ?", "words": ["can", "close", "it"]}, {"sent": "alright now you can close it .", "words": ["can", "close", "it", "you"]}, {"sent": "xxx Cathy now and tell her to close her eyes .", "words": ["to", "close", "her", "and"]}, {"sent": "Im gonna close this refrigerator now .", "words": ["this", "close", "refrigerator"]}, {"sent": "close your eyes .", "words": ["close", "your"]}, {"sent": "close them up .", "words": ["up", "close", "them"]}, {"sent": "Elana you close that cabinet .", "words": ["close", "that", "you"]}, {"sent": "we gotta close it .", "words": ["close", "it", "we"]}, {"sent": "xxx too close you can get right on top of things .", "words": ["can", "of", "you", "get", "too", "on", "close"]}, {"sent": "and me and you are very close friends .", "words": ["you", "and", "are", "me", "close"]}, {"sent": "with Eleanors hands clasped close close around all night long .", "words": ["all", "with", "long", "around", "close"]}, {"sent": "close what about the cherries ?", "words": ["the", "what", "close", "about"]}, {"sent": "and close it ?", "words": ["close", "it", "and"]}, {"sent": "well I think there are perhaps too many chocolates to close it straight away .", "words": ["think", "to", "there", "are", "it", "away", "too", "close"]}, {"sent": "but thats close .", "words": ["close", "but"]}, {"sent": "close your eyes .", "words": ["close", "your"]}, {"sent": "shes shes close to your size .", "words": ["to", "close", "your"]}, {"sent": "close .", "words": ["close"]}, {"sent": "yeah its kinda hard to close .", "words": ["to", "close", "hard"]}, {"sent": "except shes got a poorly eye hasnt she that doesnt close very well .", "words": ["eye", "close", "that", "she", "a"]}, {"sent": "close it .", "words": ["close", "it"]}, {"sent": "you youll never close it and Ill never open it .", "words": ["open", "and", "you", "it", "close"]}, {"sent": "here man close your eyes .", "words": ["here", "close", "your"]}, {"sent": "insert the cartridge and close door .", "words": ["the", "close", "door", "and"]}, {"sent": "wanna close it ?", "words": ["close", "it", "wanna"]}, {"sent": "thats right close that one now put it in this one .", "words": ["put", "this", "in", "it", "close", "that"]}, {"sent": "look at it real close .", "words": ["it", "close", "at", "look"]}, {"sent": "you putting that thing back on there and close it back up right ?", "words": ["up", "back", "you", "and", "there", "it", "on", "close", "that"]}, {"sent": "lets get your close .", "words": ["get", "close", "your"]}, {"sent": "close the bathroom door .", "words": ["the", "bathroom", "close", "door"]}, {"sent": "then close it up .", "words": ["up", "close", "then", "it"]}, {"sent": "are you almost done building because I think Josephs getting very close to naptime .", "words": ["think", "to", "you", "are", "close", "because"]}, {"sent": "close your eyes again .", "words": ["close", "your"]}, {"sent": "close the door .", "words": ["the", "close", "door"]}, {"sent": "Megan close the back door please .", "words": ["the", "close", "back", "door"]}, {"sent": "you wanna op close it ?", "words": ["close", "it", "wanna", "you"]}, {"sent": "would you like to close the door now please ?", "words": ["the", "to", "like", "you", "would", "close", "door"]}, {"sent": "close it .", "words": ["close", "it"]}, {"sent": "if you close your eyes you cant see .", "words": ["your", "if", "you", "see", "close"]}, {"sent": "the zebra likes the jungle .", "words": ["the", "zebra"]}, {"sent": "zebra .", "words": ["zebra"]}, {"sent": "is that a zebra ?", "words": ["zebra", "that", "a", "is"]}, {"sent": "zebra .", "words": ["zebra"]}, {"sent": "zebra .", "words": ["zebra"]}, {"sent": "zebra .", "words": ["zebra"]}, {"sent": "a zebra .", "words": ["zebra", "a"]}, {"sent": "a zebra .", "words": ["zebra", "a"]}, {"sent": "thats a zebra .", "words": ["zebra", "a"]}, {"sent": "a zebra .", "words": ["zebra", "a"]}, {"sent": "zebra .", "words": ["zebra"]}, {"sent": "a zebra ?", "words": ["zebra", "a"]}, {"sent": "hello zebra .", "words": ["zebra"]}, {"sent": "oh the zebra no okay .", "words": ["the", "zebra"]}, {"sent": "a zebra .", "words": ["zebra", "a"]}, {"sent": "yes an animal called a zebra .", "words": ["zebra", "an", "animal", "a"]}, {"sent": "lets not cook the zebra .", "words": ["the", "zebra", "cook", "not"]}, {"sent": "thats part of a zebra .", "words": ["of", "zebra", "a"]}, {"sent": "zebra .", "words": ["zebra"]}, {"sent": "thats quite the zebra .", "words": ["the", "zebra"]}, {"sent": "a zebra huh ?", "words": ["zebra", "a"]}, {"sent": "think theres another zebra ?", "words": ["another", "zebra", "think"]}, {"sent": "I think youll see a zebra yeah .", "words": ["zebra", "see", "think", "a"]}, {"sent": "or some people say zebra .", "words": ["zebra", "some", "say"]}, {"sent": "zebra that begins a z .", "words": ["zebra", "that", "a"]}, {"sent": "zebra .", "words": ["zebra"]}, {"sent": "thats a zebra isnt it ?", "words": ["zebra", "it", "a"]}, {"sent": "its a zebra .", "words": ["zebra", "a"]}, {"sent": "hello zebra youre the stripiest .", "words": ["the", "zebra"]}, {"sent": "and in comes the zebra .", "words": ["the", "zebra", "in", "and"]}, {"sent": "haha hi zebra .", "words": ["zebra"]}, {"sent": "zebra .", "words": ["zebra"]}, {"sent": "you heard a zebra ?", "words": ["zebra", "a", "you"]}, {"sent": "zebra ?", "words": ["zebra"]}, {"sent": "it says zebra .", "words": ["zebra", "it"]}, {"sent": "zebra colts like to run and play .", "words": ["zebra", "to", "play", "like", "and", "run"]}, {"sent": "thats a zebra .", "words": ["zebra", "a"]}, {"sent": "whos gonna play with the zebra ?", "words": ["the", "zebra", "play", "with"]}, {"sent": "do you think the zebra wants to eat some pizza ?", "words": ["the", "do", "think", "zebra", "to", "pizza", "you", "eat", "some"]}, {"sent": "snuggle giraffe snuggle zebra .", "words": ["zebra", "giraffe"]}, {"sent": "we have a zebra at home .", "words": ["zebra", "we", "have", "at", "home", "a"]}, {"sent": "I think its a nice zebra .", "words": ["zebra", "think", "nice", "a"]}, {"sent": "hello zebra .", "words": ["zebra"]}, {"sent": "how about the zebra ?", "words": ["the", "zebra", "how", "about"]}, {"sent": "zebra can visit the farm ?", "words": ["can", "zebra", "the"]}, {"sent": "a zebra .", "words": ["zebra", "a"]}, {"sent": "does the zebra gallop ?", "words": ["the", "does", "zebra"]}, {"sent": "theres mister zebra guy .", "words": ["zebra"]}, {"sent": "and heres a zebra .", "words": ["zebra", "a", "and"]}, {"sent": "there it is here well put zebra over here .", "words": ["zebra", "put", "is", "here", "there", "it", "over"]}, {"sent": "and you said you saw a zebra at the um animal nursery .", "words": ["the", "zebra", "you", "and", "animal", "at", "a"]}, {"sent": "we have a zebra .", "words": ["have", "zebra", "a", "we"]}, {"sent": "he zebra ?", "words": ["zebra", "he"]}, {"sent": "oh heres a zebra .", "words": ["zebra", "a"]}, {"sent": "thats right its a zebra .", "words": ["zebra", "a"]}, {"sent": "and whats a zebra got ?", "words": ["zebra", "a", "and"]}, {"sent": "oh the zebra only the zebra ?", "words": ["the", "zebra"]}, {"sent": "and a zebra .", "words": ["zebra", "a", "and"]}, {"sent": "hello zebra .", "words": ["zebra"]}, {"sent": "what does a zebra say ?", "words": ["zebra", "what", "does", "say", "a"]}, {"sent": "zebra .", "words": ["zebra"]}, {"sent": "its a zebra leopard giraffe .", "words": ["zebra", "giraffe", "a"]}, {"sent": "is that a zebra ?", "words": ["zebra", "that", "a", "is"]}, {"sent": "how about a zebra ?", "words": ["zebra", "how", "about", "a"]}, {"sent": "a baby zebra thats also called a fowl .", "words": ["zebra", "a"]}, {"sent": "zebra .", "words": ["zebra"]}, {"sent": "okay Abe make the zebra go to the car .", "words": ["the", "zebra", "go", "to", "car", "make"]}, {"sent": "okay the zebra .", "words": ["the", "zebra"]}, {"sent": "you the zebra would like that ?", "words": ["the", "zebra", "like", "you", "would", "that"]}, {"sent": "whered zebra go .", "words": ["zebra", "go"]}, {"sent": "Zoe Zoe zebra .", "words": ["zebra"]}, {"sent": "did we make a zebra yesterday ?", "words": ["zebra", "we", "did", "make", "a"]}, {"sent": "can you find the zebra ?", "words": ["can", "the", "zebra", "you", "find"]}, {"sent": "that is a zebra .", "words": ["zebra", "that", "a", "is"]}, {"sent": "yeah its kindof a weird looking zebra .", "words": ["zebra", "a"]}, {"sent": "we dont even have a zebra .", "words": ["zebra", "we", "have", "dont", "a"]}, {"sent": "whats wrong with the zebra ?", "words": ["the", "zebra", "with"]}, {"sent": "zebra .", "words": ["zebra"]}, {"sent": "heres the big zebra little zebra .", "words": ["the", "zebra", "big", "little"]}, {"sent": "zebra .", "words": ["zebra"]}, {"sent": "um I dont think there are lions at that one shes seen the they have zebra and they have um .", "words": ["the", "zebra", "think", "and", "have", "dont", "there", "are", "they", "at", "that"]}, {"sent": "but this for the z thats a zebra William .", "words": ["the", "zebra", "for", "this", "but", "a"]}, {"sent": "its a zebra .", "words": ["zebra", "a"]}, {"sent": "its a zebra .", "words": ["zebra", "a"]}, {"sent": "thats a zebra .", "words": ["zebra", "a"]}, {"sent": "yeah the zebra yeah I do its like that house that had a zebra in it that we see when we go to the beach ?", "words": ["the", "zebra", "do", "when", "we", "go", "to", "like", "in", "it", "house", "beach", "see", "that", "a"]}, {"sent": "next to the zebra .", "words": ["the", "to", "zebra"]}, {"sent": "zebra .", "words": ["zebra"]}, {"sent": "zebra .", "words": ["zebra"]}, {"sent": "say zebra .", "words": ["zebra", "say"]}, {"sent": "oh the zebra she didnt care about at all .", "words": ["the", "zebra", "all", "about", "at", "she"]}, {"sent": "zebra .", "words": ["zebra"]}, {"sent": "thats the one about the zebra .", "words": ["the", "zebra", "about"]}, {"sent": "zebra .", "words": ["zebra"]}, {"sent": "zebra isnt it ?", "words": ["zebra", "it"]}, {"sent": "heres a zebra .", "words": ["zebra", "a"]}, {"sent": "zebra .", "words": ["zebra"]}, {"sent": "we have a zebra .", "words": ["have", "zebra", "a", "we"]}, {"sent": "a zebra .", "words": ["zebra", "a"]}, {"sent": "isnt it funny how weve not been in this drawer for such a long time ?", "words": ["how", "for", "long", "this", "in", "it", "drawer", "not", "a"]}, {"sent": "what we do is put the things in the drawer and then we .", "words": ["the", "what", "do", "put", "is", "we", "and", "in", "drawer", "then"]}, {"sent": "dont take it outof his hands sweetie okay why dont you go put it away in the drawer .", "words": ["the", "put", "go", "you", "dont", "in", "it", "away", "his", "drawer", "take", "why"]}, {"sent": "in the bottom drawer you didnt find it ?", "words": ["the", "bottom", "you", "in", "it", "find", "drawer"]}, {"sent": "from the drawer ?", "words": ["the", "drawer"]}, {"sent": "Danny give me a package of pills in that drawer .", "words": ["drawer", "of", "in", "me", "give", "that", "a"]}, {"sent": "okay Ill put this sock away in the drawer .", "words": ["the", "put", "this", "in", "away", "sock", "drawer"]}, {"sent": "it was the drawer of the toy box closing I think .", "words": ["the", "think", "box", "of", "was", "toy", "it", "drawer"]}, {"sent": "a drawer ?", "words": ["drawer", "a"]}, {"sent": "do you wanna put Snoopy in the drawer ?", "words": ["the", "do", "put", "you", "in", "wanna", "drawer"]}, {"sent": "why is it too hard for you you can do it youre an excellent drawer .", "words": ["can", "do", "for", "is", "you", "an", "it", "hard", "too", "drawer", "why"]}, {"sent": "why dont you close the drawer .", "words": ["the", "you", "dont", "drawer", "close", "why"]}, {"sent": "in my drawer ?", "words": ["drawer", "in", "my"]}, {"sent": "push the toy drawer in a little ?", "words": ["the", "push", "little", "toy", "in", "drawer", "a"]}, {"sent": "the mackintosh thats in your drawer .", "words": ["the", "drawer", "in", "your"]}, {"sent": "I think its in the drawer with all the other lego things .", "words": ["the", "all", "think", "with", "other", "in", "drawer"]}, {"sent": "in the drawer ?", "words": ["the", "drawer", "in"]}, {"sent": "look in the drawer and see if its over there .", "words": ["the", "if", "see", "and", "there", "in", "look", "drawer", "over"]}, {"sent": "shut the drawer .", "words": ["the", "drawer"]}, {"sent": "hey look whats in this drawer .", "words": ["this", "in", "drawer", "look"]}, {"sent": "look in my top dresser drawer xxx .", "words": ["drawer", "in", "my", "look"]}, {"sent": "look in the drawer where you keep your shirt .", "words": ["the", "where", "your", "shirt", "you", "in", "look", "drawer"]}, {"sent": "and whats in this drawer ?", "words": ["this", "in", "drawer", "and"]}, {"sent": "well try in this drawer .", "words": ["this", "in", "try", "drawer"]}, {"sent": "drawer .", "words": ["drawer"]}, {"sent": "xxx your drawer .", "words": ["drawer", "your"]}, {"sent": "this is your pajamas drawer is this your pants drawer ?", "words": ["your", "is", "this", "pants", "pajamas", "drawer"]}, {"sent": "thats a drawer oh .", "words": ["drawer", "a"]}, {"sent": "I keep that in that drawer because when you have postcards we like to Bluetac them on your bedroom door dont we ?", "words": ["door", "your", "when", "we", "to", "them", "like", "you", "have", "dont", "in", "on", "drawer", "that", "because", "bedroom"]}, {"sent": "when we get them off you and hide them in the drawer and then forget to take them out .", "words": ["off", "the", "hide", "we", "to", "them", "you", "get", "and", "in", "take", "out", "drawer", "then", "when"]}, {"sent": "let me look in the drawer in here .", "words": ["the", "drawer", "here", "in", "look", "me"]}, {"sent": "lets put them back in the drawer okay ?", "words": ["the", "put", "back", "them", "in", "drawer"]}, {"sent": "drawer .", "words": ["drawer"]}, {"sent": "I think youve put the bus in the drawer where the pegs needta be .", "words": ["the", "where", "think", "put", "be", "in", "drawer", "bus"]}, {"sent": "get your little stool and then you can see inside the drawer Lara .", "words": ["can", "the", "your", "drawer", "inside", "and", "get", "little", "you", "see", "then"]}, {"sent": "well you know how to open the drawer Thomas .", "words": ["the", "how", "open", "to", "you", "drawer"]}, {"sent": "whats in that drawer ?", "words": ["drawer", "in", "that"]}, {"sent": "cmon lets go to your drawer you can show me what you have in your drawer .", "words": ["can", "what", "your", "go", "show", "to", "me", "you", "have", "in", "drawer"]}, {"sent": "thats the bottom drawer .", "words": ["the", "drawer", "bottom"]}, {"sent": "do you wanna put Daddys socks in the drawer ?", "words": ["the", "do", "put", "you", "in", "wanna", "drawer"]}, {"sent": "thats in the wrong drawer idnt it ?", "words": ["the", "drawer", "in", "it"]}, {"sent": "what did Ruby find in Maxs drawer ?", "words": ["what", "in", "did", "find", "drawer"]}, {"sent": "drawer .", "words": ["drawer"]}, {"sent": "you got to close your drawer .", "words": ["your", "to", "you", "drawer", "close"]}, {"sent": "Helen how does the how does this drawer open ?", "words": ["the", "does", "how", "open", "this", "drawer"]}, {"sent": "it must be in your room in the drawer .", "words": ["the", "your", "be", "in", "it", "room", "drawer"]}, {"sent": "wheres your pink drawer Fraser ?", "words": ["drawer", "your"]}, {"sent": "let mommy put them in the drawer .", "words": ["the", "put", "them", "in", "drawer"]}, {"sent": "I cant see it in the drawer Thomas .", "words": ["the", "drawer", "in", "it", "see"]}, {"sent": "but because theyve been in the drawer a long time .", "words": ["the", "long", "in", "but", "drawer", "because", "a"]}, {"sent": "drawer .", "words": ["drawer"]}, {"sent": "is it in the train track drawer ?", "words": ["the", "train", "is", "in", "it", "drawer"]}, {"sent": "the bottom drawer of the chest of drawers and leaning over hanging on to your mothers coat .", "words": ["the", "bottom", "your", "of", "to", "and", "on", "drawer", "over", "coat"]}, {"sent": "Momma will give us something else from the drawer .", "words": ["the", "us", "will", "drawer", "give"]}, {"sent": "xxx so weve got some tapes here in the bottom drawer .", "words": ["the", "bottom", "drawer", "so", "here", "in", "some"]}, {"sent": "which drawer ?", "words": ["drawer", "which"]}, {"sent": "shall we put some spoons in the drawer ?", "words": ["the", "put", "drawer", "we", "in", "some"]}, {"sent": "in your art drawer .", "words": ["drawer", "in", "your"]}, {"sent": "in the drawer .", "words": ["the", "drawer", "in"]}, {"sent": "yes its in the drawer isnt it ?", "words": ["the", "drawer", "in", "it"]}, {"sent": "Mummy has got a pair of scissors out of the drawer and shes going to cut the stitches .", "words": ["the", "cut", "of", "to", "scissors", "and", "out", "drawer", "a"]}, {"sent": "on the drawer .", "words": ["on", "the", "drawer"]}, {"sent": "I think we needta get a clean towel as well from the drawer .", "words": ["clean", "the", "think", "we", "get", "towel", "drawer", "a"]}, {"sent": "would you run up for mummie and get her a pair of pants outof the drawer please ?", "words": ["up", "the", "for", "of", "you", "and", "run", "get", "pants", "would", "drawer", "her", "a"]}, {"sent": "its the drawer where we keep the tapes .", "words": ["the", "drawer", "where", "we"]}, {"sent": "theyre in that bottom drawer .", "words": ["drawer", "in", "bottom", "that"]}, {"sent": "whats that peeping outof the drawer ?", "words": ["the", "drawer", "that"]}, {"sent": "thats not going in the drawer any more .", "words": ["the", "any", "in", "more", "drawer", "not"]}, {"sent": "yes I nearly had a a thing with fingers absolutely incredible she was the other side of the bedroom and I was shutting my drawer and it didnt shut and suddenly I saw a little person xxx in a flash it happened .", "words": ["the", "my", "bedroom", "of", "was", "with", "other", "and", "little", "in", "it", "drawer", "she", "a"]}, {"sent": "oh the door of the drawer .", "words": ["the", "of", "drawer", "door"]}, {"sent": "cups in that drawer .", "words": ["drawer", "in", "that"]}, {"sent": "in what we call the baby drawer .", "words": ["the", "what", "we", "in", "drawer"]}, {"sent": "okay lets get the screw Mom grab could you please grab the screw driver out of the drawer ?", "words": ["the", "of", "you", "get", "out", "drawer", "could"]}, {"sent": "now close the drawer .", "words": ["the", "drawer", "close"]}, {"sent": "look in that drawer and see if you can get some socks in that drawer .", "words": ["can", "if", "see", "and", "you", "get", "in", "look", "drawer", "that", "some"]}, {"sent": "go put them in the drawer please .", "words": ["the", "put", "go", "them", "in", "drawer"]}, {"sent": "did he find some in the drawer ?", "words": ["the", "drawer", "in", "he", "did", "find", "some"]}, {"sent": "you want ta put that back in the drawer for your mommy ?", "words": ["the", "put", "for", "your", "back", "you", "in", "drawer", "that"]}, {"sent": "and we pull the drawer out here .", "words": ["the", "we", "pull", "and", "here", "out", "drawer"]}, {"sent": "oh theres a ball in my drawer .", "words": ["my", "ball", "in", "drawer", "a"]}, {"sent": "are you opening that drawer ?", "words": ["drawer", "are", "that", "you"]}, {"sent": "but would you put the clothes that are clean in the drawer ?", "words": ["the", "clean", "put", "you", "in", "are", "would", "drawer", "that", "but"]}, {"sent": "in the drawer ?", "words": ["the", "drawer", "in"]}, {"sent": "put them back in the drawer .", "words": ["the", "put", "back", "them", "in", "drawer"]}, {"sent": "tool drawer .", "words": ["drawer"]}, {"sent": "put the piece in the drawer the bowl whatever it is .", "words": ["the", "bowl", "put", "is", "in", "it", "drawer"]}, {"sent": "go back to my night stand and look in my drawer and bring my book of matches .", "words": ["my", "back", "go", "of", "to", "stand", "bring", "and", "in", "look", "book", "drawer"]}, {"sent": "probably in the drawer .", "words": ["the", "drawer", "in"]}, {"sent": "well yes weve got a baby drawer .", "words": ["drawer", "a"]}, {"sent": "oh yeah its in that drawer there we are .", "words": ["we", "there", "in", "are", "drawer", "that"]}, {"sent": "shut the drawer .", "words": ["the", "drawer"]}, {"sent": "Im just putting Dyeishas stuff in her drawer .", "words": ["drawer", "in", "her"]}, {"sent": "its out of the drawer that one darling .", "words": ["the", "of", "out", "drawer", "that"]}, {"sent": "drawer .", "words": ["drawer"]}, {"sent": "dont close that drawer okay ?", "words": ["dont", "close", "that", "drawer"]}, {"sent": "oh thats the little handle of the cutlery drawer .", "words": ["the", "of", "drawer", "little"]}, {"sent": "and in the bottom drawer weve got mixers and diggers and things like that .", "words": ["the", "bottom", "like", "and", "in", "drawer", "that"]}, {"sent": "you press ahhah the drawer opens .", "words": ["the", "drawer", "you"]}, {"sent": "shut the drawer please .", "words": ["the", "drawer"]}, {"sent": "wheres your socks drawer ?", "words": ["drawer", "your"]}, {"sent": "no is he on the grass ?", "words": ["the", "is", "he", "on", "grass"]}, {"sent": "so youve got grass and purple crickets ?", "words": ["so", "grass", "and"]}, {"sent": "and grass and flowers .", "words": ["grass", "and"]}, {"sent": "thats right it was in the grass .", "words": ["the", "was", "in", "it", "grass"]}, {"sent": "grass is the food ?", "words": ["the", "food", "grass", "is"]}, {"sent": "put it in the grass .", "words": ["the", "put", "in", "it", "grass"]}, {"sent": "down in the meadow where the green grass grows .", "words": ["the", "where", "in", "green", "down", "grass"]}, {"sent": "slide through the grass making them shake .", "words": ["the", "slide", "them", "shake", "grass"]}, {"sent": "its just dried grass .", "words": ["grass"]}, {"sent": "Ill carry you until we get to the grass because the the cement is really dirty here huh ?", "words": ["the", "we", "is", "to", "because", "you", "carry", "get", "here", "dirty", "grass"]}, {"sent": "yeah theres grass up there isnt there ?", "words": ["up", "there", "grass"]}, {"sent": "do you remember what got onto his garden and started eating the grass ?", "words": ["the", "do", "what", "you", "and", "garden", "his", "grass"]}, {"sent": "can see it on the grass cant we ?", "words": ["can", "the", "we", "it", "on", "see", "grass"]}, {"sent": "playdough grass ?", "words": ["grass"]}, {"sent": "that is grass .", "words": ["that", "grass", "is"]}, {"sent": "theyre going to pull a plough across the the grass and plough it all over and reseed it to make a new lawn .", "words": ["the", "all", "to", "pull", "and", "it", "make", "new", "over", "grass", "a"]}, {"sent": "the grass was wet .", "words": ["the", "was", "grass", "wet"]}, {"sent": "is a is grass ?", "words": ["grass", "a", "is"]}, {"sent": "*SI1: see the grass ?", "words": ["the", "see", "grass"]}, {"sent": "itll be nice though when the weathers a little bit warmer and the grass is dryer .", "words": ["the", "dryer", "is", "be", "nice", "little", "and", "when", "grass", "a"]}, {"sent": "they eat grass what is that that baby doing ?", "words": ["what", "is", "eat", "they", "that", "grass"]}, {"sent": "he might poo on the grass ?", "words": ["on", "the", "he", "grass"]}, {"sent": "u grass .", "words": ["grass"]}, {"sent": "Matt I had remember something Matty to tell you do you remember when in the frog and toad book when when toad dropped the icecream and it got stuck on his head and he was all covered up with leaves and grass and the icecream and whatd everybody think he was ?", "words": ["the", "think", "book", "when", "with", "frog", "it", "on", "grass", "up", "all", "to", "head", "was", "you", "and", "stuck", "do", "in", "he", "his"]}, {"sent": "and grass .", "words": ["grass", "and"]}, {"sent": "so so she got a grass stain ?", "words": ["so", "she", "grass", "a"]}, {"sent": "the grass isnt very clean here .", "words": ["the", "here", "grass", "clean"]}, {"sent": "when Jesus fed five thousand men and all the women and children he first had them sit down in groups on the green grass .", "words": ["the", "all", "first", "them", "and", "in", "he", "green", "on", "sit", "down", "when", "grass"]}, {"sent": "do I like grass ?", "words": ["do", "like", "grass"]}, {"sent": "tell me about the grass .", "words": ["the", "me", "grass", "about"]}, {"sent": "what color is grass ?", "words": ["what", "grass", "is"]}, {"sent": "because he wants to live in the grass .", "words": ["the", "to", "in", "he", "grass", "because"]}, {"sent": "sheep is munching the green grass .", "words": ["the", "is", "sheep", "green", "grass"]}, {"sent": "the grass was short and green .", "words": ["the", "was", "and", "green", "grass"]}, {"sent": "and they started cutting the grass in the park didnt they ?", "words": ["the", "and", "they", "in", "park", "grass"]}, {"sent": "Mummy doesnt like grass to eat .", "words": ["eat", "to", "like", "grass"]}, {"sent": "PS Ill cut the grass when I get back .", "words": ["the", "cut", "back", "get", "when", "grass"]}, {"sent": "leave the grass there .", "words": ["the", "there", "grass"]}, {"sent": "you gonna give me something else Im raking the grass .", "words": ["the", "you", "me", "give", "grass"]}, {"sent": "is that green grass huh ?", "words": ["grass", "that", "green", "is"]}, {"sent": "slide through the grass making them shake .", "words": ["the", "slide", "them", "shake", "grass"]}, {"sent": "and can you see the grass grass .", "words": ["can", "the", "you", "and", "see", "grass"]}, {"sent": "you use a lawnmower to cut the grass with .", "words": ["the", "cut", "to", "with", "you", "grass", "a"]}, {"sent": "does he hide in the grass in the tall grass ?", "words": ["the", "does", "hide", "in", "he", "grass"]}, {"sent": "I see hes pulling the grass off and then he puts it together and then the tractor is coming and getting it .", "words": ["the", "off", "is", "and", "it", "he", "tractor", "see", "then", "grass"]}, {"sent": "youre gonna count the grass ?", "words": ["the", "grass"]}, {"sent": "thats grass from the grass box isnt it ?", "words": ["the", "it", "box", "grass"]}, {"sent": "theres the grass and the trees .", "words": ["the", "grass", "and"]}, {"sent": "green grass .", "words": ["grass", "green"]}, {"sent": "she knows how to crawl crawl quietly through the grass .", "words": ["the", "how", "to", "she", "grass"]}, {"sent": "youre cleaning the grass arent you ?", "words": ["the", "grass", "you"]}, {"sent": "grass ?", "words": ["grass"]}, {"sent": "and the grass .", "words": ["the", "grass", "and"]}, {"sent": "grass cake ?", "words": ["cake", "grass"]}, {"sent": "for cutting the grass .", "words": ["the", "for", "grass"]}, {"sent": "here is the grass ?", "words": ["here", "the", "grass", "is"]}, {"sent": "a grass .", "words": ["grass", "a"]}, {"sent": "and you and Amanda do somersaults like that dont you in the grass ?", "words": ["the", "do", "like", "you", "and", "dont", "in", "that", "grass"]}, {"sent": "it says theres still some long grass .", "words": ["some", "it", "grass", "long"]}, {"sent": "theres not any grass but there is some xxx fence .", "words": ["is", "any", "there", "grass", "some", "not", "but"]}, {"sent": "shell get her paws wet if she goes on the grass .", "words": ["the", "wet", "if", "get", "on", "her", "she", "grass"]}, {"sent": "did you leave it over there in the grass by the top ?", "words": ["the", "you", "by", "there", "in", "it", "did", "over", "grass"]}, {"sent": "sounds like somebodys cutting their grass doesnt it ?", "words": ["it", "like", "grass", "their"]}, {"sent": "he made that costume out of some grass grass .", "words": ["of", "he", "out", "some", "that", "grass"]}, {"sent": "you like playing in the grass ?", "words": ["the", "like", "you", "in", "grass"]}, {"sent": "its out on the grass .", "words": ["on", "the", "grass", "out"]}, {"sent": "cause I want to get all the rocks out of the grass .", "words": ["the", "all", "of", "to", "get", "want to", "out", "grass"]}, {"sent": "do you think hes eating the grass ?", "words": ["the", "do", "think", "you", "grass"]}, {"sent": "the kangaroos know that tasty green grass grows wherever rain has fallen .", "words": ["the", "rain", "green", "that", "grass"]}, {"sent": "moo Im here eating the grass .", "words": ["here", "the", "grass"]}, {"sent": "need some more grass there dont we ?", "words": ["need", "we", "there", "more", "dont", "some", "grass"]}, {"sent": "can you show me the lambs are eating grass ?", "words": ["can", "the", "show", "you", "are", "me", "grass"]}, {"sent": "we dont polish grass .", "words": ["dont", "grass", "we"]}, {"sent": "Alex lookit Wendys cutting the grass .", "words": ["the", "grass"]}, {"sent": "you counting the grass ?", "words": ["the", "grass", "you"]}, {"sent": "grass and the leaves .", "words": ["the", "grass", "and"]}, {"sent": "yes it does eat grass but what is it ?", "words": ["does", "what", "is", "eat", "it", "grass", "but"]}, {"sent": "yellow dandelions on the grass and yellow pansys .", "words": ["the", "and", "on", "yellow", "grass"]}, {"sent": "grass .", "words": ["grass"]}, {"sent": "theres the grass cutter .", "words": ["the", "grass"]}, {"sent": "do you know what grass tastes like ?", "words": ["do", "what", "like", "you", "grass"]}, {"sent": "walk in the grass ?", "words": ["the", "in", "walk", "grass"]}, {"sent": "do you think thats grass .", "words": ["do", "think", "grass", "you"]}, {"sent": "with you could make a fire if you had some glass and an you let and you wanted a fire you put some sticks on the grass and then you put the um um the glass by it and then and then you let the sun come come through the glass and then it starts the fire .", "words": ["the", "put", "if", "sun", "with", "then", "you", "and", "by", "an", "it", "glass", "on", "some", "could", "make", "grass", "a"]}, {"sent": "and grass ?", "words": ["grass", "and"]}, {"sent": "theres lotsof grass right .", "words": ["grass"]}, {"sent": "push it on the grass .", "words": ["the", "push", "it", "on", "grass"]}, {"sent": "oh no thats just a piece of grass .", "words": ["of", "grass", "a"]}, {"sent": "is that grass ?", "words": ["that", "grass", "is"]}, {"sent": "little boy pulling him in the grass isnt he ?", "words": ["the", "little", "in", "him", "he", "grass"]}, {"sent": "the grass is terrible xxx grass is all gorgeous xxx .", "words": ["the", "all", "grass", "is"]}, {"sent": "just some grass .", "words": ["some", "grass"]}, {"sent": "lying on the grass ?", "words": ["on", "the", "grass"]}, {"sent": "but well come and play with the flower here on the grass okay ?", "words": ["the", "flower", "play", "with", "and", "here", "grass", "on", "but"]}, {"sent": "I think its sposta be grass .", "words": ["be", "think", "grass"]}, {"sent": "what happens when we cut the grass ?", "words": ["the", "what", "cut", "we", "when", "grass"]}, {"sent": "with green grass .", "words": ["grass", "with", "green"]}, {"sent": "what about some green grass for her to stand on ?", "words": ["what", "for", "to", "stand", "green", "about", "on", "some", "her", "grass"]}, {"sent": "the kangaroos like grass do they ?", "words": ["the", "do", "like", "they", "grass"]}, {"sent": "is that a big giraffe ?", "words": ["giraffe", "is", "that", "big", "a"]}, {"sent": "has a giraffe .", "words": ["giraffe", "a"]}, {"sent": "you know how tall the giraffe is ?", "words": ["the", "how", "giraffe", "is", "you"]}, {"sent": "bugs giraffe ?", "words": ["giraffe"]}, {"sent": "a giraffe ?", "words": ["giraffe", "a"]}, {"sent": "and giraffe .", "words": ["giraffe", "and"]}, {"sent": "who do you xxx okay now youre gonna paint me a giraffe ?", "words": ["do", "paint", "giraffe", "you", "who", "me", "a"]}, {"sent": "perhaps hed like to say hello to this giraffe over here .", "words": ["giraffe", "to", "say", "like", "here", "this", "over"]}, {"sent": "a giraffe .", "words": ["giraffe", "a"]}, {"sent": "I think maybe a giraffe baby ?", "words": ["think", "giraffe", "a"]}, {"sent": "elephant and giraffe .", "words": ["giraffe", "and", "elephant"]}, {"sent": "thats a neat name for a giraffe .", "words": ["for", "giraffe", "a"]}, {"sent": "I bet the giraffe hasnt had a ride in your tractor before .", "words": ["the", "giraffe", "your", "in", "tractor", "ride", "a"]}, {"sent": "no you saw a giraffe on the television .", "words": ["the", "giraffe", "you", "on", "a"]}, {"sent": "Im sorry thats a giraffe .", "words": ["giraffe", "a"]}, {"sent": "here giraffe you turn a somersault .", "words": ["here", "giraffe", "a", "you"]}, {"sent": "um she sort looked at the giraffe weve got a lot of giraffe toys at home but she was more interested in the fact that its head came off .", "words": ["the", "off", "giraffe", "of", "was", "head", "a lot", "more", "in", "at", "home", "that", "she", "but", "a"]}, {"sent": "are you going to put the giraffe on the car ?", "words": ["the", "put", "giraffe", "to", "car", "you", "are", "on"]}, {"sent": "it is a giraffe Im telling you .", "words": ["giraffe", "is", "you", "it", "a"]}, {"sent": "see the giraffe ?", "words": ["the", "see", "giraffe"]}, {"sent": "I think Im gonna put the giraffe here like that .", "words": ["the", "think", "put", "giraffe", "like", "here", "that"]}, {"sent": "so Danny and the hen skipped along until they met a giraffe .", "words": ["the", "giraffe", "so", "and", "they", "hen", "a"]}, {"sent": "a giraffe baby ?", "words": ["giraffe", "a"]}, {"sent": "hes a pretty giraffe .", "words": ["giraffe", "pretty", "a"]}, {"sent": "is there a giraffe over there ?", "words": ["giraffe", "is", "there", "over", "a"]}, {"sent": "giraffe yes .", "words": ["giraffe"]}, {"sent": "what is that monkey and giraffe doing ?", "words": ["what", "giraffe", "is", "monkey", "and", "that"]}, {"sent": "go and see the giraffe and the penguin .", "words": ["the", "giraffe", "go", "penguin", "and", "see"]}, {"sent": "why do you think a giraffe needs long legs and long neck ?", "words": ["do", "think", "giraffe", "long", "you", "and", "why", "a"]}, {"sent": "thats a giraffe .", "words": ["giraffe", "a"]}, {"sent": "whod you see a giraffe at the zoo ?", "words": ["the", "zoo", "giraffe", "you", "at", "see", "a"]}, {"sent": "give the giraffe a ride down the hill .", "words": ["the", "giraffe", "give", "down", "ride", "a"]}, {"sent": "theres a giraffe ?", "words": ["giraffe", "a"]}, {"sent": "theres a giraffe .", "words": ["giraffe", "a"]}, {"sent": "and theres a giraffe here .", "words": ["here", "giraffe", "a", "and"]}, {"sent": "giraffe .", "words": ["giraffe"]}, {"sent": "youre going to put the giraffe in there .", "words": ["the", "put", "giraffe", "to", "there", "in"]}, {"sent": "he doesnt want the giraffe .", "words": ["the", "giraffe", "he"]}, {"sent": "are you being tall like a giraffe ?", "words": ["giraffe", "like", "you", "are", "a"]}, {"sent": "do you wanna tell me a story about the giraffe ?", "words": ["the", "do", "giraffe", "you", "wanna", "about", "me", "story", "a"]}, {"sent": "the giraffe one ?", "words": ["the", "giraffe"]}, {"sent": "the giraffe willnt fit .", "words": ["the", "fit", "giraffe"]}, {"sent": "yeah lets leave that heres a little more yogurt would you like smore leaves miss giraffe ?", "words": ["giraffe", "yogurt", "like", "little", "you", "more", "would", "that", "a"]}, {"sent": "heres a giraffe Joseph see the giraffe ?", "words": ["the", "see", "giraffe", "a"]}, {"sent": "well maybe the giraffe would like to be in there .", "words": ["the", "giraffe", "to", "be", "like", "there", "in", "would"]}, {"sent": "theres the giraffe and the monkey .", "words": ["the", "monkey", "giraffe", "and"]}, {"sent": "pardon said the giraffe .", "words": ["the", "giraffe"]}, {"sent": "it is a green giraffe .", "words": ["giraffe", "is", "it", "green", "a"]}, {"sent": "lets make a nice full tree for the giraffe to eat .", "words": ["the", "tree", "for", "giraffe", "full", "to", "nice", "eat", "make", "a"]}, {"sent": "is that Georgina the giraffe ?", "words": ["the", "giraffe", "that", "is"]}, {"sent": "its the giraffe .", "words": ["the", "giraffe"]}, {"sent": "what the giraffe ?", "words": ["the", "what", "giraffe"]}, {"sent": "giraffe yeah .", "words": ["giraffe"]}, {"sent": "giraffe ?", "words": ["giraffe"]}, {"sent": "a giraffe has a long neck .", "words": ["giraffe", "long", "a"]}, {"sent": "the giraffe with the long .", "words": ["the", "long", "giraffe", "with"]}, {"sent": "can you put the giraffe in his home ?", "words": ["can", "the", "put", "giraffe", "you", "in", "his", "home"]}, {"sent": "that is giraffe ?", "words": ["giraffe", "that", "is"]}, {"sent": "this is a giraffe .", "words": ["this", "giraffe", "a", "is"]}, {"sent": "giraffe .", "words": ["giraffe"]}, {"sent": "is this a giraffe ?", "words": ["this", "giraffe", "a", "is"]}, {"sent": "its a green giraffe isnt it ?", "words": ["it", "giraffe", "green", "a"]}, {"sent": "where is the giraffe ?", "words": ["where", "the", "giraffe", "is"]}, {"sent": "the giraffe says coocoo .", "words": ["the", "giraffe"]}, {"sent": "shall we put the giraffe in ?", "words": ["the", "put", "giraffe", "we", "in"]}, {"sent": "theres the giraffe and theres Pingu .", "words": ["the", "giraffe", "and"]}, {"sent": "lets see I think this one will build a giraffe .", "words": ["think", "giraffe", "this", "build", "will", "see", "a"]}, {"sent": "oh you want me to hold the giraffe do you ?", "words": ["the", "do", "giraffe", "hold", "to", "you", "me"]}, {"sent": "yeah and the giraffe goes in there doesnt it ?", "words": ["the", "giraffe", "and", "there", "in", "it"]}, {"sent": "a giraffe ?", "words": ["giraffe", "a"]}, {"sent": "cant get this giraffe to fit in the zoo .", "words": ["the", "zoo", "giraffe", "to", "fit", "get", "this", "in"]}, {"sent": "and thats a giraffe .", "words": ["giraffe", "a", "and"]}, {"sent": "is that your giraffe ?", "words": ["your", "giraffe", "that", "is"]}, {"sent": "lets find the giraffe .", "words": ["the", "giraffe", "find"]}, {"sent": "giraffe .", "words": ["giraffe"]}, {"sent": "Sarah first Ill be the giraffe and then you be the giraffe alright ?", "words": ["the", "giraffe", "be", "first", "and", "you", "then"]}, {"sent": "you got another giraffe downstairs .", "words": ["another", "giraffe", "you"]}, {"sent": "thats right big giraffe .", "words": ["giraffe", "big"]}, {"sent": "giraffe .", "words": ["giraffe"]}, {"sent": "you wanna just put the giraffe on the rug ?", "words": ["the", "put", "giraffe", "you", "wanna", "on"]}, {"sent": "yes well can you see the difference between a dinosaur and a giraffe ?", "words": ["can", "the", "giraffe", "you", "and", "see", "a"]}, {"sent": "thats a giraffe .", "words": ["giraffe", "a"]}, {"sent": "its just like your giraffe .", "words": ["giraffe", "your", "like"]}, {"sent": "giraffe yes .", "words": ["giraffe"]}, {"sent": "huh theres a giraffe .", "words": ["giraffe", "a"]}, {"sent": "giraffe here ?", "words": ["here", "giraffe"]}, {"sent": "theres the giraffe look .", "words": ["the", "giraffe", "look"]}, {"sent": "a giraffe .", "words": ["giraffe", "a"]}, {"sent": "well lets Im gonna make the giraffe again because I like the giraffe .", "words": ["the", "giraffe", "like", "make", "because"]}, {"sent": "right eat like the giraffe .", "words": ["eat", "the", "giraffe", "like"]}, {"sent": "oh is the giraffe coming to see us .", "words": ["the", "giraffe", "is", "to", "us", "see"]}, {"sent": "giraffe .", "words": ["giraffe"]}, {"sent": "can I see the giraffe again ?", "words": ["can", "see", "giraffe", "the"]}, {"sent": "remember when we saw that baby giraffe ?", "words": ["giraffe", "that", "when", "we"]}, {"sent": "giraffe ?", "words": ["giraffe"]}, {"sent": "whats the giraffe doing ?", "words": ["the", "giraffe"]}, {"sent": "a giraffe .", "words": ["giraffe", "a"]}, {"sent": "youve knocked the other giraffe off .", "words": ["the", "giraffe", "off", "other"]}, {"sent": "you have the giraffe ?", "words": ["have", "the", "giraffe", "you"]}, {"sent": "Fraser what colors that giraffe ?", "words": ["what", "that", "giraffe"]}, {"sent": "my balloon gasped teddy .", "words": ["my", "balloon"]}, {"sent": "a little boy found a red balloon on his way to the bus stop .", "words": ["red", "the", "to", "stop", "little", "his", "balloon", "on", "bus", "a"]}, {"sent": "and he spotted this big beautiful red balloon .", "words": ["red", "and", "this", "he", "balloon", "big"]}, {"sent": "do I want a balloon ?", "words": ["do", "balloon", "a"]}, {"sent": "and the balloon followed behind .", "words": ["the", "balloon", "behind", "and"]}, {"sent": "balloon .", "words": ["balloon"]}, {"sent": "is that your balloon ?", "words": ["your", "balloon", "that", "is"]}, {"sent": "did the balloon go up ?", "words": ["the", "up", "go", "balloon", "did"]}, {"sent": "thats a balloon theres one balloon .", "words": ["balloon", "a"]}, {"sent": "wheres the balloon ?", "words": ["the", "balloon"]}, {"sent": "did the balloon go pop ?", "words": ["the", "pop", "go", "balloon", "did"]}, {"sent": "Cathys balloon is big Jacks is even bigger but Tonys is the .", "words": ["the", "is", "balloon", "big", "but"]}, {"sent": "and when the balloons go up in the air the bear whose name is on the balloon which goes the furthest wins the race .", "words": ["the", "up", "go", "is", "which", "and", "bear", "in", "balloon", "on", "when"]}, {"sent": "and the bad boys had stolen his balloon .", "words": ["the", "bad", "and", "balloon", "his"]}, {"sent": "he was seen walking with his balloon to a bus .", "words": ["was", "to", "with", "balloon", "he", "his", "bus", "a"]}, {"sent": "there is the little girl on the balloon .", "words": ["the", "is", "little", "there", "balloon", "on"]}, {"sent": "and the little boy and the balloon went to a bakery shop .", "words": ["the", "to", "little", "and", "balloon", "a"]}, {"sent": "what would happen if a crocodile bit into this balloon ?", "words": ["what", "into", "if", "this", "balloon", "would", "a"]}, {"sent": "the balloon is not safe that way .", "words": ["the", "is", "balloon", "that", "not"]}, {"sent": "a balloon .", "words": ["balloon", "a"]}, {"sent": "wheres the balloon ?", "words": ["the", "balloon"]}, {"sent": "I said can you bring the balloon round and you didnt do it .", "words": ["can", "the", "do", "bring", "you", "and", "it", "balloon"]}, {"sent": "theres WinniethePooh with his honey and his umbrella and his balloon .", "words": ["his", "balloon", "with", "and"]}, {"sent": "a blue balloon ?", "words": ["blue", "balloon", "a"]}, {"sent": "a balloon .", "words": ["balloon", "a"]}, {"sent": "he wanted the balloon .", "words": ["the", "balloon", "he"]}, {"sent": "playing with the balloon ?", "words": ["the", "balloon", "with"]}, {"sent": "and he walked with the balloon to the bus stop .", "words": ["the", "to", "with", "stop", "and", "balloon", "he", "bus"]}, {"sent": "and the gang of boys had captured the balloon from the little boy .", "words": ["the", "of", "little", "and", "balloon"]}, {"sent": "hot air balloon ?", "words": ["hot", "balloon"]}, {"sent": "he was having an adventure in a hot air balloon .", "words": ["was", "an", "in", "he", "hot", "balloon", "a"]}, {"sent": "show mommy where the balloon is .", "words": ["the", "where", "is", "show", "balloon"]}, {"sent": "we never said you hadta go on a balloon ride .", "words": ["go", "we", "you", "balloon", "on", "ride", "a"]}, {"sent": "yellow and what color is that balloon and that frog ?", "words": ["what", "is", "and", "frog", "balloon", "yellow", "that"]}, {"sent": "mhm the balloon is broken .", "words": ["the", "balloon", "broken", "is"]}, {"sent": "Ive just broken WinniethePooh balloon .", "words": ["balloon", "broken"]}, {"sent": "youve got that very very big green balloon that Ben and Charlottes Mum gave you havent you ?", "words": ["and", "you", "that", "balloon", "green", "big"]}, {"sent": "you see a balloon ?", "words": ["see", "balloon", "a", "you"]}, {"sent": "he wanted the balloon .", "words": ["the", "balloon", "he"]}, {"sent": "and the balloon followed behind .", "words": ["the", "balloon", "behind", "and"]}, {"sent": "more interested in not blowing that balloon up arent you ?", "words": ["up", "you", "in", "more", "balloon", "that", "not"]}, {"sent": "good night light and the red balloon .", "words": ["the", "red", "light", "and", "balloon", "good"]}, {"sent": "have you got a balloon ?", "words": ["have", "balloon", "a", "you"]}, {"sent": "what color was Jwwws balloon ?", "words": ["was", "what", "balloon"]}, {"sent": "balloon yeah its pretty sad right now isnt it ?", "words": ["pretty", "sad", "it", "balloon"]}, {"sent": "but he cant see the red balloon .", "words": ["the", "red", "he", "balloon", "see", "but"]}, {"sent": "weve only got one balloon .", "words": ["balloon"]}, {"sent": "Im a brunch your balloon if you dont drink that stuff and go on in there xxx .", "words": ["your", "if", "drink", "go", "you", "and", "dont", "in", "there", "balloon", "on", "that", "a"]}, {"sent": "and one day on his way to the bus stop he found this big beautiful red balloon .", "words": ["the", "red", "to", "stop", "and", "his", "this", "big", "he", "balloon", "on", "bus"]}, {"sent": "this pink balloon goes by this one .", "words": ["this", "balloon", "by"]}, {"sent": "thats your balloon ?", "words": ["balloon", "your"]}, {"sent": "its licking the balloon .", "words": ["the", "balloon"]}, {"sent": "you wanted to take he balloon ?", "words": ["to", "you", "he", "balloon", "take"]}, {"sent": "are you gonna get the balloon ?", "words": ["the", "you", "get", "are", "balloon"]}, {"sent": "yes well your balloon will still be here .", "words": ["your", "be", "here", "balloon", "will"]}, {"sent": "its a balloon there .", "words": ["there", "balloon", "a"]}, {"sent": "he was seen walking with the balloon to a bus .", "words": ["the", "was", "to", "with", "balloon", "he", "bus", "a"]}, {"sent": "you wanted the balloon .", "words": ["the", "balloon", "you"]}, {"sent": "balloon darling .", "words": ["balloon"]}, {"sent": "your balloon but it will be gone then .", "words": ["your", "be", "it", "balloon", "will", "then", "but"]}, {"sent": "should I give Sammy the balloon ?", "words": ["the", "give", "balloon"]}, {"sent": "because theres knot in the balloon that I cannot get untied .", "words": ["the", "get", "in", "balloon", "that", "because"]}, {"sent": "he wanted the balloon to be his friend .", "words": ["the", "to", "be", "balloon", "he", "his"]}, {"sent": "she gonna be the balloon lady .", "words": ["the", "be", "balloon", "she"]}, {"sent": "oh that um isnt a balloon .", "words": ["balloon", "that", "a"]}, {"sent": "he was seen walking with the balloon to a bus .", "words": ["the", "was", "to", "with", "balloon", "he", "bus", "a"]}, {"sent": "he was because his balloon flew away didnt it ?", "words": ["was", "away", "balloon", "he", "it", "his", "because"]}, {"sent": "in the car what happened to the balloon in the car ?", "words": ["the", "what", "car", "to", "in", "balloon"]}, {"sent": "balloon .", "words": ["balloon"]}, {"sent": "you wanna take a balloon to Emilys house ?", "words": ["to", "you", "balloon", "house", "wanna", "take", "a"]}, {"sent": "can Amy have a balloon ?", "words": ["can", "have", "balloon", "a"]}, {"sent": "a balloon .", "words": ["balloon", "a"]}, {"sent": "we had a yellow balloon and what happened to it ?", "words": ["what", "we", "to", "and", "it", "balloon", "yellow", "a"]}, {"sent": "Ive got balloon .", "words": ["balloon"]}, {"sent": "that balloon popped .", "words": ["balloon", "that"]}, {"sent": "wheres that balloon ?", "words": ["balloon", "that"]}, {"sent": "and the balloon you lost was a yellow one wasnt it ?", "words": ["the", "was", "you", "and", "it", "balloon", "yellow", "a"]}, {"sent": "theres the sun and an airplane cloud heres a balloon whos in the balloon ?", "words": ["the", "airplane", "sun", "and", "an", "in", "balloon", "cloud", "a"]}, {"sent": "thats a red balloon .", "words": ["red", "balloon", "a"]}, {"sent": "theres another red balloon .", "words": ["another", "red", "balloon"]}, {"sent": "balloon .", "words": ["balloon"]}, {"sent": "after the balloon popped then what happened ?", "words": ["the", "what", "then", "balloon"]}, {"sent": "he wanted the balloon .", "words": ["the", "balloon", "he"]}, {"sent": "who went on a hot air balloon ?", "words": ["hot", "balloon", "who", "on", "a"]}, {"sent": "um the little boy and the balloon are coming to a bakery shop .", "words": ["the", "to", "and", "little", "are", "balloon", "a"]}, {"sent": "youve got some bubbles and a yellow balloon and a piece of cake and a fizzy sweet and a little Mars bar and some jelly flavored sweet oh jelly sweets .", "words": ["jelly", "cake", "bubbles", "yellow", "of", "and", "little", "balloon", "some", "a"]}, {"sent": "girls holding the balloon .", "words": ["the", "balloon"]}, {"sent": "a red balloon and a yellow balloon .", "words": ["red", "and", "balloon", "yellow", "a"]}, {"sent": "this balloon is big .", "words": ["this", "big", "balloon", "is"]}, {"sent": "oh look at your balloon .", "words": ["balloon", "at", "your", "look"]}, {"sent": "well why am I blowing up the balloon if you wanna play ?", "words": ["up", "the", "if", "play", "you", "balloon", "am", "wanna", "why"]}, {"sent": "has he got a balloon ?", "words": ["balloon", "he", "a"]}, {"sent": "well use this balloon as a great heap of sand .", "words": ["of", "this", "balloon", "a"]}, {"sent": "and the little boy is looking for his balloon everywhere .", "words": ["the", "for", "is", "little", "and", "balloon", "his"]}, {"sent": "balloon .", "words": ["balloon"]}, {"sent": "tell Rose what happened to the big TweetyBird balloon Rachel .", "words": ["the", "what", "to", "balloon", "big"]}, {"sent": "whose balloon is it ?", "words": ["it", "balloon", "is"]}, {"sent": "Derek come hold the balloon while Mia ties it .", "words": ["the", "it", "balloon", "hold"]}, {"sent": "a big red balloon .", "words": ["red", "balloon", "big", "a"]}, {"sent": "did you kiss me today ?", "words": ["me", "kiss", "did", "you"]}, {"sent": "she wants to kiss it and chew on it .", "words": ["to", "and", "it", "on", "kiss", "she"]}, {"sent": "Ross is giving the kitty a kiss now .", "words": ["the", "is", "kitty", "kiss", "a"]}, {"sent": "hes coming to kiss your nose .", "words": ["your", "to", "kiss", "nose"]}, {"sent": "right give me a kiss first .", "words": ["first", "me", "give", "kiss", "a"]}, {"sent": "and kiss Mummys hair .", "words": ["hair", "kiss", "and"]}, {"sent": "ah thats a nice kiss .", "words": ["nice", "kiss", "a"]}, {"sent": "oh that was a lovely kiss .", "words": ["kiss", "was", "that", "a"]}, {"sent": "one kiss please .", "words": ["kiss"]}, {"sent": "can mommy give you kiss too mwah .", "words": ["can", "you", "too", "give", "kiss"]}, {"sent": "kiss kiss .", "words": ["kiss"]}, {"sent": "Ill sneak a kiss .", "words": ["kiss", "a"]}, {"sent": "xxx please dont kiss me .", "words": ["dont", "me", "kiss"]}, {"sent": "are you gonna give Daddy a kiss ?", "words": ["you", "are", "give", "kiss", "a"]}, {"sent": "she got a kiss from her mama .", "words": ["her", "kiss", "she", "a"]}, {"sent": "kiss kiss kiss kiss kiss kiss .", "words": ["kiss"]}, {"sent": "is that a kiss or a star ?", "words": ["is", "star", "that", "kiss", "a"]}, {"sent": "ah that was a lovely kiss .", "words": ["kiss", "was", "that", "a"]}, {"sent": "kiss as well ?", "words": ["kiss"]}, {"sent": "a kiss .", "words": ["kiss", "a"]}, {"sent": "do you not want me to kiss you ?", "words": ["do", "to", "you", "me", "kiss", "not"]}, {"sent": "you gave her a big kiss didnt you ?", "words": ["her", "you", "big", "kiss", "a"]}, {"sent": "you kiss it .", "words": ["it", "kiss", "you"]}, {"sent": "want me to kiss it ?", "words": ["to", "me", "it", "kiss"]}, {"sent": "can you give her a kiss ?", "words": ["can", "you", "kiss", "give", "her", "a"]}, {"sent": "kiss kiss .", "words": ["kiss"]}, {"sent": "do you want me to kiss it ?", "words": ["do", "to", "you", "it", "me", "kiss"]}, {"sent": "come give me a kiss .", "words": ["me", "give", "kiss", "a"]}, {"sent": "its better when they kiss eachother .", "words": ["kiss", "better", "when", "they"]}, {"sent": "oh kiss for Lily muah .", "words": ["for", "kiss"]}, {"sent": "oh well Im going to make some kiss out of PlayDoh .", "words": ["of", "to", "make", "out", "some", "kiss"]}, {"sent": "give Daddy kiss .", "words": ["give", "kiss"]}, {"sent": "um well I kind of pretended the frog was leaping on her and she kind of she was excited about the frog after he was pretending to kiss her neck .", "words": ["the", "of", "was", "to", "kiss", "frog", "and", "he", "about", "on", "her", "she"]}, {"sent": "big kiss .", "words": ["kiss", "big"]}, {"sent": "give him a kiss .", "words": ["give", "kiss", "him", "a"]}, {"sent": "kiss oh you want to say hi to them ?", "words": ["to", "them", "say", "you", "want to", "kiss"]}, {"sent": "you give Mommy a kiss ?", "words": ["give", "kiss", "a", "you"]}, {"sent": "the hippo wanted to kiss the lion .", "words": ["the", "to", "kiss", "lion"]}, {"sent": "you want Daddy to kiss it ?", "words": ["to", "it", "kiss", "you"]}, {"sent": "give me a kiss .", "words": ["me", "give", "kiss", "a"]}, {"sent": "JJ hey can mommy have a kiss ?", "words": ["can", "have", "kiss", "a"]}, {"sent": "I saw those are very nice can you give Joseph a hug and kiss I think hes sleepy sleep .", "words": ["can", "think", "those", "sleep", "nice", "you", "and", "are", "hug", "give", "sleepy", "kiss", "a"]}, {"sent": "run and give Papa a kiss .", "words": ["and", "run", "give", "kiss", "a"]}, {"sent": "are you going to give Morag a kiss now ?", "words": ["to", "you", "are", "give", "kiss", "a"]}, {"sent": "I want a kiss .", "words": ["kiss", "a"]}, {"sent": "and they all hug and kiss .", "words": ["all", "and", "they", "hug", "kiss"]}, {"sent": "Mummy will give it a kiss .", "words": ["it", "will", "give", "kiss", "a"]}, {"sent": "with a great big hug and a kiss from me to you .", "words": ["kiss", "to", "with", "and", "you", "hug", "me", "big", "a"]}, {"sent": "are you going to give me a kiss Ross ?", "words": ["to", "you", "are", "me", "give", "kiss", "a"]}, {"sent": "give the baby a kiss .", "words": ["the", "give", "kiss", "a"]}, {"sent": "what a nice kiss .", "words": ["what", "nice", "kiss", "a"]}, {"sent": "can I have a kiss ?", "words": ["can", "have", "kiss", "a"]}, {"sent": "oh sweetheart let me kiss it .", "words": ["me", "it", "kiss"]}, {"sent": "kiss the camera ?", "words": ["the", "camera", "kiss"]}, {"sent": "let Daddy kiss it .", "words": ["it", "kiss"]}, {"sent": "do you give her a kiss .", "words": ["do", "you", "kiss", "give", "her", "a"]}, {"sent": "you want a kiss ?", "words": ["kiss", "a", "you"]}, {"sent": "kiss it better .", "words": ["better", "it", "kiss"]}, {"sent": "not a kiss .", "words": ["kiss", "not", "a"]}, {"sent": "thats not blowing a kiss .", "words": ["kiss", "not", "a"]}, {"sent": "let M Mummy kiss it better .", "words": ["better", "it", "kiss"]}, {"sent": "it looks like lotsof kiss that you put on the bottom of birthday cards doesnt it ?", "words": ["the", "put", "bottom", "of", "like", "you", "on", "it", "kiss", "that"]}, {"sent": "the babys going to kiss Daddy .", "words": ["the", "to", "kiss"]}, {"sent": "give me kiss .", "words": ["me", "give", "kiss"]}, {"sent": "a kiss ?", "words": ["kiss", "a"]}, {"sent": "can I have a kiss then ?", "words": ["can", "have", "then", "kiss", "a"]}, {"sent": "you gonna give the baby a kiss .", "words": ["the", "you", "give", "kiss", "a"]}, {"sent": "are you gonna give me a kiss ?", "words": ["you", "are", "me", "give", "kiss", "a"]}, {"sent": "or kiss you ?", "words": ["kiss", "you"]}, {"sent": "with a big kiss .", "words": ["kiss", "big", "with", "a"]}, {"sent": "give me kiss .", "words": ["me", "give", "kiss"]}, {"sent": "kiss .", "words": ["kiss"]}, {"sent": "you cant kiss my eyes .", "words": ["my", "kiss", "you"]}, {"sent": "that was a kiss silly silly .", "words": ["kiss", "was", "that", "a"]}, {"sent": "do you want me to kiss it better ?", "words": ["do", "better", "to", "you", "it", "me", "kiss"]}, {"sent": "give me a kiss .", "words": ["me", "give", "kiss", "a"]}, {"sent": "are you gonna give her a big kiss ?", "words": ["you", "big", "are", "kiss", "give", "her", "a"]}, {"sent": "oh are you blowing kiss at Purdie ?", "words": ["at", "are", "kiss", "you"]}, {"sent": "Ill get a kiss upstairs .", "words": ["get", "kiss", "a"]}, {"sent": "kiss your foot ?", "words": ["your", "foot", "kiss"]}, {"sent": "and you do a kiss .", "words": ["do", "you", "and", "kiss", "a"]}, {"sent": "can I have a big kiss ?", "words": ["can", "have", "kiss", "big", "a"]}, {"sent": "blow blow dad a kiss .", "words": ["kiss", "blow", "a"]}, {"sent": "kiss the baby .", "words": ["the", "kiss"]}, {"sent": "we hug and we kiss and cuddle .", "words": ["hug", "kiss", "and", "we"]}, {"sent": "when they finished reading Olivias mother gives her a kiss and says you know you really wear me out .", "words": ["when", "her", "and", "you", "they", "out", "me", "kiss", "a"]}, {"sent": "let me kiss it .", "words": ["me", "it", "kiss"]}, {"sent": "everybody loves a kiss from Pinky McPink .", "words": ["kiss", "a"]}, {"sent": "you give Mommy a kiss ?", "words": ["give", "kiss", "a", "you"]}, {"sent": "you want me to kiss it now ?", "words": ["to", "you", "it", "me", "kiss"]}, {"sent": "because I like a big kiss .", "words": ["like", "big", "kiss", "because", "a"]}, {"sent": "he must like getting a kiss off you Jean .", "words": ["off", "like", "you", "he", "kiss", "a"]}, {"sent": "give Daddy kiss .", "words": ["give", "kiss"]}, {"sent": "oy hafta give him a big hug now and a kiss because hes crying .", "words": ["kiss", "and", "him", "hug", "give", "big", "because", "a"]}, {"sent": "can I have a kiss please ?", "words": ["can", "have", "kiss", "a"]}, {"sent": "can I kiss them ?", "words": ["can", "kiss", "them"]}, {"sent": "cmon lemme give you a kiss on the face .", "words": ["the", "face", "you", "on", "give", "kiss", "a"]}, {"sent": "xxx a kiss ?", "words": ["kiss", "a"]}, {"sent": "lets wash off the rest of the peanutbutter .", "words": ["off", "the", "wash", "of"]}, {"sent": "this is the way we wish our face wish our face wash our face .", "words": ["the", "face", "we", "is", "wish", "our", "this", "wash"]}, {"sent": "xxx wash ourselves .", "words": ["wash"]}, {"sent": "well as far as that goes I dont wash the other one and that one is dirty .", "words": ["the", "is", "wash", "other", "and", "dont", "dirty", "that"]}, {"sent": "we should wash this really today .", "words": ["this", "wash", "we"]}, {"sent": "we gotta wash off your tray dont we ?", "words": ["off", "your", "we", "wash", "dont", "tray"]}, {"sent": "wash Nathaniels back oo .", "words": ["wash", "back"]}, {"sent": "lets wash your hands .", "words": ["your", "wash"]}, {"sent": "I think well wash that container .", "words": ["that", "think", "wash"]}, {"sent": "why dont cha wash your dollie ?", "words": ["your", "dont", "wash", "why"]}, {"sent": "Mommy is gonna wash your other foot .", "words": ["your", "is", "other", "foot", "wash"]}, {"sent": "we can wash that right off .", "words": ["can", "off", "we", "wash", "that"]}, {"sent": "pushing the bus through the car wash ?", "words": ["the", "car", "bus", "wash"]}, {"sent": "we wash ourselves .", "words": ["wash", "we"]}, {"sent": "lets just wash your hands and wipe your face .", "words": ["your", "face", "and", "wipe", "wash"]}, {"sent": "once youve had a wash .", "words": ["wash", "a"]}, {"sent": "listen let me wash your let me wash you .", "words": ["your", "listen", "you", "me", "wash"]}, {"sent": "and wash up and do all those jobs .", "words": ["up", "do", "all", "those", "and", "wash"]}, {"sent": "wash your fingers .", "words": ["your", "wash"]}, {"sent": "can you wash your face ?", "words": ["can", "your", "face", "you", "wash"]}, {"sent": "if you draw on Mummys hand shes going to hafta wash them again isnt she ?", "words": ["if", "hand", "to", "draw", "them", "you", "on", "wash", "she"]}, {"sent": "now you hafta give your hands a good wash when youve finished .", "words": ["your", "when", "you", "good", "give", "wash", "a"]}, {"sent": "this is why Mummy hasta wash your train two or three times a day .", "words": ["your", "is", "train", "this", "wash", "why", "a"]}, {"sent": "May go go wash up and Ill get your plate ready .", "words": ["up", "your", "plate", "go", "and", "get", "wash"]}, {"sent": "weve taken the wheels off the bottom of the pram to give them a wash .", "words": ["the", "off", "bottom", "of", "to", "them", "give", "wash", "a"]}, {"sent": "we gotta wash the wash .", "words": ["the", "wash", "we"]}, {"sent": "you hafta wash it first .", "words": ["it", "wash", "first", "you"]}, {"sent": "and give your shoes a wash as well ?", "words": ["your", "and", "give", "wash", "a"]}, {"sent": "here Bobby you needta wash your hands .", "words": ["here", "wash", "your", "you"]}, {"sent": "wash your bottom .", "words": ["your", "bottom", "wash"]}, {"sent": "tonight you hafta take a bath and wash your hair .", "words": ["your", "and", "you", "take", "hair", "wash", "a"]}, {"sent": "you want Mommy to wash your hair ?", "words": ["your", "to", "you", "hair", "wash"]}, {"sent": "are you going to help Mummy wash up ?", "words": ["up", "help", "to", "you", "are", "wash"]}, {"sent": "gonna wash about two loads .", "words": ["wash", "about"]}, {"sent": "wash your bib off .", "words": ["your", "bib", "wash", "off"]}, {"sent": "do you wanna wash your lips ?", "words": ["do", "your", "you", "wanna", "wash"]}, {"sent": "wash hands all done .", "words": ["all", "wash"]}, {"sent": "shall we go and wash it ?", "words": ["go", "we", "and", "it", "wash"]}, {"sent": "would you like to shall we wash your face and hands ?", "words": ["your", "face", "we", "to", "like", "you", "and", "would", "wash"]}, {"sent": "er show me how you wash your hands .", "words": ["how", "your", "show", "you", "me", "wash"]}, {"sent": "thats it just having a drink before they hafta wash up and tidy up ready for tomorrow .", "words": ["up", "for", "drink", "and", "they", "it", "wash", "a"]}, {"sent": "not wash this hand .", "words": ["this", "wash", "not", "hand"]}, {"sent": "wash my face .", "words": ["my", "wash", "face"]}, {"sent": "for Dan to wash up .", "words": ["up", "to", "for", "wash"]}, {"sent": "what are you gonna wash ?", "words": ["what", "are", "wash", "you"]}, {"sent": "Ill hafta wash it all .", "words": ["all", "it", "wash"]}, {"sent": "let me go wash this please dont put things in your mouth .", "words": ["put", "your", "go", "mouth", "dont", "this", "in", "me", "wash"]}, {"sent": "Im putting it Im gonna wash that mat .", "words": ["that", "it", "wash"]}, {"sent": "instruction wash warm .", "words": ["wash"]}, {"sent": "when we wash it its wet hair isnt it ?", "words": ["wet", "we", "wash", "it", "hair", "when"]}, {"sent": "oh you can wash it here Im just going to move you up here to the sink .", "words": ["can", "up", "the", "to", "you", "here", "it", "sink", "wash"]}, {"sent": "what do you wash your hair with ?", "words": ["what", "do", "your", "with", "you", "hair", "wash"]}, {"sent": "thats a three to four one now thats in the wash .", "words": ["the", "to", "in", "wash", "a"]}, {"sent": "wash the pear .", "words": ["the", "wash"]}, {"sent": "xxx to wash off the chair .", "words": ["off", "the", "to", "chair", "wash"]}, {"sent": "and the buckets there to wash wash wash wash .", "words": ["the", "to", "and", "there", "wash"]}, {"sent": "does Nina wash his face ?", "words": ["his", "does", "wash", "face"]}, {"sent": "wash it .", "words": ["it", "wash"]}, {"sent": "did somebody wash their clothes ?", "words": ["wash", "did", "their"]}, {"sent": "well you get in and Ill wash them .", "words": ["them", "and", "get", "you", "in", "wash"]}, {"sent": "lets wash them off .", "words": ["off", "wash", "them"]}, {"sent": "Ill wash .", "words": ["wash"]}, {"sent": "all our trousers hadta go in the wash .", "words": ["the", "all", "go", "our", "in", "wash"]}, {"sent": "what well do well have a little wash .", "words": ["what", "do", "little", "have", "wash", "a"]}, {"sent": "and you may wash off .", "words": ["off", "wash", "you", "and"]}, {"sent": "lets lets wash .", "words": ["wash"]}, {"sent": "that way we can just wash the table when were done .", "words": ["can", "the", "table", "when", "we", "wash", "that", "were"]}, {"sent": "and then wash your hands and have lunch .", "words": ["your", "and", "have", "then", "wash"]}, {"sent": "because you hafta have your hair wash brushed .", "words": ["your", "you", "have", "hair", "wash", "because"]}, {"sent": "you dont know how good it feels to wash my ears and scrub my heels .", "words": ["how", "my", "to", "you", "and", "dont", "it", "good", "wash"]}, {"sent": "what else are you going to wash ?", "words": ["what", "to", "you", "are", "wash"]}, {"sent": "but dont wash my face Mama .", "words": ["my", "face", "dont", "wash", "but"]}, {"sent": "this is the way we wash our hair wash our hair wash our hair .", "words": ["the", "we", "is", "our", "this", "hair", "wash"]}, {"sent": "and these clothes of yours probably need another wash .", "words": ["these", "need", "of", "and", "another", "wash"]}, {"sent": "think Ill wash the bike .", "words": ["the", "think", "wash"]}, {"sent": "wh when youve finished one color youve got to wash it in the water .", "words": ["the", "to", "wash", "in", "it", "water", "when"]}, {"sent": "when Mummy has finished this toast well wash your hands and face and then its time for upstairs and have your teeth brushed .", "words": ["for", "your", "when", "toast", "face", "and", "have", "this", "then", "wash"]}, {"sent": "wash your fingers darling .", "words": ["your", "wash"]}, {"sent": "why dont you wash them off ?", "words": ["off", "them", "you", "dont", "wash", "why"]}, {"sent": "you needta wash under the table as well ?", "words": ["the", "table", "you", "under", "wash"]}, {"sent": "and theyll go in another cooler wash later wont they ?", "words": ["go", "and", "another", "they", "in", "wash"]}, {"sent": "BowWow has had a wash .", "words": ["wash", "a"]}, {"sent": "well hafta wash it later .", "words": ["it", "wash"]}, {"sent": "let Mommy wash your face and your hands .", "words": ["your", "wash", "face", "and"]}, {"sent": "go throw this in the wash for me okay ?", "words": ["the", "for", "throw", "go", "this", "in", "me", "wash"]}, {"sent": "Thomas come and wash your hands please .", "words": ["your", "wash", "and"]}, {"sent": "when you wash your windows .", "words": ["your", "when", "wash", "you"]}, {"sent": "you can have a wash and get down .", "words": ["can", "you", "have", "and", "get", "down", "wash", "a"]}, {"sent": "shall I wash my hands ?", "words": ["my", "wash"]}, {"sent": "first I heard of it was Daddy asking me to wash your hands because they were covered in dead mouse .", "words": ["were", "your", "of", "was", "to", "first", "they", "in", "it", "mouse", "me", "wash", "because"]}, {"sent": "go wash your hands and your face .", "words": ["your", "face", "go", "and", "wash"]}, {"sent": "we can wash your hands .", "words": ["can", "wash", "your", "we"]}, {"sent": "lets just wash your hands .", "words": ["your", "wash"]}, {"sent": "are you ready wash up now ?", "words": ["up", "are", "wash", "you"]}, {"sent": "well Purdie uses her paws to wash her face after she has had a meal .", "words": ["face", "to", "wash", "her", "she", "a"]}, {"sent": "think Id better get you a pinnie and you could wash up everyday .", "words": ["up", "better", "think", "you", "get", "and", "could", "wash", "a"]}, {"sent": "we hafta wash these for sure after were done huh ?", "words": ["these", "for", "we", "wash", "were"]}, {"sent": "now Im gonna wash your leg .", "words": ["your", "wash", "leg"]}, {"sent": "you gonna wash it with that ?", "words": ["with", "wash", "you", "it", "that"]}, {"sent": "the turtle is hers the budgie is his and the dog is .", "words": ["the", "is", "hers", "and", "his", "turtle", "dog"]}, {"sent": "look hers another bed .", "words": ["hers", "bed", "another", "look"]}, {"sent": "mommy xxx hers on too .", "words": ["on", "too", "hers"]}, {"sent": "Mummy will roll hers through it .", "words": ["will", "it", "hers"]}, {"sent": "why do you needta make dolly wear hers ?", "words": ["do", "hers", "you", "make", "why"]}, {"sent": "well Kipper cant find hers .", "words": ["hers", "find"]}, {"sent": "hers took the wrong turn .", "words": ["the", "hers"]}, {"sent": "let her do hers and then you can do yours .", "words": ["can", "do", "and", "hers", "you", "then", "her"]}, {"sent": "so she was alright until my aunt Eva give her hers .", "words": ["my", "was", "so", "hers", "give", "her", "she"]}, {"sent": "the turtle is hers the budgie is his and the dog is .", "words": ["the", "is", "hers", "and", "his", "turtle", "dog"]}, {"sent": "Mommy has hers on .", "words": ["on", "hers"]}, {"sent": "Mummys eating hers .", "words": ["hers"]}, {"sent": "and Mommy put hers on a napkin too .", "words": ["put", "hers", "and", "too", "on", "napkin", "a"]}, {"sent": "but the other wee girl only got hers done a month beforehand .", "words": ["the", "hers", "other", "but", "a"]}, {"sent": "mhm thats hers .", "words": ["hers"]}, {"sent": "Mummys going to go upstairs for hers because I feel very cold .", "words": ["for", "go", "to", "hers", "cold", "because"]}, {"sent": "see now mummy got hers .", "words": ["see", "hers"]}, {"sent": "I dont wanna hear hers xxx .", "words": ["hers", "dont", "wanna", "hear"]}, {"sent": "the budgie is his the turtle is hers and the dog is .", "words": ["the", "is", "hers", "and", "his", "turtle", "dog"]}, {"sent": "hers is blue .", "words": ["blue", "is", "hers"]}, {"sent": "youre a big girl because Ewww can do hers by herself .", "words": ["can", "do", "hers", "by", "big", "because", "a"]}, {"sent": "the turtle is hers the budgie is his and the dog .", "words": ["the", "is", "hers", "and", "his", "turtle", "dog"]}, {"sent": "yeah thats why mommy clips hers to her shirt because its right near my mouth .", "words": ["my", "to", "shirt", "mouth", "because", "hers", "her", "why"]}, {"sent": "the turtle is hers the budgie is his and the dog is .", "words": ["the", "is", "hers", "and", "his", "turtle", "dog"]}, {"sent": "you can still xxx Perris finished with hers .", "words": ["can", "hers", "with", "you"]}, {"sent": "leave hers here okay ?", "words": ["here", "hers"]}, {"sent": "the turtle is hers the budgie is his and the dog is .", "words": ["the", "is", "hers", "and", "his", "turtle", "dog"]}, {"sent": "the turtle is hers the budgie is his and the dog is .", "words": ["the", "is", "hers", "and", "his", "turtle", "dog"]}, {"sent": "does she have any animals or particular favorites of hers ?", "words": ["does", "of", "any", "hers", "have", "she"]}, {"sent": "the turtle is hers the budgie is his and the dog is .", "words": ["the", "is", "hers", "and", "his", "turtle", "dog"]}, {"sent": "shall Mummy put hers on ?", "words": ["on", "put", "hers"]}, {"sent": "you got to keep it on look mommys wearing hers .", "words": ["to", "you", "hers", "it", "look", "on"]}, {"sent": "no hers is like new .", "words": ["new", "like", "is", "hers"]}, {"sent": "no she already has hers in now .", "words": ["in", "she", "hers"]}, {"sent": "is hers like this ?", "words": ["hers", "this", "like", "is"]}, {"sent": "well you mash hers .", "words": ["hers", "you"]}, {"sent": "mommys gonna put hers on too see ?", "words": ["put", "hers", "too", "on", "see"]}, {"sent": "oh look what Kalie did to hers too .", "words": ["what", "to", "hers", "did", "look", "too"]}, {"sent": "hm see how do I test hers .", "words": ["see", "how", "do", "hers"]}, {"sent": "MrsStouffer calls hers Scrubbybear .", "words": ["hers"]}, {"sent": "thats definitely a hers teacup .", "words": ["hers", "a"]}, {"sent": "and have hers and the baby can sit there like Amy does when we have our dinner .", "words": ["the", "can", "does", "we", "like", "hers", "have", "and", "there", "our", "sit", "when"]}, {"sent": "didnt you eat chawrz you cant aks for hers .", "words": ["eat", "hers", "for", "you"]}, {"sent": "because it was hers when she was little .", "words": ["was", "hers", "little", "it", "when", "she", "because"]}, {"sent": "hers ?", "words": ["hers"]}, {"sent": "its not hers .", "words": ["not", "hers"]}, {"sent": "Kathryn likes to have hers on back to front .", "words": ["back", "to", "hers", "have", "on"]}, {"sent": "the turtle is hers the budgie is his and the dog is .", "words": ["the", "is", "hers", "and", "his", "turtle", "dog"]}, {"sent": "I dont know where I got hers either but when I first got her and she was very poorly she was on lotsof tablets .", "words": ["where", "was", "first", "her", "hers", "and", "dont", "on", "when", "she", "but"]}, {"sent": "just dropped hers on her knee .", "words": ["on", "knee", "her", "hers"]}, {"sent": "move the others away from her so she just has hers .", "words": ["the", "so", "hers", "away", "her", "she"]}, {"sent": "how do you know it was hers ?", "words": ["do", "how", "was", "you", "hers", "it"]}, {"sent": "Mummys just finishing hers .", "words": ["hers"]}, {"sent": "Mummy got hers .", "words": ["hers"]}, {"sent": "its not hers .", "words": ["not", "hers"]}, {"sent": "eh dont take hers .", "words": ["dont", "take", "hers"]}, {"sent": "and mummy will find it at work and think its hers .", "words": ["think", "work", "find", "and", "hers", "it", "at", "will"]}, {"sent": "she done that with two of hers .", "words": ["of", "with", "hers", "that", "she"]}, {"sent": "wee Holly next door shes lost hers .", "words": ["hers", "door"]}, {"sent": "Im sure you have much better ideas than I do like Samantha has hers .", "words": ["better", "do", "like", "you", "have", "hers", "much"]}, {"sent": "them two are hers .", "words": ["are", "them", "hers"]}, {"sent": "Mommy is sure your juice tastes much better than hers anyway .", "words": ["better", "your", "is", "hers", "juice", "much"]}, {"sent": "were having our lunch and Purdies having hers .", "words": ["hers", "were", "and", "our"]}, {"sent": "those are hers .", "words": ["are", "those", "hers"]}, {"sent": "maybe Carolined like some tomato with hers .", "words": ["some", "with", "like", "hers"]}, {"sent": "and Mummy put hers on the floor when the telephone rang and when I came off the telephone Purdie had licked all the cream right into the inside .", "words": ["the", "off", "put", "all", "into", "inside", "hers", "and", "on", "when", "telephone"]}, {"sent": "most of hers are like baby dolls .", "words": ["of", "are", "like", "hers"]}, {"sent": "she wont keep hers in .", "words": ["in", "she", "hers"]}, {"sent": "go and get Amy a toy to play with if youre going to to nick hers .", "words": ["if", "go", "to", "play", "with", "and", "get", "hers", "toy", "a"]}, {"sent": "oh no Mum hasnt got hers on .", "words": ["on", "hers"]}, {"sent": "one of hers to play with and one for you to play with .", "words": ["for", "of", "to", "play", "with", "hers", "and", "you"]}, {"sent": "the turtle is hers the budgie is his and the dog is .", "words": ["the", "is", "hers", "and", "his", "turtle", "dog"]}, {"sent": "thats hers .", "words": ["hers"]}, {"sent": "mommy made hers fall down too .", "words": ["too", "down", "fall", "hers"]}, {"sent": "hers are mostly missing most of the flaps at this point .", "words": ["the", "of", "hers", "this", "are", "at"]}, {"sent": "shes drinking hers .", "words": ["hers"]}, {"sent": "she ate all a hers .", "words": ["hers", "all", "she", "a"]}, {"sent": "great are you just like mommy now because mommys holding hers too .", "words": ["like", "hers", "you", "are", "too", "because"]}, {"sent": "Mummy has eaten all hers as well .", "words": ["all", "hers"]}, {"sent": "she folded hers and she has see the blue in the middle there ?", "words": ["the", "and", "hers", "blue", "in", "there", "see", "she"]}, {"sent": "no if you dont want her messing with your buttons you dont mash hers .", "words": ["your", "if", "with", "you", "hers", "dont", "her"]}, {"sent": "because when we go on our holiday Purdie has got to go on hers Thomas .", "words": ["go", "we", "to", "our", "hers", "on", "when", "because"]}, {"sent": "but anyway sometimes Ill go down to her bed to nurse her and sometimes Ill bring her up to the bed and sometimes Ill have the energy to put her back on hers and other times she just stays in bed .", "words": ["up", "the", "put", "back", "go", "to", "bed", "bring", "and", "have", "hers", "other", "in", "on", "down", "her", "she", "but"]}, {"sent": "hm kinda like hers though .", "words": ["like", "hers"]}, {"sent": "let her do hers and you do yours .", "words": ["do", "you", "and", "hers", "her"]}, {"sent": "Mommy has hers too .", "words": ["too", "hers"]}, {"sent": "the turtle is hers the budgie is his and the dog is .", "words": ["the", "is", "hers", "and", "his", "turtle", "dog"]}, {"sent": "hers is called a chiffonier .", "words": ["a", "is", "hers"]}, {"sent": "hers are his are I think hers are hazel but maybe their blue hazel or something .", "words": ["think", "their", "hers", "blue", "are", "his", "but"]}, {"sent": "its not hers .", "words": ["not", "hers"]}, {"sent": "try and build one like hers .", "words": ["like", "hers", "and", "build", "try"]}, {"sent": "oh when Mummy dropped hers it was an accident .", "words": ["was", "hers", "an", "it", "when"]}, {"sent": "the turtle is hers the budgie is his and the dog is .", "words": ["the", "is", "hers", "and", "his", "turtle", "dog"]}, {"sent": "we just leave our toaster out and grandma puts hers away in the cupboard .", "words": ["the", "we", "our", "hers", "and", "in", "away", "out"]}, {"sent": "Tanisha popped hers .", "words": ["hers"]}, {"sent": "no dis is hers .", "words": ["hers", "is"]}, {"sent": "the turtle is hers the budgie is his and the dog is .", "words": ["the", "is", "hers", "and", "his", "turtle", "dog"]}, {"sent": "oh babys eating hers .", "words": ["hers"]}, {"sent": "when I did hers .", "words": ["when", "did", "hers"]}, {"sent": "well theres well thats hers with the white hair .", "words": ["the", "white", "with", "hers", "hair"]}, {"sent": "yeah dont try to push it down you might break it .", "words": ["break", "push", "to", "you", "dont", "it", "down", "try", "try to"]}, {"sent": "lets try to keep it on your pants okay ?", "words": ["your", "to", "it", "pants", "on", "try", "try to"]}, {"sent": "I try to give you cheese and you dont like any of them .", "words": ["try", "of", "to", "any", "like", "them", "you", "and", "dont", "give", "cheese", "try to"]}, {"sent": "now dont try to do that .", "words": ["do", "to", "dont", "that", "try", "try to"]}, {"sent": "can you try to do that ?", "words": ["can", "do", "to", "you", "that", "try", "try to"]}, {"sent": "try to eat her .", "words": ["try", "to", "eat", "her", "try to"]}, {"sent": "we gotta try to find a page with Woody on it .", "words": ["we", "to", "with", "it", "find", "on", "try", "try to", "a"]}, {"sent": "and sometimes Ill sit down and try to play with him and he just hes like Im doing this Mom like get away .", "words": ["to", "play", "with", "like", "and", "get", "this", "him", "away", "he", "sit", "down", "try", "try to"]}, {"sent": "Ill try to pick it up .", "words": ["up", "pick", "to", "it", "try", "try to"]}, {"sent": "and occasionally if she wants them dressed she comes and is like mom help and shell have me try to put on whatever it is .", "words": ["help", "put", "if", "is", "to", "them", "like", "and", "have", "it", "on", "me", "try to", "try", "she"]}, {"sent": "always try to remember that you should leave your boots outside the house when you come home .", "words": ["the", "try", "your", "when", "to", "outside", "you", "boots", "house", "that", "try to", "home"]}, {"sent": "you wanna try to eat it with a spoon ?", "words": ["to", "with", "you", "eat", "it", "spoon", "wanna", "try", "try to", "a"]}, {"sent": "try to remember .", "words": ["to", "try", "try to"]}, {"sent": "we tried to go to on to on to Riverside and try to bring my tricycle .", "words": ["my", "go", "we", "to", "bring", "and", "tricycle", "on", "try", "try to"]}, {"sent": "now that you have the noisiest toy out well try to see if youll talk for us .", "words": ["the", "try", "for", "if", "to", "you", "have", "us", "toy", "out", "talk", "see", "that", "try to"]}, {"sent": "do you think I should try to finish it up were basically done with it its just .", "words": ["up", "do", "think", "to", "with", "finish", "you", "it", "try", "try to", "were"]}, {"sent": "well let me fix it or try to .", "words": ["to", "fix", "it", "me", "try", "try to"]}, {"sent": "wouldja like to try to do that ?", "words": ["do", "to", "like", "that", "try", "try to"]}, {"sent": "try to find something yellow .", "words": ["to", "find", "yellow", "try", "try to"]}, {"sent": "no no dont try to nurse on the side of the box .", "words": ["the", "box", "of", "to", "dont", "on", "try", "try to"]}, {"sent": "now you wanna try to pop that out at that end .", "words": ["pop", "to", "you", "that", "at", "wanna", "out", "try", "try to"]}, {"sent": "they try to smell the fox .", "words": ["the", "to", "they", "try", "try to"]}, {"sent": "Im gonna try to get my guys out .", "words": ["my", "to", "get", "out", "try", "try to"]}, {"sent": "yeah yeah hes gonna try to poke Cricter .", "words": ["to", "try", "try to"]}, {"sent": "but um when we when we read books he generally does try to eat them still .", "words": ["does", "try", "we", "to", "them", "eat", "he", "read", "when", "try to", "but"]}, {"sent": "try to eat a little more of this .", "words": ["of", "to", "little", "eat", "this", "more", "try", "try to", "a"]}, {"sent": "so he pinched giant Babybears nose to try to stop him snoring .", "words": ["to", "so", "stop", "him", "he", "try", "try to", "nose"]}, {"sent": "a couple times she would grab the stuffed animals or I would try to .", "words": ["the", "to", "would", "try to", "try", "she", "a"]}, {"sent": "and it hasta be correc no I never try to break the you know the train of thought .", "words": ["the", "break", "train", "of", "to", "be", "and", "you", "it", "try", "try to"]}, {"sent": "lets put away some of this stuff and you can try to stand up .", "words": ["can", "up", "put", "of", "to", "stand", "and", "you", "this", "away", "some", "try", "try to"]}, {"sent": "okay let us try to color just on the paper .", "words": ["the", "to", "paper", "us", "on", "try", "try to"]}, {"sent": "uh momma try to push I cant do it .", "words": ["do", "push", "to", "it", "try", "try to"]}, {"sent": "does he ever try to like .", "words": ["does", "to", "like", "he", "try", "try to"]}, {"sent": "lets try to make the ice cream Violet .", "words": ["the", "ice cream", "try", "to", "ice", "make", "try to"]}, {"sent": "I have to try to learn to control it .", "words": ["to", "have", "it", "try", "try to"]}, {"sent": "Ross try to remember .", "words": ["to", "try", "try to"]}, {"sent": "try to control stuff .", "words": ["to", "try", "try to"]}, {"sent": "you try to keep your mouth to your self .", "words": ["your", "to", "mouth", "you", "try", "try to"]}, {"sent": "we can try to have lunch .", "words": ["can", "we", "to", "have", "try", "try to"]}, {"sent": "wanna try to do something ?", "words": ["do", "to", "wanna", "try", "try to"]}, {"sent": "can you try to make the body .", "words": ["can", "the", "try", "to", "you", "make", "try to"]}, {"sent": "lets all try to remember okay ?", "words": ["to", "all", "try", "try to"]}, {"sent": "does she try to well actually she tried to read .", "words": ["does", "to", "read", "try to", "try", "she"]}, {"sent": "try to get her in shes got her arms here .", "words": ["try", "to", "get", "here", "in", "her", "try to"]}, {"sent": "banan at least could I try to some custard pudding .", "words": ["to", "pudding", "at", "some", "could", "try", "try to"]}, {"sent": "well that like I said that is a um its a visual reception thing so I try to cycle through a number of .", "words": ["try", "is", "of", "so", "to", "like", "that", "try to", "a"]}, {"sent": "dont try to spit up honey .", "words": ["up", "to", "dont", "try", "try to"]}, {"sent": "did you try to shake it ?", "words": ["to", "shake", "you", "it", "did", "try", "try to"]}, {"sent": "try to click it on .", "words": ["to", "it", "on", "try", "try to"]}, {"sent": "and I try to to make sure I go all the way .", "words": ["the", "all", "try", "go", "to", "and", "make", "try to"]}, {"sent": "now try to shoot that one .", "words": ["to", "that", "try", "try to"]}, {"sent": "I think Im gonna try to find some toys in this book .", "words": ["think", "to", "this", "in", "find", "book", "some", "try", "try to"]}, {"sent": "hafta keep moving about to try to keep warm .", "words": ["to", "try", "try to", "about"]}, {"sent": "now we hafta try to get three in a row .", "words": ["we", "to", "get", "in", "try", "try to", "a"]}, {"sent": "well I think what we should do is try to remember what we took apart last and put that together first .", "words": ["what", "think", "do", "put", "is", "we", "to", "last", "first", "and", "that", "try", "try to"]}, {"sent": "lets try to set that one up .", "words": ["up", "try", "to", "that", "try to"]}, {"sent": "and then did you try to get him to play with anything else besides the foods ?", "words": ["the", "to", "play", "with", "you", "and", "get", "him", "did", "then", "try", "try to"]}, {"sent": "now you try to screw the black one on here .", "words": ["the", "to", "you", "black", "here", "on", "try", "try to"]}, {"sent": "try to think about it try to remember .", "words": ["think", "to", "it", "about", "try", "try to"]}, {"sent": "you should probably sit on the potty and try to poop .", "words": ["the", "to", "you", "and", "potty", "on", "sit", "try", "try to"]}, {"sent": "I think Joseph would like to use the little one lets let him have it hes crawling over to try to get it you wanna draw ?", "words": ["the", "think", "try", "to", "draw", "like", "little", "have", "get", "him", "it", "you", "wanna", "would", "over", "try to"]}, {"sent": "it looks like youre gonna try to finish them .", "words": ["to", "them", "like", "finish", "it", "try", "try to"]}, {"sent": "I think Ill try to do that while youre getting your things ready .", "words": ["do", "think", "your", "to", "that", "try", "try to"]}, {"sent": "you wanna try to pull up ?", "words": ["up", "to", "pull", "you", "wanna", "try", "try to"]}, {"sent": "here can you can you try to put it on ?", "words": ["can", "put", "to", "you", "here", "it", "on", "try", "try to"]}, {"sent": "why dont you try to do them too ?", "words": ["do", "to", "them", "you", "dont", "too", "try", "try to", "why"]}, {"sent": "lets try to give the baby something to drink .", "words": ["the", "drink", "to", "give", "try", "try to"]}, {"sent": "what game what game did you just try to play ?", "words": ["what", "game", "to", "play", "you", "did", "try", "try to"]}, {"sent": "why dont you try to hit the ceiling ?", "words": ["the", "to", "you", "dont", "hit", "try", "try to", "why"]}, {"sent": "um just try to play with her much like you were playing at home if you guys were at home instead of here and if you were playing with her toys .", "words": ["try", "if", "of", "to", "play", "with", "like", "much", "you", "here", "and", "at", "her", "try to", "were", "home"]}, {"sent": "but if you try to catch him .", "words": ["if", "to", "catch", "you", "him", "try", "try to", "but"]}, {"sent": "and then Ill Ill try to move to something else in another month .", "words": ["to", "and", "another", "in", "then", "try", "try to"]}, {"sent": "well just try to eat a little bit of xxx then sweetheart .", "words": ["of", "to", "little", "eat", "then", "try", "try to", "a"]}, {"sent": "so I was gonna try to start inducing a nap around nine thirtyish .", "words": ["was", "so", "to", "around", "try", "try to", "a"]}, {"sent": "right what does he try to do ?", "words": ["what", "does", "do", "to", "he", "try", "try to"]}, {"sent": "dont you try to t .", "words": ["to", "you", "dont", "try", "try to"]}, {"sent": "try to think about it how did we do it the other time ?", "words": ["the", "do", "think", "how", "we", "to", "other", "it", "did", "about", "try", "try to"]}, {"sent": "and and so Ill do it for him and every so often hell try to flip the blocks but hes more interested in just kinda flipping the pages and .", "words": ["the", "do", "for", "so", "to", "and", "him", "it", "more", "every", "in", "try", "try to", "but"]}, {"sent": "lemme try to pick you up .", "words": ["up", "pick", "to", "you", "try", "try to"]}, {"sent": "you wanna try to eat some more of your little fruit ?", "words": ["your", "of", "to", "you", "eat", "little", "more", "wanna", "some", "try", "try to"]}, {"sent": "I think Ill try to get you there .", "words": ["think", "to", "you", "get", "there", "try", "try to"]}, {"sent": "now you you try to fit some of these .", "words": ["these", "of", "to", "fit", "you", "some", "try", "try to"]}, {"sent": "okay xxx wanna try to write you um card or you want me to help you ?", "words": ["help", "to", "write", "you", "wanna", "me", "try", "try to"]}, {"sent": "wanna try to get up ?", "words": ["up", "to", "get", "wanna", "try", "try to"]}, {"sent": "try to say it .", "words": ["to", "say", "it", "try", "try to"]}, {"sent": "and at times shell try to stick the bottle towards their mouth .", "words": ["the", "stick", "their", "to", "mouth", "and", "at", "bottle", "try", "try to"]}, {"sent": "you need some teeth before you can try to eat something like that .", "words": ["can", "need", "to", "like", "you", "eat", "that", "some", "try", "try to"]}, {"sent": "somebody might try to call us .", "words": ["to", "try", "try to", "us"]}, {"sent": "you can try to hook one up to it .", "words": ["can", "up", "to", "you", "it", "try", "try to"]}, {"sent": "dont even try to pull that past me anymore .", "words": ["try", "to", "pull", "dont", "me", "that", "try to"]}, {"sent": "well funny enough I try to get her .", "words": ["try", "to", "get", "her", "try to"]}, {"sent": "oh poor dolly .", "words": ["poor"]}, {"sent": "poor little cat .", "words": ["cat", "little", "poor"]}, {"sent": "probably poor blood tired blood .", "words": ["tired", "poor"]}, {"sent": "poor dolly got hurt .", "words": ["hurt", "poor"]}, {"sent": "poor dolly .", "words": ["poor"]}, {"sent": "so poor ?", "words": ["so", "poor"]}, {"sent": "oh my poor eye .", "words": ["my", "eye", "poor"]}, {"sent": "poor child .", "words": ["poor"]}, {"sent": "poor Zita .", "words": ["poor"]}, {"sent": "poor squirrel .", "words": ["squirrel", "poor"]}, {"sent": "poor PoohBear .", "words": ["poor"]}, {"sent": "well poor poor Mandys crying .", "words": ["poor"]}, {"sent": "poor knee .", "words": ["knee", "poor"]}, {"sent": "oh poor Mummy .", "words": ["poor"]}, {"sent": "that poor little girl .", "words": ["that", "little", "poor"]}, {"sent": "poor dolly .", "words": ["poor"]}, {"sent": "oy poor Fraser .", "words": ["poor"]}, {"sent": "poor Eric hes crying .", "words": ["poor"]}, {"sent": "och poor Denver .", "words": ["poor"]}, {"sent": "poor Toby .", "words": ["poor"]}, {"sent": "poor Terrence with a sheep standing on him .", "words": ["with", "poor", "him", "sheep", "on", "a"]}, {"sent": "poor dog .", "words": ["dog", "poor"]}, {"sent": "youre hammering the poor lady .", "words": ["the", "poor"]}, {"sent": "poor little boy with hiccups .", "words": ["with", "little", "poor"]}, {"sent": "why poor Charlotte ?", "words": ["why", "poor"]}, {"sent": "poor old Gerry .", "words": ["old", "poor"]}, {"sent": "and poor Mummy was working .", "words": ["was", "poor", "and"]}, {"sent": "and the poor steam roller and and Lofty are tiny arent they ?", "words": ["the", "tiny", "poor", "and", "they", "are"]}, {"sent": "poor Maisy ?", "words": ["poor"]}, {"sent": "we said poor dada .", "words": ["poor", "we"]}, {"sent": "oh poor grandma xxx shell be shoveling and shoveling and shoveling .", "words": ["be", "and", "poor"]}, {"sent": "poor Peggy .", "words": ["poor"]}, {"sent": "poor Fraser .", "words": ["poor"]}, {"sent": "yeah poor sweetie .", "words": ["poor"]}, {"sent": "poor Fraser .", "words": ["poor"]}, {"sent": "oh poor Snoopy .", "words": ["poor"]}, {"sent": "oh poor Jess .", "words": ["poor"]}, {"sent": "your poor your poor Mums been sat on that train for ages and its not gone anywhere Thomas .", "words": ["for", "your", "train", "poor", "and", "on", "that", "not"]}, {"sent": "so what about this poor guys face ?", "words": ["what", "face", "so", "poor", "this", "about"]}, {"sent": "oh poor lorry .", "words": ["poor"]}, {"sent": "whos a poor little love .", "words": ["poor", "love", "little", "a"]}, {"sent": "poor Pooh hes gonna have to climb a honey tree or something huh ?", "words": ["tree", "to", "poor", "have", "climb", "a"]}, {"sent": "poor poor Thomas .", "words": ["poor"]}, {"sent": "what about the poor giraffe ?", "words": ["the", "what", "giraffe", "poor", "about"]}, {"sent": "poor Mummy .", "words": ["poor"]}, {"sent": "poor dada .", "words": ["poor"]}, {"sent": "now but look at poor wee Susan .", "words": ["poor", "at", "but", "look"]}, {"sent": "poor gingerbread man .", "words": ["poor"]}, {"sent": "look at the poor driver .", "words": ["the", "poor", "at", "look"]}, {"sent": "poor dada .", "words": ["poor"]}, {"sent": "you poor baby .", "words": ["poor", "you"]}, {"sent": "poor Babybear .", "words": ["poor"]}, {"sent": "poor teddy .", "words": ["poor"]}, {"sent": "this poor guy .", "words": ["this", "poor"]}, {"sent": "poor poor .", "words": ["poor"]}, {"sent": "and theres poor Susan and Susan .", "words": ["poor", "and"]}, {"sent": "poor TinkyWinkys on the floor .", "words": ["on", "the", "poor"]}, {"sent": "poor cat .", "words": ["cat", "poor"]}, {"sent": "poor me .", "words": ["me", "poor"]}, {"sent": "poor Sabina kept looking at Charlie as we as Judy and I oohed and aahed over Charlies blue lips and Sabina kept saying .", "words": ["we", "and", "poor", "blue", "at", "over"]}, {"sent": "oh poor Mummy .", "words": ["poor"]}, {"sent": "oh my poor poor Thomas .", "words": ["my", "poor"]}, {"sent": "poor Henry .", "words": ["poor"]}, {"sent": "poor Daddy .", "words": ["poor"]}, {"sent": "oh poor xxx .", "words": ["poor"]}, {"sent": "oh wheres poor Tinkeywinkey ?", "words": ["poor"]}, {"sent": "poor baby you just threw him down on the ground .", "words": ["the", "you", "poor", "him", "on", "down"]}, {"sent": "then the catll get up and the poor dogll just be laying there .", "words": ["the", "up", "be", "and", "get", "poor", "there", "then"]}, {"sent": "poor Bertie .", "words": ["poor"]}, {"sent": "my poor thing .", "words": ["my", "poor"]}, {"sent": "poor Mandys crying .", "words": ["poor"]}, {"sent": "poor lady .", "words": ["poor"]}, {"sent": "aw that poor girls in lotsof pieces now .", "words": ["in", "that", "poor"]}, {"sent": "aw poor Thomas xxx .", "words": ["poor"]}, {"sent": "oh what about that poor truck ?", "words": ["what", "truck", "poor", "about", "that"]}, {"sent": "poor James .", "words": ["poor"]}, {"sent": "ah ah poor Mummy .", "words": ["poor"]}, {"sent": "oh what happened to this poor baby ?", "words": ["to", "what", "this", "poor"]}, {"sent": "poor old crow .", "words": ["old", "poor"]}, {"sent": "aw that poor little boy .", "words": ["that", "little", "poor"]}, {"sent": "oh poor BobbyBear .", "words": ["poor"]}, {"sent": "poor little duck on the floor .", "words": ["the", "little", "poor", "on", "duck"]}, {"sent": "oh poor bear .", "words": ["bear", "poor"]}, {"sent": "poor little baby .", "words": ["little", "poor"]}, {"sent": "oh poor dolly .", "words": ["poor"]}, {"sent": "poor Maisy hasnt got a hand .", "words": ["hand", "a", "poor"]}, {"sent": "they were so poor .", "words": ["they", "so", "were", "poor"]}, {"sent": "poor Paul .", "words": ["poor"]}, {"sent": "oh this poor dollie .", "words": ["this", "poor"]}, {"sent": "you poor thing .", "words": ["poor", "you"]}, {"sent": "poor ducks they got their head stuck .", "words": ["their", "head", "poor", "they", "stuck"]}, {"sent": "poor Jwww .", "words": ["poor"]}, {"sent": "poor old Thomas .", "words": ["old", "poor"]}, {"sent": "poor baby .", "words": ["poor"]}, {"sent": "poor Alice .", "words": ["poor"]}, {"sent": "aw poor lobster .", "words": ["poor"]}, {"sent": "poor Mummy .", "words": ["poor"]}, {"sent": "poor VerityClaire just crying and crying isnt she ?", "words": ["she", "and", "poor"]}, {"sent": "poor Ethy .", "words": ["poor"]}, {"sent": "oh poor baby poor baby .", "words": ["poor"]}, {"sent": "Dwww had a dress .", "words": ["dress", "a"]}, {"sent": "the blue dress or the red dress ?", "words": ["the", "red", "dress", "blue"]}, {"sent": "a dress maybe .", "words": ["dress", "a"]}, {"sent": "thats why shes wearing a dress .", "words": ["dress", "why", "a"]}, {"sent": "shes has a dress she has a dress on .", "words": ["on", "dress", "she", "a"]}, {"sent": "is your dress staying on Nina ?", "words": ["dress", "on", "your", "is"]}, {"sent": "right the flamingo is wearing a dress .", "words": ["the", "dress", "a", "is"]}, {"sent": "youve got to dress smartly when we go down to the dining room .", "words": ["the", "when", "go", "we", "to", "room", "down", "dress"]}, {"sent": "I can make this off a dress is just so .", "words": ["can", "off", "is", "dress", "so", "this", "make", "a"]}, {"sent": "do wear fancy underwear under your dress .", "words": ["dress", "do", "under", "your"]}, {"sent": "your dress is sliding up .", "words": ["dress", "up", "your", "is"]}, {"sent": "and now Im gonna put a new dress on this doll .", "words": ["put", "doll", "and", "this", "on", "new", "dress", "a"]}, {"sent": "please dress me .", "words": ["me", "dress"]}, {"sent": "their jobs because they only have uh two sets of clothing a clothing for school and clothing for dress .", "words": ["for", "their", "of", "school", "and", "have", "they", "dress", "because", "a"]}, {"sent": "what colors are in your dress honey ?", "words": ["what", "your", "in", "are", "dress"]}, {"sent": "silly ways I like to dress .", "words": ["to", "dress", "like"]}, {"sent": "dress dress dress .", "words": ["dress"]}, {"sent": "is that your dress ?", "words": ["your", "dress", "that", "is"]}, {"sent": "no dress ?", "words": ["dress"]}, {"sent": "Fraser did you see Cwwws dress ?", "words": ["see", "dress", "did", "you"]}, {"sent": "lets take the dress off the dolly before we give her a bath .", "words": ["the", "off", "give", "we", "her", "take", "dress", "a"]}, {"sent": "can you tell her what your dress looked like ?", "words": ["can", "what", "your", "dress", "like", "you", "her"]}, {"sent": "oh now youre gonna go get K Blitzens dress ?", "words": ["get", "dress", "go"]}, {"sent": "did you have a new dress ?", "words": ["you", "have", "did", "new", "dress", "a"]}, {"sent": "you havent got a dress .", "words": ["dress", "a", "you"]}, {"sent": "oh thats a beautiful dress .", "words": ["dress", "a"]}, {"sent": "do you like that dress ?", "words": ["do", "like", "you", "that", "dress"]}, {"sent": "can you tell me what color the dress is Becky ?", "words": ["can", "what", "the", "is", "you", "me", "dress"]}, {"sent": "youd better dress him quickly .", "words": ["better", "him", "dress"]}, {"sent": "theres that dress here .", "words": ["dress", "here", "that"]}, {"sent": "there are in fact were gonna get a new bin and were gonna put all the dress up things together .", "words": ["the", "all", "put", "up", "dress", "and", "get", "there", "in", "are", "new", "were", "a"]}, {"sent": "mommy doesnt dress you anymore do I ?", "words": ["do", "dress", "you"]}, {"sent": "thats a pretty dress aswell isnt it ?", "words": ["dress", "it", "pretty", "a"]}, {"sent": "let us dress the little boy .", "words": ["the", "us", "dress", "little"]}, {"sent": "you hafta put her dress on .", "words": ["put", "her", "you", "on", "dress"]}, {"sent": "oh isnt it a pretty dress Aran ?", "words": ["dress", "it", "pretty", "a"]}, {"sent": "and this bunnys got a blue dress on .", "words": ["blue", "and", "this", "on", "dress", "a"]}, {"sent": "but youknow and this you could wear this as a summer dress aswell .", "words": ["you", "and", "this", "could", "dress", "but", "a"]}, {"sent": "do you wanna take her dress off as well ?", "words": ["off", "do", "dress", "you", "wanna", "take", "her"]}, {"sent": "I wanna show I wanna show Dada your summer dress because I think its a bit big .", "words": ["think", "your", "show", "big", "wanna", "dress", "because", "a"]}, {"sent": "it might be a good day for shorts or or leggings or even a dress if you feel like it .", "words": ["for", "shorts", "if", "be", "like", "you", "it", "good", "dress", "a"]}, {"sent": "huh a dress .", "words": ["dress", "a"]}, {"sent": "look at that dress .", "words": ["dress", "at", "that", "look"]}, {"sent": "oop can you dress the boy ?", "words": ["can", "the", "dress", "you"]}, {"sent": "I think Ill hafta take it back and buy buy you a dress .", "words": ["think", "back", "dress", "and", "you", "it", "take", "buy", "a"]}, {"sent": "eight pound to get my dress dry cleaned .", "words": ["my", "to", "dry", "get", "dress"]}, {"sent": "too cold for her dress is it ?", "words": ["for", "is", "her", "it", "too", "dress", "cold"]}, {"sent": "the way I take spots off a dress is just so .", "words": ["the", "off", "is", "so", "take", "dress", "a"]}, {"sent": "when we went to a fancy dress party dressed as AndyPandy and xxx .", "words": ["when", "we", "to", "party", "and", "dress", "a"]}, {"sent": "well I would have thought really that he wouldnt have bothered with the dress .", "words": ["the", "dress", "with", "have", "he", "would", "that"]}, {"sent": "youll see me Im wearing a pink dress .", "words": ["see", "me", "dress", "a"]}, {"sent": "you wanna hold the egg while I dress you ?", "words": ["the", "egg", "hold", "you", "wanna", "dress"]}, {"sent": "I was thinking we could put this pretty dress on .", "words": ["put", "we", "was", "this", "pretty", "on", "could", "dress"]}, {"sent": "we brought Annes we brought babys dress down earlier didnt we ?", "words": ["down", "dress", "we"]}, {"sent": "I dont think youre going to get the dress on Nicole .", "words": ["the", "think", "to", "get", "dont", "on", "dress"]}, {"sent": "let us let us dress this kid here .", "words": ["here", "us", "dress", "this"]}, {"sent": "oh its very hard trying to dress you when youre in this mood Thomas .", "words": ["when", "to", "you", "this", "in", "hard", "dress"]}, {"sent": "Devons gonna help you dress your dolls okay ?", "words": ["your", "help", "dress", "you"]}, {"sent": "we need someone to dress up as the Easter bunny .", "words": ["up", "the", "bunny", "need", "we", "to", "dress"]}, {"sent": "does daddy call this the best dress book ever ?", "words": ["the", "does", "this", "book", "dress"]}, {"sent": "thats a pretty dress isnt it ?", "words": ["dress", "it", "pretty", "a"]}, {"sent": "oh the one in the green dress ?", "words": ["the", "in", "dress", "green"]}, {"sent": "who do you dress up as ?", "words": ["up", "do", "you", "who", "dress"]}, {"sent": "thats your dress youre going to try on for the wedding .", "words": ["the", "try", "your", "for", "to", "on", "dress"]}, {"sent": "its a dress .", "words": ["dress", "a"]}, {"sent": "is that because youre climbing all over me and Im wearing a blue dress ?", "words": ["all", "is", "dress", "and", "blue", "that", "me", "over", "because", "a"]}, {"sent": "and your dress .", "words": ["dress", "your", "and"]}, {"sent": "she is wearing quite a snazzy dress I think .", "words": ["think", "is", "dress", "she", "a"]}, {"sent": "that ladys trying on a new dress .", "words": ["that", "on", "new", "dress", "a"]}, {"sent": "you wanna dress her ?", "words": ["her", "dress", "wanna", "you"]}, {"sent": "we went to get measured for her dress .", "words": ["for", "we", "dress", "to", "get", "her"]}, {"sent": "oh what happened to her clothes to her dress ?", "words": ["to", "what", "her", "dress"]}, {"sent": "whats on Mommys dress ?", "words": ["on", "dress"]}, {"sent": "when I dress you upstairs I know exactly xxx in one place .", "words": ["dress", "in", "when", "you"]}, {"sent": "so she likes to dress and undress her baby doll .", "words": ["to", "so", "her", "doll", "and", "dress", "she"]}, {"sent": "put the dress on her .", "words": ["the", "put", "her", "on", "dress"]}, {"sent": "and her dress too .", "words": ["dress", "too", "her", "and"]}, {"sent": "do you wanna put her dress back on ?", "words": ["do", "put", "back", "dress", "you", "wanna", "on", "her"]}, {"sent": "something to dress and to eat ah haha .", "words": ["eat", "to", "dress", "and"]}, {"sent": "and whos that with the pink dress ?", "words": ["the", "with", "and", "that", "dress"]}, {"sent": "while youre eating those Mummys going to iron her dress .", "words": ["dress", "to", "her", "those"]}, {"sent": "you hafta take her dress off .", "words": ["off", "her", "you", "take", "dress"]}, {"sent": "the blue spotty dress aswell ?", "words": ["the", "dress", "blue"]}, {"sent": "its a lady wearing a long dress .", "words": ["dress", "long", "a"]}, {"sent": "ooh its blowing your dress .", "words": ["dress", "your"]}, {"sent": "dress up clothes .", "words": ["up", "dress"]}, {"sent": "we could bring that in for dress up .", "words": ["up", "for", "we", "dress", "bring", "in", "could", "that"]}, {"sent": "and grandma can take it with her dress on and then the puppy dog in there .", "words": ["can", "the", "dress", "with", "her", "and", "there", "in", "it", "puppy", "on", "then", "take", "dog"]}, {"sent": "with mothers white dress .", "words": ["dress", "white", "with"]}, {"sent": "should put a dress on now .", "words": ["on", "put", "dress", "a"]}, {"sent": "goes on this dress here .", "words": ["on", "this", "here", "dress"]}, {"sent": "who gave you that dress ?", "words": ["dress", "that", "you", "who"]}, {"sent": "I think it must have been the summer because the little girl has got a summer dress on hasnt she ?", "words": ["the", "think", "little", "have", "it", "on", "dress", "she", "because", "a"]}, {"sent": "got a pretty dress on hasnt she ?", "words": ["pretty", "on", "dress", "she", "a"]}, {"sent": "dress .", "words": ["dress"]}, {"sent": "and it looks like shes wearing her Mummys dress because its far too big for her .", "words": ["for", "her", "like", "and", "big", "it", "too", "dress", "because"]}, {"sent": "are you thinking about that song about the red dress ?", "words": ["the", "red", "dress", "you", "are", "about", "that"]}, {"sent": "can Mummy dress her ?", "words": ["can", "her", "dress"]}, {"sent": "a dress for you .", "words": ["for", "dress", "you", "a"]}, {"sent": "wait for it to go dry ?", "words": ["wait", "for", "go", "to", "dry", "it"]}, {"sent": "well we hafta dry off your head first .", "words": ["off", "your", "we", "dry", "head", "first"]}, {"sent": "youre eating the dry cornflakes off the coasters .", "words": ["the", "dry", "off"]}, {"sent": "sugar hasta be kept dry .", "words": ["be", "dry"]}, {"sent": "how long does it take to dry ?", "words": ["does", "how", "to", "dry", "long", "it", "take"]}, {"sent": "and its dry so shes bringing it in .", "words": ["so", "dry", "and", "in", "it"]}, {"sent": "let me make sure the plates dry because the color from the Smarties will run off .", "words": ["the", "off", "dry", "run", "will", "me", "make", "because"]}, {"sent": "dry up ?", "words": ["up", "dry"]}, {"sent": "I think youve got some dry skin .", "words": ["some", "think", "dry"]}, {"sent": "it will dry very quickly if you just leave it there .", "words": ["if", "dry", "you", "there", "it", "will"]}, {"sent": "thats dry or .", "words": ["dry"]}, {"sent": "is it dry ?", "words": ["dry", "it", "is"]}, {"sent": "because everything in the trucks dry isnt it .", "words": ["the", "dry", "in", "it", "because"]}, {"sent": "and then you got nice cream on and nice talcum powder so you smell lovely and fresh and you feel nice and dry .", "words": ["ice cream", "so", "dry", "nice", "you", "and", "on", "then"]}, {"sent": "were gonna get a dry one .", "words": ["get", "dry", "were", "a"]}, {"sent": "you hold her up to him and maybe hell dry it .", "words": ["up", "hold", "to", "dry", "you", "and", "him", "it", "her"]}, {"sent": "go and dry your hands please .", "words": ["dry", "your", "and", "go"]}, {"sent": "well Ill stick him under and you can dry him .", "words": ["can", "stick", "dry", "you", "and", "under", "him"]}, {"sent": "wet foot dry foot low foot .", "words": ["wet", "foot", "dry"]}, {"sent": "hes not peeing hes just sitting there dry yeah those pants arent dry .", "words": ["those", "dry", "there", "pants", "not"]}, {"sent": "they hafta dry off ?", "words": ["off", "they", "dry"]}, {"sent": "whats that for to keep us dry ?", "words": ["for", "to", "dry", "us", "that"]}, {"sent": "umbrellas nice isnt it when its raining because it keeps your hair dry .", "words": ["your", "dry", "nice", "it", "hair", "when", "because"]}, {"sent": "hes all dry .", "words": ["all", "dry"]}, {"sent": "and when he finishes eating he helps his mother to wash and dry the dishes .", "words": ["the", "to", "dry", "wash", "and", "he", "his", "when"]}, {"sent": "alright well let me dry them off .", "words": ["off", "me", "dry", "them"]}, {"sent": "because it was a bit crumbly and dry before wasnt it ?", "words": ["was", "dry", "and", "it", "because", "a"]}, {"sent": "because its dry skin .", "words": ["dry", "because"]}, {"sent": "wet and dry .", "words": ["wet", "dry", "and"]}, {"sent": "youve been dry for days and days now .", "words": ["dry", "for", "and"]}, {"sent": "because otherwise it will be very dry .", "words": ["be", "dry", "it", "will", "because"]}, {"sent": "these willnt keep her feet very dry will they ?", "words": ["these", "dry", "they", "will", "her"]}, {"sent": "when they have finished washing they dry .", "words": ["have", "they", "dry", "when"]}, {"sent": "is she all dry ?", "words": ["all", "dry", "she", "is"]}, {"sent": "dry your hands .", "words": ["dry", "your"]}, {"sent": "shall we leave that to dry now ?", "words": ["to", "dry", "that", "we"]}, {"sent": "ask Ethan to dry it would you ?", "words": ["to", "dry", "you", "it", "would"]}, {"sent": "dry foot .", "words": ["dry", "foot"]}, {"sent": "did we keep dry ?", "words": ["dry", "did", "we"]}, {"sent": "lets just get you dry now .", "words": ["get", "dry", "you"]}, {"sent": "are you going to dry them off for her ?", "words": ["off", "for", "to", "dry", "them", "you", "are", "her"]}, {"sent": "dry your hands on the towel please .", "words": ["the", "your", "dry", "towel", "on"]}, {"sent": "shall we just dry our hands on the towel ?", "words": ["the", "we", "dry", "our", "towel", "on"]}, {"sent": "has it got dry ?", "words": ["dry", "it"]}, {"sent": "and dry your legs .", "words": ["dry", "your", "and"]}, {"sent": "xxx dry .", "words": ["dry"]}, {"sent": "and then you coat the apple in hot runny toffee and then you leave them to dry .", "words": ["the", "to", "dry", "them", "you", "and", "in", "hot", "apple", "then", "coat"]}, {"sent": "must be very dry .", "words": ["be", "dry"]}, {"sent": "dry hair .", "words": ["dry", "hair"]}, {"sent": "those are dry .", "words": ["dry", "are", "those"]}, {"sent": "lets get your head dry .", "words": ["get", "head", "dry", "your"]}, {"sent": "let it dry .", "words": ["dry", "it"]}, {"sent": "now dry your hands .", "words": ["dry", "your"]}, {"sent": "dry your diaper ?", "words": ["diaper", "dry", "your"]}, {"sent": "youre to leave them to dry .", "words": ["to", "dry", "them"]}, {"sent": "just a minute Ill get a dry on too wait want a dry one ?", "words": ["wait", "dry", "get", "too", "on", "a"]}, {"sent": "otherwise the penll go dry .", "words": ["the", "dry", "go"]}, {"sent": "then well get some tissue and we can dry it .", "words": ["can", "tissue", "we", "dry", "and", "get", "it", "some", "then"]}, {"sent": "wet foot dry foot .", "words": ["wet", "foot", "dry"]}, {"sent": "let me in under the mushroom to dry out and rest until the rain stops .", "words": ["the", "rain", "to", "dry", "and", "under", "in", "out", "me"]}, {"sent": "well thats where you put your things to dry .", "words": ["where", "put", "your", "to", "dry", "you"]}, {"sent": "shall we dry them ?", "words": ["dry", "them", "we"]}, {"sent": "put it to dry .", "words": ["to", "put", "it", "dry"]}, {"sent": "you have them dry .", "words": ["have", "dry", "them", "you"]}, {"sent": "to keep them nice and dry .", "words": ["to", "dry", "nice", "them", "and"]}, {"sent": "to dry ?", "words": ["to", "dry"]}, {"sent": "first we hafta dry dolly .", "words": ["dry", "first", "we"]}, {"sent": "we needta wait for the glue to dry now .", "words": ["the", "wait", "for", "we", "to", "dry", "glue"]}, {"sent": "lets dry your face with this .", "words": ["your", "face", "dry", "with", "this"]}, {"sent": "where is it gonna hang to dry ?", "words": ["where", "is", "to", "dry", "it"]}, {"sent": "are you dry ?", "words": ["dry", "are", "you"]}, {"sent": "hes dry and hes wet .", "words": ["wet", "dry", "and"]}, {"sent": "is what dry ?", "words": ["what", "dry", "is"]}, {"sent": "or are these dry ones ?", "words": ["these", "dry", "are"]}, {"sent": "it turned out its very good that its raining because all the little plants are very dry .", "words": ["the", "all", "dry", "little", "it", "are", "good", "out", "that", "because"]}, {"sent": "dry your hair now .", "words": ["dry", "hair", "your"]}, {"sent": "you could dry him .", "words": ["could", "dry", "him", "you"]}, {"sent": "look this ducks dry .", "words": ["this", "dry", "look"]}, {"sent": "let them dry out .", "words": ["dry", "them", "out"]}, {"sent": "no its dry paint .", "words": ["dry", "paint"]}, {"sent": "you want Daddy to take you and dry you off ?", "words": ["off", "to", "dry", "and", "you", "take"]}, {"sent": "what do you dry ?", "words": ["what", "do", "dry", "you"]}, {"sent": "I think its sposta be a dry powder but its .", "words": ["think", "be", "dry", "but", "a"]}, {"sent": "so we kept quite dry .", "words": ["so", "dry", "we"]}, {"sent": "well leave it on the window ledge in the kitchen to dry .", "words": ["the", "to", "dry", "in", "it", "kitchen", "on", "window"]}, {"sent": "dry your hands .", "words": ["dry", "your"]}, {"sent": "dry .", "words": ["dry"]}, {"sent": "oh itll be dry when we do your legs .", "words": ["do", "your", "we", "be", "dry", "when"]}, {"sent": "and Daddy is gonna go get you a dry diaper .", "words": ["go", "is", "diaper", "dry", "you", "get", "and", "a"]}, {"sent": "well what are you going to dry your hands on now youve thrown the towel down ?", "words": ["the", "what", "your", "to", "dry", "you", "towel", "are", "on", "down"]}, {"sent": "those are dry Fraser .", "words": ["dry", "are", "those"]}, {"sent": "otherwise youre not going to get dry are you ?", "words": ["to", "dry", "you", "get", "are", "not"]}, {"sent": "and we are gonna dry his chest ?", "words": ["we", "dry", "and", "are", "his"]}, {"sent": "and dry your back .", "words": ["dry", "your", "back", "and"]}, {"sent": "xxx oh no no clean it dry it .", "words": ["clean", "dry", "it"]}, {"sent": "are they slimy or are they dry ?", "words": ["they", "dry", "are"]}, {"sent": "the pens will dry up and they wont work will they ?", "words": ["the", "up", "work", "dry", "and", "they", "will"]}, {"sent": "Mummy has got to go and dry her hair in a minute .", "words": ["go", "to", "dry", "and", "in", "hair", "her", "a"]}, {"sent": "nice dry diaper .", "words": ["diaper", "dry", "nice"]}, {"sent": "then we are gonna dry his mimi .", "words": ["we", "dry", "are", "his", "then"]}, {"sent": "sun in your eye ?", "words": ["in", "sun", "your", "eye"]}, {"sent": "oh the baby sun .", "words": ["the", "sun"]}, {"sent": "Ill wear the sun on my legs that run .", "words": ["the", "my", "sun", "run", "on", "that"]}, {"sent": "up came the sun .", "words": ["up", "the", "sun"]}, {"sent": "heres half of the sun .", "words": ["of", "the", "sun"]}, {"sent": "can you put the sun where it belongs ?", "words": ["can", "the", "put", "where", "sun", "you", "it"]}, {"sent": "no look at that sun what color is it ?", "words": ["what", "look", "is", "sun", "it", "at", "that"]}, {"sent": "the sun .", "words": ["the", "sun"]}, {"sent": "the sun was just sunning up the little white house .", "words": ["the", "up", "was", "sun", "little", "house", "white"]}, {"sent": "you can see the sun shining on Sues house .", "words": ["can", "the", "sun", "you", "house", "on", "see"]}, {"sent": "sun is shining .", "words": ["sun", "is"]}, {"sent": "you cant really see today because there isnt enough sun .", "words": ["sun", "you", "there", "see", "because"]}, {"sent": "Ill wear the sun .", "words": ["the", "sun"]}, {"sent": "the sun .", "words": ["the", "sun"]}, {"sent": "the first sunny day I was up here I checked to see if the sun shone out the back here .", "words": ["the", "up", "if", "back", "was", "to", "first", "sun", "here", "out", "see"]}, {"sent": "and finally the baby fairys brought glitter from sun beams and moon beams and the tiniest fairy of all borrowed a star from the sky .", "words": ["the", "all", "moon", "sky", "of", "star", "sun", "and", "a"]}, {"sent": "windy windy morning and the sun was shining bright .", "words": ["the", "was", "sun", "windy", "and"]}, {"sent": "but I think after a few minutes when the sun has .", "words": ["the", "think", "sun", "when", "but", "a"]}, {"sent": "look at the sun shining now .", "words": ["the", "sun", "at", "look"]}, {"sent": "warm from the sun just picked .", "words": ["the", "sun"]}, {"sent": "you dont want the sun to go away .", "words": ["the", "go", "to", "sun", "you", "dont", "away"]}, {"sent": "sun shine xxx pray for all .", "words": ["all", "sun", "for"]}, {"sent": "where is the sun ?", "words": ["where", "the", "sun", "is"]}, {"sent": "it is such a beautiful place and the crops grow so well that there is little for the Ferian people to do but enjoy themselves in the sun .", "words": ["the", "do", "for", "is", "so", "to", "sun", "and", "little", "there", "in", "it", "that", "but", "a"]}, {"sent": "there is a sun .", "words": ["there", "sun", "a", "is"]}, {"sent": "the sun is over there .", "words": ["the", "is", "sun", "there", "over"]}, {"sent": "you always say that to the sun .", "words": ["the", "to", "sun", "say", "you", "that"]}, {"sent": "shes got sun written on it .", "words": ["on", "sun", "it"]}, {"sent": "out came the sun .", "words": ["the", "sun", "out"]}, {"sent": "do you not wanna color the sun in yellow first ?", "words": ["the", "do", "sun", "first", "you", "in", "wanna", "yellow", "not"]}, {"sent": "okay Ill give the sun some feet .", "words": ["the", "some", "give", "sun"]}, {"sent": "thats the sun .", "words": ["the", "sun"]}, {"sent": "thats the sun .", "words": ["the", "sun"]}, {"sent": "where has the sun gone ?", "words": ["where", "the", "sun"]}, {"sent": "is that gonna stop the sun coming in .", "words": ["the", "is", "sun", "stop", "in", "that"]}, {"sent": "oh look at the sun shining now .", "words": ["the", "sun", "at", "look"]}, {"sent": "when the sun is really shining huh ?", "words": ["the", "sun", "when", "is"]}, {"sent": "the moon went over the sun didnt it ?", "words": ["the", "moon", "sun", "it", "over"]}, {"sent": "its the sun .", "words": ["the", "sun"]}, {"sent": "theres the kitty warming his fur in the sun .", "words": ["the", "sun", "in", "kitty", "his"]}, {"sent": "I cant believe all that sun out there .", "words": ["all", "sun", "there", "out", "that"]}, {"sent": "there well smokes coming outof this chimney yeah and heres the sun see except it got bigger .", "words": ["the", "sun", "and", "there", "this", "it", "see"]}, {"sent": "its a sun shining and Summertime its sun .", "words": ["sun", "and", "a"]}, {"sent": "where is the sun shining ?", "words": ["where", "the", "sun", "is"]}, {"sent": "because youre gonna be going to the sun .", "words": ["the", "to", "be", "sun", "because"]}, {"sent": "how do you spell sun ?", "words": ["do", "how", "sun", "you"]}, {"sent": "you got sun ?", "words": ["sun", "you"]}, {"sent": "is it the sun making you sleepy is the sun making Thomas sleepy ?", "words": ["the", "is", "sun", "you", "it", "sleepy"]}, {"sent": "sun .", "words": ["sun"]}, {"sent": "yes thats the sun isnt it ?", "words": ["the", "sun", "it"]}, {"sent": "and the sun .", "words": ["the", "sun", "and"]}, {"sent": "its going to dry in the sun .", "words": ["the", "to", "dry", "sun", "in"]}, {"sent": "sometimes Olivia likes to bask in the sun .", "words": ["the", "to", "in", "sun"]}, {"sent": "are you trying to get out of the sun ?", "words": ["the", "of", "to", "sun", "you", "get", "are", "out"]}, {"sent": "the sun was really bright that day .", "words": ["the", "was", "sun", "that"]}, {"sent": "you wanna color the sun ?", "words": ["the", "sun", "wanna", "you"]}, {"sent": "usually when the sun shines and it rains we get a rainbow .", "words": ["the", "we", "sun", "and", "get", "it", "when", "a"]}, {"sent": "now you put the sun together how ?", "words": ["the", "how", "put", "sun", "you"]}, {"sent": "the sun isnt strong enough .", "words": ["the", "sun"]}, {"sent": "the sun and the duck .", "words": ["the", "sun", "duck", "and"]}, {"sent": "the sun .", "words": ["the", "sun"]}, {"sent": "do you see sun or clouds ?", "words": ["see", "do", "sun", "you"]}, {"sent": "look let us make a sun up here .", "words": ["up", "sun", "here", "us", "look", "make", "a"]}, {"sent": "look at the sun look there it is the sun look .", "words": ["the", "look", "is", "sun", "there", "it", "at"]}, {"sent": "the sun .", "words": ["the", "sun"]}, {"sent": "yeah the sun came back it was raining .", "words": ["the", "back", "was", "sun", "it"]}, {"sent": "but plants grow from warmth and from sun and from water .", "words": ["water", "sun", "but", "and"]}, {"sent": "I only moved them because the sun was so bright .", "words": ["the", "was", "sun", "so", "them", "because"]}, {"sent": "I think that magpies taken the sun as well .", "words": ["the", "think", "that", "sun"]}, {"sent": "it keeps them from the sun .", "words": ["the", "sun", "it", "them"]}, {"sent": "I need the umbrella though so I dont get too much sun .", "words": ["the", "need", "so", "sun", "get", "dont", "too", "much"]}, {"sent": "I wanna sit on my hill with the trees and the sun .", "words": ["the", "my", "sun", "with", "and", "wanna", "on", "sit"]}, {"sent": "oh Thomas where has the sun gone now ?", "words": ["where", "the", "sun"]}, {"sent": "out came the sun and dried up all the rain .", "words": ["the", "up", "all", "rain", "sun", "and", "out"]}, {"sent": "its the sun and its got a great big smile on its face .", "words": ["the", "face", "sun", "and", "on", "big", "smile", "a"]}, {"sent": "oh oh oh Im not gonna erase the sun .", "words": ["the", "sun", "not"]}, {"sent": "thats where the balloon and the sun are .", "words": ["where", "the", "sun", "and", "are", "balloon"]}, {"sent": "a sun .", "words": ["sun", "a"]}, {"sent": "no I didnt sun say ten thirty .", "words": ["sun", "say"]}, {"sent": "u is for umbrella to shade chipmunks mother from the sun .", "words": ["the", "for", "is", "to", "sun"]}, {"sent": "comes the sun and dries up all the rain .", "words": ["the", "all", "up", "rain", "sun", "and"]}, {"sent": "the sun isnt shining in that window .", "words": ["the", "sun", "in", "window", "that"]}, {"sent": "can you see the sun shining Fraser ?", "words": ["can", "the", "sun", "you", "see"]}, {"sent": "sun .", "words": ["sun"]}, {"sent": "oh let the sun shine in .", "words": ["the", "in", "sun"]}, {"sent": "now I send you the rays of the sun .", "words": ["the", "of", "sun", "you"]}, {"sent": "oh mister sun sun mister golden sun good sun Violet .", "words": ["sun", "good"]}, {"sent": "and enjoy the sun .", "words": ["the", "sun", "and"]}, {"sent": "and the sun got higher .", "words": ["the", "sun", "and"]}, {"sent": "see the sun outside ?", "words": ["the", "see", "sun", "outside"]}, {"sent": "see the sun ?", "words": ["the", "see", "sun"]}, {"sent": "its nice when the sun shines isnt it ?", "words": ["the", "sun", "nice", "it", "when"]}, {"sent": "thats the sun .", "words": ["the", "sun"]}, {"sent": "Sarahs got Sarahs got a swimming costume on and it says sun .", "words": ["sun", "and", "it", "on", "a"]}, {"sent": "oh look at the sun there .", "words": ["the", "look", "sun", "there", "at"]}, {"sent": "its a quick drawing of the sun .", "words": ["of", "the", "sun", "a"]}, {"sent": "setting sun and the moon and theres the moon .", "words": ["the", "sun", "moon", "and"]}, {"sent": "it has flowers growing on a garden and a picture of the sun .", "words": ["the", "of", "sun", "picture", "and", "garden", "it", "on", "a"]}, {"sent": "yes was it the sun aswell ?", "words": ["the", "was", "sun", "it"]}, {"sent": "remember sometimes you wake up even before the sun comes up .", "words": ["up", "the", "sun", "you", "wake"]}, {"sent": "you can see anything here are are covered by snow and ice .", "words": ["can", "snow", "you", "here", "by", "and", "are", "ice", "see"]}, {"sent": "oh I like ice cream .", "words": ["ice", "ice cream", "like"]}, {"sent": "ice cream .", "words": ["ice", "ice cream"]}, {"sent": "Ive never even had apple ice cream .", "words": ["ice", "apple", "ice cream"]}, {"sent": "want some ice cream ?", "words": ["ice", "some", "ice cream"]}, {"sent": "do you want vanilla ice cream with chocolate syrup on it ?", "words": ["do", "ice cream", "chocolate", "vanilla", "with", "you", "on", "it", "ice"]}, {"sent": "okay what kind of ice cream can I have ?", "words": ["can", "what", "ice cream", "of", "have", "ice"]}, {"sent": "I know someone who can get us some ice cream .", "words": ["can", "ice cream", "get", "us", "who", "ice", "some"]}, {"sent": "poor ice cream man .", "words": ["ice", "ice cream", "poor"]}, {"sent": "you see the ice the snow ?", "words": ["the", "snow", "you", "ice", "see"]}, {"sent": "theres not real ice cream though thats just a toy .", "words": ["ice cream", "toy", "ice", "not", "a"]}, {"sent": "oh this ice cream too ?", "words": ["ice", "this", "ice cream", "too"]}, {"sent": "can I have some ice cream ?", "words": ["can", "ice cream", "have", "ice", "some"]}, {"sent": "I cant see an ice cream van .", "words": ["ice", "see", "an", "ice cream"]}, {"sent": "hes ice skating .", "words": ["ice"]}, {"sent": "youre going ice skating .", "words": ["ice"]}, {"sent": "is this the ice cream ?", "words": ["the", "ice cream", "is", "this", "ice"]}, {"sent": "and the thing is sometimes I think he might have thought it wasnt worth stopping here because whenever he stopped nobody came out to buy ice cream .", "words": ["the", "ice cream", "think", "is", "to", "and", "have", "here", "it", "he", "out", "ice", "buy", "because"]}, {"sent": "what kind of ice cream do you like best ?", "words": ["what", "do", "ice cream", "of", "like", "you", "ice"]}, {"sent": "so Ill ice them .", "words": ["ice", "so", "them"]}, {"sent": "I dont know what ice vanilla food is .", "words": ["what", "is", "vanilla", "dont", "food", "ice"]}, {"sent": "how about for lunch you had ice cream ?", "words": ["ice cream", "how", "for", "you", "about", "ice"]}, {"sent": "ice cream cone .", "words": ["ice", "ice cream"]}, {"sent": "more ice cream ?", "words": ["ice", "ice cream", "more"]}, {"sent": "feel that ice cream .", "words": ["ice", "ice cream", "that"]}, {"sent": "ice cream cone .", "words": ["ice", "ice cream"]}, {"sent": "yeah say ice cream cone .", "words": ["ice", "ice cream", "say"]}, {"sent": "I xxx ice cubes in it .", "words": ["ice", "in", "it"]}, {"sent": "it was a little bit messy thats the way ice cream is on a hot day .", "words": ["the", "ice cream", "is", "was", "little", "on", "it", "hot", "ice", "a"]}, {"sent": "its hot today Maisys having an ice cold drink lemonade .", "words": ["drink", "hot", "an", "ice", "cold"]}, {"sent": "do you want an ice cube Max ?", "words": ["ice", "do", "an", "you"]}, {"sent": "ice cream .", "words": ["ice", "ice cream"]}, {"sent": "Daddy went out to buy you an ice cream once didnt he ?", "words": ["ice cream", "to", "you", "an", "he", "out", "ice", "buy"]}, {"sent": "I heard you had an ice cream cone at the zoo .", "words": ["the", "zoo", "ice cream", "you", "an", "at", "ice"]}, {"sent": "ice its an ice cube tray .", "words": ["ice", "an", "tray"]}, {"sent": "you dont need ice cream .", "words": ["ice cream", "need", "you", "dont", "ice"]}, {"sent": "where else do you get ice cream ?", "words": ["where", "do", "ice cream", "you", "get", "ice"]}, {"sent": "then we would need what kinda truck to clean the ice ?", "words": ["clean", "what", "the", "need", "we", "to", "truck", "would", "ice", "then"]}, {"sent": "theres the ice cream .", "words": ["the", "ice cream", "ice"]}, {"sent": "weve had ice lollies .", "words": ["ice"]}, {"sent": "the ice cream one .", "words": ["the", "ice cream", "ice"]}, {"sent": "and youve had a choc ice at lunchtime .", "words": ["ice", "at", "a", "and"]}, {"sent": "yeah do you do you feel happy when you eat ice cream ?", "words": ["do", "ice cream", "happy", "you", "eat", "ice", "when"]}, {"sent": "ice cream ice cream and cake ?", "words": ["ice", "ice cream", "cake", "and"]}, {"sent": "hes filling up on ice cream .", "words": ["up", "ice cream", "on", "ice"]}, {"sent": "do we need some ice on something ?", "words": ["do", "need", "we", "on", "ice", "some"]}, {"sent": "xxx get your ice cubes ?", "words": ["get", "ice", "your"]}, {"sent": "you know you know how dey put de ice mattress .", "words": ["ice", "how", "put", "you"]}, {"sent": "ice cubes .", "words": ["ice"]}, {"sent": "I think maybe we need to put some ice on that William .", "words": ["think", "put", "need", "we", "need to", "to", "on", "ice", "some", "that"]}, {"sent": "ice boat .", "words": ["ice", "boat"]}, {"sent": "oh well we might have some cherries then if you have the ice cream .", "words": ["the", "ice cream", "if", "we", "you", "have", "ice", "some", "then"]}, {"sent": "you want ice cream .", "words": ["ice", "ice cream", "you"]}, {"sent": "oh ice cream .", "words": ["ice", "ice cream"]}, {"sent": "thats tasty ice cream .", "words": ["ice", "ice cream"]}, {"sent": "what flavor do you think that ice cream is ?", "words": ["what", "do", "think", "ice cream", "is", "you", "ice", "that"]}, {"sent": "just sit him on there if you want him to sit on ice .", "words": ["if", "to", "ice", "you", "there", "him", "on", "sit"]}, {"sent": "well we need some ice creams .", "words": ["ice", "some", "need", "we"]}, {"sent": "youve drawn the cone and then the ice cream and then the sticky .", "words": ["the", "ice cream", "sticky", "and", "ice", "then"]}, {"sent": "is that the ice cream van ?", "words": ["the", "ice cream", "is", "ice", "that"]}, {"sent": "yeah maybe Ive given her an ice cream cone before .", "words": ["ice", "ice cream", "an", "her"]}, {"sent": "eat the bottom of the cone the ice cream will fall down .", "words": ["the", "ice cream", "bottom", "of", "will", "fall", "eat", "ice", "down"]}, {"sent": "its a really big ice rink in London isnt it .", "words": ["in", "it", "ice", "big", "a"]}, {"sent": "do you want an ice cream or a lollipop ?", "words": ["do", "ice cream", "you", "lollipop", "an", "ice", "a"]}, {"sent": "you gonna put it in the ice refrigerator ?", "words": ["the", "put", "refrigerator", "you", "in", "it", "ice"]}, {"sent": "mommy gets ice cream .", "words": ["ice", "ice cream"]}, {"sent": "theres no ice cream cake .", "words": ["ice", "ice cream", "cake"]}, {"sent": "she doesnt eat ice cream .", "words": ["eat", "ice cream", "ice", "she"]}, {"sent": "I want an ice cream .", "words": ["ice", "ice cream", "an"]}, {"sent": "you were ice skating .", "words": ["ice", "were", "you"]}, {"sent": "could the ice cream van not come round like our ice cream man does ?", "words": ["the", "does", "ice cream", "like", "our", "ice", "could", "not"]}, {"sent": "xxx theyre getting ice cream cones ?", "words": ["ice", "ice cream"]}, {"sent": "and you like ice lolly pops dont you ?", "words": ["like", "you", "and", "dont", "ice"]}, {"sent": "in the ice box .", "words": ["the", "in", "box", "ice"]}, {"sent": "which ice cube ?", "words": ["ice", "which"]}, {"sent": "ice cream cone .", "words": ["ice", "ice cream"]}, {"sent": "i is for ice cream .", "words": ["ice cream", "i", "for", "is", "ice"]}, {"sent": "well think about jelly ice cream jelly and ice cream .", "words": ["jelly", "ice cream", "think", "and", "about", "ice"]}, {"sent": "um the cupcakes the cheese the ice cream .", "words": ["the", "ice cream", "ice", "cheese"]}, {"sent": "thats the tune that our ice cream man plays isnt it ?", "words": ["the", "ice cream", "our", "it", "ice", "that"]}, {"sent": "I know we I know weve not as many cakes to ice as we made but weve still got a good few without icing on .", "words": ["we", "to", "on", "good", "ice", "not", "but", "a"]}, {"sent": "ice cream .", "words": ["ice", "ice cream"]}, {"sent": "I eat sardines and ice cream I feel grouchy all day .", "words": ["all", "ice cream", "and", "eat", "ice"]}, {"sent": "is that an ice cream ?", "words": ["ice cream", "is", "an", "ice", "that"]}, {"sent": "look at that all these these rings are all made up of ice .", "words": ["up", "all", "these", "look", "of", "are", "at", "ice", "that"]}, {"sent": "well they werent really lolly pops they were more like those ice pop things .", "words": ["pop", "those", "like", "they", "more", "ice", "were"]}, {"sent": "xxx you havent had ice cream yet .", "words": ["ice", "ice cream", "you"]}, {"sent": "if you eat anything cold like ice cream or anything sweet .", "words": ["ice cream", "if", "like", "you", "eat", "ice", "cold"]}, {"sent": "ooh what a big ice cream .", "words": ["what", "ice cream", "ice", "big", "a"]}, {"sent": "what should we put on our ice cream ?", "words": ["what", "put", "ice cream", "we", "ice", "our", "on"]}, {"sent": "what do you have do you have an ice cream cone ?", "words": ["what", "do", "ice cream", "you", "have", "an", "ice"]}, {"sent": "I dont know what ice vanilla is .", "words": ["what", "is", "vanilla", "dont", "ice"]}, {"sent": "whos the ice for ?", "words": ["the", "ice", "for"]}, {"sent": "and theres some ice as well .", "words": ["ice", "some", "and"]}, {"sent": "no ice cream bus .", "words": ["ice", "ice cream", "bus"]}, {"sent": "ice cream .", "words": ["ice", "ice cream"]}, {"sent": "wheres the ice cream ?", "words": ["the", "ice cream", "ice"]}, {"sent": "heres a ice cream .", "words": ["ice", "ice cream", "a"]}, {"sent": "the ice ?", "words": ["the", "ice"]}, {"sent": "at breakfast Cheeky used her spoon to tap a tune on her milk glass .", "words": ["milk", "to", "spoon", "glass", "at", "on", "her", "a"]}, {"sent": "Abe what would have happened if Mommy would have just filled your glass to here ?", "words": ["what", "your", "if", "to", "have", "here", "glass", "would"]}, {"sent": "would you like a glass of water Jason ?", "words": ["water", "of", "like", "you", "glass", "would", "a"]}, {"sent": "a glass ?", "words": ["glass", "a"]}, {"sent": "we left a glass of sherry and a mince pie .", "words": ["we", "of", "and", "glass", "a"]}, {"sent": "the drink in the glass goes fizzywizzy whizz .", "words": ["the", "in", "glass", "drink"]}, {"sent": "and you had a big glass of lemonade didnt you ?", "words": ["of", "you", "and", "glass", "big", "a"]}, {"sent": "tell me about that glass .", "words": ["me", "that", "glass", "about"]}, {"sent": "why is there glass on the floor .", "words": ["the", "is", "there", "glass", "on", "why"]}, {"sent": "if this glass if these glasses were filled not with water but chocolate milk the ones that you made equal which glass would you choose to have your chocolate milk out of ?", "words": ["the", "of", "water", "milk", "which", "not", "with", "glasses", "have", "glass", "that", "were", "these", "if", "to", "you", "would", "your", "chocolate", "this", "out", "but"]}, {"sent": "glass in the door in the window .", "words": ["the", "in", "glass", "window", "door"]}, {"sent": "you went to see them blow glass ?", "words": ["to", "them", "you", "glass", "see", "blow"]}, {"sent": "do you want that cold tea form lunch or do you want a glass of orange ?", "words": ["do", "of", "you", "orange", "glass", "that", "cold", "a"]}, {"sent": "dats why da glass is dere xxx hes making another noise .", "words": ["another", "glass", "why", "is"]}, {"sent": "where is your glass ?", "words": ["where", "your", "glass", "is"]}, {"sent": "I think theres some in that glass .", "words": ["think", "in", "glass", "some", "that"]}, {"sent": "well he doesnt want a glass if hes having a coffee uhn does he ?", "words": ["does", "if", "coffee", "he", "glass", "a"]}, {"sent": "I ordered some more glass .", "words": ["some", "more", "glass"]}, {"sent": "glass in your mouth ?", "words": ["in", "mouth", "your", "glass"]}, {"sent": "because I had a big glass table .", "words": ["table", "glass", "big", "because", "a"]}, {"sent": "because its going into the glass .", "words": ["the", "glass", "into", "because"]}, {"sent": "shall I get Snoopy another glass from the kitchen ?", "words": ["the", "get", "another", "glass", "kitchen"]}, {"sent": "and wheres the glass ?", "words": ["the", "glass", "and"]}, {"sent": "oh youve got beautiful glass beads .", "words": ["glass", "beads"]}, {"sent": "theres a piece of glass .", "words": ["of", "glass", "a"]}, {"sent": "would you like well get another ladybird glass because youre a good girl .", "words": ["like", "you", "get", "another", "glass", "good", "would", "because", "a"]}, {"sent": "but you can see Daddys banging on the glass because look can you see all those marks on the glass .", "words": ["can", "the", "all", "those", "because", "you", "glass", "look", "on", "see", "but"]}, {"sent": "oh youve not had glass in your mouth Thomas .", "words": ["your", "mouth", "in", "glass", "not"]}, {"sent": "your glass ?", "words": ["your", "glass"]}, {"sent": "some tea in a glass too .", "words": ["in", "glass", "too", "some", "a"]}, {"sent": "so theres a glass .", "words": ["so", "glass", "a"]}, {"sent": "huh theres your glass of milk .", "words": ["of", "milk", "your", "glass"]}, {"sent": "conservatories have got glass windows .", "words": ["have", "glass"]}, {"sent": "how did you break the glass ?", "words": ["the", "how", "you", "did", "glass", "break"]}, {"sent": "do you wanna have a glass of juice at my house ?", "words": ["do", "my", "of", "you", "have", "house", "glass", "wanna", "at", "juice", "a"]}, {"sent": "why is there a banana in a glass on the mantlepiece ?", "words": ["the", "is", "there", "banana", "in", "glass", "on", "why", "a"]}, {"sent": "Paddingtons mouth began to water as he peered through the glass .", "words": ["the", "to", "mouth", "he", "glass", "water"]}, {"sent": "a glass of wine ?", "words": ["of", "glass", "a"]}, {"sent": "glass can break honey .", "words": ["can", "break", "glass"]}, {"sent": "and give us the glass .", "words": ["the", "and", "us", "glass", "give"]}, {"sent": "were not allowed to touch the fragile things but do you like to give the green glass sculpture a hug ?", "words": ["the", "do", "to", "like", "touch", "you", "glass", "green", "hug", "give", "were", "not", "but", "a"]}, {"sent": "youre going to pour some tea in Snoopys glass ?", "words": ["to", "pour", "in", "glass", "some"]}, {"sent": "and you drink that in those nice glass .", "words": ["those", "drink", "nice", "you", "and", "in", "glass", "that"]}, {"sent": "I wouldnt mind a nice glass of wine though please .", "words": ["of", "nice", "glass", "a"]}, {"sent": "glass things sometimes break .", "words": ["break", "glass"]}, {"sent": "look I made it into a little glass .", "words": ["into", "little", "it", "glass", "look", "a"]}, {"sent": "its a glass measuring cup isnt it ?", "words": ["cup", "it", "glass", "a"]}, {"sent": "then Ill have a glass .", "words": ["have", "then", "glass", "a"]}, {"sent": "thats glass .", "words": ["glass"]}, {"sent": "that that glass is dirty Thomas .", "words": ["dirty", "that", "glass", "is"]}, {"sent": "here Ill give you a glass of milk .", "words": ["of", "you", "here", "glass", "milk", "give", "a"]}, {"sent": "look inside of the church and in some churches and synagogues there are beautiful stained glass windows .", "words": ["the", "of", "church", "inside", "and", "there", "in", "are", "glass", "look", "some"]}, {"sent": "lets see if Linda can bring in a glass of something to drink .", "words": ["can", "if", "drink", "of", "to", "bring", "in", "glass", "see", "a"]}, {"sent": "hes trodden on some glass and cut his foot .", "words": ["cut", "and", "his", "foot", "glass", "on", "some"]}, {"sent": "sliding glass door .", "words": ["glass", "door"]}, {"sent": "Thomas youre bending those glass .", "words": ["those", "glass"]}, {"sent": "well you dont need your glass .", "words": ["your", "need", "you", "dont", "glass"]}, {"sent": "you want a glass ?", "words": ["glass", "a", "you"]}, {"sent": "not glass ?", "words": ["glass", "not"]}, {"sent": "drinking glass .", "words": ["glass"]}, {"sent": "a glass ?", "words": ["glass", "a"]}, {"sent": "its a glass of water .", "words": ["of", "water", "glass", "a"]}, {"sent": "in this short fat glass .", "words": ["this", "in", "glass"]}, {"sent": "what kinda glass do you want ?", "words": ["what", "do", "glass", "you"]}, {"sent": "he poured the water from the glass right ?", "words": ["the", "he", "glass", "water"]}, {"sent": "are you looking at Purdie through the glass ?", "words": ["the", "you", "are", "glass", "at"]}, {"sent": "blue glass on these .", "words": ["on", "these", "glass", "blue"]}, {"sent": "do you remember he blew glass ?", "words": ["do", "he", "glass", "you"]}, {"sent": "thats the glass jar .", "words": ["the", "jar", "glass"]}, {"sent": "do these two glasses put together have as much cup water as this blue glass had all by itself ?", "words": ["cup", "do", "put", "these", "all", "glasses", "have", "this", "blue", "by", "glass", "water", "much"]}, {"sent": "she doesnt know how to drink outof a glass .", "words": ["how", "drink", "to", "glass", "she", "a"]}, {"sent": "Mommas gonna buy you a looking glass .", "words": ["glass", "buy", "a", "you"]}, {"sent": "its so strange to look over and see that creature in the glass bottle .", "words": ["the", "to", "so", "and", "that", "in", "glass", "look", "see", "bottle", "over"]}, {"sent": "would you like a glass of water ?", "words": ["water", "of", "like", "you", "glass", "would", "a"]}, {"sent": "oh I did break that glass didnt I .", "words": ["that", "glass", "break", "did"]}, {"sent": "a lady said what have you done to your glass ?", "words": ["what", "your", "to", "you", "have", "glass", "a"]}, {"sent": "and what about the glass of sherry ?", "words": ["the", "what", "of", "and", "glass", "about"]}, {"sent": "you drink your juice thats in your glass .", "words": ["your", "drink", "you", "in", "glass", "juice"]}, {"sent": "they are made of glass .", "words": ["of", "they", "are", "glass"]}, {"sent": "and she said look at all that broken glass .", "words": ["all", "and", "broken", "at", "glass", "look", "that", "she"]}, {"sent": "are those your glass slippers ?", "words": ["your", "are", "those", "glass"]}, {"sent": "we left a glass of sherry and a mince pie for Santa .", "words": ["for", "we", "of", "and", "glass", "a"]}, {"sent": "thats the glass .", "words": ["the", "glass"]}, {"sent": "well give you some in a glass .", "words": ["you", "in", "glass", "some", "give", "a"]}, {"sent": "now be careful with that glass .", "words": ["be", "with", "glass", "careful", "that"]}, {"sent": "oh you got Daddys glass of iced coffee .", "words": ["of", "coffee", "glass", "you"]}, {"sent": "and its a glass jar isnt it ?", "words": ["and", "it", "glass", "jar", "a"]}, {"sent": "the next day the prince announced he wanted to marry the girl who had lost her glass slipper .", "words": ["the", "to", "he", "glass", "who", "slipper", "her"]}, {"sent": "the marble when it hit the glass the glass dish I thought it threw you threw it against the window .", "words": ["the", "you", "dish", "it", "glass", "window", "hit", "when"]}, {"sent": "you can stand right next to the glass and see under the water .", "words": ["can", "the", "stand", "to", "you", "and", "under", "glass", "water", "see"]}, {"sent": "well maybe well well ask um the waitress to bring you a glass of water .", "words": ["the", "of", "to", "bring", "you", "glass", "water", "a"]}, {"sent": "because when we water one theres a hole in the bottom of the glass .", "words": ["the", "bottom", "we", "of", "in", "glass", "water", "when", "because", "a"]}, {"sent": "Thomas disappears in this glass .", "words": ["this", "in", "glass"]}, {"sent": "you mustnt touch the glass .", "words": ["the", "glass", "touch", "you"]}, {"sent": "you take this and you and uh you put it in a glass and you drink it .", "words": ["put", "drink", "and", "you", "this", "in", "it", "glass", "take", "a"]}, {"sent": "its the the glass sweetie jar .", "words": ["the", "jar", "glass"]}, {"sent": "move come on did you wipe the bottom of your glass ?", "words": ["the", "bottom", "your", "of", "you", "did", "glass", "on", "wipe"]}, {"sent": "xxx my tights were down to there .", "words": ["my", "tights", "to", "there", "down", "were"]}, {"sent": "those are her tights .", "words": ["her", "are", "those", "tights"]}, {"sent": "so next Tuesday Charles was in tights .", "words": ["was", "so", "in", "tights"]}, {"sent": "that would go with your tights wouldnt it ?", "words": ["your", "tights", "go", "with", "it", "would", "that"]}, {"sent": "shall we take your tights off aswell ?", "words": ["off", "your", "tights", "we", "take"]}, {"sent": "can we put your tights back on because youre getting cold .", "words": ["can", "put", "your", "tights", "back", "we", "on", "cold", "because"]}, {"sent": "youve got your skirt tucked in in your tights .", "words": ["in", "your", "tights"]}, {"sent": "but its cold outside so you needta wear tights in the winter dont you ?", "words": ["the", "tights", "outside", "so", "you", "dont", "in", "cold", "but"]}, {"sent": "xxx tights .", "words": ["tights"]}, {"sent": "how is your tights wet ?", "words": ["wet", "how", "your", "tights", "is"]}, {"sent": "just take your tights off .", "words": ["your", "take", "tights", "off"]}, {"sent": "shall we put some pretty white socks on or tights ?", "words": ["put", "tights", "we", "pretty", "on", "some", "white"]}, {"sent": "wow your tights are dirty .", "words": ["dirty", "are", "your", "tights"]}, {"sent": "put more tights in .", "words": ["in", "put", "more", "tights"]}, {"sent": "goodness sake youve come out in your tights .", "words": ["in", "your", "tights", "out"]}, {"sent": "you got pink tights and white tights .", "words": ["white", "tights", "and", "you"]}, {"sent": "what happened to your tights ?", "words": ["to", "what", "your", "tights"]}, {"sent": "wheres the rest of your tights gone ?", "words": ["the", "of", "your", "tights"]}, {"sent": "she looks like she has got tights socks and shoes doesnt she ?", "words": ["tights", "she", "like", "and"]}, {"sent": "yeah but Ill bring the camera and Ill get a picture of you because youll look nice in your black tights .", "words": ["the", "camera", "your", "tights", "of", "picture", "nice", "bring", "because", "and", "get", "you", "in", "black", "look", "but", "a"]}, {"sent": "you have tights ?", "words": ["have", "tights", "you"]}, {"sent": "youre losing your tights .", "words": ["your", "tights"]}, {"sent": "well we cant really get yours your tights off honey .", "words": ["off", "your", "tights", "we", "get"]}, {"sent": "how about if we just put it on top of your tights ?", "words": ["how", "put", "your", "if", "tights", "we", "of", "it", "about", "on"]}, {"sent": "shed a lovely wee skirt on with black woolly tights and I thought .", "words": ["tights", "with", "and", "black", "on", "a"]}, {"sent": "I thought they were tights .", "words": ["they", "were", "tights"]}, {"sent": "those are Olivias tights .", "words": ["are", "those", "tights"]}, {"sent": "red theyre red tights .", "words": ["red", "tights"]}, {"sent": "shes sitting in her tights .", "words": ["in", "her", "tights"]}, {"sent": "and you have pink tights on .", "words": ["tights", "you", "have", "and", "on"]}, {"sent": "xxx xxx hm you have teddy bears on your tights xxx ?", "words": ["your", "tights", "you", "have", "on"]}, {"sent": "next Tuesday Charles was in tights .", "words": ["was", "in", "tights"]}, {"sent": "we hafta pull your tights up a little ?", "words": ["up", "tights", "your", "we", "pull", "little", "a"]}, {"sent": "like Lilys friend said and you know theyre all in little leotards and tights and everything and her friend said that she had to go to the bathroom and the woman was like youre supposed to go to the bathroom before you go to class .", "words": ["the", "all", "tights", "go", "to", "was", "like", "you", "and", "little", "in", "that", "bathroom", "her", "she"]}, {"sent": "hats and a necklace and some sunglasses and tights and a bow and a shirt .", "words": ["necklace", "tights", "shirt", "and", "some", "a"]}, {"sent": "okay these tights .", "words": ["these", "tights"]}, {"sent": "and your tights .", "words": ["your", "tights", "and"]}, {"sent": "now tights ?", "words": ["tights"]}, {"sent": "be great with those tights willnt it ?", "words": ["tights", "those", "be", "with", "it"]}, {"sent": "those are the tights that got ever so muddy at the EnchantedForest arent they ?", "words": ["the", "those", "tights", "so", "they", "are", "at", "that"]}, {"sent": "got tights on .", "words": ["on", "tights"]}, {"sent": "its slippy on the floor isnt it in tights .", "words": ["the", "tights", "in", "it", "on"]}, {"sent": "these tights are too small for you now arent they ?", "words": ["these", "for", "tights", "you", "they", "are", "too"]}, {"sent": "just take your tights off then and nothing else .", "words": ["off", "your", "tights", "and", "then", "take"]}, {"sent": "shes got a big thick blue woolly jumper on and big thick tights and snowboots hasnt she ?", "words": ["tights", "blue", "and", "on", "big", "she", "a"]}, {"sent": "will you stand to let me pull your tights up .", "words": ["up", "your", "tights", "stand", "to", "you", "pull", "will", "me"]}, {"sent": "got water on your tights ?", "words": ["on", "your", "tights", "water"]}, {"sent": "with some tights ?", "words": ["some", "with", "tights"]}, {"sent": "are you keeping your tights on or not ?", "words": ["your", "tights", "you", "are", "on", "not"]}, {"sent": "tights off Darling .", "words": ["off", "tights"]}, {"sent": "do you wanna take these tights off ?", "words": ["off", "do", "these", "tights", "you", "wanna", "take"]}, {"sent": "I like your sparkly tights .", "words": ["your", "like", "tights"]}, {"sent": "why are your tights off ?", "words": ["off", "your", "tights", "are", "why"]}, {"sent": "why are your tights off pumpkin ?", "words": ["off", "your", "tights", "are", "pumpkin", "why"]}, {"sent": "why did you take your tights off ?", "words": ["off", "your", "tights", "you", "did", "take", "why"]}, {"sent": "these are your new tights .", "words": ["these", "your", "tights", "are", "new"]}, {"sent": "I like your sparkly tights .", "words": ["your", "like", "tights"]}, {"sent": "show Dada your sparkly tights .", "words": ["show", "your", "tights"]}, {"sent": "your tights are falling down .", "words": ["down", "are", "your", "tights"]}, {"sent": "wanna wear tights ?", "words": ["tights", "wanna"]}, {"sent": "white tights ?", "words": ["white", "tights"]}, {"sent": "you know its so warm I dont think we need tights .", "words": ["think", "need", "tights", "we", "so", "you", "dont"]}, {"sent": "well bring the white tights along if it gets cold .", "words": ["the", "tights", "if", "bring", "it", "white", "cold"]}, {"sent": "um those are yesterdays tights cause we got we got up kinda early this morning for an oil change before coming here .", "words": ["up", "for", "those", "tights", "we", "here", "this", "an", "are"]}, {"sent": "thats your tights .", "words": ["your", "tights"]}, {"sent": "dont do that to your tights .", "words": ["do", "your", "tights", "to", "dont", "that"]}, {"sent": "well let him walk on the carpet instead of your tights .", "words": ["the", "your", "tights", "of", "him", "on", "walk"]}, {"sent": "Ive got the Playdoh on my tights Jonathan Philip .", "words": ["the", "on", "tights", "my"]}, {"sent": "shall I help you with your pee because your tights are very tight ?", "words": ["help", "your", "tights", "with", "you", "are", "because"]}, {"sent": "I got you your tights .", "words": ["your", "tights", "you"]}, {"sent": "ballet day and we hafta tonight we hafta get out your tights .", "words": ["your", "tights", "we", "and", "get", "out"]}, {"sent": "no you cant put tights on over your pants .", "words": ["put", "tights", "your", "you", "pants", "on", "over"]}, {"sent": "Laras tights .", "words": ["tights"]}, {"sent": "Laras tights and Dadaws tights .", "words": ["tights", "and"]}, {"sent": "do you wanna take your tights off so they dont get wet ?", "words": ["off", "do", "wet", "your", "tights", "so", "you", "get", "they", "dont", "wanna", "take"]}, {"sent": "can you put your tights on or shall I do it ?", "words": ["can", "do", "put", "your", "tights", "you", "it", "on"]}, {"sent": "Amys got white tights too .", "words": ["white", "tights", "too"]}, {"sent": "thats Mummys tights .", "words": ["tights"]}, {"sent": "like Mummys tights Thomass socks are in the net bag so they dont get lost .", "words": ["the", "tights", "so", "like", "get", "they", "in", "are", "dont"]}, {"sent": "are you having a problem with your tights there ?", "words": ["your", "tights", "with", "you", "there", "are", "a"]}, {"sent": "could you not get your tights on ?", "words": ["your", "tights", "you", "get", "on", "could", "not"]}, {"sent": "her tights ?", "words": ["her", "tights"]}, {"sent": "tights .", "words": ["tights"]}, {"sent": "ah now the girl is putting on her tights blue tights .", "words": ["the", "tights", "is", "blue", "on", "her"]}, {"sent": "and when shes put on her tights she puts on her jersey and I dont know what that is a p dress pinafore dress .", "words": ["what", "put", "tights", "is", "dress", "her", "and", "dont", "that", "on", "when", "she", "a"]}, {"sent": "xxx are you having trouble with your tights .", "words": ["your", "tights", "with", "you", "are"]}, {"sent": "when we went for our walk .", "words": ["for", "when", "we", "our", "walk"]}, {"sent": "lets go for a little walk .", "words": ["for", "go", "little", "walk", "a"]}, {"sent": "alright they can walk in and out then .", "words": ["can", "and", "they", "in", "out", "then", "walk"]}, {"sent": "and we are gonna walk a long way .", "words": ["we", "long", "and", "are", "walk", "a"]}, {"sent": "wanna go for a little walk ?", "words": ["for", "go", "little", "wanna", "walk", "a"]}, {"sent": "and then after weve been there we can walk all the way through Rusholme to where Daddy works .", "words": ["can", "all", "the", "where", "we", "to", "and", "there", "then", "walk"]}, {"sent": "she takes it for a walk .", "words": ["for", "it", "walk", "she", "a"]}, {"sent": "so hes walk walk walk .", "words": ["so", "walk"]}, {"sent": "he lined the walk with old pillows .", "words": ["the", "old", "with", "he", "walk"]}, {"sent": "I think youre probably talking about our front door because the other day when we went for a walk you insisted when we came back that we posted some letters through our front door .", "words": ["the", "think", "for", "a", "we", "back", "our", "other", "you", "walk", "that", "about", "some", "when", "because", "door"]}, {"sent": "William Im gonna walk away from whining .", "words": ["away", "walk"]}, {"sent": "we can walk and talk .", "words": ["can", "we", "and", "talk", "walk"]}, {"sent": "youre going for a walk ?", "words": ["for", "walk", "a"]}, {"sent": "can she walk walk out the door and shut the door ?", "words": ["can", "the", "and", "out", "walk", "she", "door"]}, {"sent": "is that the teddy bear going for a walk ?", "words": ["the", "for", "is", "bear", "walk", "that", "a"]}, {"sent": "so you could ride the train over to the bridge and drop the people off that way they could walk over the top .", "words": ["the", "off", "train", "so", "to", "you", "and", "they", "that", "walk", "drop", "over", "could", "ride"]}, {"sent": "walk walk walk walk walk this is what its supposed to be but Im helping her .", "words": ["what", "is", "to", "be", "her", "this", "walk", "but"]}, {"sent": "what did you walk into ?", "words": ["what", "into", "you", "did", "walk"]}, {"sent": "animals walk two by two dont they ?", "words": ["dont", "walk", "they", "by"]}, {"sent": "hes gotta be really strong to walk a truck doesnt he ?", "words": ["to", "be", "truck", "he", "walk", "a"]}, {"sent": "walk walk walk .", "words": ["walk"]}, {"sent": "lets go for a walk .", "words": ["for", "walk", "a", "go"]}, {"sent": "oh hes ready to walk .", "words": ["to", "walk"]}, {"sent": "youre going to walk walk walk huh ?", "words": ["to", "walk"]}, {"sent": "Henry walk to Mommy .", "words": ["to", "walk"]}, {"sent": "do deckchairs walk on their own normally ?", "words": ["on", "do", "walk", "their"]}, {"sent": "Ill walk through the front door thank you .", "words": ["the", "walk", "you", "door"]}, {"sent": "walk walk walk walk walk walk walk .", "words": ["walk"]}, {"sent": "are you not big much to walk up it yet ?", "words": ["up", "to", "much", "you", "walk", "are", "it", "big", "not"]}, {"sent": "when he was watching something walk by .", "words": ["when", "was", "by", "he", "walk"]}, {"sent": "when you walk for awhile you get hungry .", "words": ["for", "when", "hungry", "you", "get", "walk"]}, {"sent": "come on walk walk walk .", "words": ["on", "walk"]}, {"sent": "can he walk ?", "words": ["can", "walk", "he"]}, {"sent": "do you want to go for a walk with Daddy down at cemetery ?", "words": ["do", "for", "go", "to", "with", "you", "want to", "at", "down", "walk", "a"]}, {"sent": "does she walk like a penguin ?", "words": ["does", "penguin", "like", "walk", "she", "a"]}, {"sent": "its a small room and it has something on it and you walk on it .", "words": ["and", "you", "it", "room", "on", "walk", "a"]}, {"sent": "xxx we can walk around .", "words": ["can", "around", "walk", "we"]}, {"sent": "they all wanna live right by the pond so they dont hafta walk far to drink some nice cool water .", "words": ["the", "all", "drink", "so", "to", "nice", "by", "they", "dont", "wanna", "water", "some", "walk"]}, {"sent": "yeah you got to walk her feet see .", "words": ["to", "you", "walk", "see", "her"]}, {"sent": "then Ill help you walk .", "words": ["then", "walk", "you", "help"]}, {"sent": "wanna walk ?", "words": ["walk", "wanna"]}, {"sent": "its a long walk isnt it ?", "words": ["it", "walk", "long", "a"]}, {"sent": "one two you hafta go slow or dollie wont walk .", "words": ["go", "walk", "slow", "you"]}, {"sent": "do you want him to walk or run ?", "words": ["do", "to", "you", "run", "him", "walk"]}, {"sent": "lets go walk Davey walk .", "words": ["walk", "go"]}, {"sent": "were gonna walk this way .", "words": ["this", "walk", "were"]}, {"sent": "you know how you get um mazes in gardens where you can go for a walk and get lost ?", "words": ["where", "can", "how", "for", "go", "you", "get", "and", "in", "walk", "a"]}, {"sent": "I should just walk to the door .", "words": ["the", "to", "walk", "door"]}, {"sent": "wheres he gone for a walk to ?", "words": ["for", "to", "he", "walk", "a"]}, {"sent": "walk yeah .", "words": ["walk"]}, {"sent": "theyre thirsty after that big walk .", "words": ["walk", "big", "that", "thirsty"]}, {"sent": "once once weve had our lunch well go for a walk then wont we ?", "words": ["for", "go", "we", "our", "then", "walk", "a"]}, {"sent": "I guess you just like to walk it .", "words": ["to", "like", "you", "it", "walk"]}, {"sent": "the ones you walk on ?", "words": ["the", "on", "walk", "you"]}, {"sent": "lets walk to the beach .", "words": ["the", "to", "beach", "walk"]}, {"sent": "just suppose if um you walk into the bathroom .", "words": ["the", "if", "into", "you", "bathroom", "walk"]}, {"sent": "almost walk .", "words": ["walk"]}, {"sent": "not talking about a walk .", "words": ["walk", "not", "about", "a"]}, {"sent": "walk walk .", "words": ["walk"]}, {"sent": "you wanna walk over here and see the water ?", "words": ["the", "you", "here", "and", "walk", "wanna", "water", "see", "over"]}, {"sent": "and well go outside for a walk .", "words": ["for", "go", "outside", "and", "walk", "a"]}, {"sent": "Noonoos gone for a walk .", "words": ["for", "walk", "a"]}, {"sent": "perhaps the Teletubbies would like to go for a walk in the farmyard .", "words": ["the", "for", "go", "to", "like", "in", "would", "walk", "a"]}, {"sent": "is your doggy going for a walk ?", "words": ["for", "your", "is", "walk", "a"]}, {"sent": "ya hafta walk it over .", "words": ["over", "it", "walk"]}, {"sent": "walk walk walk walk walk walk walk .", "words": ["walk"]}, {"sent": "oh Pepper and Bouncy take a walk around the Jellyworld and find a box .", "words": ["the", "box", "find", "and", "around", "take", "walk", "a"]}, {"sent": "go take a walk ?", "words": ["walk", "take", "a", "go"]}, {"sent": "can he walk now ?", "words": ["can", "walk", "he"]}, {"sent": "walk .", "words": ["walk"]}, {"sent": "and then theyre gonna walk .", "words": ["then", "walk", "and"]}, {"sent": "we are gonna turn around and walk back .", "words": ["back", "we", "and", "are", "around", "walk"]}, {"sent": "he doesnt walk forward very well .", "words": ["walk", "he"]}, {"sent": "hows he gonna walk ?", "words": ["walk", "he"]}, {"sent": "she taught them to walk in a lines to come when they were called and to keep a safe distance from bikes and scooters and other things with wheels .", "words": ["when", "to", "them", "with", "and", "other", "they", "in", "walk", "she", "were", "a"]}, {"sent": "they walk dont they ?", "words": ["dont", "they", "walk"]}, {"sent": "maybe for a walk or to the park .", "words": ["the", "for", "to", "park", "walk", "a"]}, {"sent": "then when we went out for a walk last night before seven oclock Bens bedroom curtains were closed .", "words": ["for", "bedroom", "we", "last", "walk", "out", "then", "when", "were", "a"]}, {"sent": "has gone for a walk with her .", "words": ["for", "with", "walk", "her", "a"]}, {"sent": "if you walk it wont be so bad .", "words": ["if", "be", "so", "bad", "you", "it", "walk"]}, {"sent": "you walk so well .", "words": ["so", "walk", "you"]}, {"sent": "walk in .", "words": ["in", "walk"]}, {"sent": "shall we go and post that letter and have a little walk ?", "words": ["go", "we", "and", "have", "little", "walk", "that", "a"]}, {"sent": "lets get Chantilly to walk .", "words": ["get", "to", "walk"]}, {"sent": "let us go for a walk .", "words": ["for", "go", "us", "walk", "a"]}, {"sent": "walk walk walk walk walk walk .", "words": ["walk"]}, {"sent": "oh the snakes are going for another walk ?", "words": ["the", "for", "another", "are", "walk"]}, {"sent": "wanna walk feet ?", "words": ["walk", "wanna"]}, {"sent": "lets just walk to the end of the road .", "words": ["the", "to", "walk", "of"]}, {"sent": "oh Mummy might like to go for a walk .", "words": ["for", "go", "to", "like", "walk", "a"]}, {"sent": "he doesnt walk properly .", "words": ["walk", "he"]}, {"sent": "are we going to go for a walk in a bit ?", "words": ["for", "go", "we", "to", "in", "are", "walk", "a"]}, {"sent": "babies dont walk Jwww do they ?", "words": ["dont", "do", "walk", "they"]}, {"sent": "oh well thats easy to walk .", "words": ["to", "walk"]}, {"sent": "hows he gonna walk if he has no feet ?", "words": ["walk", "he", "if"]}, {"sent": "no thats the path or perhaps thats a cement walk where you walk up to the house so you wont get on the grass .", "words": ["the", "where", "up", "to", "so", "you", "get", "house", "on", "walk", "grass", "a"]}, {"sent": "right Sue doesnt Sue have them in her garden ?", "words": ["them", "have", "garden", "in", "her"]}, {"sent": "or garden shears .", "words": ["garden"]}, {"sent": "still digging his garden ?", "words": ["his", "garden"]}, {"sent": "I like the warm weather very much but all the things in the garden make me sneeze .", "words": ["the", "all", "like", "in", "garden", "make", "me", "much", "but"]}, {"sent": "ah God did you not just take them out to the garden and hose them down ?", "words": ["the", "to", "them", "you", "and", "hose", "garden", "did", "out", "down", "take", "not"]}, {"sent": "garden .", "words": ["garden"]}, {"sent": "Ive got some some garden rubbish .", "words": ["some", "garden"]}, {"sent": "you were helping me so much when we were pulling weeds down at the garden the other day .", "words": ["the", "when", "we", "so", "much", "you", "other", "garden", "at", "me", "down", "were"]}, {"sent": "do you do it in the garden ?", "words": ["the", "do", "you", "in", "it", "garden"]}, {"sent": "the only cure for her illness was a rare plant and it grew in a witchs garden .", "words": ["the", "for", "was", "and", "in", "it", "garden", "plant", "her", "a"]}, {"sent": "she has gone for a walk round the garden .", "words": ["the", "for", "garden", "walk", "she", "a"]}, {"sent": "and she wondered slowly in her Wellingtons along the garden .", "words": ["the", "and", "in", "garden", "her", "she"]}, {"sent": "do you wanna go in the back garden now ?", "words": ["the", "do", "back", "go", "you", "in", "garden", "wanna"]}, {"sent": "they could share the garden couldnt they ?", "words": ["the", "share", "they", "garden", "could"]}, {"sent": "well just dry out the garden rubbish and then have a bonfire .", "words": ["the", "dry", "and", "have", "garden", "out", "then", "a"]}, {"sent": "yes in your back garden .", "words": ["garden", "in", "your", "back"]}, {"sent": "theyve gone into the garden now hasnt he ?", "words": ["the", "garden", "he", "into"]}, {"sent": "he hadta dig really deep into the garden to put the drains on .", "words": ["the", "put", "into", "to", "garden", "he", "on"]}, {"sent": "thats what we clean the garden furniture with when we finally get around to cleaning it .", "words": ["clean", "what", "the", "we", "to", "with", "get", "garden", "it", "around", "when"]}, {"sent": "theres some more in the garden but theyre not quite ripe are they ?", "words": ["the", "they", "more", "garden", "in", "are", "some", "not", "but"]}, {"sent": "in the garden .", "words": ["the", "garden", "in"]}, {"sent": "on the gar on the grass in the garden .", "words": ["the", "in", "garden", "on", "grass"]}, {"sent": "talking of Purdie I can see her right at the bottom of our garden .", "words": ["can", "the", "bottom", "of", "our", "garden", "at", "see", "her"]}, {"sent": "you could say I was using a trowel in the garden .", "words": ["the", "was", "say", "you", "in", "garden", "could", "a"]}, {"sent": "they ran around the garden again and again .", "words": ["the", "and", "they", "garden", "around"]}, {"sent": "lets have a look at this in the garden .", "words": ["the", "have", "this", "in", "at", "garden", "look", "a"]}, {"sent": "do you have flowers like this in your garden ?", "words": ["do", "your", "like", "you", "have", "this", "in", "garden"]}, {"sent": "go away from my pretty green garden said I .", "words": ["my", "go", "garden", "away", "pretty", "green"]}, {"sent": "yes he is digging in the garden huh ?", "words": ["the", "is", "garden", "in", "he"]}, {"sent": "in the in the garden ?", "words": ["the", "garden", "in"]}, {"sent": "has she got a tree in her garden ?", "words": ["tree", "in", "garden", "her", "she", "a"]}, {"sent": "I think you and Daddy saw some cosmos in the garden .", "words": ["the", "think", "and", "you", "in", "garden", "some"]}, {"sent": "working in the garden .", "words": ["the", "garden", "in"]}, {"sent": "now our Mummy got to hang some washing on the line in the garden and .", "words": ["the", "to", "our", "and", "in", "garden", "on", "some"]}, {"sent": "wanna go and play in the garden ?", "words": ["the", "go", "play", "and", "in", "garden", "wanna"]}, {"sent": "and its almost getting too big for its pot so well hafta dig it in the um in the garden somewhere .", "words": ["the", "for", "so", "and", "in", "it", "garden", "too", "big"]}, {"sent": "there are no weeds in your garden are there ?", "words": ["your", "there", "garden", "in", "are"]}, {"sent": "worm at the bottom of the garden ?", "words": ["the", "bottom", "of", "garden", "at"]}, {"sent": "opposites in the garden .", "words": ["the", "garden", "in"]}, {"sent": "and next week were going to plant some seeds in your garden arent we ?", "words": ["your", "we", "to", "and", "in", "garden", "some", "plant", "were"]}, {"sent": "oh hes just doing some work in his garden .", "words": ["work", "in", "garden", "his", "some"]}, {"sent": "whats Daddy doing in the garden ?", "words": ["the", "garden", "in"]}, {"sent": "where was Swwws garden ?", "words": ["where", "was", "garden"]}, {"sent": "just like outside downstairs by the garden ?", "words": ["the", "outside", "like", "by", "garden"]}, {"sent": "what are they gonna grow in their garden ?", "words": ["what", "their", "they", "in", "are", "garden"]}, {"sent": "in the garden .", "words": ["the", "garden", "in"]}, {"sent": "hes pulling weeds outof his garden .", "words": ["his", "garden"]}, {"sent": "the rabbitgarden was surely the most beautiful garden .", "words": ["the", "was", "garden"]}, {"sent": "we might go in the garden this afternoon if it doesnt rain .", "words": ["the", "if", "rain", "go", "we", "this", "in", "garden", "it"]}, {"sent": "but hers was actually an old pond built in the garden wasnt it ?", "words": ["old", "the", "was", "hers", "an", "in", "garden", "it", "but"]}, {"sent": "uhhuh Im going to go out and water the garden .", "words": ["the", "go", "to", "and", "garden", "out", "water"]}, {"sent": "flying through the garden ?", "words": ["the", "garden"]}, {"sent": "perhaps he has had his tea and hes playing the garden .", "words": ["the", "and", "garden", "he", "his"]}, {"sent": "Eleanor look in the garden at the big magpie .", "words": ["the", "look", "garden", "in", "at", "big"]}, {"sent": "theres a mess in the garden now isnt there ?", "words": ["the", "there", "garden", "in", "a"]}, {"sent": "whos this in the garden ?", "words": ["the", "this", "in", "garden"]}, {"sent": "for the garden ?", "words": ["the", "garden", "for"]}, {"sent": "shall we go in the garden ?", "words": ["the", "go", "we", "in", "garden"]}, {"sent": "wanna go in the garden ?", "words": ["the", "go", "garden", "in", "wanna"]}, {"sent": "this is the garden .", "words": ["the", "this", "garden", "is"]}, {"sent": "the maid was in the garden hanging up the clothes .", "words": ["the", "up", "was", "in", "garden"]}, {"sent": "and it goes in the garden right ?", "words": ["the", "and", "garden", "in", "it"]}, {"sent": "have some fun in the garden in summer with that .", "words": ["the", "with", "have", "in", "garden", "some", "that"]}, {"sent": "good thing about this garden is you dont get messy do you ?", "words": ["do", "is", "you", "get", "this", "garden", "dont", "about", "good"]}, {"sent": "were going to have lunch and then well go to the garden centre and come back and wake daddy up .", "words": ["the", "up", "back", "go", "to", "and", "have", "garden", "wake", "then", "were"]}, {"sent": "and then I went down to the garden centre on the DoaghRoad .", "words": ["the", "to", "and", "garden", "on", "then", "down"]}, {"sent": "out in the garden .", "words": ["the", "garden", "in", "out"]}, {"sent": "I think it was a lady in the garden hanging out the clothes .", "words": ["the", "think", "was", "in", "it", "garden", "out", "a"]}, {"sent": "these things we hafta wait until we have our garden built .", "words": ["these", "wait", "we", "our", "have", "garden"]}, {"sent": "round and round the garden .", "words": ["the", "garden", "and"]}, {"sent": "go away from my pretty green garden said I .", "words": ["my", "go", "garden", "away", "pretty", "green"]}, {"sent": "mother picked flowers in the garden .", "words": ["the", "garden", "in"]}, {"sent": "go in the garden centre ?", "words": ["the", "garden", "in", "go"]}, {"sent": "arent they in the garden ?", "words": ["the", "they", "in", "garden"]}, {"sent": "youre not going on the garden .", "words": ["on", "the", "garden", "not"]}, {"sent": "can you see the garden up there ?", "words": ["can", "the", "up", "you", "there", "garden", "see"]}, {"sent": "did you put it in the garden ?", "words": ["the", "put", "you", "in", "it", "garden", "did"]}, {"sent": "would you like to come and play in my garden ?", "words": ["my", "to", "play", "like", "and", "you", "in", "garden", "would"]}, {"sent": "Jasmine held back her tears as her father left the palace garden .", "words": ["the", "garden", "her", "back"]}, {"sent": "what you gonna do in the garden ?", "words": ["the", "what", "do", "you", "in", "garden"]}, {"sent": "we see bees outside in the garden .", "words": ["the", "we", "outside", "in", "garden", "see"]}, {"sent": "Mary Mary quite contrary how does your garden grow .", "words": ["garden", "does", "how", "your"]}, {"sent": "go and play in the garden .", "words": ["the", "go", "play", "and", "in", "garden"]}, {"sent": "oh how cute is that garden is so nice looking .", "words": ["how", "is", "cute", "so", "nice", "garden", "that"]}, {"sent": "round and round the garden .", "words": ["the", "garden", "and"]}, {"sent": "I every time I went through our dining room to go to the garden I took a banana and one day my mother reckons she gave me five .", "words": ["the", "my", "go", "to", "our", "and", "banana", "garden", "room", "every", "me", "she", "a"]}, {"sent": "shall we have a look at these blackbirds that weve just seen at the bottom of the garden ?", "words": ["the", "these", "bottom", "we", "of", "have", "garden", "at", "look", "that", "a"]}, {"sent": "in the garden where we useta live .", "words": ["the", "where", "we", "garden", "in"]}, {"sent": "theyre looking at all of our garden huh ?", "words": ["all", "of", "our", "garden", "at"]}, {"sent": "who came to play with you in your new garden ?", "words": ["your", "to", "play", "with", "you", "in", "garden", "who", "new"]}, {"sent": "black bags for household and green bags for the garden .", "words": ["the", "for", "and", "black", "garden", "green"]}, {"sent": "daddy bunny grows weeds in his garden .", "words": ["his", "garden", "in", "bunny"]}, {"sent": "ah the garden looks an awful mess Thomas .", "words": ["the", "garden", "an"]}, {"sent": "theyre not like the little birds we get in the garden are they ?", "words": ["the", "we", "like", "little", "get", "they", "in", "garden", "are", "not"]}, {"sent": "strawberries fresh from the garden .", "words": ["the", "garden"]}, {"sent": "its hard to see because weve got an outside light at the bottom of the house but it shines on the patio in the garden .", "words": ["the", "bottom", "light", "to", "outside", "of", "an", "it", "house", "on", "hard", "at", "but", "see", "in", "garden", "because"]}, {"sent": "weve put the lid on because when were sitting out here in the garden there might be flys and wasps and buzzy bees .", "words": ["the", "put", "were", "be", "and", "here", "there", "in", "garden", "out", "on", "when", "because"]}, {"sent": "have you seen the snail in the garden have you ?", "words": ["the", "you", "have", "in", "garden"]}, {"sent": "well Mommy doesnt really like the music class either but were gonna go today so we can see the other babies and so we can shake the eggs .", "words": ["the", "can", "go", "we", "so", "like", "shake", "other", "and", "see", "were", "but"]}, {"sent": "shake them up .", "words": ["up", "them", "shake"]}, {"sent": "shake shake shake shake shake shake shake .", "words": ["shake"]}, {"sent": "shake it .", "words": ["it", "shake"]}, {"sent": "makes you shake doesnt it ?", "words": ["it", "shake", "you"]}, {"sent": "well shake on it .", "words": ["on", "it", "shake"]}, {"sent": "shake it .", "words": ["it", "shake"]}, {"sent": "shake shake shake shake shake the man .", "words": ["the", "shake"]}, {"sent": "feel it shake shake shake .", "words": ["it", "shake"]}, {"sent": "shake it .", "words": ["it", "shake"]}, {"sent": "trying to get that pizza out gotta shake it out .", "words": ["to", "shake", "pizza", "get", "it", "out", "that"]}, {"sent": "shake .", "words": ["shake"]}, {"sent": "shake shake shake shake shake shake shake .", "words": ["shake"]}, {"sent": "shake shake shake shake shake shake shake .", "words": ["shake"]}, {"sent": "xxx gonna go in there and shake the plant anyway .", "words": ["the", "go", "shake", "and", "there", "in", "plant"]}, {"sent": "I have a shake sweetie .", "words": ["have", "shake", "a"]}, {"sent": "shake shake ?", "words": ["shake"]}, {"sent": "Ill shake it and you put it in .", "words": ["put", "shake", "you", "and", "in", "it"]}, {"sent": "please not shake the table its gonna tip over .", "words": ["the", "table", "shake", "over", "not"]}, {"sent": "shake shake shake shake shake .", "words": ["shake"]}, {"sent": "Ozzie and Pelican shake feet when they meet .", "words": ["they", "when", "shake", "and"]}, {"sent": "that one doesnt shake only this one .", "words": ["this", "that", "shake"]}, {"sent": "shake it shake it shake it .", "words": ["it", "shake"]}, {"sent": "do you wanna shake the popcorn ?", "words": ["the", "do", "shake", "popcorn", "you", "wanna"]}, {"sent": "shake shake shake shake shake shake shake shake .", "words": ["shake"]}, {"sent": "shake it .", "words": ["it", "shake"]}, {"sent": "shake him shake it .", "words": ["him", "it", "shake"]}, {"sent": "shake shake shake shake shake .", "words": ["shake"]}, {"sent": "stand there and shake your head when Daddy tells you to do something .", "words": ["do", "your", "stand", "head", "to", "shake", "and", "you", "there", "when"]}, {"sent": "shake shake shake .", "words": ["shake"]}, {"sent": "shake .", "words": ["shake"]}, {"sent": "then were shake sharing arent we ?", "words": ["then", "were", "shake", "we"]}, {"sent": "whoa shake shake shake .", "words": ["shake"]}, {"sent": "shake shake shake .", "words": ["shake"]}, {"sent": "shake a shake a shake .", "words": ["shake", "a"]}, {"sent": "shake shake shake .", "words": ["shake"]}, {"sent": "shake shake shake .", "words": ["shake"]}, {"sent": "can you shake it with me ?", "words": ["can", "with", "shake", "you", "it", "me"]}, {"sent": "shake shake shake .", "words": ["shake"]}, {"sent": "shake shake shake shake shake shake shake .", "words": ["shake"]}, {"sent": "shake shake shake .", "words": ["shake"]}, {"sent": "shake .", "words": ["shake"]}, {"sent": "just shake it .", "words": ["it", "shake"]}, {"sent": "shake shake shake .", "words": ["shake"]}, {"sent": "shake shake shake .", "words": ["shake"]}, {"sent": "and he would shake me ei its your turn to move .", "words": ["your", "to", "shake", "and", "he", "would", "me"]}, {"sent": "Ill shake you upside down .", "words": ["down", "shake", "you"]}, {"sent": "shake it .", "words": ["it", "shake"]}, {"sent": "shake them shake them shake them .", "words": ["them", "shake"]}, {"sent": "shake .", "words": ["shake"]}, {"sent": "yeah were gonna hafta shake it to get them to all go to the bottom .", "words": ["the", "all", "bottom", "go", "to", "them", "shake", "get", "it", "were"]}, {"sent": "shake shake shake shake shake the mouse .", "words": ["the", "shake", "mouse"]}, {"sent": "that makes noise when you shake it .", "words": ["when", "shake", "you", "it", "that"]}, {"sent": "give me shake my hand ?", "words": ["my", "hand", "shake", "me", "give"]}, {"sent": "shake shake shake shake shake .", "words": ["shake"]}, {"sent": "shake it shake it shake it .", "words": ["it", "shake"]}, {"sent": "hey dont shake the baby .", "words": ["the", "dont", "shake"]}, {"sent": "shake shake shake shake .", "words": ["shake"]}, {"sent": "shake them out .", "words": ["them", "shake", "out"]}, {"sent": "shake shake shake shake shake shake shake .", "words": ["shake"]}, {"sent": "dont shake your head at me all the time .", "words": ["the", "all", "your", "head", "shake", "dont", "at", "me"]}, {"sent": "just shake Rosie about a bit and itll come out .", "words": ["shake", "and", "out", "about", "a"]}, {"sent": "shake shake shake it shake it shake shake it .", "words": ["it", "shake"]}, {"sent": "lemme see you shake yours .", "words": ["see", "shake", "you"]}, {"sent": "shake shake shake .", "words": ["shake"]}, {"sent": "I give my whole self a shake shake shake and turn myself about .", "words": ["my", "shake", "and", "myself", "about", "give", "a"]}, {"sent": "shake you upside down .", "words": ["down", "shake", "you"]}, {"sent": "shake a leg .", "words": ["shake", "leg", "a"]}, {"sent": "shake my egg shake my new egg .", "words": ["egg", "my", "new", "shake"]}, {"sent": "shake shake shake .", "words": ["shake"]}, {"sent": "shake hands .", "words": ["shake"]}, {"sent": "and I just left him xxx shake him xxx .", "words": ["him", "shake", "and"]}, {"sent": "no dont shake it .", "words": ["dont", "it", "shake"]}, {"sent": "Im gonna put you in a bag and shake you up .", "words": ["up", "put", "shake", "and", "you", "in", "a"]}, {"sent": "well shake it and let me hear .", "words": ["shake", "hear", "and", "it", "me"]}, {"sent": "shake shake shake .", "words": ["shake"]}, {"sent": "and shake my feet like that .", "words": ["my", "shake", "like", "and", "that"]}, {"sent": "okay and then when youre gonna walk were gonna shake it and then hopefully we wont run into any bears .", "words": ["when", "into", "we", "any", "shake", "and", "run", "it", "then", "walk", "were"]}, {"sent": "right I just needta shake this out .", "words": ["this", "shake", "out"]}, {"sent": "shake shake shake shake .", "words": ["shake"]}, {"sent": "shake shake shake shake shake .", "words": ["shake"]}, {"sent": "shake shake shake shake .", "words": ["shake"]}, {"sent": "shake .", "words": ["shake"]}, {"sent": "shake shake shake shake shake .", "words": ["shake"]}, {"sent": "were going to shake the numbers up .", "words": ["the", "up", "to", "shake", "were"]}, {"sent": "shake my hand shake my hand .", "words": ["my", "shake", "hand"]}, {"sent": "shake shake .", "words": ["shake"]}, {"sent": "shake .", "words": ["shake"]}, {"sent": "you shake them ?", "words": ["them", "shake", "you"]}, {"sent": "oh dont shake it .", "words": ["dont", "it", "shake"]}, {"sent": "shake shake shake .", "words": ["shake"]}, {"sent": "shake shake shake shake .", "words": ["shake"]}, {"sent": "shake shake shake .", "words": ["shake"]}, {"sent": "a big heavy pudding .", "words": ["pudding", "heavy", "big", "a"]}, {"sent": "its heavy .", "words": ["heavy"]}, {"sent": "a bulldozer can push very heavy loads with its strong blade .", "words": ["can", "push", "with", "heavy", "a"]}, {"sent": "ostriches are too heavy to fly .", "words": ["to", "heavy", "are", "too"]}, {"sent": "youre too heavy thats right .", "words": ["heavy", "too"]}, {"sent": "theyre useta move heavy trees and .", "words": ["heavy", "and"]}, {"sent": "pull it its heavy .", "words": ["heavy", "it", "pull"]}, {"sent": "I think Id be too heavy for Annas knee .", "words": ["think", "for", "knee", "be", "too", "heavy"]}, {"sent": "your eyes look heavy .", "words": ["heavy", "your", "look"]}, {"sent": "theyre pretty heavy .", "words": ["heavy", "pretty"]}, {"sent": "ThomasTheTankEngines too heavy for you .", "words": ["heavy", "for", "you", "too"]}, {"sent": "trains pretty heavy isnt it ?", "words": ["heavy", "it", "pretty"]}, {"sent": "how heavy is it ?", "words": ["how", "it", "heavy", "is"]}, {"sent": "really heavy ?", "words": ["heavy"]}, {"sent": "heavy ?", "words": ["heavy"]}, {"sent": "these instruments are too heavy to carry around carry around so they are already on the stage .", "words": ["the", "these", "around", "to", "so", "carry", "they", "are", "too", "on", "heavy"]}, {"sent": "see how heavy it is .", "words": ["how", "is", "it", "see", "heavy"]}, {"sent": "is that heavy that ?", "words": ["heavy", "that", "is"]}, {"sent": "you are getting heavy .", "words": ["heavy", "are", "you"]}, {"sent": "yes you are getting kind of heavy .", "words": ["of", "heavy", "are", "you"]}, {"sent": "but they were youknow they werent that heavy .", "words": ["they", "but", "heavy", "that", "were"]}, {"sent": "you big heavy pudding .", "words": ["pudding", "heavy", "big", "you"]}, {"sent": "and thats too heavy honey .", "words": ["too", "heavy", "and"]}, {"sent": "its a bit too heavy .", "words": ["too", "heavy", "a"]}, {"sent": "youre heavy handed .", "words": ["heavy"]}, {"sent": "youre too heavy .", "words": ["heavy", "too"]}, {"sent": "ah heavy heavy documented sources or I mean is it a leader in na .", "words": ["is", "in", "it", "heavy", "a"]}, {"sent": "right well Ill carry them through because theyre a bit too heavy .", "words": ["them", "carry", "too", "heavy", "because", "a"]}, {"sent": "its too heavy ?", "words": ["heavy", "too"]}, {"sent": "that one is not as heavy .", "words": ["heavy", "that", "not", "is"]}, {"sent": "too heavy .", "words": ["heavy", "too"]}, {"sent": "is all that stuff heavy ?", "words": ["all", "heavy", "that", "is"]}, {"sent": "were very heavy on the starch .", "words": ["on", "the", "heavy", "were"]}, {"sent": "daddys bags so heavy .", "words": ["so", "heavy"]}, {"sent": "oh is that heavy ?", "words": ["heavy", "that", "is"]}, {"sent": "I would have thought with that heavy load behind it would have been ever so hard to chug up the hill .", "words": ["up", "the", "so", "to", "with", "behind", "have", "it", "hard", "would", "heavy", "that"]}, {"sent": "but you dont feel that heavy .", "words": ["you", "dont", "heavy", "that", "but"]}, {"sent": "theyre awfully heavy .", "words": ["heavy"]}, {"sent": "oh too heavy .", "words": ["heavy", "too"]}, {"sent": "well yes its heavy .", "words": ["heavy"]}, {"sent": "is he not heavy ?", "words": ["heavy", "he", "not", "is"]}, {"sent": "that must be very heavy .", "words": ["be", "heavy", "that"]}, {"sent": "heavy .", "words": ["heavy"]}, {"sent": "its really heavy Thomas .", "words": ["heavy"]}, {"sent": "its heavy .", "words": ["heavy"]}, {"sent": "shes much too heavy for you to carry .", "words": ["for", "to", "you", "carry", "too", "heavy", "much"]}, {"sent": "you havent had a big poo but you must be quite wet because that nappies very big and heavy .", "words": ["wet", "be", "because", "you", "and", "that", "heavy", "big", "but", "a"]}, {"sent": "oof oof oof youre a heavy little train Percy .", "words": ["heavy", "train", "little", "a"]}, {"sent": "EddietheElephant would be too heavy .", "words": ["be", "too", "heavy", "would"]}, {"sent": "shes too heavy .", "words": ["heavy", "too"]}, {"sent": "something very heavy while you were asleep .", "words": ["asleep", "heavy", "were", "you"]}, {"sent": "and an oak kitchen youknow a real heavy wood .", "words": ["and", "an", "kitchen", "heavy", "a"]}, {"sent": "thats some heavy duty jumping .", "words": ["some", "heavy"]}, {"sent": "its heavy .", "words": ["heavy"]}, {"sent": "heavy lorrys .", "words": ["heavy"]}, {"sent": "heavy with the .", "words": ["the", "heavy", "with"]}, {"sent": "youre too heavy arent you ?", "words": ["heavy", "you", "too"]}, {"sent": "shes not very heavy is she ?", "words": ["she", "heavy", "not", "is"]}, {"sent": "that guys gonna be too heavy for him .", "words": ["for", "be", "him", "too", "heavy", "that"]}, {"sent": "Im not heavy am I ?", "words": ["heavy", "not", "am"]}, {"sent": "its pretty heavy isnt it ?", "words": ["heavy", "it", "pretty"]}, {"sent": "youre pre getting pretty heavy you know .", "words": ["heavy", "pretty", "you"]}, {"sent": "youre too heavy .", "words": ["heavy", "too"]}, {"sent": "oh thats a heavy book .", "words": ["book", "heavy", "a"]}, {"sent": "or maybe hes sitting down because the sacks are quite heavy .", "words": ["the", "are", "heavy", "down", "because"]}, {"sent": "heavy ?", "words": ["heavy"]}, {"sent": "youre getting a bit heavy for this .", "words": ["this", "heavy", "for", "a"]}, {"sent": "because its quite a heavy heavy sort of vehicle .", "words": ["of", "heavy", "because", "a"]}, {"sent": "you do it too heavy .", "words": ["do", "you", "it", "too", "heavy"]}, {"sent": "youre a bit heavy for that now .", "words": ["heavy", "for", "that", "a"]}, {"sent": "because hes a bit heavy .", "words": ["heavy", "because", "a"]}, {"sent": "dont youre too heavy .", "words": ["dont", "heavy", "too"]}, {"sent": "its heavy ?", "words": ["heavy"]}, {"sent": "let me take off that heavy shirt .", "words": ["off", "heavy", "shirt", "me", "take", "that"]}, {"sent": "its going to be heavy this basket .", "words": ["to", "be", "this", "basket", "heavy"]}, {"sent": "oh heavy heavy heavy oh hes such a heavy boy hes such a heavy boy .", "words": ["heavy", "a"]}, {"sent": "it looks heavy to me .", "words": ["to", "heavy", "it", "me"]}, {"sent": "this is heavy .", "words": ["this", "heavy", "is"]}, {"sent": "because theyre quite heavy arent they ?", "words": ["they", "heavy", "because"]}, {"sent": "ooh the vase is very heavy isnt it ?", "words": ["the", "heavy", "it", "is"]}, {"sent": "its a heavy load isnt it ?", "words": ["heavy", "it", "a"]}, {"sent": "a bulldozer can push very heavy loads with its strong blade .", "words": ["can", "push", "with", "heavy", "a"]}, {"sent": "its very heavy .", "words": ["heavy"]}, {"sent": "I think its very heavy uhn do you ?", "words": ["do", "think", "heavy", "you"]}, {"sent": "its heavy .", "words": ["heavy"]}, {"sent": "is that post sack very heavy PostmanPat ?", "words": ["heavy", "that", "is"]}, {"sent": "am I too heavy ?", "words": ["heavy", "am", "too"]}, {"sent": "its a good job she has got a scooter and she can ride home because now the letters are very heavy to carry arent they ?", "words": ["can", "the", "to", "and", "carry", "they", "are", "good", "home", "heavy", "ride", "she", "because", "a"]}, {"sent": "big heavy girl .", "words": ["heavy", "big"]}, {"sent": "oh too heavy ?", "words": ["heavy", "too"]}, {"sent": "oh its much too heavy its very heavy .", "words": ["heavy", "much", "too"]}, {"sent": "I think its a little bit heavy for you really isnt it ?", "words": ["think", "for", "little", "you", "it", "heavy", "a"]}, {"sent": "no Alex youre too heavy .", "words": ["heavy", "too"]}, {"sent": "want something heavy in your hand ?", "words": ["in", "heavy", "your", "hand"]}, {"sent": "shes very heavy for her size isnt she Becky ?", "words": ["heavy", "for", "her", "she"]}, {"sent": "we have a very heavy discussion okay very heavy important discussion .", "words": ["have", "heavy", "a", "we"]}, {"sent": "pick up those heavy parcels .", "words": ["pick", "heavy", "up", "those"]}, {"sent": "no youre too heavy for Mommy going up the hill .", "words": ["up", "the", "for", "too", "heavy"]}, {"sent": "its heavy .", "words": ["heavy"]}, {"sent": "an alligator .", "words": ["an", "alligator"]}, {"sent": "so he says to him see you later alligator .", "words": ["to", "so", "you", "him", "he", "alligator", "see"]}, {"sent": "like an alligator .", "words": ["an", "like", "alligator"]}, {"sent": "thats kinda silly does grandpa Zevenbergen have a pet alligator ?", "words": ["have", "does", "alligator", "a"]}, {"sent": "no no no a alligator .", "words": ["alligator", "a"]}, {"sent": "and what about that alligator ?", "words": ["what", "and", "alligator", "about", "that"]}, {"sent": "oh thats an alligator .", "words": ["an", "alligator"]}, {"sent": "hes an alligator snapping turtle .", "words": ["turtle", "an", "alligator"]}, {"sent": "hes scared of the alligator ?", "words": ["of", "the", "alligator", "scared"]}, {"sent": "does that look like an alligator ?", "words": ["does", "like", "an", "alligator", "look", "that"]}, {"sent": "uhhum two because theres a turtle and an alligator .", "words": ["and", "an", "alligator", "turtle", "because", "a"]}, {"sent": "alligoogoo the alligator ?", "words": ["the", "alligator"]}, {"sent": "while I color in this alligator .", "words": ["this", "in", "alligator"]}, {"sent": "alligator .", "words": ["alligator"]}, {"sent": "an alligator ?", "words": ["an", "alligator"]}, {"sent": "like the lady is riding on the alligator .", "words": ["the", "is", "like", "alligator", "on"]}, {"sent": "Id rather be the alligator .", "words": ["the", "be", "alligator"]}, {"sent": "yeah I think that is an alligator .", "words": ["think", "is", "an", "alligator", "that"]}, {"sent": "wanna be the alligator ?", "words": ["the", "be", "alligator", "wanna"]}, {"sent": "and the birds reading and the alligator the alligator is reading .", "words": ["the", "alligator", "is", "and"]}, {"sent": "he didnt wanna be eaten by the alligator did he ?", "words": ["the", "be", "by", "he", "alligator", "wanna", "did"]}, {"sent": "this is called an alligator clip .", "words": ["this", "an", "alligator", "is"]}, {"sent": "alligator yep .", "words": ["alligator"]}, {"sent": "alligator zebra lion tiger walrus .", "words": ["zebra", "lion", "alligator", "tiger"]}, {"sent": "in the meantime you dont wanna fix the alligator .", "words": ["the", "fix", "you", "dont", "in", "alligator", "wanna"]}, {"sent": "and look because that girls that girls in that alligator .", "words": ["and", "in", "alligator", "look", "that", "because"]}, {"sent": "like alligator .", "words": ["like", "alligator"]}, {"sent": "the alligator clip .", "words": ["the", "alligator"]}, {"sent": "whats an alligator ?", "words": ["an", "alligator"]}, {"sent": "a is for alligator .", "words": ["for", "alligator", "is", "a"]}, {"sent": "did he say see you later alligator ?", "words": ["say", "you", "he", "did", "alligator", "see"]}, {"sent": "oh back alligator ?", "words": ["alligator", "back"]}, {"sent": "a for alligator .", "words": ["for", "alligator", "a"]}, {"sent": "Richard Scarrys alligator is called Smiley .", "words": ["alligator", "is"]}, {"sent": "mm I knew xxx theres the alligator .", "words": ["the", "alligator"]}, {"sent": "Im gonna put a whale on the farm and an alligator on the farm .", "words": ["the", "put", "and", "an", "alligator", "on", "a"]}, {"sent": "its supposed to be an alligator isnt it ?", "words": ["to", "be", "an", "it", "alligator"]}, {"sent": "so maybe sometime when you have something that you wanna do you say to Joe you say to Gabriel see you later alligator .", "words": ["do", "when", "so", "to", "say", "you", "have", "alligator", "wanna", "see", "that"]}, {"sent": "see you later alligator .", "words": ["see", "alligator", "you"]}, {"sent": "is the alligator sitting under the bridge ?", "words": ["the", "under", "alligator", "is"]}, {"sent": "maybe an alligator .", "words": ["an", "alligator"]}, {"sent": "get your alligator right there look get your alligator .", "words": ["your", "get", "there", "alligator", "look"]}, {"sent": "alligator begins with an a .", "words": ["an", "with", "alligator", "a"]}, {"sent": "alligator ?", "words": ["alligator"]}, {"sent": "an alligator .", "words": ["an", "alligator"]}, {"sent": "thats why angry ants advanced across an apple an acorn an apricot an axe an angle worm an alligator aunt Alices airplane avenue A xxx and all of Arizona .", "words": ["all", "airplane", "of", "and", "an", "alligator", "apple", "why"]}, {"sent": "Ill be the big alligator .", "words": ["the", "be", "big", "alligator"]}, {"sent": "is my alligator in the show too ?", "words": ["the", "my", "is", "show", "in", "alligator", "too"]}, {"sent": "now the camels with the alligator .", "words": ["the", "with", "alligator"]}, {"sent": "see you later alligator .", "words": ["see", "alligator", "you"]}, {"sent": "I want a alligator nose .", "words": ["nose", "alligator", "a"]}, {"sent": "alligator .", "words": ["alligator"]}, {"sent": "the basket is brown the grass is green and Tom the alligator is green .", "words": ["the", "brown", "is", "and", "basket", "alligator", "green", "grass"]}, {"sent": "that says a for alligator .", "words": ["for", "that", "alligator", "a"]}, {"sent": "alligator .", "words": ["alligator"]}, {"sent": "it looks as if its an alligator .", "words": ["an", "it", "if", "alligator"]}, {"sent": "oh alligator .", "words": ["alligator"]}, {"sent": "it is a alligator alligator .", "words": ["it", "alligator", "a", "is"]}, {"sent": "this is a different alligator .", "words": ["this", "alligator", "a", "is"]}, {"sent": "Im an alligator and I eat children .", "words": ["eat", "an", "alligator", "and"]}, {"sent": "thats an alligator .", "words": ["an", "alligator"]}, {"sent": "the lady with the alligator purse .", "words": ["the", "with", "alligator", "purse"]}, {"sent": "see you later alligator ?", "words": ["see", "alligator", "you"]}, {"sent": "alligator .", "words": ["alligator"]}, {"sent": "the alligator was gonna have a party ?", "words": ["the", "was", "party", "have", "alligator", "a"]}, {"sent": "an alligator walks away from her daddy ?", "words": ["an", "away", "her", "alligator"]}, {"sent": "alligator .", "words": ["alligator"]}, {"sent": "read Mama theres alligator Mama .", "words": ["read", "alligator"]}, {"sent": "look there is an alligator .", "words": ["is", "there", "an", "alligator", "look"]}, {"sent": "let me be the alligator once alright ?", "words": ["the", "me", "be", "alligator"]}, {"sent": "you wanna see the alligator again .", "words": ["the", "you", "alligator", "wanna", "see"]}, {"sent": "did you invite Bwwws alligator ?", "words": ["did", "alligator", "you"]}, {"sent": "alligator .", "words": ["alligator"]}, {"sent": "okay its an alligator .", "words": ["an", "alligator"]}, {"sent": "it belongs with the alligator .", "words": ["the", "it", "with", "alligator"]}, {"sent": "whats the alligator doing ?", "words": ["the", "alligator"]}, {"sent": "and theres alligator too and hippo .", "words": ["too", "alligator", "and"]}, {"sent": "an alligator .", "words": ["an", "alligator"]}, {"sent": "alligator .", "words": ["alligator"]}, {"sent": "it was as big as an alligator wasnt it ?", "words": ["was", "an", "it", "alligator", "big"]}, {"sent": "see you later alligator .", "words": ["see", "alligator", "you"]}, {"sent": "alligator .", "words": ["alligator"]}, {"sent": "Im gonna put the alligator clip its gonna take .", "words": ["the", "put", "take", "alligator"]}, {"sent": "goops youre tangled up whoops okay Im gonna take the alligator clip off .", "words": ["up", "the", "off", "alligator", "take"]}, {"sent": "so the alligator did that to me .", "words": ["the", "so", "to", "alligator", "did", "me", "that"]}, {"sent": "what about see you later alligator ?", "words": ["what", "you", "alligator", "about", "see"]}, {"sent": "Nathaniel made a marvelous picture of an alligator and a train under a bridge .", "words": ["train", "of", "picture", "and", "under", "an", "alligator", "a"]}, {"sent": "that is an alligator .", "words": ["an", "that", "alligator", "is"]}, {"sent": "but this is Aaron the alligator .", "words": ["the", "is", "this", "alligator", "but"]}, {"sent": "night tell nanny see her later alligator .", "words": ["see", "her", "alligator"]}, {"sent": "and then the alligator clip .", "words": ["the", "then", "alligator", "and"]}, {"sent": "thats right alligator .", "words": ["alligator"]}, {"sent": "yep theres a alligator .", "words": ["alligator", "a"]}, {"sent": "alligator .", "words": ["alligator"]}, {"sent": "alligator apple .", "words": ["apple", "alligator"]}, {"sent": "alligator .", "words": ["alligator"]}, {"sent": "like an alligator .", "words": ["an", "like", "alligator"]}, {"sent": "alligator .", "words": ["alligator"]}, {"sent": "yup push .", "words": ["push"]}, {"sent": "now let us push it up and turn it on .", "words": ["up", "push", "and", "us", "it", "on"]}, {"sent": "push hard .", "words": ["push", "hard"]}, {"sent": "push .", "words": ["push"]}, {"sent": "okay push .", "words": ["push"]}, {"sent": "push it push it up push .", "words": ["up", "it", "push"]}, {"sent": "push it .", "words": ["it", "push"]}, {"sent": "um yeah push and pull kinda stuff .", "words": ["push", "pull", "and"]}, {"sent": "push that push that corner .", "words": ["that", "push"]}, {"sent": "dont push the buttons .", "words": ["the", "dont", "push"]}, {"sent": "push hard .", "words": ["push", "hard"]}, {"sent": "push it real hard .", "words": ["it", "push", "hard"]}, {"sent": "thats right you push the go button but first you have to push the on off button .", "words": ["the", "off", "push", "go", "to", "first", "you", "have", "on", "button", "but"]}, {"sent": "push the button here .", "words": ["the", "here", "push", "button"]}, {"sent": "now push the button down and take a picture .", "words": ["the", "push", "picture", "and", "take", "down", "button", "a"]}, {"sent": "not more push .", "words": ["more", "push", "not"]}, {"sent": "push it right in there .", "words": ["there", "in", "it", "push"]}, {"sent": "okay you push the clown in .", "words": ["the", "in", "push", "you"]}, {"sent": "push your legs .", "words": ["your", "push"]}, {"sent": "lorry will crash if you push it under there .", "words": ["push", "if", "you", "under", "there", "it", "will"]}, {"sent": "push down .", "words": ["down", "push"]}, {"sent": "wanna push it in ?", "words": ["in", "it", "push", "wanna"]}, {"sent": "okay now you can push .", "words": ["can", "push", "you"]}, {"sent": "the wants the car to push the boat .", "words": ["the", "push", "car", "to", "boat"]}, {"sent": "push push .", "words": ["push"]}, {"sent": "there it goes on push a little harder .", "words": ["push", "little", "there", "it", "on", "a"]}, {"sent": "push it .", "words": ["it", "push"]}, {"sent": "push it .", "words": ["it", "push"]}, {"sent": "and push .", "words": ["push", "and"]}, {"sent": "push with both feet .", "words": ["push", "with"]}, {"sent": "push it down a little bit more Playdoh at the bottom .", "words": ["the", "bottom", "push", "little", "more", "it", "at", "down", "a"]}, {"sent": "lets push it back in .", "words": ["in", "it", "push", "back"]}, {"sent": "did you push the chair in ?", "words": ["the", "push", "chair", "you", "in", "did"]}, {"sent": "yeah theres lotsof room because look just push it up like that .", "words": ["up", "push", "like", "it", "room", "look", "that", "because"]}, {"sent": "and you cant go out and push it .", "words": ["push", "go", "you", "and", "it", "out"]}, {"sent": "push back .", "words": ["push", "back"]}, {"sent": "push it in there .", "words": ["there", "in", "it", "push"]}, {"sent": "brumm and the fenderd push it outof the way .", "words": ["the", "it", "push", "and"]}, {"sent": "why dont you push your to drawer in a little bit honey .", "words": ["your", "push", "to", "you", "little", "dont", "in", "drawer", "why", "a"]}, {"sent": "push on his head .", "words": ["on", "his", "head", "push"]}, {"sent": "push it in .", "words": ["in", "it", "push"]}, {"sent": "push push .", "words": ["push"]}, {"sent": "okay push it on there baby .", "words": ["on", "there", "it", "push"]}, {"sent": "push the man .", "words": ["the", "push"]}, {"sent": "push that button .", "words": ["that", "push", "button"]}, {"sent": "because you can push it up and down the garden .", "words": ["can", "up", "the", "push", "you", "and", "garden", "it", "down", "because"]}, {"sent": "push him in .", "words": ["in", "him", "push"]}, {"sent": "can we push it ?", "words": ["can", "it", "push", "we"]}, {"sent": "well if you push .", "words": ["push", "if", "you"]}, {"sent": "push it all gone .", "words": ["all", "all gone", "it", "push"]}, {"sent": "let us push it back out there .", "words": ["push", "back", "us", "there", "it", "out"]}, {"sent": "push this side forwards .", "words": ["this", "push"]}, {"sent": "push it you gotta be very strong .", "words": ["be", "it", "push", "you"]}, {"sent": "n push it .", "words": ["it", "push"]}, {"sent": "push it a little bit harder .", "words": ["it", "push", "little", "a"]}, {"sent": "okay so you have to push it you have to take it .", "words": ["push", "so", "to", "you", "have", "it", "take"]}, {"sent": "and then you kinda push with your other foot .", "words": ["push", "your", "with", "you", "and", "other", "foot", "then"]}, {"sent": "oh we hafta push hard dont we ?", "words": ["dont", "push", "hard", "we"]}, {"sent": "how are you gonna push them ?", "words": ["how", "push", "them", "you", "are"]}, {"sent": "push it up her arms .", "words": ["up", "her", "it", "push"]}, {"sent": "thats fine now you hafta just move the lever push the lever the other way .", "words": ["the", "push", "fine", "other", "you"]}, {"sent": "push it in gently Darling .", "words": ["in", "it", "push"]}, {"sent": "push .", "words": ["push"]}, {"sent": "push it down .", "words": ["down", "it", "push"]}, {"sent": "push .", "words": ["push"]}, {"sent": "just push em .", "words": ["push"]}, {"sent": "you gonna push the pedal ah no I dont think so because we cant drive it in the house .", "words": ["the", "think", "push", "we", "so", "you", "drive", "dont", "in", "it", "house", "because"]}, {"sent": "lets push them in .", "words": ["in", "push", "them"]}, {"sent": "push the orange down .", "words": ["the", "down", "orange", "push"]}, {"sent": "let me show you you hafta push this .", "words": ["push", "show", "you", "this", "me"]}, {"sent": "well shall Mummy hold Granny and you push Granny out ?", "words": ["push", "hold", "you", "and", "out"]}, {"sent": "do you want Mommy to push it ?", "words": ["do", "push", "to", "you", "it"]}, {"sent": "you go push you do it you push this push it up .", "words": ["up", "do", "push", "go", "you", "this", "it"]}, {"sent": "push it through the hole .", "words": ["the", "it", "push"]}, {"sent": "push xxx that piece in look .", "words": ["in", "that", "push", "look"]}, {"sent": "push on the scales .", "words": ["on", "the", "push"]}, {"sent": "how about if I just push it ?", "words": ["how", "push", "if", "it", "about"]}, {"sent": "you can push the train you dont have you dont hafta use the .", "words": ["can", "the", "push", "train", "you", "have", "dont"]}, {"sent": "Fraser look push this .", "words": ["this", "push", "look"]}, {"sent": "a world of outdoor stands and push carts gold and silver ornaments books old furniture fruit vegetables people .", "words": ["old", "push", "of", "and", "a"]}, {"sent": "wanna push the truck .", "words": ["the", "truck", "push", "wanna"]}, {"sent": "push .", "words": ["push"]}, {"sent": "you hafta push the tongue in .", "words": ["the", "push", "you", "in", "tongue"]}, {"sent": "push this down .", "words": ["this", "down", "push"]}, {"sent": "okay if he doesnt get off Ill push him off .", "words": ["off", "push", "if", "get", "him", "he"]}, {"sent": "dont push that .", "words": ["dont", "that", "push"]}, {"sent": "lets push it .", "words": ["it", "push"]}, {"sent": "push .", "words": ["push"]}, {"sent": "push it over .", "words": ["over", "it", "push"]}, {"sent": "take one hand take one hand and push .", "words": ["take", "push", "and", "hand"]}, {"sent": "push hard .", "words": ["push", "hard"]}, {"sent": "*IN2: wanna push them in ?", "words": ["in", "push", "wanna", "them"]}, {"sent": "push .", "words": ["push"]}, {"sent": "push it .", "words": ["it", "push"]}, {"sent": "push him in .", "words": ["in", "him", "push"]}, {"sent": "like that push it in .", "words": ["push", "like", "in", "it", "that"]}, {"sent": "oh its going to push you off the settee isnt it ?", "words": ["off", "the", "push", "to", "you", "it"]}, {"sent": "shell feed them shell dress them you know shell push them in her stroller shell tell me that shes putting them nightnight .", "words": ["push", "them", "her", "you", "in", "that", "stroller", "feed", "me", "dress"]}, {"sent": "please let me outof this sock AnneRussell .", "words": ["me", "this", "sock"]}, {"sent": "wheres the dolls other sock ?", "words": ["the", "sock", "other"]}, {"sent": "should we put on your sock ?", "words": ["put", "your", "we", "sock", "on"]}, {"sent": "Thomas youve got one sock on and one sock off .", "words": ["on", "off", "sock", "and"]}, {"sent": "one sock on .", "words": ["on", "sock"]}, {"sent": "xxx put your sock back on or leave it off ?", "words": ["off", "put", "your", "back", "it", "sock", "on"]}, {"sent": "one sock .", "words": ["sock"]}, {"sent": "another another sock off please .", "words": ["another", "off", "sock"]}, {"sent": "slipper and a sock off ?", "words": ["off", "and", "sock", "slipper", "a"]}, {"sent": "pull the sock .", "words": ["the", "sock", "pull"]}, {"sent": "yeah you look funny with a sock on your head Fraser .", "words": ["your", "head", "with", "you", "sock", "look", "on", "a"]}, {"sent": "one sock off .", "words": ["off", "sock"]}, {"sent": "is this a sock too ?", "words": ["is", "this", "sock", "too", "a"]}, {"sent": "sit sit just there and let me put your sock on .", "words": ["put", "your", "and", "there", "sock", "on", "me", "sit"]}, {"sent": "what happened to Andys sock ?", "words": ["to", "what", "sock"]}, {"sent": "you took off his sock .", "words": ["off", "his", "sock", "you"]}, {"sent": "Awww wheres your other sock darling ?", "words": ["your", "sock", "other"]}, {"sent": "you have a sock on .", "words": ["you", "have", "sock", "on", "a"]}, {"sent": "no you cant sock your wife .", "words": ["your", "sock", "you"]}, {"sent": "can you take the sock off ?", "words": ["can", "the", "off", "you", "sock", "take"]}, {"sent": "sock oh .", "words": ["sock"]}, {"sent": "naughty sock .", "words": ["naughty", "sock"]}, {"sent": "green sock .", "words": ["green", "sock"]}, {"sent": "look Mummy has found another sock .", "words": ["another", "sock", "look"]}, {"sent": "shall I put your sock on ?", "words": ["on", "put", "your", "sock"]}, {"sent": "the duck has taken his sock off ?", "words": ["the", "off", "duck", "sock", "his"]}, {"sent": "why do you want your sock ?", "words": ["do", "your", "you", "sock", "why"]}, {"sent": "look at this a footprint on the bottom of your sock .", "words": ["the", "look", "bottom", "your", "of", "this", "sock", "at", "on", "a"]}, {"sent": "wheres your sock ?", "words": ["your", "sock"]}, {"sent": "just come and put this sock on please .", "words": ["put", "and", "this", "sock", "on"]}, {"sent": "whatve you got on your sock ?", "words": ["on", "your", "sock", "you"]}, {"sent": "wheres your other sock gone ?", "words": ["your", "sock", "other"]}, {"sent": "theres sand all over your sock too .", "words": ["all", "your", "sock", "too", "over"]}, {"sent": "you want me to take my sock off .", "words": ["off", "my", "to", "you", "sock", "me", "take"]}, {"sent": "you think I wanna kiss a sock mouth ?", "words": ["think", "mouth", "you", "wanna", "sock", "kiss", "a"]}, {"sent": "we could have made something easy outof a sock big sock or something .", "words": ["we", "have", "sock", "could", "big", "a"]}, {"sent": "sock .", "words": ["sock"]}, {"sent": "well now what are you gonna do now you have one sock on and one sock off .", "words": ["off", "what", "do", "you", "have", "and", "are", "sock", "on"]}, {"sent": "is that Frasers smelly sock ?", "words": ["that", "sock", "is"]}, {"sent": "its not Mummys sock .", "words": ["not", "sock"]}, {"sent": "its a sock ?", "words": ["sock", "a"]}, {"sent": "xxx your sock back on .", "words": ["on", "back", "your", "sock"]}, {"sent": "should I put her sock on ?", "words": ["on", "put", "her", "sock"]}, {"sent": "look at your sock .", "words": ["at", "your", "sock", "look"]}, {"sent": "yeah thats Naimas sock .", "words": ["sock"]}, {"sent": "he doesnt look like he has any sock on it .", "words": ["any", "like", "it", "he", "sock", "look", "on"]}, {"sent": "where is your sock ?", "words": ["where", "your", "sock", "is"]}, {"sent": "the green sock .", "words": ["the", "green", "sock"]}, {"sent": "sock ?", "words": ["sock"]}, {"sent": "that sock doesnt belong in the wipes box .", "words": ["the", "box", "in", "sock", "that"]}, {"sent": "mustnt tear your sock .", "words": ["your", "tear", "sock"]}, {"sent": "you kicked off the sock ?", "words": ["off", "the", "sock", "you"]}, {"sent": "wheres the sock ?", "words": ["the", "sock"]}, {"sent": "one sock is on and one socks on .", "words": ["on", "and", "sock", "is"]}, {"sent": "its a show with a sock .", "words": ["show", "with", "sock", "a"]}, {"sent": "does the sock belong to Daddy ?", "words": ["the", "does", "to", "sock"]}, {"sent": "that was your playing sock wasnt it .", "words": ["your", "was", "it", "sock", "that"]}, {"sent": "you got a little hole in your sock ?", "words": ["your", "little", "you", "in", "sock", "a"]}, {"sent": "wheres Mamas other sock ?", "words": ["sock", "other"]}, {"sent": "he decides to wash the sock .", "words": ["the", "to", "he", "sock", "wash"]}, {"sent": "now lets get that other sock on finally .", "words": ["other", "get", "sock", "on", "that"]}, {"sent": "sock .", "words": ["sock"]}, {"sent": "someones in the lion sock .", "words": ["the", "in", "lion", "sock"]}, {"sent": "sock it to me sock it to me sock it to me sock it to me .", "words": ["to", "me", "it", "sock"]}, {"sent": "are you cooking your sock ?", "words": ["are", "your", "sock", "you"]}, {"sent": "how come I found a sock ?", "words": ["how", "sock", "a"]}, {"sent": "what a smelly sock .", "words": ["what", "sock", "a"]}, {"sent": "a sock .", "words": ["sock", "a"]}, {"sent": "youve got one giraffes leg in your sock and one up your shorts .", "words": ["up", "shorts", "your", "and", "in", "leg", "sock"]}, {"sent": "where did the sock go ?", "words": ["where", "the", "go", "did", "sock"]}, {"sent": "no we needta find her other sock .", "words": ["we", "other", "sock", "find", "her"]}, {"sent": "yep heres the other sock .", "words": ["the", "sock", "other"]}, {"sent": "thats a sock .", "words": ["sock", "a"]}, {"sent": "how would he get food on his sock ?", "words": ["how", "get", "his", "food", "he", "sock", "would", "on"]}, {"sent": "is that a sock ?", "words": ["sock", "that", "a", "is"]}, {"sent": "wheres my other sock gone ?", "words": ["my", "sock", "other"]}, {"sent": "you have a sock on your foot already .", "words": ["your", "you", "have", "foot", "sock", "on", "a"]}, {"sent": "let that sock go .", "words": ["that", "sock", "go"]}, {"sent": "wheres her other sock gone ?", "words": ["her", "sock", "other"]}, {"sent": "what color sock do you want for your other hand ?", "words": ["what", "do", "for", "your", "hand", "you", "other", "sock"]}, {"sent": "one sock is on and one sock is off .", "words": ["off", "is", "and", "sock", "on"]}, {"sent": "your sock .", "words": ["your", "sock"]}, {"sent": "put the sock in it ?", "words": ["the", "put", "in", "it", "sock"]}, {"sent": "its become a hugging sock now .", "words": ["sock", "a"]}, {"sent": "I wish I had another sock .", "words": ["another", "sock", "wish"]}, {"sent": "is that your sock coming off ?", "words": ["off", "your", "is", "sock", "that"]}, {"sent": "lets put this foot into this sock .", "words": ["put", "into", "this", "foot", "sock"]}, {"sent": "sock .", "words": ["sock"]}, {"sent": "Nina has a sock missing .", "words": ["sock", "a"]}, {"sent": "thats sock thing .", "words": ["sock"]}, {"sent": "Mummys putting her sock on her .", "words": ["on", "her", "sock"]}, {"sent": "you took off your sock ?", "words": ["off", "your", "sock", "you"]}, {"sent": "you want me to put that sock on ?", "words": ["put", "to", "you", "sock", "on", "me", "that"]}, {"sent": "thats a sock hang it up as if it was Christmas waiting for FatherChristmas on the mantlepiece .", "words": ["up", "the", "for", "if", "was", "it", "sock", "on", "a"]}, {"sent": "xxx you take that over towards the sock drawer .", "words": ["the", "you", "that", "sock", "drawer", "take", "over"]}, {"sent": "are you taking your sock off ?", "words": ["off", "your", "you", "are", "sock"]}, {"sent": "no what are you sposta put inside your sock ?", "words": ["what", "put", "your", "inside", "you", "are", "sock"]}, {"sent": "arent you gonna put a sock on ?", "words": ["put", "you", "sock", "on", "a"]}, {"sent": "you cant put the sock on after the shoe .", "words": ["the", "put", "you", "shoe", "sock", "on"]}, {"sent": "Jwww dont be so loud .", "words": ["dont", "be", "so", "loud"]}, {"sent": "not too loud .", "words": ["not", "loud", "too"]}, {"sent": "is it gonna be loud ?", "words": ["be", "it", "loud", "is"]}, {"sent": "you can talk loud .", "words": ["can", "loud", "talk", "you"]}, {"sent": "sounds very very loud I think it sounds far too loud .", "words": ["think", "it", "loud", "too"]}, {"sent": "Daddy does have a loud voice .", "words": ["have", "does", "loud", "a"]}, {"sent": "say I am loud .", "words": ["say", "am", "loud"]}, {"sent": "not so loud .", "words": ["so", "not", "loud"]}, {"sent": "the aeroplanes sound loud this morning dont they ?", "words": ["the", "this", "dont", "they", "loud"]}, {"sent": "that was loud .", "words": ["was", "that", "loud"]}, {"sent": "go on you say it loud loud .", "words": ["go", "say", "you", "it", "on", "loud"]}, {"sent": "those are gonna be really loud .", "words": ["be", "are", "those", "loud"]}, {"sent": "such loud gun xxx .", "words": ["loud"]}, {"sent": "just a little bit loud .", "words": ["loud", "little", "a"]}, {"sent": "it sounds very loud .", "words": ["it", "loud"]}, {"sent": "it sounds loud .", "words": ["it", "loud"]}, {"sent": "sing it loud darling .", "words": ["sing", "it", "loud"]}, {"sent": "is that too loud do you think ?", "words": ["do", "think", "is", "you", "too", "that", "loud"]}, {"sent": "it goes way too loud .", "words": ["it", "loud", "too"]}, {"sent": "I think this Spot ones a loud one .", "words": ["this", "think", "loud", "a"]}, {"sent": "thats loud Liz .", "words": ["loud"]}, {"sent": "he can talk so loud ?", "words": ["can", "so", "he", "talk", "loud"]}, {"sent": "the cat sat asleep by the side of the fire the mistress snored loud as a pig Jack took up his fiddle by Janies desire and struck up a bit of a jig .", "words": ["the", "up", "of", "by", "pig", "cat", "asleep", "and", "his", "loud", "a"]}, {"sent": "say it loud .", "words": ["it", "say", "loud"]}, {"sent": "say it out loud three .", "words": ["it", "say", "loud", "out"]}, {"sent": "it was loud .", "words": ["was", "it", "loud"]}, {"sent": "its a loud truck huh ?", "words": ["truck", "loud", "a"]}, {"sent": "shh Brieanna not so loud .", "words": ["so", "not", "loud"]}, {"sent": "loud .", "words": ["loud"]}, {"sent": "a loud motorcycle .", "words": ["motorcycle", "loud", "a"]}, {"sent": "suddenly a loud loud noise made him jump .", "words": ["jump", "him", "loud", "a"]}, {"sent": "but it is also loud .", "words": ["it", "loud", "but", "is"]}, {"sent": "loud and clear and strong .", "words": ["loud", "and"]}, {"sent": "loud shoes .", "words": ["loud"]}, {"sent": "whats the opposite of loud ?", "words": ["the", "of", "loud"]}, {"sent": "and his gun made a loud bang .", "words": ["his", "loud", "a", "and"]}, {"sent": "talk loud I can hardly hear you .", "words": ["can", "you", "hear", "talk", "loud"]}, {"sent": "loud .", "words": ["loud"]}, {"sent": "does that sound loud ?", "words": ["does", "that", "loud"]}, {"sent": "making noise yes does that seem loud to you ?", "words": ["does", "to", "you", "that", "loud"]}, {"sent": "say it loud uhn Im black and Im proud .", "words": ["say", "and", "black", "it", "loud"]}, {"sent": "one day Pooh hears a loud buzzing noise .", "words": ["loud", "a"]}, {"sent": "very loud .", "words": ["loud"]}, {"sent": "you can speak loud .", "words": ["can", "loud", "you"]}, {"sent": "okay now remember its going to make loud noise .", "words": ["to", "make", "loud"]}, {"sent": "sing it loud .", "words": ["sing", "it", "loud"]}, {"sent": "dont talk so loud please xxx .", "words": ["dont", "so", "loud", "talk"]}, {"sent": "loud claps of thunder .", "words": ["of", "loud"]}, {"sent": "loud .", "words": ["loud"]}, {"sent": "it sounds very loud .", "words": ["it", "loud"]}, {"sent": "definitely very loud when we did that .", "words": ["when", "we", "did", "that", "loud"]}, {"sent": "and Isabel doesnt like loud noises .", "words": ["like", "loud", "and"]}, {"sent": "loud .", "words": ["loud"]}, {"sent": "if they raise their voices I mean you know what would be really loud for them I can hear them .", "words": ["can", "what", "for", "if", "their", "be", "them", "you", "hear", "they", "would", "loud"]}, {"sent": "thats pretty loud isnt it .", "words": ["it", "pretty", "loud"]}, {"sent": "loud noise .", "words": ["loud"]}, {"sent": "is it a bit loud ?", "words": ["it", "loud", "a", "is"]}, {"sent": "in a voice loud enough for Percy and James to hear .", "words": ["for", "to", "and", "hear", "in", "loud", "a"]}, {"sent": "because if its loud I cant hear you .", "words": ["if", "you", "hear", "loud", "because"]}, {"sent": "and loud .", "words": ["loud", "and"]}, {"sent": "oh that made a loud bang .", "words": ["that", "loud", "a"]}, {"sent": "I dont need you to be loud .", "words": ["need", "to", "be", "you", "dont", "loud"]}, {"sent": "their shoes are loud .", "words": ["are", "their", "loud"]}, {"sent": "peeppeep very loud .", "words": ["loud"]}, {"sent": "that was a big loud kiss .", "words": ["was", "that", "kiss", "big", "loud", "a"]}, {"sent": "talk loud .", "words": ["loud", "talk"]}, {"sent": "its a bit loud .", "words": ["loud", "a"]}, {"sent": "hes a loud sleeper .", "words": ["loud", "a"]}, {"sent": "yeah makes a loud noise when you throw it .", "words": ["throw", "you", "it", "when", "loud", "a"]}, {"sent": "she wont hear you how ever loud you shout .", "words": ["how", "you", "hear", "she", "loud"]}, {"sent": "talk loud .", "words": ["loud", "talk"]}, {"sent": "you count loud .", "words": ["loud", "you"]}, {"sent": "uhoh its loud whats that loud sound truck ?", "words": ["truck", "that", "loud"]}, {"sent": "loud noises mean were in trouble .", "words": ["in", "were", "loud"]}, {"sent": "okay loud Anthony .", "words": ["loud"]}, {"sent": "Brittany dont be so loud .", "words": ["dont", "be", "so", "loud"]}, {"sent": "didnt you tell me there was a loud hyena there ?", "words": ["was", "you", "there", "me", "loud", "a"]}, {"sent": "and uh have loud music .", "words": ["have", "loud", "and"]}, {"sent": "goodness me thats loud .", "words": ["me", "loud"]}, {"sent": "it was loud that wasnt it ?", "words": ["was", "it", "that", "loud"]}, {"sent": "he was loud wasnt he ?", "words": ["was", "he", "loud"]}, {"sent": "say it nice and loud so I can hear you .", "words": ["can", "so", "nice", "say", "and", "hear", "you", "it", "loud"]}, {"sent": "I I heard that loud noise yes I certainly did .", "words": ["did", "that", "loud"]}, {"sent": "loud yes its loud .", "words": ["loud"]}, {"sent": "somebody turned the tv on too loud didnt they ?", "words": ["the", "they", "tv", "too", "on", "loud"]}, {"sent": "so very loud puppy dog he is .", "words": ["is", "so", "loud", "he", "puppy", "dog"]}, {"sent": "was there a really loud bang ?", "words": ["was", "loud", "a", "there"]}, {"sent": "is that one loud too ?", "words": ["too", "that", "loud", "is"]}, {"sent": "talk loud .", "words": ["loud", "talk"]}, {"sent": "then I realized shed gotten in the habit talking loud .", "words": ["the", "in", "then", "loud"]}, {"sent": "bet these are loud .", "words": ["these", "are", "loud"]}, {"sent": "whats the opposite of loud ?", "words": ["the", "of", "loud"]}, {"sent": "its too loud ?", "words": ["loud", "too"]}, {"sent": "because it was so loud .", "words": ["was", "so", "it", "loud", "because"]}, {"sent": "uhhuh dont turn it all the way or itll be too loud .", "words": ["the", "all", "be", "dont", "it", "too", "loud"]}, {"sent": "Roms loud tractor .", "words": ["tractor", "loud"]}, {"sent": "how does a loud car sound ?", "words": ["does", "how", "car", "loud", "a"]}, {"sent": "her whistle was as loud as a dozen teakettles .", "words": ["was", "her", "loud", "a"]}, {"sent": "is he getting your nose ?", "words": ["your", "nose", "he", "is"]}, {"sent": "nose .", "words": ["nose"]}, {"sent": "not to pick your nose .", "words": ["your", "pick", "to", "not", "nose"]}, {"sent": "give her a Kleenex and then and then she can blow her nose .", "words": ["can", "blow", "nose", "then", "and", "give", "her", "she", "a"]}, {"sent": "nose .", "words": ["nose"]}, {"sent": "oh does Anchor have a funny nose ?", "words": ["have", "does", "nose", "a"]}, {"sent": "Dexter gets nose bleeds sometimes and he knows what to do .", "words": ["what", "do", "to", "and", "he", "nose"]}, {"sent": "put his nose .", "words": ["his", "put", "nose"]}, {"sent": "look eyes nose mouth .", "words": ["nose", "mouth", "look"]}, {"sent": "lets clean your nose youve got a very dirty nose .", "words": ["clean", "your", "nose", "dirty", "a"]}, {"sent": "wheres his nose ?", "words": ["his", "nose"]}, {"sent": "does you nose itch ?", "words": ["nose", "does", "you"]}, {"sent": "try this nose .", "words": ["this", "try", "nose"]}, {"sent": "xxx nose .", "words": ["nose"]}, {"sent": "wheres Williams nose ?", "words": ["nose"]}, {"sent": "wheres the nose on the hat ?", "words": ["the", "on", "hat", "nose"]}, {"sent": "think you got some vegetables on your nose .", "words": ["think", "your", "you", "on", "some", "nose"]}, {"sent": "wheres the doggys nose ?", "words": ["the", "nose"]}, {"sent": "theres his nose .", "words": ["his", "nose"]}, {"sent": "under your nose .", "words": ["under", "your", "nose"]}, {"sent": "Purdie might not wanna talk to you if you said she has got a big nose .", "words": ["if", "nose", "to", "you", "she", "talk", "wanna", "big", "not", "a"]}, {"sent": "look at my nose look at my nose .", "words": ["nose", "my", "at", "look"]}, {"sent": "nose to nose .", "words": ["to", "nose"]}, {"sent": "blow your nose .", "words": ["your", "blow", "nose"]}, {"sent": "are you pressing Mummys nose ?", "words": ["nose", "are", "you"]}, {"sent": "xxx gonna take Aidans nose .", "words": ["take", "nose"]}, {"sent": "your nose is running .", "words": ["your", "is", "nose"]}, {"sent": "for pulling Thomass nose off .", "words": ["off", "for", "nose"]}, {"sent": "and your nose .", "words": ["nose", "your", "and"]}, {"sent": "or her nose .", "words": ["her", "nose"]}, {"sent": "on his nose ?", "words": ["on", "his", "nose"]}, {"sent": "put your nose in one .", "words": ["in", "put", "your", "nose"]}, {"sent": "look at his nose .", "words": ["his", "nose", "at", "look"]}, {"sent": "round your nose .", "words": ["your", "nose"]}, {"sent": "heres the nose .", "words": ["the", "nose"]}, {"sent": "your nose please .", "words": ["your", "nose"]}, {"sent": "crusty nose .", "words": ["nose"]}, {"sent": "shall I put his nose back ?", "words": ["his", "put", "back", "nose"]}, {"sent": "lets see you nose .", "words": ["nose", "see", "you"]}, {"sent": "what about the nose growing yeah ?", "words": ["the", "what", "about", "nose"]}, {"sent": "getting bit of a runny nose arent you ?", "words": ["of", "nose", "you", "a"]}, {"sent": "you have a mask on your nose ?", "words": ["your", "nose", "you", "have", "on", "a"]}, {"sent": "I dont know where your nose drops are baby .", "words": ["where", "your", "dont", "are", "nose"]}, {"sent": "youve put it on Purdies nose .", "words": ["on", "put", "it", "nose"]}, {"sent": "you have a long nose elephant .", "words": ["long", "you", "have", "elephant", "a", "nose"]}, {"sent": "wheres his nose ?", "words": ["his", "nose"]}, {"sent": "the nose .", "words": ["the", "nose"]}, {"sent": "oh thats the nose where .", "words": ["the", "where", "nose"]}, {"sent": "youve got a yucky nose havent you ?", "words": ["nose", "yucky", "you", "a"]}, {"sent": "he squirted Sally on the nose .", "words": ["on", "the", "he", "nose"]}, {"sent": "he really has got a big nose hasnt he ?", "words": ["nose", "big", "he", "a"]}, {"sent": "oh we hafta get a yucky nose .", "words": ["we", "yucky", "get", "a", "nose"]}, {"sent": "then kissed my puppet on the nose .", "words": ["the", "my", "on", "then", "nose"]}, {"sent": "how about your nose your nose .", "words": ["how", "your", "about", "nose"]}, {"sent": "you wouldnt like it if someone held your nose right ?", "words": ["your", "if", "like", "you", "it", "nose"]}, {"sent": "oh look at that nose .", "words": ["nose", "at", "that", "look"]}, {"sent": "you hurt your your nose and your hair ?", "words": ["your", "hurt", "and", "you", "hair", "nose"]}, {"sent": "plane a big nose ?", "words": ["nose", "big", "a"]}, {"sent": "mommy has to wipe your nose .", "words": ["to", "wipe", "your", "nose"]}, {"sent": "this one with the big red nose .", "words": ["the", "red", "with", "this", "big", "nose"]}, {"sent": "what do you d what do you clean your nose with ?", "words": ["clean", "what", "do", "your", "with", "you", "nose"]}, {"sent": "nose .", "words": ["nose"]}, {"sent": "can I just wipe your nose ?", "words": ["can", "wipe", "your", "nose"]}, {"sent": "nose yes .", "words": ["nose"]}, {"sent": "nose no ?", "words": ["nose"]}, {"sent": "what about wiping your nose ?", "words": ["what", "your", "about", "nose"]}, {"sent": "I think she bit his nose .", "words": ["his", "think", "she", "nose"]}, {"sent": "has he got a big nose ?", "words": ["nose", "big", "he", "a"]}, {"sent": "a baby elephant plays music with her nose .", "words": ["with", "elephant", "her", "a", "nose"]}, {"sent": "what kind of nose does he have ?", "words": ["what", "does", "of", "have", "he", "nose"]}, {"sent": "nose nose nose .", "words": ["nose"]}, {"sent": "and this is going to peck at your nose .", "words": ["your", "is", "to", "and", "this", "at", "nose"]}, {"sent": "how about Daddys nose ?", "words": ["how", "about", "nose"]}, {"sent": "how about I bite your nose ?", "words": ["how", "your", "about", "bite", "nose"]}, {"sent": "theres her nose right there .", "words": ["there", "her", "nose"]}, {"sent": "oh you think he thought thats why he cried because he thought you were gonna use the tissue to wipe his nose ?", "words": ["the", "think", "were", "tissue", "to", "you", "he", "his", "wipe", "why", "because", "nose"]}, {"sent": "you always pick that nose .", "words": ["pick", "nose", "that", "you"]}, {"sent": "the snowman have a nose ?", "words": ["the", "have", "snowman", "a", "nose"]}, {"sent": "you got something in your nose .", "words": ["nose", "in", "your", "you"]}, {"sent": "youre hugging it on its nose now rather than its tail are you ?", "words": ["you", "it", "are", "on", "nose"]}, {"sent": "big long nose ?", "words": ["big", "long", "nose"]}, {"sent": "does your nose come off ?", "words": ["off", "does", "your", "nose"]}, {"sent": "wash your nose okay get some Kleenex .", "words": ["your", "get", "some", "wash", "nose"]}, {"sent": "oh you want me to wipe your nose ?", "words": ["your", "to", "you", "me", "wipe", "nose"]}, {"sent": "xxx funny nose xxx a carrot .", "words": ["a", "nose"]}, {"sent": "stuffy nose .", "words": ["nose"]}, {"sent": "money on the nose .", "words": ["money", "the", "on", "nose"]}, {"sent": "is that your little nose there ?", "words": ["your", "is", "little", "there", "that", "nose"]}, {"sent": "and a nose .", "words": ["nose", "a", "and"]}, {"sent": "let us start with the nose .", "words": ["the", "us", "with", "nose"]}, {"sent": "its much better when your nose isnt running isnt it ?", "words": ["better", "your", "when", "it", "much", "nose"]}, {"sent": "on your nose .", "words": ["on", "your", "nose"]}, {"sent": "your nose is red .", "words": ["red", "your", "is", "nose"]}, {"sent": "rub your nose .", "words": ["your", "nose"]}, {"sent": "where is your nose ?", "words": ["where", "nose", "your", "is"]}, {"sent": "I cant let that stay in your nose .", "words": ["your", "in", "stay", "that", "nose"]}, {"sent": "I think thats a wolf .", "words": ["think", "wolf", "a"]}, {"sent": "yeah but where is the wolf sleeping ?", "words": ["where", "the", "wolf", "is", "but"]}, {"sent": "and say nyah come here mister wolf .", "words": ["here", "wolf", "say", "and"]}, {"sent": "look a wolf .", "words": ["wolf", "a", "look"]}, {"sent": "hes wearing his wolf suit .", "words": ["his", "wolf"]}, {"sent": "a wolf is chasing me .", "words": ["me", "wolf", "is", "a"]}, {"sent": "now you Im the wolf .", "words": ["the", "wolf", "you"]}, {"sent": "its the wolf pretending to be Grandma .", "words": ["the", "to", "wolf", "be"]}, {"sent": "Im tired a dat wolf .", "words": ["wolf", "a", "tired"]}, {"sent": "uncle wolf didnt have chicken stew that night .", "words": ["have", "chicken", "wolf", "that"]}, {"sent": "thats a wolf Im sorry wolf .", "words": ["wolf", "a"]}, {"sent": "whos afraid of the big red wolf .", "words": ["the", "red", "wolf", "of", "big"]}, {"sent": "say wolf .", "words": ["wolf", "say"]}, {"sent": "what happened on Peter and the wolf story ?", "words": ["the", "what", "wolf", "and", "on", "story"]}, {"sent": "quick as a winkle and the third little pig put the lid back on the kettle and that was the end of the wolf .", "words": ["the", "put", "wolf", "back", "of", "was", "pig", "little", "and", "on", "that", "a"]}, {"sent": "it was a wolf .", "words": ["wolf", "was", "it", "a"]}, {"sent": "and before she knew what had happened the wolf turned and ran off .", "words": ["the", "what", "off", "wolf", "and", "she"]}, {"sent": "the wolf pups are friendly I wont be alone .", "words": ["the", "be", "wolf", "are"]}, {"sent": "growled the wolf with a big bad smile .", "words": ["the", "wolf", "with", "bad", "big", "smile", "a"]}, {"sent": "whats a wolf ?", "words": ["wolf", "a"]}, {"sent": "the wolf is gonna eat the cars .", "words": ["the", "wolf", "eat", "is"]}, {"sent": "wolf a wolf .", "words": ["wolf", "a"]}, {"sent": "Im the big bag wolf .", "words": ["the", "wolf", "big"]}, {"sent": "its chocolate chip the wolf licked her lip .", "words": ["the", "chocolate", "her", "wolf"]}, {"sent": "wolf .", "words": ["wolf"]}, {"sent": "remember what the wolf says ?", "words": ["the", "what", "wolf"]}, {"sent": "its a wolf .", "words": ["wolf", "a"]}, {"sent": "thats the wolf pretending to be Grandma isnt it ?", "words": ["the", "wolf", "to", "be", "it"]}, {"sent": "the wolf went in .", "words": ["the", "in", "wolf"]}, {"sent": "cats have ants bird cries wolf and remember the other one ?", "words": ["the", "wolf", "bird", "and", "have", "other"]}, {"sent": "Daddy wolf and the wolf cub .", "words": ["the", "wolf", "and"]}, {"sent": "whos afraid of the big bad wolf tra la la la la .", "words": ["the", "wolf", "of", "bad", "big"]}, {"sent": "its a wolf .", "words": ["wolf", "a"]}, {"sent": "the wolf however followed the pair .", "words": ["the", "wolf"]}, {"sent": "yeah big bad wolf .", "words": ["wolf", "big", "bad"]}, {"sent": "and the wolf says Ill huff .", "words": ["the", "wolf", "and"]}, {"sent": "that says wolf .", "words": ["wolf", "that"]}, {"sent": "after the wolf had a chance to catch his breath .", "words": ["the", "wolf", "to", "catch", "his", "a"]}, {"sent": "shes scared of the wolf .", "words": ["of", "the", "wolf", "scared"]}, {"sent": "think the big bad wolf broke it huh ?", "words": ["the", "think", "wolf", "bad", "it", "big"]}, {"sent": "because I think the Grandma and the wolf when he was pretending to be the Grandma called everyone dear didnt he ?", "words": ["the", "think", "wolf", "was", "to", "be", "and", "he", "when", "because"]}, {"sent": "whats gonna happen to the wolf ?", "words": ["the", "to", "wolf"]}, {"sent": "whos afraid of the big bad wolf ?", "words": ["the", "wolf", "of", "bad", "big"]}, {"sent": "help help the big bad wolf is coming .", "words": ["the", "help", "wolf", "is", "bad", "big"]}, {"sent": "when the wolf came though the wood he saw Porker at the window .", "words": ["the", "wolf", "he", "at", "window", "when"]}, {"sent": "the wolf ran home to his kitchen and began to ?", "words": ["the", "wolf", "to", "and", "kitchen", "his", "home"]}, {"sent": "the hunter came in from outside and he was very cross with that big bad wolf wasnt he ?", "words": ["the", "wolf", "outside", "was", "with", "bad", "and", "in", "big", "he", "that"]}, {"sent": "where does the wolf go first ?", "words": ["where", "does", "the", "wolf", "go", "first"]}, {"sent": "lets find the wolf .", "words": ["the", "wolf", "find"]}, {"sent": "the night Max wore his wolf suit and made mischief of one kind .", "words": ["the", "wolf", "of", "and", "his"]}, {"sent": "timber wolf wuhoh .", "words": ["wolf"]}, {"sent": "yeah bricks and then what does the wolf do ?", "words": ["the", "what", "does", "wolf", "do", "and", "then"]}, {"sent": "that is a wolf .", "words": ["wolf", "that", "a", "is"]}, {"sent": "a wolf .", "words": ["wolf", "a"]}, {"sent": "you think Im gonna read the wolf chicken stew .", "words": ["the", "think", "wolf", "you", "read", "chicken"]}, {"sent": "ooh those are very fast animals a wolf .", "words": ["fast", "wolf", "those", "are", "a"]}, {"sent": "wolf is this your wolf ?", "words": ["this", "wolf", "your", "is"]}, {"sent": "lift the latch and come in said wolf .", "words": ["the", "in", "wolf", "and"]}, {"sent": "the roof the roof and a wolf .", "words": ["the", "wolf", "and", "roof", "a"]}, {"sent": "the big big wolf .", "words": ["the", "wolf", "big"]}, {"sent": "a wolf to eat ?", "words": ["eat", "to", "wolf", "a"]}, {"sent": "does the wolf has a name ?", "words": ["the", "does", "wolf", "a"]}, {"sent": "she seen the wolf did she ?", "words": ["the", "did", "wolf", "she"]}, {"sent": "thats a Daddy wolf and a baby wolf .", "words": ["wolf", "and", "a"]}, {"sent": "and the big bad wolf takes him .", "words": ["the", "wolf", "bad", "and", "him", "big"]}, {"sent": "I hope that big bad wolf doesnt come back .", "words": ["wolf", "back", "bad", "that", "big"]}, {"sent": "do you know what the wolf says ?", "words": ["the", "do", "what", "wolf", "you"]}, {"sent": "its a wolf .", "words": ["wolf", "a"]}, {"sent": "what does wolf say ?", "words": ["what", "does", "say", "wolf"]}, {"sent": "and this is the day when they all the wolf elders are having a meeting .", "words": ["the", "all", "wolf", "is", "and", "this", "they", "are", "when", "a"]}, {"sent": "its the wolf .", "words": ["the", "wolf"]}, {"sent": "I bet hes not even a wolf I bet hes a fox .", "words": ["wolf", "not", "a"]}, {"sent": "wore his wolf suit .", "words": ["his", "wolf"]}, {"sent": "it was the wolf of course and he was hungrier than ever .", "words": ["the", "wolf", "of", "was", "and", "it", "he"]}, {"sent": "kinda like a wolf .", "words": ["wolf", "like", "a"]}, {"sent": "it looks like a wolf .", "words": ["wolf", "it", "like", "a"]}, {"sent": "you think its a wolf ?", "words": ["think", "wolf", "a", "you"]}, {"sent": "before long a hungry wolf knocked .", "words": ["wolf", "hungry", "long", "a"]}, {"sent": "the wolf sprang from the hot water ran f far away .", "words": ["the", "wolf", "hot", "away", "water"]}, {"sent": "the wolf .", "words": ["the", "wolf"]}, {"sent": "whats wolf doing ?", "words": ["wolf"]}, {"sent": "are you the big bad wolf ?", "words": ["the", "wolf", "bad", "you", "are", "big"]}, {"sent": "big bad wolf .", "words": ["wolf", "big", "bad"]}, {"sent": "you wanna be the wolf ?", "words": ["the", "wolf", "be", "you", "wanna"]}, {"sent": "wolf .", "words": ["wolf"]}, {"sent": "so the wolf began .", "words": ["the", "so", "wolf"]}, {"sent": "replied the wolf .", "words": ["the", "wolf"]}, {"sent": "what about the wolf ?", "words": ["the", "what", "wolf", "about"]}, {"sent": "whats the wolf doing here ?", "words": ["the", "wolf", "here"]}, {"sent": "look at that Big Bad wolf is he dead ?", "words": ["wolf", "look", "is", "he", "at", "that"]}, {"sent": "the story of little red riding hood and the wolf .", "words": ["the", "red", "wolf", "of", "little", "and", "story"]}, {"sent": "and whatd the wolf say ?", "words": ["the", "wolf", "say", "and"]}, {"sent": "the night Max wore his wolf suit and made mischief of one kind .", "words": ["the", "wolf", "of", "and", "his"]}, {"sent": "the wolf came and he blew down the house of straw and the house of wood .", "words": ["the", "wolf", "of", "and", "he", "house", "down"]}, {"sent": "but within seconds the wolf let out a deep deafening scream while the three little pigs danced for joy .", "words": ["the", "wolf", "for", "little", "out", "but", "a"]}, {"sent": "who doesnt vote for duh wolf ?", "words": ["wolf", "for", "who"]}, {"sent": "because we go and feed ducks all the time so .", "words": ["the", "all", "go", "we", "so", "and", "feed", "because"]}, {"sent": "what will you feed us ?", "words": ["what", "you", "us", "feed", "will"]}, {"sent": "do you wanna feed her ?", "words": ["do", "you", "wanna", "feed", "her"]}, {"sent": "feed Sandy with the spoon too honey .", "words": ["the", "feed", "with", "spoon", "too"]}, {"sent": "feed the baby .", "words": ["the", "feed"]}, {"sent": "do ya wanna feed the baby ?", "words": ["the", "do", "wanna", "feed"]}, {"sent": "dont feed her .", "words": ["dont", "her", "feed"]}, {"sent": "you could feed it to Dandog couldnt you ?", "words": ["to", "you", "it", "feed", "could"]}, {"sent": "you wanna feed the baby pizza ?", "words": ["the", "pizza", "you", "wanna", "feed"]}, {"sent": "just like we feed you ?", "words": ["feed", "like", "you", "we"]}, {"sent": "no you dont feed the baby .", "words": ["the", "dont", "feed", "you"]}, {"sent": "what did they feed on ?", "words": ["what", "they", "did", "feed", "on"]}, {"sent": "we can feed hm what should we who should we feed ?", "words": ["can", "what", "we", "feed", "who"]}, {"sent": "no Judy said to feed him just once a day .", "words": ["to", "him", "a", "feed"]}, {"sent": "think its time for the babys feed uhn yeah ?", "words": ["the", "think", "for", "feed"]}, {"sent": "you wanna feed the baby ?", "words": ["the", "feed", "wanna", "you"]}, {"sent": "and there was one time we were down here he was interested and he wanted to pretend to feed the doll but this time he could care less .", "words": ["the", "we", "was", "to", "doll", "and", "here", "there", "this", "he", "feed", "pretend", "down", "were", "but", "could"]}, {"sent": "oh you going to feed baby ?", "words": ["to", "feed", "you"]}, {"sent": "can you can we feed the baby ?", "words": ["can", "the", "we", "you", "feed"]}, {"sent": "who are you going to feed now ?", "words": ["to", "you", "are", "feed", "who"]}, {"sent": "should we feed the the cows ?", "words": ["the", "feed", "we"]}, {"sent": "Im not sure if hes trying to feed the baby or just like hey look or let me hit you on the head with this object .", "words": ["the", "look", "if", "to", "head", "with", "like", "you", "this", "feed", "on", "me", "hit", "not"]}, {"sent": "who shall we feed with it ?", "words": ["we", "with", "it", "feed", "who"]}, {"sent": "well we gotta let Mommy wash it before I can feed you because my hands are all dirty .", "words": ["can", "all", "my", "we", "you", "it", "are", "dirty", "feed", "wash", "because"]}, {"sent": "are you gonna feed her ?", "words": ["feed", "are", "her", "you"]}, {"sent": "feed the baby .", "words": ["the", "feed"]}, {"sent": "because youve watched Mummy feed her this morning havent you ?", "words": ["you", "this", "feed", "her", "because"]}, {"sent": "I want the feed trough .", "words": ["the", "feed"]}, {"sent": "are you gonna feed the doll ?", "words": ["the", "doll", "you", "are", "feed"]}, {"sent": "you can feed your dollies too .", "words": ["can", "your", "you", "feed", "too"]}, {"sent": "can you feed her some more ?", "words": ["can", "you", "more", "feed", "some", "her"]}, {"sent": "you can feed it to this bunny .", "words": ["can", "bunny", "to", "you", "this", "it", "feed"]}, {"sent": "I think she wants to feed herself like you do .", "words": ["do", "think", "to", "like", "you", "feed", "she"]}, {"sent": "are you gonna feed his ears ?", "words": ["his", "feed", "are", "you"]}, {"sent": "Jackie would you come and feed your baby ?", "words": ["your", "would", "and", "you", "feed"]}, {"sent": "you gonna feed her ?", "words": ["feed", "her", "you"]}, {"sent": "feed the baby .", "words": ["the", "feed"]}, {"sent": "I tried I tried to get her to feed the baby and no interest in the baby .", "words": ["the", "to", "and", "get", "in", "feed", "her"]}, {"sent": "you can feed her .", "words": ["can", "feed", "her", "you"]}, {"sent": "if you wanna feed her you can .", "words": ["can", "if", "you", "wanna", "feed", "her"]}, {"sent": "what are we going to feed him ?", "words": ["what", "we", "to", "him", "are", "feed"]}, {"sent": "feed the bird xxx thank you .", "words": ["the", "bird", "you", "feed"]}, {"sent": "gonna feed the baby again ?", "words": ["the", "feed"]}, {"sent": "maybe you wanna feed her a little juice or milk .", "words": ["you", "little", "wanna", "feed", "milk", "juice", "her", "a"]}, {"sent": "are you going to feed the doggy ?", "words": ["the", "to", "you", "are", "feed"]}, {"sent": "can I just feed Purdie please Thomas ?", "words": ["can", "feed"]}, {"sent": "who do you want me to feed ?", "words": ["do", "feed", "to", "you", "who", "me"]}, {"sent": "mummy feed who ?", "words": ["who", "feed"]}, {"sent": "do you think they had ducks to feed ?", "words": ["do", "think", "to", "you", "they", "feed"]}, {"sent": "and he wouldnt let Stuart feed him .", "words": ["feed", "he", "him", "and"]}, {"sent": "and then if I do the same thing shes sposta say thank you if I pretend you know feed her something .", "words": ["the", "do", "if", "say", "and", "you", "same", "feed", "pretend", "then", "her"]}, {"sent": "feed her .", "words": ["her", "feed"]}, {"sent": "pick some grass to feed the horse .", "words": ["the", "pick", "to", "feed", "horse", "some", "grass"]}, {"sent": "can you feed the baby ?", "words": ["can", "the", "feed", "you"]}, {"sent": "feed you ?", "words": ["you", "feed"]}, {"sent": "a fire engine can pump water from a street hydrant to feed its hoses .", "words": ["can", "feed", "to", "water", "street", "a"]}, {"sent": "wanna read the other book now hm or shall we feed the baby ?", "words": ["the", "we", "other", "read", "wanna", "feed", "book"]}, {"sent": "you tried to feed a moth this morning didnt you with raisins .", "words": ["to", "with", "you", "this", "feed", "a"]}, {"sent": "can we feed WinnieThePooh ?", "words": ["can", "feed", "we"]}, {"sent": "I think you can take them out of the house and get them to feed all of the animals .", "words": ["can", "the", "think", "all", "of", "to", "them", "you", "and", "get", "house", "out", "feed", "take"]}, {"sent": "some bananas to feed the elephant .", "words": ["the", "to", "elephant", "feed", "some"]}, {"sent": "Im wondering whether to put him down or whether hed take his feed .", "words": ["put", "to", "him", "feed", "his", "down", "take"]}, {"sent": "she likes to feed feed her babydolls at home .", "words": ["feed", "to", "at", "her", "she", "home"]}, {"sent": "you feed the squirrels outside huh ?", "words": ["the", "outside", "feed", "you"]}, {"sent": "want mommy to feed her ?", "words": ["to", "her", "feed"]}, {"sent": "how could you feed him and have another child and do everything ?", "words": ["do", "how", "you", "and", "have", "him", "another", "feed", "could"]}, {"sent": "you want me to feed the baby ?", "words": ["the", "to", "you", "feed", "me"]}, {"sent": "you feed the baby ?", "words": ["the", "feed", "you"]}, {"sent": "are you going to feed the baby ?", "words": ["the", "to", "you", "are", "feed"]}, {"sent": "and what happened last time we went to feed the ducks ?", "words": ["the", "what", "we", "to", "last", "and", "feed"]}, {"sent": "now lets see what can we feed you ?", "words": ["can", "what", "we", "you", "feed", "see"]}, {"sent": "I could feed it to him ?", "words": ["to", "him", "it", "feed", "could"]}, {"sent": "you really know how to feed her that bottle very well .", "words": ["how", "to", "you", "that", "feed", "bottle", "her"]}, {"sent": "feed the animals .", "words": ["the", "feed"]}, {"sent": "you feed him ?", "words": ["feed", "him", "you"]}, {"sent": "well never get you back in if we feed you now .", "words": ["if", "back", "we", "you", "get", "in", "feed"]}, {"sent": "what are we gonna feed baby ?", "words": ["what", "feed", "are", "we"]}, {"sent": "she went to a petting zoo last weekend with her grandparents so she got to feed some birds and things like that .", "words": ["zoo", "to", "last", "so", "with", "like", "and", "that", "feed", "some", "her", "she", "a"]}, {"sent": "xxx youre trying to feed him and hes like rejecting are there any .", "words": ["to", "any", "like", "and", "there", "him", "are", "feed"]}, {"sent": "you going to feed the gorilla a cookie ?", "words": ["the", "cookie", "to", "you", "feed", "a"]}, {"sent": "I needta feed him .", "words": ["him", "feed"]}, {"sent": "oh you wanna feed the baby ?", "words": ["the", "feed", "wanna", "you"]}, {"sent": "did you feed dolly ?", "words": ["feed", "did", "you"]}, {"sent": "did you feed them a hamburger ?", "words": ["them", "you", "hamburger", "did", "feed", "a"]}, {"sent": "did you feed bread to the ducks yesterday Thomas ?", "words": ["the", "bread", "to", "you", "did", "feed"]}, {"sent": "didnt you feed the goats ?", "words": ["the", "feed", "you"]}, {"sent": "Peters Mummy doesnt hafta feed her him .", "words": ["him", "her", "feed"]}, {"sent": "they dont have a bottle so you can feed him .", "words": ["can", "so", "you", "have", "dont", "they", "him", "feed", "bottle", "a"]}, {"sent": "you want your Mummy to feed you juice .", "words": ["your", "to", "you", "feed", "juice"]}, {"sent": "can I feed one to the kitty ?", "words": ["can", "the", "to", "kitty", "feed"]}, {"sent": "why dont you feed the baby up on top of the table ?", "words": ["the", "up", "table", "of", "you", "dont", "feed", "on", "why"]}, {"sent": "I would imagine its the big bird gone to get some worms to feed the baby birds in the nest .", "words": ["the", "feed", "to", "bird", "get", "in", "would", "some", "big"]}, {"sent": "do you wanna feed the baby ?", "words": ["the", "do", "you", "wanna", "feed"]}, {"sent": "who else shall we feed Nina ?", "words": ["feed", "we", "who"]}, {"sent": "do you wanna feed mommy ?", "words": ["do", "feed", "wanna", "you"]}, {"sent": "I was just going to feed the front lawn .", "words": ["to", "was", "the", "feed"]}, {"sent": "so we could feed the ducks .", "words": ["the", "we", "so", "feed", "could"]}, {"sent": "wanna feed the baby ?", "words": ["the", "wanna", "feed"]}, {"sent": "feed the baby ?", "words": ["the", "feed"]}, {"sent": "why dont you sing it ?", "words": ["you", "sing", "dont", "it", "why"]}, {"sent": "sing it again .", "words": ["sing", "it"]}, {"sent": "and you had a little sing song a nursery didnt you ?", "words": ["little", "sing", "you", "and", "a"]}, {"sent": "come on sing .", "words": ["on", "sing"]}, {"sent": "want ta sing guitar ?", "words": ["sing"]}, {"sent": "do you sing a song while you do that ?", "words": ["do", "you", "sing", "that", "a"]}, {"sent": "you sing it for me because you know the words better than I do .", "words": ["the", "better", "do", "for", "you", "sing", "it", "me", "because"]}, {"sent": "sing with you ?", "words": ["sing", "with", "you"]}, {"sent": "we can sing happy birthday to her .", "words": ["can", "happy", "we", "to", "sing", "her"]}, {"sent": "shall we sing the star song ?", "words": ["sing", "the", "star", "we"]}, {"sent": "you sing that for Dimitra .", "words": ["sing", "for", "that", "you"]}, {"sent": "well you can sing it to me if you like .", "words": ["can", "if", "to", "like", "you", "sing", "it", "me"]}, {"sent": "would you like to sing another song ?", "words": ["to", "like", "you", "sing", "another", "would"]}, {"sent": "sing sing sing along .", "words": ["sing"]}, {"sent": "we could sing OldMcDonald couldnt you with all these animals ?", "words": ["all", "these", "we", "with", "you", "sing", "could"]}, {"sent": "when do we sing about left and right ?", "words": ["do", "we", "and", "sing", "about", "when"]}, {"sent": "do you wanna sing one of your songs ?", "words": ["do", "your", "of", "you", "sing", "wanna"]}, {"sent": "oh we could sing a song about a farm couldnt we ?", "words": ["we", "sing", "about", "could", "a"]}, {"sent": "can you sing ?", "words": ["can", "sing", "you"]}, {"sent": "lad you sing it as well .", "words": ["sing", "it", "you"]}, {"sent": "are you gonna sing a song ?", "words": ["sing", "are", "a", "you"]}, {"sent": "what are they gonna sing ?", "words": ["sing", "what", "are", "they"]}, {"sent": "you wouldnt sing AuntPolly had a Dolly and you wouldnt sing the wheels on the bus .", "words": ["the", "and", "sing", "you", "on", "bus", "a"]}, {"sent": "you going to sing PostmanPat to me ?", "words": ["sing", "to", "me", "you"]}, {"sent": "can we sing a minute ?", "words": ["can", "sing", "a", "we"]}, {"sent": "so what did we sing ?", "words": ["what", "we", "so", "sing", "did"]}, {"sent": "why arent you going to sing it ?", "words": ["to", "you", "sing", "it", "why"]}, {"sent": "can you sing that huh ?", "words": ["can", "sing", "that", "you"]}, {"sent": "can you sing me Baa Baa Black Sheep ?", "words": ["can", "me", "sing", "you"]}, {"sent": "because she talk a lot and she sing a lot right ?", "words": ["a lot", "and", "sing", "talk", "she", "because", "a"]}, {"sent": "is everybody going to sing to you ?", "words": ["sing", "to", "you", "is"]}, {"sent": "do you sing pretty songs to mommy ?", "words": ["do", "to", "you", "sing", "pretty"]}, {"sent": "Lara sing it first then mummy sing it .", "words": ["sing", "then", "it", "first"]}, {"sent": "you sing .", "words": ["sing", "you"]}, {"sent": "theyre the trucks that sing pop goes the diesel .", "words": ["the", "sing", "that", "pop"]}, {"sent": "will you sing a song so we can listen to you can lis listen to you sing ?", "words": ["can", "listen", "we", "so", "to", "you", "sing", "will", "a"]}, {"sent": "Ill sing it with Po .", "words": ["sing", "it", "with"]}, {"sent": "youre not gonna sing .", "words": ["sing", "not"]}, {"sent": "play the piano sing the abcs for mommy .", "words": ["the", "play", "for", "sing"]}, {"sent": "whatre you going to sing ?", "words": ["sing", "to", "you"]}, {"sent": "is that the way we sing it ?", "words": ["the", "we", "is", "sing", "it", "that"]}, {"sent": "how about if you sing your song TopherColumbus ?", "words": ["how", "your", "if", "you", "sing", "about"]}, {"sent": "what are you gonna sing this time ?", "words": ["what", "you", "sing", "this", "are"]}, {"sent": "lets sing it again .", "words": ["sing", "it"]}, {"sent": "you sing jingle bells .", "words": ["sing", "you"]}, {"sent": "sing a song .", "words": ["sing", "a"]}, {"sent": "did you sing happy birthday ?", "words": ["sing", "did", "happy", "you"]}, {"sent": "sing xxx .", "words": ["sing"]}, {"sent": "what do we sing at a birthday ?", "words": ["what", "do", "we", "sing", "at", "a"]}, {"sent": "did you sing happy birthday to Dada ?", "words": ["happy", "to", "you", "sing", "did"]}, {"sent": "and a cake and well sing happy birthday to Violet .", "words": ["happy", "cake", "to", "and", "sing", "a"]}, {"sent": "can you sing it Megan ?", "words": ["can", "it", "sing", "you"]}, {"sent": "what kind of song can you sing me ?", "words": ["can", "what", "of", "you", "sing", "me"]}, {"sent": "can you sing do a deer .", "words": ["can", "do", "deer", "you", "sing", "a"]}, {"sent": "hold the baby and sing it .", "words": ["the", "hold", "and", "sing", "it"]}, {"sent": "somebody come and sing this song .", "words": ["sing", "this", "and"]}, {"sent": "okay when you finish up we can sing chico .", "words": ["up", "can", "we", "finish", "you", "sing", "when"]}, {"sent": "can you sing it ?", "words": ["can", "it", "sing", "you"]}, {"sent": "when you blow it shall we sing happy birthday to the year two thousand ?", "words": ["the", "happy", "we", "to", "you", "sing", "it", "when", "blow"]}, {"sent": "can you sing that song ?", "words": ["can", "sing", "that", "you"]}, {"sent": "you sing happy birthday ?", "words": ["sing", "happy", "you"]}, {"sent": "are you going to sing along ?", "words": ["sing", "to", "are", "you"]}, {"sent": "I dont think I can sing Thomas because of this cold .", "words": ["can", "think", "of", "sing", "dont", "this", "cold", "because"]}, {"sent": "do you wanna sing some more ?", "words": ["do", "you", "sing", "more", "wanna", "some"]}, {"sent": "that you sing at play school .", "words": ["school", "play", "you", "sing", "at", "that"]}, {"sent": "Ill play the guitar and you sing this time okay ?", "words": ["the", "play", "you", "sing", "and", "this"]}, {"sent": "sing the whole sing first .", "words": ["sing", "the", "first"]}, {"sent": "you wanna sing the Swedish chef ?", "words": ["sing", "the", "wanna", "you"]}, {"sent": "you can sing happy birthday .", "words": ["can", "sing", "happy", "you"]}, {"sent": "you can sing your own songs .", "words": ["can", "sing", "your", "you"]}, {"sent": "what else do we sing there do you remember any songs ?", "words": ["what", "do", "we", "any", "you", "sing", "there"]}, {"sent": "may be you can sing me a song ?", "words": ["can", "be", "you", "sing", "me", "a"]}, {"sent": "I can sing but as for as that goes being sexy .", "words": ["can", "for", "sing", "that", "but"]}, {"sent": "here that xxx you xxx sing to me .", "words": ["to", "you", "here", "sing", "me", "that"]}, {"sent": "sing the horse song .", "words": ["sing", "the", "horse"]}, {"sent": "can you sing to her ?", "words": ["can", "to", "you", "sing", "her"]}, {"sent": "shall we sing it ?", "words": ["sing", "it", "we"]}, {"sent": "so come on clap your hands and sing .", "words": ["your", "clap", "so", "and", "sing", "on"]}, {"sent": "I dont remember the other part you sing it .", "words": ["the", "you", "sing", "dont", "other", "it"]}, {"sent": "xxx sing her a lullaby ?", "words": ["sing", "her", "a"]}, {"sent": "cmon sing it now .", "words": ["sing", "it"]}, {"sent": "and you press it and it says sing sing sing along .", "words": ["sing", "it", "you", "and"]}, {"sent": "thats what you sing into .", "words": ["sing", "what", "into", "you"]}, {"sent": "are you gonna sing with mommy ?", "words": ["sing", "are", "with", "you"]}, {"sent": "oh well you sing if you want .", "words": ["sing", "if", "you"]}, {"sent": "are you going to sing Baa baa black sheep ?", "words": ["to", "you", "sing", "black", "are", "sheep"]}, {"sent": "Miss Janet and Miss Cheryl sing that ?", "words": ["sing", "that", "and"]}, {"sent": "when she started to sing harry almost bit her hand but he bit the leg of the piano instead .", "words": ["the", "when", "hand", "of", "to", "sing", "he", "leg", "her", "she", "but"]}, {"sent": "let us sing a pretty song .", "words": ["sing", "us", "pretty", "a"]}, {"sent": "sing Rupert the bear for our Mum then .", "words": ["the", "for", "our", "sing", "bear", "then"]}, {"sent": "and what do you do when you sing that ?", "words": ["what", "do", "you", "and", "sing", "that", "when"]}, {"sent": "sing the next one .", "words": ["sing", "the"]}, {"sent": "can you sing a song ?", "words": ["can", "sing", "a", "you"]}, {"sent": "can you sing her the happybirthday song ?", "words": ["can", "the", "you", "sing", "her"]}, {"sent": "are you going to sing your a b c now ?", "words": ["your", "to", "you", "sing", "are", "a"]}, {"sent": "may be he can sing a little song .", "words": ["can", "be", "little", "sing", "he", "a"]}, {"sent": "sing that one .", "words": ["sing", "that"]}, {"sent": "and then we sing .", "words": ["sing", "then", "we", "and"]}, {"sent": "when the pie was open the birds began to sing wasnt that a dainty dish to set before the king ?", "words": ["the", "open", "was", "to", "sing", "dish", "that", "when", "a"]}, {"sent": "can you sing it ?", "words": ["can", "it", "sing", "you"]}, {"sent": "theyre fine .", "words": ["fine"]}, {"sent": "okay fine .", "words": ["fine"]}, {"sent": "its fine to stack them up but not to throw books .", "words": ["up", "throw", "fine", "to", "them", "not", "but"]}, {"sent": "that piece is just fine .", "words": ["fine", "that", "is"]}, {"sent": "ride a cockhorse to BanburyCross to see a fine lady upon a white horse .", "words": ["fine", "to", "horse", "see", "white", "ride", "a"]}, {"sent": "thats fine then .", "words": ["then", "fine"]}, {"sent": "thats fine thank you yes .", "words": ["you", "fine"]}, {"sent": "thats fine .", "words": ["fine"]}, {"sent": "thats fine .", "words": ["fine"]}, {"sent": "thats fine .", "words": ["fine"]}, {"sent": "well that was fine whatever .", "words": ["was", "that", "fine"]}, {"sent": "fine .", "words": ["fine"]}, {"sent": "thats fine with me .", "words": ["me", "with", "fine"]}, {"sent": "the wheels are fine .", "words": ["the", "are", "fine"]}, {"sent": "yah thats fine .", "words": ["fine"]}, {"sent": "Im fine Alysa .", "words": ["fine"]}, {"sent": "yep thats fine .", "words": ["fine"]}, {"sent": "you were doing just fine .", "words": ["fine", "were", "you"]}, {"sent": "two and a half three four weeks somewhere around there so um Ill let you know if I feel like theres a time where either we cant do it here and we hafta go somewhere else or maybe well just go upstairs thats probably fine because um .", "words": ["where", "do", "if", "we", "go", "so", "fine", "like", "and", "you", "there", "here", "it", "around", "because", "a"]}, {"sent": "fine thanks .", "words": ["fine"]}, {"sent": "it looks fine to me .", "words": ["to", "me", "it", "fine"]}, {"sent": "thats fine .", "words": ["fine"]}, {"sent": "thats fine .", "words": ["fine"]}, {"sent": "um ah if this is oh the way she normally is thats fine .", "words": ["the", "if", "is", "fine", "this", "she"]}, {"sent": "fine .", "words": ["fine"]}, {"sent": "oh thats fine .", "words": ["fine"]}, {"sent": "youre doing fine look at all those carrots and raisins .", "words": ["all", "those", "fine", "and", "at", "carrots", "look"]}, {"sent": "and hes fine now isnt he ?", "words": ["he", "fine", "and"]}, {"sent": "which is fine .", "words": ["fine", "which", "is"]}, {"sent": "fine .", "words": ["fine"]}, {"sent": "so the batteries are working fine .", "words": ["the", "so", "are", "fine"]}, {"sent": "thats fine .", "words": ["fine"]}, {"sent": "thats fine .", "words": ["fine"]}, {"sent": "a fine castle me thinks .", "words": ["me", "fine", "a"]}, {"sent": "do you feel fine ?", "words": ["do", "fine", "you"]}, {"sent": "fine xxx firemen on a fire engine going to a fire .", "words": ["on", "to", "a", "fine"]}, {"sent": "yeah thats fine because the last one we had shrunk pretty well so .", "words": ["the", "we", "fine", "so", "last", "pretty", "because"]}, {"sent": "fine one thirty is fine then .", "words": ["then", "is", "fine"]}, {"sent": "youre fine .", "words": ["fine"]}, {"sent": "I know you dont like this but youre fine .", "words": ["fine", "like", "you", "dont", "this", "but"]}, {"sent": "well if he walked in his own doorway thats fine because he lives there .", "words": ["if", "fine", "there", "in", "he", "his", "because"]}, {"sent": "oh okay I think its fine .", "words": ["think", "fine"]}, {"sent": "oh that was fine .", "words": ["was", "that", "fine"]}, {"sent": "it works on the big table just fine .", "words": ["the", "table", "fine", "it", "on", "big"]}, {"sent": "and she was fine the whole time .", "words": ["the", "fine", "was", "and", "she"]}, {"sent": "that sounds fine .", "words": ["that", "fine"]}, {"sent": "shes fine if .", "words": ["if", "fine"]}, {"sent": "okay fine so well bring the things there .", "words": ["the", "fine", "so", "bring", "there"]}, {"sent": "thats one of the reasons like for a long time we werent really worried about the eating thing because she was kind of off the charts in ster in terms of her gross motor fine motor language like everything seemed to be ahead of where it should be so I was like obviously we dont have a stunting issue here .", "words": ["the", "off", "fine", "of", "like", "about", "where", "be", "have", "it", "she", "for", "was", "to", "so", "long", "here", "dont", "because", "a", "we", "in", "her"]}, {"sent": "that ones fine .", "words": ["that", "fine"]}, {"sent": "fine .", "words": ["fine"]}, {"sent": "youre fine my dear .", "words": ["my", "fine"]}, {"sent": "so I guess if you wanna if he wants to go out and get some water or if hes fine just were gonna do a quick interview .", "words": ["water", "do", "if", "go", "fine", "so", "to", "you", "and", "get", "he", "wanna", "out", "some", "were", "a"]}, {"sent": "theyre both fine .", "words": ["fine"]}, {"sent": "okay thats fine .", "words": ["fine"]}, {"sent": "well get on fine xxx you Thomas .", "words": ["get", "on", "you", "fine"]}, {"sent": "theyre fine .", "words": ["fine"]}, {"sent": "very very fine rain .", "words": ["rain", "fine"]}, {"sent": "its totally fine so .", "words": ["so", "fine"]}, {"sent": "xxx fine three bites .", "words": ["fine"]}, {"sent": "fine yeah do .", "words": ["do", "fine"]}, {"sent": "no youre fine you were absolutely fine this is actually .", "words": ["is", "fine", "you", "this", "were"]}, {"sent": "say Im fine .", "words": ["say", "fine"]}, {"sent": "she looks fine now .", "words": ["she", "fine"]}, {"sent": "fine thankyou .", "words": ["fine"]}, {"sent": "alright well look it grew back shes fine .", "words": ["it", "fine", "back", "look"]}, {"sent": "fine .", "words": ["fine"]}, {"sent": "fine .", "words": ["fine"]}, {"sent": "thats fine .", "words": ["fine"]}, {"sent": "youre feeling fine arent you ?", "words": ["you", "fine"]}, {"sent": "um w glass a water will be just fine .", "words": ["fine", "be", "glass", "water", "will", "a"]}, {"sent": "thats fine you can go over there .", "words": ["can", "go", "fine", "you", "there", "over"]}, {"sent": "and they both had very very fine fair hair to begin with .", "words": ["fine", "to", "with", "and", "they", "hair"]}, {"sent": "okay fine .", "words": ["fine"]}, {"sent": "thats fine .", "words": ["fine"]}, {"sent": "thats fine .", "words": ["fine"]}, {"sent": "thats fine .", "words": ["fine"]}, {"sent": "thats fine by me .", "words": ["me", "by", "fine"]}, {"sent": "yes thats fine .", "words": ["fine"]}, {"sent": "fine .", "words": ["fine"]}, {"sent": "xxx yeah its fine .", "words": ["fine"]}, {"sent": "fine .", "words": ["fine"]}, {"sent": "hats fine then .", "words": ["then", "fine"]}, {"sent": "fine .", "words": ["fine"]}, {"sent": "thats fine with me says puppy splashing is the best game of all .", "words": ["the", "all", "game", "is", "fine", "of", "with", "puppy", "me"]}, {"sent": "thats fine .", "words": ["fine"]}, {"sent": "fine .", "words": ["fine"]}, {"sent": "hes pretty hes got a lot of mo mo um fine motor coordination .", "words": ["fine", "of", "a lot", "pretty", "a"]}, {"sent": "no thats fine yeah mhm .", "words": ["fine"]}, {"sent": "thats fine .", "words": ["fine"]}, {"sent": "its fine as long as hes quiet .", "words": ["quiet", "long", "fine"]}, {"sent": "well you was doing a fine job .", "words": ["was", "fine", "a", "you"]}, {"sent": "youre doing fine .", "words": ["fine"]}, {"sent": "yes theyre fine now .", "words": ["fine"]}, {"sent": "this is fine .", "words": ["this", "fine", "is"]}, {"sent": "okay fine .", "words": ["fine"]}, {"sent": "everything is fine lately huh .", "words": ["fine", "is"]}, {"sent": "you think would make a fine quantity for all of us .", "words": ["all", "think", "for", "fine", "of", "you", "us", "would", "make", "a"]}, {"sent": "aye thirty is fine as long as its no more than thirty .", "words": ["more", "fine", "long", "is"]}, {"sent": "were having a very quiet day today arent we ?", "words": ["were", "we", "quiet", "a"]}, {"sent": "called the very quiet cricket .", "words": ["the", "quiet"]}, {"sent": "you cant keep quiet for five minutes anyway .", "words": ["for", "quiet", "you"]}, {"sent": "xxx quiet .", "words": ["quiet"]}, {"sent": "yeah sleepy sheep two is singing a quiet lullaby .", "words": ["is", "sheep", "quiet", "sleepy", "a"]}, {"sent": "so quiet .", "words": ["so", "quiet"]}, {"sent": "how come were being quiet ?", "words": ["how", "were", "quiet"]}, {"sent": "thatll keep it quiet wont it ?", "words": ["it", "quiet"]}, {"sent": "quiet .", "words": ["quiet"]}, {"sent": "always quiet when Thomas isnt here .", "words": ["here", "when", "quiet"]}, {"sent": "we hafta be quiet .", "words": ["be", "quiet", "we"]}, {"sent": "and a quiet old lady was whispering .", "words": ["old", "was", "and", "quiet", "a"]}, {"sent": "one quiet lady whispering hush .", "words": ["quiet"]}, {"sent": "its very quiet now isnt it ?", "words": ["it", "quiet"]}, {"sent": "why do we hafta be quiet ?", "words": ["do", "we", "be", "quiet", "why"]}, {"sent": "thats keeping us quiet isnt it ?", "words": ["us", "it", "quiet"]}, {"sent": "tell your sister to be quiet .", "words": ["to", "be", "your", "quiet"]}, {"sent": "he has gone very quiet .", "words": ["he", "quiet"]}, {"sent": "better be quiet .", "words": ["better", "be", "quiet"]}, {"sent": "youre very quiet .", "words": ["quiet"]}, {"sent": "youre very quiet .", "words": ["quiet"]}, {"sent": "quiet .", "words": ["quiet"]}, {"sent": "man that dog needs to be quiet .", "words": ["to", "be", "quiet", "that", "dog"]}, {"sent": "no a quiet few moments means a quiet few moments without having a finger put down your ear thank you .", "words": ["put", "your", "you", "finger", "quiet", "ear", "down", "a"]}, {"sent": "you are too quiet .", "words": ["too", "are", "quiet", "you"]}, {"sent": "quiet .", "words": ["quiet"]}, {"sent": "one quiet old lady oopsy .", "words": ["old", "quiet"]}, {"sent": "youre so quiet .", "words": ["so", "quiet"]}, {"sent": "hey quiet .", "words": ["quiet"]}, {"sent": "if were quiet and gentle with her she might like to join in .", "words": ["if", "to", "with", "her", "like", "and", "in", "quiet", "gentle", "were", "she"]}, {"sent": "quiet .", "words": ["quiet"]}, {"sent": "Im gonna be quiet and be good .", "words": ["be", "good", "quiet", "and"]}, {"sent": "be quiet darling .", "words": ["be", "quiet"]}, {"sent": "this is a sign that says quiet .", "words": ["is", "this", "quiet", "that", "a"]}, {"sent": "the quiet cricket .", "words": ["the", "quiet"]}, {"sent": "be quiet in there .", "words": ["be", "in", "quiet", "there"]}, {"sent": "one quiet old lady whispering hush .", "words": ["old", "quiet"]}, {"sent": "be quiet .", "words": ["be", "quiet"]}, {"sent": "and then theyll be quiet .", "words": ["be", "then", "quiet", "and"]}, {"sent": "you said youd give anybody ten dollars who would be quiet for two minutes .", "words": ["for", "be", "would", "you", "quiet", "who", "give"]}, {"sent": "I dont hafta be quiet .", "words": ["dont", "be", "quiet"]}, {"sent": "mama whispers be quiet house be quiet mouse be quiet dog be quiet cat be quiet owl and be quiet Nicky .", "words": ["be", "owl", "and", "cat", "house", "quiet", "dog", "mouse"]}, {"sent": "youve got to learn to be quiet Thomas .", "words": ["to", "be", "quiet"]}, {"sent": "Ill be very quiet so they wont hear .", "words": ["be", "so", "hear", "they", "quiet"]}, {"sent": "youre not quiet like a mouse .", "words": ["like", "quiet", "not", "mouse", "a"]}, {"sent": "oh shes very quiet yeah .", "words": ["quiet"]}, {"sent": "but be quiet and dont wake daddy up .", "words": ["up", "be", "and", "dont", "quiet", "wake", "but"]}, {"sent": "oh that was quiet .", "words": ["was", "that", "quiet"]}, {"sent": "I mean peace and quiet .", "words": ["quiet", "and"]}, {"sent": "quiet .", "words": ["quiet"]}, {"sent": "better be quiet then .", "words": ["better", "then", "be", "quiet"]}, {"sent": "but now its quiet .", "words": ["quiet", "but"]}, {"sent": "quiet .", "words": ["quiet"]}, {"sent": "are you gonna go quiet on me today ?", "words": ["go", "you", "are", "quiet", "on", "me"]}, {"sent": "right come on when the house is quiet we needta do some things Thomas .", "words": ["the", "do", "is", "we", "house", "quiet", "on", "some", "when"]}, {"sent": "oh quiet .", "words": ["quiet"]}, {"sent": "nice and quiet .", "words": ["nice", "quiet", "and"]}, {"sent": "while Mummy and Daddy have a nice quiet day with no Warren .", "words": ["nice", "with", "and", "have", "quiet", "a"]}, {"sent": "be quiet .", "words": ["be", "quiet"]}, {"sent": "what do you say when its quiet ?", "words": ["what", "do", "say", "you", "quiet", "when"]}, {"sent": "if you just keep walking hell be quiet .", "words": ["be", "if", "quiet", "you"]}, {"sent": "have you gone quiet on me ?", "words": ["you", "have", "quiet", "on", "me"]}, {"sent": "you are quiet arent you ?", "words": ["are", "quiet", "you"]}, {"sent": "its a quiet fire whistle .", "words": ["quiet", "a"]}, {"sent": "hi sorry to bother you again the dinosaurs are working out pretty good they seem to be getting along and you were right that the other one keeps them quiet .", "words": ["the", "to", "be", "them", "you", "and", "they", "that", "are", "pretty", "other", "out", "good", "quiet", "were"]}, {"sent": "would you are you interested in sitting on your bed and reading a book and having some quiet time ?", "words": ["your", "bed", "you", "and", "in", "are", "quiet", "would", "on", "book", "some", "a"]}, {"sent": "be quiet cant do that on the stairs .", "words": ["the", "do", "be", "quiet", "on", "stairs", "that"]}, {"sent": "thats about the only time your quiet Thomas .", "words": ["the", "quiet", "your", "about"]}, {"sent": "its a very quiet ambulance .", "words": ["quiet", "a"]}, {"sent": "I want you to be quiet for five minutes .", "words": ["for", "to", "be", "you", "quiet"]}, {"sent": "gone quiet havent you ?", "words": ["quiet", "you"]}, {"sent": "sure are quiet .", "words": ["are", "quiet"]}, {"sent": "we said quiet or we said .", "words": ["quiet", "we"]}, {"sent": "goodness youre so quiet .", "words": ["so", "quiet"]}, {"sent": "its quite quiet isnt it ?", "words": ["it", "quiet"]}, {"sent": "its so quiet here with everybody gone isnt it ?", "words": ["so", "with", "here", "it", "quiet"]}, {"sent": "quiet darling .", "words": ["quiet"]}, {"sent": "its okay if youre quiet just xxx just talk to it just xxx .", "words": ["if", "to", "it", "talk", "quiet"]}, {"sent": "the builder was having a quiet lunch and then that happened ?", "words": ["the", "was", "and", "quiet", "then", "that", "a"]}, {"sent": "you want some quiet ?", "words": ["some", "quiet", "you"]}, {"sent": "were a quiet bunch .", "words": ["were", "quiet", "a"]}, {"sent": "Naimas being very quiet .", "words": ["quiet"]}, {"sent": "Rafael be quiet .", "words": ["be", "quiet"]}, {"sent": "Mummys on the telephone now so be quiet .", "words": ["the", "be", "so", "quiet", "on", "telephone"]}, {"sent": "be quiet .", "words": ["be", "quiet"]}, {"sent": "it is quiet .", "words": ["it", "quiet", "is"]}, {"sent": "thats only so Christopher will stay quiet .", "words": ["will", "so", "stay", "quiet"]}, {"sent": "six quiet dogs .", "words": ["quiet"]}, {"sent": "see the quiet old lady ?", "words": ["the", "see", "old", "quiet"]}, {"sent": "are we being quiet today ?", "words": ["are", "quiet", "we"]}, {"sent": "if youre just very quiet shell stay longer .", "words": ["stay", "if", "quiet"]}, {"sent": "are you interested in sitting on your bed and reading a book and having some quiet time ?", "words": ["your", "bed", "you", "and", "in", "are", "quiet", "on", "book", "some", "a"]}, {"sent": "its quiet now .", "words": ["quiet"]}, {"sent": "uhhuh that was a quiet please .", "words": ["was", "that", "quiet", "a"]}, {"sent": "whatre you so quiet about ?", "words": ["so", "about", "quiet", "you"]}, {"sent": "isnt she being quiet ?", "words": ["she", "quiet"]}, {"sent": "be quiet .", "words": ["be", "quiet"]}, {"sent": "couldnt you play with a nice quiet toy instead of the bang bang ?", "words": ["the", "of", "nice", "play", "with", "you", "toy", "quiet", "a"]}, {"sent": "are you always real quiet in the car too when we ride ?", "words": ["the", "we", "car", "ride", "you", "in", "are", "quiet", "too", "when"]}, {"sent": "do you wanna help Mummy tidy up the bedroom ?", "words": ["up", "do", "help", "the", "you", "wanna", "bedroom"]}, {"sent": "have the bedroom just here look .", "words": ["the", "have", "here", "look", "bedroom"]}, {"sent": "is this Jwwws bedroom ?", "words": ["this", "bedroom", "is"]}, {"sent": "is it in Mummys bedroom do you think ?", "words": ["do", "think", "is", "you", "in", "it", "bedroom"]}, {"sent": "is he in your bedroom ?", "words": ["your", "is", "in", "he", "bedroom"]}, {"sent": "yours is in your bedroom isnt it ?", "words": ["your", "is", "in", "it", "bedroom"]}, {"sent": "when were upstairs we usually have the bedroom window open .", "words": ["the", "we", "open", "have", "window", "when", "were", "bedroom"]}, {"sent": "they gonna go to sleep in the bedroom ?", "words": ["the", "go", "sleep", "to", "they", "in", "bedroom"]}, {"sent": "in your bedroom ?", "words": ["in", "your", "bedroom"]}, {"sent": "its time to go in the bedroom .", "words": ["the", "go", "to", "in", "bedroom"]}, {"sent": "what do we have in your bedroom ?", "words": ["what", "do", "your", "we", "have", "in", "bedroom"]}, {"sent": "you want me to take you to your bedroom ?", "words": ["your", "to", "you", "me", "take", "bedroom"]}, {"sent": "in the kitchen or the bedroom ?", "words": ["the", "in", "kitchen", "bedroom"]}, {"sent": "do you wanna look in Frasers bedroom ?", "words": ["do", "you", "in", "wanna", "look", "bedroom"]}, {"sent": "is that her um shes half the jewelery in her bedroom in that bag there .", "words": ["the", "is", "there", "in", "that", "her", "bedroom"]}, {"sent": "youve got a taperecorder in your bedroom .", "words": ["in", "bedroom", "your", "a"]}, {"sent": "yeah this is Jwwws bedroom .", "words": ["this", "bedroom", "is"]}, {"sent": "what did he do when he went into the bedroom ?", "words": ["the", "what", "do", "into", "he", "did", "when", "bedroom"]}, {"sent": "now weve just got to do the bed in Mummy and Daddys bedroom .", "words": ["the", "do", "to", "bed", "and", "in", "bedroom"]}, {"sent": "whats happening in the bedroom ?", "words": ["the", "in", "bedroom"]}, {"sent": "and Mummys friends were staying in Mummys bedroom and Mummy was sleeping on the mattress on the floor in the lounge .", "words": ["the", "was", "and", "in", "on", "were", "bedroom"]}, {"sent": "you came in Mummys bedroom and stopped her going to sleep didnt you ?", "words": ["sleep", "to", "and", "you", "in", "her", "bedroom"]}, {"sent": "will we go up to your bedroom ?", "words": ["up", "your", "go", "we", "to", "will", "bedroom"]}, {"sent": "why dont you nip upstairs to your bedroom quickly and get Superman .", "words": ["your", "to", "you", "and", "dont", "get", "why", "bedroom"]}, {"sent": "where is the bedroom ?", "words": ["where", "the", "bedroom", "is"]}, {"sent": "do you come up and play up in your bedroom by yourself ?", "words": ["up", "do", "yourself", "your", "play", "and", "you", "by", "in", "bedroom"]}, {"sent": "lets go look for it in the bedroom .", "words": ["the", "for", "go", "in", "it", "look", "bedroom"]}, {"sent": "were in your bedroom now arent we ?", "words": ["your", "we", "in", "were", "bedroom"]}, {"sent": "in the bedroom ?", "words": ["the", "in", "bedroom"]}, {"sent": "xxx bedroom and her own bathroom and all .", "words": ["all", "and", "bathroom", "her", "bedroom"]}, {"sent": "thats our bedroom isnt it ?", "words": ["it", "bedroom", "our"]}, {"sent": "sitting at the table or go up to your bedroom or something to do it .", "words": ["the", "up", "do", "your", "table", "go", "to", "it", "at", "bedroom"]}, {"sent": "you wanna go in the bedroom ?", "words": ["the", "go", "you", "in", "wanna", "bedroom"]}, {"sent": "always in your bedroom xxx .", "words": ["in", "your", "bedroom"]}, {"sent": "its in the uh bedroom right there you see it ?", "words": ["the", "you", "there", "in", "it", "see", "bedroom"]}, {"sent": "no dont put it in your bedroom you may walk on it .", "words": ["put", "your", "you", "dont", "in", "it", "on", "walk", "bedroom"]}, {"sent": "Kates got it in her bedroom ?", "words": ["in", "it", "her", "bedroom"]}, {"sent": "I took them outof Mommys bedroom .", "words": ["them", "bedroom"]}, {"sent": "mhm thats right put the bed in the bedroom .", "words": ["the", "put", "bed", "in", "bedroom"]}, {"sent": "theyre in my bedroom arent they ?", "words": ["they", "in", "my", "bedroom"]}, {"sent": "then she went upstairs into the bedroom .", "words": ["the", "into", "then", "she", "bedroom"]}, {"sent": "youre going to your bedroom .", "words": ["to", "your", "bedroom"]}, {"sent": "big bedroom window .", "words": ["window", "big", "bedroom"]}, {"sent": "wanna look in the window in your bedroom ?", "words": ["the", "your", "in", "wanna", "look", "window", "bedroom"]}, {"sent": "in the bedroom .", "words": ["the", "in", "bedroom"]}, {"sent": "in your bedroom ?", "words": ["in", "your", "bedroom"]}, {"sent": "in your bedroom .", "words": ["in", "your", "bedroom"]}, {"sent": "Mummy saw this in her bedroom and she said to Grandma oh please could I take the monkey back with me because Thomas likes monkeys .", "words": ["the", "back", "to", "monkey", "with", "and", "this", "in", "take", "me", "could", "her", "she", "because", "bedroom"]}, {"sent": "who do you usually see when youre having your nappie changed in the bedroom ?", "words": ["the", "do", "your", "you", "in", "who", "see", "when", "bedroom"]}, {"sent": "wanna go in Mummys and Daddys bedroom ?", "words": ["go", "and", "in", "wanna", "bedroom"]}, {"sent": "wheres Frasers bedroom ?", "words": ["bedroom"]}, {"sent": "and he sits on the door knob of your bedroom .", "words": ["the", "bedroom", "your", "of", "and", "he", "on", "door"]}, {"sent": "now you take it up in your bedroom .", "words": ["up", "your", "you", "in", "it", "take", "bedroom"]}, {"sent": "in Ellies bedroom or Johns bedroom ?", "words": ["in", "bedroom"]}, {"sent": "wheres his bedroom ?", "words": ["his", "bedroom"]}, {"sent": "are you going to get a ManUnited bedroom ?", "words": ["bedroom", "to", "you", "get", "are", "a"]}, {"sent": "if to call her go in the bedroom .", "words": ["the", "if", "go", "to", "in", "her", "bedroom"]}, {"sent": "your bedroom ceiling ?", "words": ["your", "bedroom"]}, {"sent": "oh yeah he has a rug in his bedroom .", "words": ["in", "he", "his", "a", "bedroom"]}, {"sent": "do you wanna go to your bedroom ?", "words": ["do", "your", "go", "to", "you", "wanna", "bedroom"]}, {"sent": "is this is this her bedroom ?", "words": ["this", "bedroom", "her", "is"]}, {"sent": "Daddys in the bedroom .", "words": ["the", "in", "bedroom"]}, {"sent": "whats that doing in your bedroom ?", "words": ["your", "in", "that", "bedroom"]}, {"sent": "thats your bedroom tidy .", "words": ["your", "bedroom"]}, {"sent": "I am in mummy and daddys bedroom .", "words": ["in", "bedroom", "am", "and"]}, {"sent": "back in your bedroom .", "words": ["in", "your", "back", "bedroom"]}, {"sent": "what color have you got in your bedroom ?", "words": ["what", "your", "you", "have", "in", "bedroom"]}, {"sent": "into the bedroom .", "words": ["the", "into", "bedroom"]}, {"sent": "words name things in the bedroom .", "words": ["the", "in", "bedroom"]}, {"sent": "a bedroom .", "words": ["bedroom", "a"]}, {"sent": "let us go bedroom .", "words": ["us", "bedroom", "go"]}, {"sent": "and what color are they in your bedroom ?", "words": ["what", "your", "and", "they", "in", "are", "bedroom"]}, {"sent": "they were four bedroom houses .", "words": ["they", "were", "bedroom"]}, {"sent": "did she ask you to go in your bedroom ?", "words": ["your", "go", "to", "you", "in", "did", "she", "bedroom"]}, {"sent": "we had rabbit in Mummys bedroom didnt we ?", "words": ["in", "bedroom", "we"]}, {"sent": "and thats his bedroom .", "words": ["his", "bedroom", "and"]}, {"sent": "we can have a bedroom there .", "words": ["can", "we", "have", "there", "a", "bedroom"]}, {"sent": "do you want Mummy to send you to your bedroom ?", "words": ["do", "your", "to", "you", "bedroom"]}, {"sent": "youre going to hafta stay in this bedroom all the time .", "words": ["the", "all", "to", "this", "in", "stay", "bedroom"]}, {"sent": "bring the diaper from the bedroom and put it on in here .", "words": ["the", "put", "diaper", "bring", "and", "here", "in", "it", "on", "bedroom"]}, {"sent": "the ceiling got cracked in their bedroom didnt it ?", "words": ["the", "their", "in", "it", "bedroom"]}, {"sent": "oh Ill put them in my bedroom .", "words": ["put", "my", "them", "in", "bedroom"]}, {"sent": "Maisy closes her bedroom curtains .", "words": ["her", "bedroom"]}, {"sent": "you want me to see your bedroom .", "words": ["your", "see", "to", "you", "me", "bedroom"]}, {"sent": "in the bedroom okay some people we have a tv in our bedroom dont we .", "words": ["the", "we", "our", "have", "dont", "in", "tv", "some", "a", "bedroom"]}, {"sent": "in the bedroom .", "words": ["the", "in", "bedroom"]}, {"sent": "and wheres Jwwws bedroom ?", "words": ["bedroom", "and"]}, {"sent": "do you wanna go up to your bedroom then ?", "words": ["up", "do", "your", "go", "to", "you", "wanna", "then", "bedroom"]}, {"sent": "xxx do her bedroom up and stencils and stuff .", "words": ["up", "do", "and", "her", "bedroom"]}, {"sent": "mhm for the bedroom .", "words": ["the", "for", "bedroom"]}, {"sent": "what you doing in Johns bedroom ?", "words": ["what", "in", "bedroom", "you"]}, {"sent": "when did you go into Nicoles bedroom ?", "words": ["into", "go", "you", "did", "when", "bedroom"]}, {"sent": "have you got a lorry in your bedroom ?", "words": ["bedroom", "your", "you", "have", "in", "a"]}, {"sent": "Ive got to clean the bedroom xxx .", "words": ["clean", "to", "bedroom", "the"]}, {"sent": "switch out the bedroom light ?", "words": ["the", "light", "out", "bedroom"]}, {"sent": "the bedroom ?", "words": ["the", "bedroom"]}, {"sent": "you want me to take you to your bedroom ?", "words": ["your", "to", "you", "me", "take", "bedroom"]}, {"sent": "in your bedroom .", "words": ["in", "your", "bedroom"]}, {"sent": "dont go and walk in the bedroom .", "words": ["the", "go", "and", "dont", "in", "walk", "bedroom"]}, {"sent": "how many more have you got to knock over ?", "words": ["how", "to", "you", "have", "more", "knock", "over"]}, {"sent": "knock knock .", "words": ["knock"]}, {"sent": "didja knock your head on that ?", "words": ["your", "head", "knock", "on", "that"]}, {"sent": "well what do you mean they knock you out ?", "words": ["what", "do", "you", "they", "knock", "out"]}, {"sent": "and you knock them down .", "words": ["them", "you", "and", "knock", "down"]}, {"sent": "knock .", "words": ["knock"]}, {"sent": "knock knock knock .", "words": ["knock"]}, {"sent": "Ill knock them over .", "words": ["over", "knock", "them"]}, {"sent": "dont knock that over .", "words": ["over", "dont", "knock", "that"]}, {"sent": "knock knock .", "words": ["knock"]}, {"sent": "thats not enough you have to make a tower and then knock it down one two .", "words": ["to", "you", "have", "and", "knock", "it", "then", "down", "make", "not", "a"]}, {"sent": "so he hasta go to the door and knock on the door .", "words": ["the", "go", "to", "so", "and", "knock", "he", "on", "door"]}, {"sent": "whys he gonna knock her over ?", "words": ["over", "her", "knock", "he"]}, {"sent": "Im gonna knock on the door .", "words": ["on", "the", "knock", "door"]}, {"sent": "see if it can knock that one off .", "words": ["can", "off", "if", "it", "knock", "see", "that"]}, {"sent": "knock knock whos there ?", "words": ["there", "knock"]}, {"sent": "do you think hes gonna knock that .", "words": ["do", "think", "you", "knock", "that"]}, {"sent": "knock knock knock .", "words": ["knock"]}, {"sent": "youre going to knock your cornflakes over .", "words": ["over", "to", "knock", "your"]}, {"sent": "you dont wanna knock it down ?", "words": ["you", "dont", "knock", "it", "wanna", "down"]}, {"sent": "no you cant knock it down they dont knock it down on the tellie when they play football uhn do they ?", "words": ["the", "do", "play", "you", "they", "dont", "it", "knock", "on", "down", "when"]}, {"sent": "now xxx you run up just knock on the door and give it to her okay ?", "words": ["up", "the", "to", "you", "and", "run", "knock", "it", "on", "give", "her", "door"]}, {"sent": "but everyone will hafta knock their prices down once these Sainsburys and things come wont they ?", "words": ["these", "their", "and", "they", "knock", "will", "down", "but"]}, {"sent": "knock knock knock knock .", "words": ["knock"]}, {"sent": "did the dog knock her over ?", "words": ["the", "knock", "did", "over", "her", "dog"]}, {"sent": "knock it over .", "words": ["it", "knock", "over"]}, {"sent": "I dont wanna touch these ones incase I knock it down .", "words": ["these", "touch", "dont", "knock", "it", "wanna", "down"]}, {"sent": "ohdear yes that was such a bad knock she had .", "words": ["was", "bad", "knock", "that", "she", "a"]}, {"sent": "because you cant knock it over .", "words": ["you", "it", "knock", "over", "because"]}, {"sent": "Ill build a high one and you can knock it down .", "words": ["can", "you", "and", "knock", "build", "it", "high", "down", "a"]}, {"sent": "well we havent built them up so you cant knock them down can you ?", "words": ["up", "can", "we", "so", "them", "you", "knock", "down"]}, {"sent": "youll knock that cup of tea over .", "words": ["cup", "of", "knock", "over", "that"]}, {"sent": "do not knock down my block wall .", "words": ["do", "my", "block", "knock", "down", "not"]}, {"sent": "knock knock knock knock .", "words": ["knock"]}, {"sent": "dont knock the tub over then .", "words": ["the", "dont", "knock", "then", "over"]}, {"sent": "knock knock .", "words": ["knock"]}, {"sent": "you hafta knock him back a bit .", "words": ["back", "you", "him", "knock", "a"]}, {"sent": "you knock it down ?", "words": ["it", "down", "knock", "you"]}, {"sent": "because youd knock people down wouldnt you ?", "words": ["down", "knock", "because", "you"]}, {"sent": "because youre gonna knock that over .", "words": ["over", "knock", "that", "because"]}, {"sent": "oh and then when the friends knock at the door we open it up ?", "words": ["the", "up", "we", "open", "and", "knock", "it", "at", "then", "when", "door"]}, {"sent": "ya gotta knock over ya gotta knock over my block ?", "words": ["my", "knock", "over", "block"]}, {"sent": "can you knock them over ?", "words": ["can", "them", "you", "knock", "over"]}, {"sent": "well if theres two of them you can knock them down cant you ?", "words": ["can", "if", "of", "them", "you", "knock", "down"]}, {"sent": "hey dont knock it over .", "words": ["dont", "knock", "over", "it"]}, {"sent": "are you in the closet knock knock .", "words": ["the", "closet", "you", "in", "are", "knock"]}, {"sent": "knock it in .", "words": ["it", "in", "knock"]}, {"sent": "youll knock that down .", "words": ["down", "knock", "that"]}, {"sent": "moo moo buzz buzz pop pop clop cockadoodledoo woo woo dibble dibble dop tick tock knock knock boom boom splat last a whisper whisper and that is that .", "words": ["pop", "is", "last", "and", "knock", "that", "a"]}, {"sent": "you knock at the door .", "words": ["the", "you", "knock", "at", "door"]}, {"sent": "did he knock it over ?", "words": ["it", "knock", "he", "did", "over"]}, {"sent": "or knock it down .", "words": ["it", "down", "knock"]}, {"sent": "knock at the door .", "words": ["the", "knock", "door", "at"]}, {"sent": "knock knock knock .", "words": ["knock"]}, {"sent": "knock .", "words": ["knock"]}, {"sent": "hes gonna knock on the door .", "words": ["on", "the", "knock", "door"]}, {"sent": "dont knock it down .", "words": ["dont", "down", "knock", "it"]}, {"sent": "knock knock .", "words": ["knock"]}, {"sent": "did you knock that over ?", "words": ["you", "that", "knock", "did", "over"]}, {"sent": "you wont have white wings if you knock your Ribena over .", "words": ["your", "if", "you", "have", "knock", "white", "over"]}, {"sent": "its gonna get knocked over before I even get a chance to knock it over .", "words": ["to", "get", "knock", "it", "over", "a"]}, {"sent": "knock .", "words": ["knock"]}, {"sent": "youre gonna knock at the door .", "words": ["the", "knock", "door", "at"]}, {"sent": "when you knock on peoples doors they open the door .", "words": ["the", "open", "you", "they", "knock", "on", "when", "door"]}, {"sent": "Will be careful if you knock that over the cookies gonna go bye .", "words": ["the", "if", "go", "be", "you", "knock", "careful", "over", "that"]}, {"sent": "let me knock it down .", "words": ["me", "down", "knock", "it"]}, {"sent": "knock knock .", "words": ["knock"]}, {"sent": "somebodys gonna knock these all over ?", "words": ["all", "these", "knock", "over"]}, {"sent": "I think shes gonna come knock on the door .", "words": ["the", "think", "knock", "on", "door"]}, {"sent": "well stack them up here and well knock knock them down .", "words": ["up", "them", "and", "here", "knock", "down"]}, {"sent": "knock knock knock .", "words": ["knock"]}, {"sent": "no Sefi didnt knock on the door this time .", "words": ["the", "this", "knock", "on", "door"]}, {"sent": "you wont knock them down .", "words": ["down", "knock", "them", "you"]}, {"sent": "knock knock knock knock .", "words": ["knock"]}, {"sent": "oh dont knock it over .", "words": ["dont", "knock", "over", "it"]}, {"sent": "you knock it down with your hand .", "words": ["your", "hand", "with", "you", "it", "knock", "down"]}, {"sent": "knock knock knock knock knock .", "words": ["knock"]}, {"sent": "knock knock .", "words": ["knock"]}, {"sent": "youre gonna knock it over .", "words": ["it", "knock", "over"]}, {"sent": "mike dont knock that down .", "words": ["dont", "down", "knock", "that"]}, {"sent": "dont knock it over .", "words": ["dont", "knock", "over", "it"]}, {"sent": "knock him over oh there he went over .", "words": ["there", "him", "knock", "he", "over"]}, {"sent": "dont knock those clothes over please .", "words": ["over", "dont", "knock", "those"]}, {"sent": "knock .", "words": ["knock"]}, {"sent": "are you gonna knock at the door ?", "words": ["the", "you", "knock", "are", "at", "door"]}, {"sent": "knock knock ?", "words": ["knock"]}, {"sent": "knock .", "words": ["knock"]}, {"sent": "big tower you gonna knock it down ?", "words": ["you", "it", "knock", "down", "big"]}, {"sent": "knock knock knock knock .", "words": ["knock"]}, {"sent": "knock knock knock .", "words": ["knock"]}, {"sent": "okay knock it down .", "words": ["it", "down", "knock"]}, {"sent": "knock on the door ?", "words": ["on", "the", "knock", "door"]}, {"sent": "you wanna knock push the car and knock the cups over ?", "words": ["the", "push", "car", "you", "and", "knock", "wanna", "over"]}, {"sent": "okay hum knock knock .", "words": ["knock"]}, {"sent": "shes got a front door to knock at to come in .", "words": ["to", "in", "knock", "at", "a", "door"]}, {"sent": "okay well see who can knock down the most .", "words": ["can", "the", "knock", "who", "see", "down"]}, {"sent": "untie this shoe .", "words": ["this", "shoe"]}, {"sent": "one two tie your shoe .", "words": ["your", "shoe"]}, {"sent": "all hes got is a shoe .", "words": ["all", "shoe", "a", "is"]}, {"sent": "tie that shoe .", "words": ["that", "shoe"]}, {"sent": "where is Daddys shoe where is Daddys shoe .", "words": ["where", "shoe", "is"]}, {"sent": "shoe .", "words": ["shoe"]}, {"sent": "you gonna put the shoe on the babys foot ?", "words": ["the", "put", "shoe", "you", "foot", "on"]}, {"sent": "where is Daddys shoe .", "words": ["where", "shoe", "is"]}, {"sent": "is that a shoe ?", "words": ["shoe", "that", "a", "is"]}, {"sent": "oh my shoe .", "words": ["my", "shoe"]}, {"sent": "shoe fell off again .", "words": ["off", "shoe"]}, {"sent": "put on this shoe .", "words": ["on", "this", "put", "shoe"]}, {"sent": "shoe ?", "words": ["shoe"]}, {"sent": "and we hafta pick our shoes up at the shoe maker .", "words": ["up", "the", "we", "pick", "our", "and", "shoe", "at"]}, {"sent": "shoe a shoe shoe shi shoe shi shi shi .", "words": ["a", "shoe"]}, {"sent": "Dillon wheres your shoe ?", "words": ["your", "shoe"]}, {"sent": "that is a shoe .", "words": ["shoe", "that", "a", "is"]}, {"sent": "heres a shoe .", "words": ["shoe", "a"]}, {"sent": "where your shoe ?", "words": ["where", "your", "shoe"]}, {"sent": "is that your shoe ?", "words": ["your", "that", "shoe", "is"]}, {"sent": "no shoe ?", "words": ["shoe"]}, {"sent": "no that stays on so you can put his shoe back on .", "words": ["can", "put", "back", "so", "you", "his", "shoe", "on", "that"]}, {"sent": "let me tie your shoe laces here .", "words": ["here", "me", "your", "shoe"]}, {"sent": "oh a shoe .", "words": ["shoe", "a"]}, {"sent": "I just tied that shoe .", "words": ["that", "shoe"]}, {"sent": "white shoe and whats that color ?", "words": ["white", "that", "and", "shoe"]}, {"sent": "that is a shoe .", "words": ["shoe", "that", "a", "is"]}, {"sent": "cobbler cobbler mend my shoe .", "words": ["my", "shoe"]}, {"sent": "see that shoe ?", "words": ["see", "that", "shoe"]}, {"sent": "did you take off your shoe ?", "words": ["off", "your", "you", "shoe", "did", "take"]}, {"sent": "have you got my shoe Amysquamy ?", "words": ["have", "my", "shoe", "you"]}, {"sent": "well do one two buckle my shoe in in a minute .", "words": ["do", "my", "shoe", "in", "a"]}, {"sent": "his shoe .", "words": ["his", "shoe"]}, {"sent": "here is a shoe .", "words": ["here", "shoe", "a", "is"]}, {"sent": "you wanna put the shoe you wanna take this one off .", "words": ["the", "off", "put", "shoe", "you", "this", "wanna", "take"]}, {"sent": "tie your shoe .", "words": ["your", "shoe"]}, {"sent": "and they shoe you the different snowflakes it might look like .", "words": ["the", "like", "you", "shoe", "they", "and", "it", "look"]}, {"sent": "thats a shoe .", "words": ["shoe", "a"]}, {"sent": "leave your shoe on .", "words": ["on", "your", "shoe"]}, {"sent": "shoe .", "words": ["shoe"]}, {"sent": "because he dont have a shoe so he cannot go outside right ?", "words": ["go", "so", "outside", "shoe", "have", "dont", "he", "because", "a"]}, {"sent": "a shoe .", "words": ["shoe", "a"]}, {"sent": "I think its stuck round your shoe .", "words": ["your", "think", "stuck", "shoe"]}, {"sent": "wheres the babys shoe ?", "words": ["the", "shoe"]}, {"sent": "and the heel of that shoe because my shoe fell off the heel of the other one caught that and theres a big black sort of rubber mark from the shoe of my heel .", "words": ["the", "off", "my", "of", "shoe", "and", "other", "big", "black", "that", "because", "a"]}, {"sent": "taking my shoe off .", "words": ["off", "my", "shoe"]}, {"sent": "look you you lost your shoe even in this .", "words": ["your", "shoe", "you", "this", "in", "look"]}, {"sent": "we did lose his shoe on the horse didnt we ?", "words": ["the", "we", "shoe", "horse", "on", "did", "his"]}, {"sent": "I think that is a hat that is a shoe .", "words": ["think", "is", "shoe", "hat", "that", "a"]}, {"sent": "how do we sing the shoe song ?", "words": ["the", "do", "how", "we", "shoe", "sing"]}, {"sent": "put the shoe in here .", "words": ["the", "put", "shoe", "here", "in"]}, {"sent": "you dont want the shoe off .", "words": ["the", "off", "shoe", "you", "dont"]}, {"sent": "can you get the other shoe ?", "words": ["can", "the", "other", "get", "you", "shoe"]}, {"sent": "her shoe did she drop her shoe ?", "words": ["shoe", "drop", "did", "her", "she"]}, {"sent": "you see the shoe ?", "words": ["the", "see", "shoe", "you"]}, {"sent": "well there only is one shoe Thomas .", "words": ["there", "shoe", "is"]}, {"sent": "got the shoe on ?", "words": ["the", "on", "shoe"]}, {"sent": "oh thats her shoe isnt it ?", "words": ["it", "her", "shoe"]}, {"sent": "is that your toes shoe ?", "words": ["your", "that", "shoe", "is"]}, {"sent": "you dont swing your shoe around like that .", "words": ["your", "swing", "like", "shoe", "you", "dont", "around", "that"]}, {"sent": "is what color is Mickeys shoe ?", "words": ["what", "shoe", "is"]}, {"sent": "I dont think a monkey knows how to tie up a shoe .", "words": ["up", "think", "how", "to", "monkey", "shoe", "dont", "a"]}, {"sent": "what color is his shoe ?", "words": ["his", "what", "shoe", "is"]}, {"sent": "one two tie my shoe .", "words": ["my", "shoe"]}, {"sent": "tie this shoe ?", "words": ["this", "shoe"]}, {"sent": "thats a big shoe for Noddy .", "words": ["for", "big", "shoe", "a"]}, {"sent": "I did step on the tinkertoy with my shoe yes .", "words": ["the", "my", "with", "shoe", "did", "on"]}, {"sent": "well see this shows you how you can untie the shoe .", "words": ["can", "the", "how", "you", "shoe", "this", "see"]}, {"sent": "wheres her shoe ?", "words": ["her", "shoe"]}, {"sent": "can you tie a shoe ?", "words": ["can", "shoe", "a", "you"]}, {"sent": "where is Daddys shoe .", "words": ["where", "shoe", "is"]}, {"sent": "we found your shoe .", "words": ["your", "shoe", "we"]}, {"sent": "wheres the other baby shoe ?", "words": ["the", "shoe", "other"]}, {"sent": "you like your shoe ?", "words": ["your", "like", "shoe", "you"]}, {"sent": "theres another shoe .", "words": ["another", "shoe"]}, {"sent": "there is a shoe tied all together .", "words": ["all", "is", "shoe", "there", "a"]}, {"sent": "no shoe ?", "words": ["shoe"]}, {"sent": "Im threading a shoe with a shoe lace .", "words": ["with", "shoe", "a"]}, {"sent": "oh lets get your shoe on .", "words": ["get", "on", "your", "shoe"]}, {"sent": "the baby has a shoe on her foot .", "words": ["the", "shoe", "foot", "on", "her", "a"]}, {"sent": "does babys shoe go on ba .", "words": ["go", "does", "on", "shoe"]}, {"sent": "and one on each shoe .", "words": ["on", "shoe", "each", "and"]}, {"sent": "is the ladybird trying to get your shoe ?", "words": ["the", "your", "is", "to", "shoe", "get"]}, {"sent": "shoe .", "words": ["shoe"]}, {"sent": "you dont wanna eat my shoe .", "words": ["my", "you", "eat", "dont", "shoe", "wanna"]}, {"sent": "you wanna tie her shoe ?", "words": ["shoe", "her", "wanna", "you"]}, {"sent": "dollys shoe ?", "words": ["shoe"]}, {"sent": "what do you wanna do with your shoe ?", "words": ["what", "do", "your", "with", "you", "shoe", "wanna"]}, {"sent": "you can take your shoe off .", "words": ["can", "off", "your", "shoe", "you", "take"]}, {"sent": "wanna tie this shoe ?", "words": ["this", "wanna", "shoe"]}, {"sent": "just like your shoe .", "words": ["your", "like", "shoe"]}, {"sent": "shoe .", "words": ["shoe"]}, {"sent": "Daddys shoe Daddys shoe gets squeaky .", "words": ["shoe"]}, {"sent": "oh okay you wanna put the shoe on ?", "words": ["the", "put", "shoe", "you", "wanna", "on"]}, {"sent": "your shoe has come off .", "words": ["off", "your", "shoe"]}, {"sent": "this is the shoe for her babydoll .", "words": ["the", "for", "is", "shoe", "this", "her"]}, {"sent": "he can go in the shoe .", "words": ["can", "the", "go", "shoe", "in", "he"]}, {"sent": "Lara trod on mummys shoe .", "words": ["on", "shoe"]}, {"sent": "what do you want your shoe off for ?", "words": ["off", "what", "do", "for", "your", "shoe", "you"]}, {"sent": "wheres your shoe ?", "words": ["your", "shoe"]}, {"sent": "its yucky .", "words": ["yucky"]}, {"sent": "theyre not yucky carrots .", "words": ["yucky", "not", "carrots"]}, {"sent": "no not in your mouth that is yucky .", "words": ["your", "is", "yucky", "mouth", "in", "that", "not"]}, {"sent": "Fraser thats yucky .", "words": ["yucky"]}, {"sent": "its yucky Fraser .", "words": ["yucky"]}, {"sent": "who calls you yucky ?", "words": ["yucky", "you", "who"]}, {"sent": "are you yucky ?", "words": ["yucky", "are", "you"]}, {"sent": "yucky .", "words": ["yucky"]}, {"sent": "dont touch the yucky Stef .", "words": ["the", "dont", "yucky", "touch"]}, {"sent": "yucky honey really yuck .", "words": ["yucky"]}, {"sent": "yucky .", "words": ["yucky"]}, {"sent": "that apples yucky isnt it ?", "words": ["yucky", "it", "that"]}, {"sent": "xxx oh you think rats are yucky too ?", "words": ["think", "yucky", "you", "are", "too"]}, {"sent": "its not yucky .", "words": ["yucky", "not"]}, {"sent": "I would think that tastes kinda yucky .", "words": ["yucky", "think", "that", "would"]}, {"sent": "hey k this is yucky .", "words": ["this", "yucky", "is"]}, {"sent": "not yucky .", "words": ["yucky", "not"]}, {"sent": "yucky .", "words": ["yucky"]}, {"sent": "yucky .", "words": ["yucky"]}, {"sent": "thats yucky .", "words": ["yucky"]}, {"sent": "yucky .", "words": ["yucky"]}, {"sent": "its an old green bean Ill go throw it away yucky .", "words": ["old", "throw", "go", "yucky", "an", "it", "away", "green"]}, {"sent": "you think the bats are yucky ?", "words": ["the", "think", "yucky", "you", "are"]}, {"sent": "yucky .", "words": ["yucky"]}, {"sent": "its yucky ?", "words": ["yucky"]}, {"sent": "yucky wasnt it ?", "words": ["yucky", "it"]}, {"sent": "yucky .", "words": ["yucky"]}, {"sent": "oh yucky glasses look at his glasses .", "words": ["yucky", "glasses", "at", "look", "his"]}, {"sent": "oh its a yucky .", "words": ["yucky", "a"]}, {"sent": "yucky .", "words": ["yucky"]}, {"sent": "why is it yucky ?", "words": ["yucky", "it", "why", "is"]}, {"sent": "that looks yucky .", "words": ["yucky", "that"]}, {"sent": "thats yucky .", "words": ["yucky"]}, {"sent": "yucky .", "words": ["yucky"]}, {"sent": "no that is yucky .", "words": ["yucky", "that", "is"]}, {"sent": "yucky yucky .", "words": ["yucky"]}, {"sent": "thats yucky now .", "words": ["yucky"]}, {"sent": "thats yucky .", "words": ["yucky"]}, {"sent": "a bit yucky isnt it ?", "words": ["yucky", "it", "a"]}, {"sent": "xxx okay thats a yucky piece .", "words": ["yucky", "a"]}, {"sent": "yeah lets put the yucky monkey up on the table .", "words": ["the", "up", "put", "table", "monkey", "yucky", "on"]}, {"sent": "stuff is all yucky .", "words": ["all", "yucky", "is"]}, {"sent": "yucky .", "words": ["yucky"]}, {"sent": "this ones yucky .", "words": ["this", "yucky"]}, {"sent": "thats yucky .", "words": ["yucky"]}, {"sent": "its yucky outside strictly yucky .", "words": ["outside", "yucky"]}, {"sent": "yucky yucky .", "words": ["yucky"]}, {"sent": "weve got yucky stuff like that on some of our plants havent we ?", "words": ["we", "of", "yucky", "like", "our", "on", "some", "that"]}, {"sent": "that would be kind of yucky .", "words": ["of", "be", "yucky", "would", "that"]}, {"sent": "yucky .", "words": ["yucky"]}, {"sent": "is it yucky ?", "words": ["yucky", "it", "is"]}, {"sent": "oh thats too yucky .", "words": ["yucky", "too"]}, {"sent": "youd get all yucky wouldnt you ?", "words": ["get", "all", "yucky", "you"]}, {"sent": "because its yucky its all messy and yucky .", "words": ["all", "yucky", "because", "and"]}, {"sent": "yucky or gunky ?", "words": ["yucky"]}, {"sent": "yucky .", "words": ["yucky"]}, {"sent": "yucky yeah .", "words": ["yucky"]}, {"sent": "yucky flies .", "words": ["yucky"]}, {"sent": "yeah its yucky isnt it ?", "words": ["yucky", "it"]}, {"sent": "that ones yucky I think .", "words": ["yucky", "that", "think"]}, {"sent": "that is yucky .", "words": ["yucky", "that", "is"]}, {"sent": "felttips get yucky dont they ?", "words": ["get", "dont", "yucky", "they"]}, {"sent": "yucky .", "words": ["yucky"]}, {"sent": "how did it get yucky ?", "words": ["how", "yucky", "get", "it", "did"]}, {"sent": "yucky Fraser .", "words": ["yucky"]}, {"sent": "not in your mouth yucky .", "words": ["your", "yucky", "mouth", "in", "not"]}, {"sent": "youre yucky now and Ive just changed your nappie .", "words": ["yucky", "your", "and"]}, {"sent": "yucky yucky .", "words": ["yucky"]}, {"sent": "ew that feels yucky .", "words": ["yucky", "that"]}, {"sent": "thats yucky .", "words": ["yucky"]}, {"sent": "yucky .", "words": ["yucky"]}, {"sent": "I dunno maybe that shop over there that yucky shop that smells like smoke .", "words": ["yucky", "like", "there", "that", "over"]}, {"sent": "oh the bats and the rats are both yucky ?", "words": ["the", "yucky", "are", "and"]}, {"sent": "this is yucky .", "words": ["this", "yucky", "is"]}, {"sent": "thats yucky .", "words": ["yucky"]}, {"sent": "those are yucky .", "words": ["yucky", "are", "those"]}, {"sent": "yucky .", "words": ["yucky"]}, {"sent": "yucky .", "words": ["yucky"]}, {"sent": "yucky with the candy no no no no .", "words": ["the", "yucky", "candy", "with"]}, {"sent": "theyre yucky ?", "words": ["yucky"]}, {"sent": "yucky gooey you ?", "words": ["yucky", "you"]}, {"sent": "yucky boy .", "words": ["yucky"]}, {"sent": "is it yucky ?", "words": ["yucky", "it", "is"]}, {"sent": "its salty and yucky .", "words": ["yucky", "and"]}, {"sent": "have you got a yucky nose ?", "words": ["yucky", "you", "have", "a", "nose"]}, {"sent": "it is yucky .", "words": ["yucky", "it", "is"]}, {"sent": "what yucky ones ?", "words": ["what", "yucky"]}, {"sent": "sandwiches yucky ?", "words": ["yucky"]}, {"sent": "theyre all yucky yucky now because he should have picked them ages ago .", "words": ["all", "yucky", "them", "have", "he", "because"]}, {"sent": "now youre hands are gonna be all yucky .", "words": ["all", "be", "yucky", "are"]}, {"sent": "youre all yucky .", "words": ["all", "yucky"]}, {"sent": "you hafta wear special clothes when you go down there because its really yucky down there and .", "words": ["go", "yucky", "you", "and", "there", "down", "when", "because"]}, {"sent": "yucky .", "words": ["yucky"]}, {"sent": "yucky .", "words": ["yucky"]}, {"sent": "thanks for finding the yucky on the deck .", "words": ["the", "yucky", "for", "on"]}, {"sent": "ha er no its yucky now .", "words": ["yucky"]}, {"sent": "yucky .", "words": ["yucky"]}, {"sent": "yucky in his cell ?", "words": ["his", "in", "yucky"]}, {"sent": "do you wanna wake dad up ?", "words": ["up", "do", "you", "wanna", "wake"]}, {"sent": "you can wake up now I think .", "words": ["can", "up", "think", "you", "wake"]}, {"sent": "your babys going to wake up in a little while .", "words": ["up", "your", "to", "little", "in", "wake", "a"]}, {"sent": "Po its wake up time .", "words": ["up", "wake"]}, {"sent": "go and shout daddy to wake up .", "words": ["up", "go", "to", "and", "wake"]}, {"sent": "what does the mommy say when they all wake up ?", "words": ["the", "what", "does", "all", "up", "say", "they", "wake", "when"]}, {"sent": "you didnt wake up .", "words": ["up", "wake", "you"]}, {"sent": "did you wake her up ?", "words": ["up", "you", "did", "wake", "her"]}, {"sent": "wake up time school time .", "words": ["up", "wake", "school"]}, {"sent": "I hadta wake Ross up I hadta wake Mark up for that .", "words": ["up", "wake", "for", "that"]}, {"sent": "hm wake up ?", "words": ["up", "wake"]}, {"sent": "and then after you wake up what do you do ?", "words": ["up", "what", "do", "you", "and", "wake", "then"]}, {"sent": "wake up .", "words": ["up", "wake"]}, {"sent": "what if you go to bed at Bretts house and you wake up at our house .", "words": ["up", "what", "if", "go", "to", "bed", "you", "and", "our", "house", "at", "wake"]}, {"sent": "remember sometimes I come in and wake you up in the morning and Im like Matty wake up wake up and youre still sleeping .", "words": ["up", "the", "like", "you", "and", "in", "wake"]}, {"sent": "but didnt we wake up this morning and stop pulling stop pulling did we wake up this morning and go trick or treating ?", "words": ["up", "we", "go", "stop", "and", "this", "did", "wake", "but"]}, {"sent": "go and wake Baby up .", "words": ["up", "wake", "and", "go"]}, {"sent": "be quiet so you dont wake Tom .", "words": ["be", "so", "you", "dont", "quiet", "wake"]}, {"sent": "you know youre gonna wake up at about ten oclock and tell me youre hungry .", "words": ["up", "me", "hungry", "you", "and", "about", "at", "wake"]}, {"sent": "wake up dolly .", "words": ["up", "wake"]}, {"sent": "ssh we cant wake up the .", "words": ["up", "wake", "the", "we"]}, {"sent": "did Daddy wake you up Fraser ?", "words": ["up", "wake", "did", "you"]}, {"sent": "wake up wake up .", "words": ["up", "wake"]}, {"sent": "you know Eleanor youre not sposta wake come out of your room .", "words": ["your", "of", "you", "room", "out", "wake", "not"]}, {"sent": "trying to wake you up with this .", "words": ["up", "to", "with", "you", "this", "wake"]}, {"sent": "tell her to wake up .", "words": ["up", "to", "her", "wake"]}, {"sent": "what do you do if you wake up in the middle of the night ?", "words": ["up", "what", "do", "the", "if", "of", "you", "in", "wake"]}, {"sent": "breakfast is cooking theres the sun hey wake up the days begun .", "words": ["the", "up", "is", "sun", "wake"]}, {"sent": "dont wake them up .", "words": ["up", "dont", "them", "wake"]}, {"sent": "we can play with the children yes when you wake up .", "words": ["can", "the", "up", "we", "play", "with", "you", "wake", "when"]}, {"sent": "wake up ?", "words": ["up", "wake"]}, {"sent": "you dont wanna wake her up .", "words": ["up", "you", "dont", "wanna", "wake", "her"]}, {"sent": "did you wake up on the in the morning ?", "words": ["up", "the", "you", "in", "did", "on", "wake"]}, {"sent": "when you wake up in the morning its a quarter to one .", "words": ["up", "the", "to", "you", "in", "wake", "when", "a"]}, {"sent": "wake up frog .", "words": ["up", "wake", "frog"]}, {"sent": "so when you wake up in the morning it will be right there .", "words": ["up", "the", "so", "be", "you", "there", "in", "it", "will", "wake", "when"]}, {"sent": "is it time to wake up ?", "words": ["up", "is", "to", "it", "wake"]}, {"sent": "did you wake up ?", "words": ["up", "wake", "did", "you"]}, {"sent": "did you wake me up with a tickle there ?", "words": ["up", "me", "with", "tickle", "you", "there", "did", "wake", "a"]}, {"sent": "when you wake up in the morning .", "words": ["up", "the", "you", "in", "wake", "when"]}, {"sent": "yes I think youre only doing it to wake Purdie up and annoy Purdie .", "words": ["up", "think", "to", "and", "it", "wake"]}, {"sent": "Im gonna turn the page I dont want him to wake up .", "words": ["the", "up", "to", "dont", "him", "wake"]}, {"sent": "Im going to wake you up now .", "words": ["up", "to", "wake", "you"]}, {"sent": "dont wake her up .", "words": ["up", "dont", "her", "wake"]}, {"sent": "and then you take it out when you wake up ?", "words": ["up", "when", "you", "and", "it", "out", "wake", "then", "take"]}, {"sent": "does that wake you up ?", "words": ["up", "does", "you", "wake", "that"]}, {"sent": "youll wake him up .", "words": ["up", "wake", "him"]}, {"sent": "I hadta wake her up didnt I ?", "words": ["up", "wake", "her"]}, {"sent": "one day Miffy wake up .", "words": ["up", "wake"]}, {"sent": "you go to Ginas class you know when you wake up in the afternoon ?", "words": ["up", "the", "go", "to", "you", "in", "wake", "when"]}, {"sent": "I think Daddy better wake up to drink it Anne .", "words": ["up", "better", "think", "drink", "to", "it", "wake"]}, {"sent": "good night sleep tight wake up bright in the morning light do whats right with all your might .", "words": ["up", "the", "do", "all", "your", "sleep", "light", "with", "in", "good", "wake"]}, {"sent": "what did she say to you when she came to wake you up this morning ?", "words": ["up", "what", "to", "say", "you", "this", "did", "wake", "when", "she"]}, {"sent": "what time are you going to wake up darling ?", "words": ["up", "what", "to", "you", "are", "wake"]}, {"sent": "when you wake up daddy and I will be .", "words": ["up", "be", "and", "you", "will", "wake", "when"]}, {"sent": "wake up .", "words": ["up", "wake"]}, {"sent": "wake up .", "words": ["up", "wake"]}, {"sent": "so when we wake up on Christmas morning .", "words": ["up", "we", "so", "on", "wake", "when"]}, {"sent": "wake up .", "words": ["up", "wake"]}, {"sent": "I hafta go to a wake tonight .", "words": ["to", "wake", "a", "go"]}, {"sent": "you dont want anybody to wake up your dolly ?", "words": ["up", "your", "to", "you", "dont", "wake"]}, {"sent": "if you went to sleep in the water it would soon go cold and youd wake up really freezing wouldnt you ?", "words": ["the", "up", "if", "go", "sleep", "to", "would", "you", "and", "in", "it", "water", "wake", "cold"]}, {"sent": "tell him to wake up .", "words": ["up", "to", "him", "wake"]}, {"sent": "wake up .", "words": ["up", "wake"]}, {"sent": "you wont wake her .", "words": ["wake", "her", "you"]}, {"sent": "work pay wake Purdie up and Daddy .", "words": ["up", "wake", "work", "and"]}, {"sent": "I dont want you to wake him up either .", "words": ["up", "to", "you", "dont", "him", "wake"]}, {"sent": "did you wake when did you wake up ?", "words": ["up", "you", "did", "wake", "when"]}, {"sent": "baby angels watch me wake .", "words": ["me", "watch", "wake"]}, {"sent": "wake up wake up wake up .", "words": ["up", "wake"]}, {"sent": "in the morning when you wake up .", "words": ["the", "up", "you", "in", "wake", "when"]}, {"sent": "I wake up every morning I greet the shining sun .", "words": ["up", "the", "sun", "every", "wake"]}, {"sent": "sometimes you wake up and you say nursie .", "words": ["up", "say", "and", "you", "wake"]}, {"sent": "dont shout and wake grandma up .", "words": ["up", "dont", "wake", "and"]}, {"sent": "what time does Eleanor wake up now ?", "words": ["up", "what", "does", "wake"]}, {"sent": "she doesnt wake her Mama up .", "words": ["up", "wake", "her", "she"]}, {"sent": "wake up ?", "words": ["up", "wake"]}, {"sent": "from which he would not wake up for a long while .", "words": ["up", "for", "long", "he", "would", "wake", "which", "not", "a"]}, {"sent": "he wont wake up tomorrow ?", "words": ["up", "wake", "he"]}, {"sent": "dont wake Mommy up .", "words": ["up", "dont", "wake"]}, {"sent": "dont wake Daddy up will you ?", "words": ["up", "you", "dont", "will", "wake"]}, {"sent": "wake up .", "words": ["up", "wake"]}, {"sent": "wake up wake up wake up .", "words": ["up", "wake"]}, {"sent": "oh wake up .", "words": ["up", "wake"]}, {"sent": "dont wake you up again .", "words": ["up", "dont", "wake", "you"]}, {"sent": "you didnt wake up this morning yet did you ?", "words": ["up", "you", "this", "did", "wake"]}, {"sent": "time to wake up .", "words": ["up", "to", "wake"]}, {"sent": "so when we wake up well see all the presents at first thing in the morning .", "words": ["up", "all", "the", "we", "see", "so", "first", "in", "at", "wake", "when"]}, {"sent": "mustnt wake you mustnt wake Jwww up Fraser .", "words": ["up", "wake", "you"]}, {"sent": "you wake up at twelve oclock ?", "words": ["up", "wake", "at", "you"]}, {"sent": "dont wake her up ?", "words": ["up", "dont", "her", "wake"]}, {"sent": "did the bird wake you up ?", "words": ["the", "up", "bird", "you", "did", "wake"]}, {"sent": "wake up wake up wake up you .", "words": ["up", "wake", "you"]}, {"sent": "are you going to wake Sooty up ?", "words": ["up", "to", "you", "are", "wake"]}, {"sent": "she told Jeremy to wake him up ?", "words": ["up", "to", "him", "wake", "she"]}, {"sent": "I think itll be mouldy by the time they wake up .", "words": ["the", "up", "think", "be", "by", "they", "wake"]}, {"sent": "okay is it time to wake up ?", "words": ["up", "is", "to", "it", "wake"]}, {"sent": "did you wake up in the night ?", "words": ["up", "the", "you", "in", "did", "wake"]}, {"sent": "had I better wake up some of these other people ?", "words": ["up", "better", "these", "of", "other", "wake", "some"]}, {"sent": "the slide is broken .", "words": ["the", "slide", "broken", "is"]}, {"sent": "careful youre gonna slide all over the place .", "words": ["the", "all", "slide", "careful", "over"]}, {"sent": "stand with the donkey slide with the sheep .", "words": ["the", "slide", "stand", "with", "sheep", "donkey"]}, {"sent": "but the slide was for your second birthday wasnt it ?", "words": ["the", "slide", "for", "your", "was", "it", "but"]}, {"sent": "wheres the slide going ?", "words": ["the", "slide"]}, {"sent": "yes it came with the slide xxx .", "words": ["the", "it", "with", "slide"]}, {"sent": "you liked the slide ?", "words": ["the", "slide", "you"]}, {"sent": "lets see if it will go down the slide .", "words": ["the", "slide", "if", "go", "it", "will", "see", "down"]}, {"sent": "youre not scared of the slide at PlanetPlay are you ?", "words": ["the", "slide", "of", "scared", "you", "are", "at", "not"]}, {"sent": "slide .", "words": ["slide"]}, {"sent": "he wasnt allowed to play on the slide ?", "words": ["the", "slide", "to", "play", "he", "on"]}, {"sent": "yeah slide .", "words": ["slide"]}, {"sent": "how about mister slide box .", "words": ["how", "slide", "box", "about"]}, {"sent": "it looks like the little mouse is sending parcels down the slide .", "words": ["the", "slide", "is", "like", "little", "it", "down", "mouse"]}, {"sent": "wheres the other end of the slide ?", "words": ["the", "of", "slide", "other"]}, {"sent": "cant go on the slide .", "words": ["on", "the", "slide", "go"]}, {"sent": "wanna see the slide again ?", "words": ["the", "see", "slide", "wanna"]}, {"sent": "put it in there slide him in the little .", "words": ["the", "put", "slide", "little", "there", "in", "it", "him"]}, {"sent": "have a play on the swings and the slide .", "words": ["the", "slide", "play", "and", "have", "on", "a"]}, {"sent": "we can set some aeroplanes off down the slide .", "words": ["can", "off", "the", "slide", "we", "some", "down"]}, {"sent": "slide oh slide .", "words": ["slide"]}, {"sent": "and he told me you needed something special to slide on it .", "words": ["slide", "to", "you", "and", "it", "he", "on", "me"]}, {"sent": "yeah they climb up and then they slide down .", "words": ["up", "slide", "down", "and", "they", "then", "climb"]}, {"sent": "you sit on your slide dont you ?", "words": ["slide", "your", "you", "dont", "on", "sit"]}, {"sent": "he can slide around .", "words": ["can", "slide", "he", "around"]}, {"sent": "but Mom can I go on the slide .", "words": ["can", "the", "slide", "go", "on", "but"]}, {"sent": "you made a slide ?", "words": ["slide", "a", "you"]}, {"sent": "*SI1: wanna come and slide ?", "words": ["slide", "wanna", "and"]}, {"sent": "whoosh down the slide .", "words": ["the", "down", "slide"]}, {"sent": "wanna go down the slide Amy ?", "words": ["the", "slide", "go", "wanna", "down"]}, {"sent": "dont slide .", "words": ["dont", "slide"]}, {"sent": "he comes around t the back of the slide .", "words": ["the", "slide", "back", "of", "he", "around"]}, {"sent": "its got a slide .", "words": ["slide", "a"]}, {"sent": "did you slide off then ?", "words": ["off", "slide", "you", "did", "then"]}, {"sent": "no thats not a slide .", "words": ["slide", "not", "a"]}, {"sent": "you can see the children going down the slide can you ?", "words": ["can", "the", "slide", "you", "see", "down"]}, {"sent": "is the man gonna go down the slide ?", "words": ["the", "slide", "go", "is", "down"]}, {"sent": "slide your chair over .", "words": ["over", "slide", "your", "chair"]}, {"sent": "you know we could build it and then we could slide it underneath after we build it .", "words": ["slide", "we", "then", "you", "and", "it", "build", "could"]}, {"sent": "theyre all on the slide .", "words": ["on", "all", "slide", "the"]}, {"sent": "do you think some of these animals would like to go down the slide ?", "words": ["the", "do", "think", "these", "slide", "go", "of", "to", "like", "you", "would", "some", "down"]}, {"sent": "is that like your slide ?", "words": ["slide", "your", "is", "like", "that"]}, {"sent": "youre not meant to put babies down the slide .", "words": ["the", "put", "slide", "to", "down", "not"]}, {"sent": "*SI1: wanna jump off the slide ?", "words": ["off", "the", "slide", "jump", "wanna"]}, {"sent": "shes going down the slide .", "words": ["the", "down", "slide"]}, {"sent": "under the slide ?", "words": ["the", "under", "slide"]}, {"sent": "why is there a slide ?", "words": ["slide", "is", "there", "why", "a"]}, {"sent": "well we can just slide the barriers underneath like that look .", "words": ["can", "the", "slide", "we", "like", "look", "that"]}, {"sent": "sit down now the mommys the slide .", "words": ["the", "sit", "down", "slide"]}, {"sent": "grandma came down the slide followed by grandpa .", "words": ["the", "down", "slide", "by"]}, {"sent": "was there was there a slide on the boat ?", "words": ["the", "slide", "was", "there", "on", "boat", "a"]}, {"sent": "did you ever slide ?", "words": ["slide", "did", "you"]}, {"sent": "they wanna go on the slide .", "words": ["the", "slide", "go", "they", "wanna", "on"]}, {"sent": "what happened on the slide ?", "words": ["on", "what", "slide", "the"]}, {"sent": "mummy will hafta help baby up the slide .", "words": ["the", "up", "help", "slide", "will"]}, {"sent": "back off the side and slide down .", "words": ["off", "the", "slide", "back", "and", "down"]}, {"sent": "I dont think so its a slide thats the slide .", "words": ["the", "think", "slide", "so", "dont", "a"]}, {"sent": "was there a slide there ?", "words": ["was", "slide", "a", "there"]}, {"sent": "are they gonna slide down the sunflower ?", "words": ["the", "slide", "they", "are", "down"]}, {"sent": "yeah they can slide .", "words": ["can", "they", "slide"]}, {"sent": "thats a ladder but its not a slide .", "words": ["slide", "ladder", "not", "but", "a"]}, {"sent": "she shell go down the swing but shes not as she likes to climb up things and she recently fell off the slide so shell want so shell wanna slide down with you now because she you know she fell off she hit her head .", "words": ["the", "off", "go", "swing", "not", "slide", "with", "down", "she", "up", "to", "so", "head", "and", "you", "because", "climb", "wanna", "hit", "her", "but"]}, {"sent": "slide and what kind of slide is it ?", "words": ["what", "slide", "is", "of", "and", "it"]}, {"sent": "and they had a big metal slide .", "words": ["slide", "and", "they", "big", "a"]}, {"sent": "Bea look at the slide and the swing .", "words": ["the", "slide", "look", "swing", "and", "at"]}, {"sent": "climb up the ladder and slide down the slide .", "words": ["up", "the", "slide", "down", "and", "ladder", "climb"]}, {"sent": "does that one fit on the slide ?", "words": ["the", "does", "slide", "fit", "on", "that"]}, {"sent": "are they on a slide .", "words": ["slide", "they", "are", "on", "a"]}, {"sent": "and weve put a swing and a slide in the garden .", "words": ["the", "put", "slide", "swing", "and", "in", "garden", "a"]}, {"sent": "coming down the slide on her tummy isnt she ?", "words": ["the", "slide", "tummy", "on", "down", "her", "she"]}, {"sent": "the proper slide .", "words": ["the", "slide"]}, {"sent": "he wants to slide over ?", "words": ["over", "to", "slide", "he"]}, {"sent": "well you want ta come and fix the slide again ?", "words": ["the", "slide", "fix", "and", "you"]}, {"sent": "going down the slide okay .", "words": ["the", "down", "slide"]}, {"sent": "that could be a slide look .", "words": ["slide", "be", "look", "could", "that", "a"]}, {"sent": "heres the slide .", "words": ["the", "slide"]}, {"sent": "down the slide .", "words": ["the", "down", "slide"]}, {"sent": "did you go on the slide ?", "words": ["the", "slide", "go", "you", "did", "on"]}, {"sent": "and Mummy and even Mummy went on the slide .", "words": ["on", "the", "slide", "and"]}, {"sent": "your slide is in the shed .", "words": ["the", "slide", "your", "is", "in"]}, {"sent": "Boos blankie what did you do with the blanket on the slide ?", "words": ["the", "what", "do", "slide", "with", "you", "on", "did", "blanket"]}, {"sent": "if you dont slide down Ill beat your hand .", "words": ["slide", "your", "if", "hand", "you", "dont", "down"]}, {"sent": "its a slide .", "words": ["slide", "a"]}, {"sent": "now stop falling off the slide baby .", "words": ["off", "the", "slide", "stop"]}, {"sent": "they wanna go on the slide .", "words": ["the", "slide", "go", "they", "wanna", "on"]}, {"sent": "down the slide thats right .", "words": ["the", "down", "slide"]}, {"sent": "slide ?", "words": ["slide"]}, {"sent": "okay slide back alright slide back a little bit .", "words": ["slide", "little", "back", "a"]}, {"sent": "mummys not allowed on the slide ?", "words": ["on", "the", "slide", "not"]}, {"sent": "what about the slide ?", "words": ["the", "what", "slide", "about"]}, {"sent": "slide .", "words": ["slide"]}, {"sent": "slide it all the way in .", "words": ["the", "all", "slide", "in", "it"]}, {"sent": "you hafta move that hand and let it slide .", "words": ["slide", "hand", "and", "you", "it", "that"]}, {"sent": "they came down the slide .", "words": ["the", "they", "down", "slide"]}, {"sent": "just slide it round .", "words": ["it", "slide"]}, {"sent": "slide thats right Maisys whoosh down the slide .", "words": ["the", "down", "slide"]}, {"sent": "right at the back of your slide .", "words": ["the", "slide", "your", "back", "of", "at"]}, {"sent": "oh that boys going up the slide .", "words": ["up", "the", "slide", "that"]}, {"sent": "pretty soon .", "words": ["pretty"]}, {"sent": "that was a pretty tall one .", "words": ["pretty", "was", "that", "a"]}, {"sent": "oh so pretty .", "words": ["so", "pretty"]}, {"sent": "she looks very pretty .", "words": ["pretty", "she"]}, {"sent": "youre a pretty big boy arent you ?", "words": ["big", "pretty", "you", "a"]}, {"sent": "that ones pretty good I guess .", "words": ["pretty", "that", "good"]}, {"sent": "pretty .", "words": ["pretty"]}, {"sent": "is not it pretty ?", "words": ["it", "pretty", "not", "is"]}, {"sent": "mm pretty .", "words": ["pretty"]}, {"sent": "it mustve been real pretty huh ?", "words": ["it", "pretty"]}, {"sent": "boy those ears are pretty big .", "words": ["pretty", "big", "are", "those"]}, {"sent": "pretty funny isnt he ?", "words": ["he", "pretty"]}, {"sent": "its pretty .", "words": ["pretty"]}, {"sent": "that is pretty .", "words": ["pretty", "that", "is"]}, {"sent": "thats pretty cool .", "words": ["pretty"]}, {"sent": "pretty wild arent they ?", "words": ["they", "pretty"]}, {"sent": "how pretty .", "words": ["how", "pretty"]}, {"sent": "we should probably make some up pretty soon .", "words": ["up", "we", "pretty", "some", "make"]}, {"sent": "that is pretty .", "words": ["pretty", "that", "is"]}, {"sent": "theyre really pretty .", "words": ["pretty"]}, {"sent": "alrighty okay it looks like Im doing pretty good .", "words": ["it", "pretty", "like", "good"]}, {"sent": "theyre pretty straightforward but .", "words": ["pretty", "but"]}, {"sent": "you see the pretty flowers Morgan ?", "words": ["the", "see", "pretty", "you"]}, {"sent": "thats a pretty good treasure .", "words": ["pretty", "good", "a"]}, {"sent": "that is a pretty neat suitcase .", "words": ["pretty", "that", "a", "is"]}, {"sent": "that is pretty .", "words": ["pretty", "that", "is"]}, {"sent": "is that a pretty color ?", "words": ["pretty", "that", "a", "is"]}, {"sent": "and she likes she Imean shes pretty interested in the animal book .", "words": ["the", "and", "in", "animal", "pretty", "book", "she"]}, {"sent": "oh I need some juice Im pretty thirsty .", "words": ["need", "thirsty", "pretty", "some", "juice"]}, {"sent": "sounds like you had a pretty exciting time at the pet shop too right ?", "words": ["the", "like", "you", "pretty", "at", "too", "a"]}, {"sent": "oh youre doing a pretty good job .", "words": ["pretty", "good", "a"]}, {"sent": "you like pretty flower plants ?", "words": ["flower", "pretty", "like", "you"]}, {"sent": "pretty good .", "words": ["pretty", "good"]}, {"sent": "its pretty disgusting you know ?", "words": ["pretty", "you"]}, {"sent": "he is trying pretty hard .", "words": ["pretty", "he", "hard", "is"]}, {"sent": "I bet youll make one of these at school pretty soon ?", "words": ["these", "of", "school", "pretty", "at", "make"]}, {"sent": "thats pretty funny .", "words": ["pretty"]}, {"sent": "oh I see you got the raisins that was pretty good .", "words": ["the", "was", "you", "pretty", "good", "see", "that"]}, {"sent": "eat well my pretty chicken he cried .", "words": ["my", "eat", "pretty", "he", "chicken"]}, {"sent": "thats very pretty .", "words": ["pretty"]}, {"sent": "a lot of them look pretty real its just .", "words": ["of", "a lot", "them", "pretty", "look", "a"]}, {"sent": "I dont know then I think he could throw them pretty far though he probably wont throw them on the ice .", "words": ["the", "think", "throw", "ice", "them", "dont", "he", "pretty", "on", "then", "could"]}, {"sent": "that is pretty .", "words": ["pretty", "that", "is"]}, {"sent": "very pretty ladybirds .", "words": ["pretty"]}, {"sent": "that was pretty confusing Peter .", "words": ["pretty", "was", "that"]}, {"sent": "so theres a pretty strong .", "words": ["so", "pretty", "a"]}, {"sent": "you think the pretty lady in blue will save you from mean Mommy ?", "words": ["the", "think", "you", "blue", "in", "pretty", "will"]}, {"sent": "go ahead make a pretty picture .", "words": ["go", "picture", "pretty", "make", "a"]}, {"sent": "nice pretty spring pictures .", "words": ["nice", "pretty"]}, {"sent": "soccer sounds like its gonna be pretty fun .", "words": ["be", "pretty", "like"]}, {"sent": "that is pretty good .", "words": ["pretty", "that", "good", "is"]}, {"sent": "in fact like we were youknow worried about her protein level for a while because she just loves fruits and veggies so she recognizes pretty much all of them .", "words": ["all", "for", "we", "of", "so", "her", "like", "much", "and", "them", "in", "pretty", "about", "were", "she", "because", "a"]}, {"sent": "she can write pretty good .", "words": ["can", "write", "pretty", "good", "she"]}, {"sent": "no no I I pretty well knew what as you were spreading things out which she would show some interest in and which should .", "words": ["what", "show", "which", "you", "and", "in", "pretty", "out", "would", "some", "were", "she"]}, {"sent": "see how pretty ?", "words": ["see", "how", "pretty"]}, {"sent": "you are pretty .", "words": ["are", "pretty", "you"]}, {"sent": "theres the scooter oh I drew a pretty good scooter .", "words": ["the", "pretty", "good", "a"]}, {"sent": "pretty flowers .", "words": ["pretty"]}, {"sent": "arent they pretty ?", "words": ["they", "pretty"]}, {"sent": "thats a pretty book .", "words": ["book", "pretty", "a"]}, {"sent": "youre so pretty .", "words": ["so", "pretty"]}, {"sent": "youre pretty proud arent you .", "words": ["pretty", "you"]}, {"sent": "isnt it a pretty tree yawned mama bear quite drowsily .", "words": ["tree", "bear", "it", "pretty", "a"]}, {"sent": "oh its a pretty pretty its a pretty pretty jewelry .", "words": ["pretty", "a"]}, {"sent": "if ours dont look good yours dont look so pretty .", "words": ["if", "so", "dont", "pretty", "good", "look"]}, {"sent": "hey Kip you think were gonna be pretty good buddies ?", "words": ["think", "be", "you", "pretty", "good", "were"]}, {"sent": "isnt that a pretty necklace ?", "words": ["pretty", "necklace", "that", "a"]}, {"sent": "are they pretty good ?", "words": ["they", "are", "pretty", "good"]}, {"sent": "okay well great I think thats pretty much it unless you wanted to comment .", "words": ["think", "to", "you", "it", "pretty", "much"]}, {"sent": "okay well let us let us write you wanna write your name on it so everybody knows that pretty map is yours ?", "words": ["your", "is", "so", "write", "you", "us", "it", "pretty", "wanna", "on", "that"]}, {"sent": "hes pretty silly .", "words": ["pretty"]}, {"sent": "shes coming over pretty soon huh ?", "words": ["pretty", "over"]}, {"sent": "is that pretty ?", "words": ["pretty", "that", "is"]}, {"sent": "pretty much .", "words": ["pretty", "much"]}, {"sent": "we dont really havent seen too many like all together but there are some that are pretty loud .", "words": ["all", "we", "like", "dont", "there", "are", "pretty", "too", "some", "that", "loud", "but"]}, {"sent": "pretty soon ?", "words": ["pretty"]}, {"sent": "it looks so pretty .", "words": ["so", "it", "pretty"]}, {"sent": "how pretty .", "words": ["how", "pretty"]}, {"sent": "oh you like that on top of her head cause that makes it is that like a pretty bow ?", "words": ["is", "of", "head", "her", "like", "you", "it", "pretty", "on", "that", "a"]}, {"sent": "oh pretty pretty belly .", "words": ["pretty"]}, {"sent": "what a pretty mirror .", "words": ["what", "pretty", "a"]}, {"sent": "it does look pretty bad down there .", "words": ["does", "bad", "there", "it", "pretty", "look", "down"]}, {"sent": "pretty youll make blue .", "words": ["make", "pretty", "blue"]}, {"sent": "oh those would hafta be pretty big spears .", "words": ["those", "be", "pretty", "would", "big"]}, {"sent": "youre pretty silly .", "words": ["pretty"]}, {"sent": "make a pretty picture for Sissie .", "words": ["for", "picture", "pretty", "make", "a"]}, {"sent": "pretty noisy .", "words": ["pretty", "noisy"]}, {"sent": "thats pretty good .", "words": ["pretty", "good"]}, {"sent": "very pretty like that .", "words": ["that", "pretty", "like"]}, {"sent": "they look like wood sticks and Im pretty sure as a baby you had them and liked them .", "words": ["them", "like", "and", "you", "they", "pretty", "look", "a"]}, {"sent": "pretty pretty ?", "words": ["pretty"]}, {"sent": "you both throw it pretty well dont ya ?", "words": ["throw", "you", "dont", "it", "pretty"]}, {"sent": "what a pretty picture you gonna do some more ?", "words": ["what", "do", "picture", "you", "more", "pretty", "some", "a"]}, {"sent": "look a pretty green .", "words": ["pretty", "green", "a", "look"]}, {"sent": "look how pretty the garden looks when its wet .", "words": ["the", "wet", "how", "garden", "pretty", "look", "when"]}, {"sent": "for decoration to look pretty .", "words": ["to", "for", "pretty", "look"]}, {"sent": "oh thats pretty .", "words": ["pretty"]}, {"sent": "was she ugly or pretty ?", "words": ["was", "pretty", "she"]}, {"sent": "did she say pretty ?", "words": ["pretty", "say", "did", "she"]}, {"sent": "thats for when you spill things .", "words": ["for", "when", "spill", "you"]}, {"sent": "dont spill okay ?", "words": ["dont", "spill"]}, {"sent": "watch it doesnt spill darling .", "words": ["it", "watch", "spill"]}, {"sent": "whaddya spill ?", "words": ["spill"]}, {"sent": "dont spill it .", "words": ["dont", "it", "spill"]}, {"sent": "where did I spill .", "words": ["where", "did", "spill"]}, {"sent": "boy that was really a spill Laura .", "words": ["was", "that", "spill", "a"]}, {"sent": "she didnt spill any soda on the on the table .", "words": ["the", "soda", "table", "any", "spill", "on", "she"]}, {"sent": "dont spill it out .", "words": ["dont", "it", "spill", "out"]}, {"sent": "dont you spill it .", "words": ["dont", "it", "spill", "you"]}, {"sent": "I wont spill it .", "words": ["it", "spill"]}, {"sent": "careful else youll spill it .", "words": ["it", "spill", "careful"]}, {"sent": "I put juice she gawn spill every day .", "words": ["put", "spill", "every", "juice", "she"]}, {"sent": "please remove your hand so it doesnt spill .", "words": ["your", "hand", "so", "it", "spill"]}, {"sent": "and Ill Ill probably have a cup with a lid on so I dont spill it .", "words": ["cup", "so", "with", "and", "have", "dont", "it", "spill", "on", "a"]}, {"sent": "and you even set it on the floor flat sothat it does not spill .", "words": ["the", "does", "you", "and", "it", "spill", "on", "not"]}, {"sent": "well occasionally you spill your juice but it doesnt stop you having juice does it ?", "words": ["does", "your", "stop", "you", "it", "spill", "juice", "but"]}, {"sent": "did ya spill it ?", "words": ["spill", "did ya", "it", "did"]}, {"sent": "did we spill it ?", "words": ["spill", "it", "did", "we"]}, {"sent": "be careful dont spill it .", "words": ["be", "dont", "it", "spill", "careful"]}, {"sent": "thats a good idea you might spill .", "words": ["you", "spill", "good", "a"]}, {"sent": "did you see me spill that ?", "words": ["me", "you", "spill", "did", "see", "that"]}, {"sent": "sip that up before you spill it .", "words": ["up", "you", "it", "spill", "that"]}, {"sent": "I better call the oil spill experts .", "words": ["the", "better", "spill"]}, {"sent": "did you spill it ?", "words": ["spill", "it", "did", "you"]}, {"sent": "dont spill those apples sweetie .", "words": ["dont", "those", "spill"]}, {"sent": "did it spill ?", "words": ["spill", "it", "did"]}, {"sent": "be careful so you dont spill it Honey .", "words": ["be", "so", "you", "dont", "it", "spill", "careful"]}, {"sent": "dont spill it .", "words": ["dont", "it", "spill"]}, {"sent": "where did you spill it ?", "words": ["where", "you", "it", "did", "spill"]}, {"sent": "a little drop spill .", "words": ["spill", "drop", "little", "a"]}, {"sent": "you didnt spill anything ?", "words": ["spill", "you"]}, {"sent": "watch you dont spill it .", "words": ["watch", "you", "dont", "it", "spill"]}, {"sent": "uhoh dont spill the cereal .", "words": ["the", "dont", "cereal", "spill"]}, {"sent": "you cant spill it you hafta keep it up or you hafta close it .", "words": ["up", "you", "it", "spill", "close"]}, {"sent": "spill .", "words": ["spill"]}, {"sent": "Im not going to move it in case I spill it .", "words": ["to", "in", "it", "spill", "not"]}, {"sent": "wait a minute dont spill it .", "words": ["wait", "dont", "it", "spill", "a"]}, {"sent": "Fraser be careful with that drink because if you spill it everywhere Ill be cross .", "words": ["if", "drink", "be", "with", "you", "it", "spill", "careful", "that", "because"]}, {"sent": "dont spill it .", "words": ["dont", "it", "spill"]}, {"sent": "but youre gonna spill it Odera .", "words": ["it", "spill", "but"]}, {"sent": "yeah now it wont spill .", "words": ["it", "spill"]}, {"sent": "dont spill your juice .", "words": ["dont", "juice", "your", "spill"]}, {"sent": "dont spill it .", "words": ["dont", "it", "spill"]}, {"sent": "she asked him to spill it on the floor ?", "words": ["the", "to", "him", "it", "spill", "on", "she"]}, {"sent": "because youre likely to spill it .", "words": ["to", "it", "spill", "because"]}, {"sent": "did he spill water ?", "words": ["spill", "he", "did", "water"]}, {"sent": "its about an oil spill .", "words": ["an", "spill", "about"]}, {"sent": "careful dont spill it .", "words": ["dont", "it", "spill", "careful"]}, {"sent": "weve got to be careful because theyre going to spill all over the place arent they ?", "words": ["the", "all", "to", "be", "they", "spill", "careful", "over", "because"]}, {"sent": "did Mom spill some ?", "words": ["spill", "some", "did"]}, {"sent": "did you ever spill anything ?", "words": ["spill", "did", "you"]}, {"sent": "you dont you dont spill it .", "words": ["dont", "it", "spill", "you"]}, {"sent": "ohno did it spill all over the table ?", "words": ["the", "all", "table", "it", "spill", "did", "over"]}, {"sent": "sit up now so you dont spill it on your skirt .", "words": ["up", "your", "so", "you", "dont", "it", "spill", "on", "sit"]}, {"sent": "did I spill my milk over my eye ?", "words": ["my", "eye", "spill", "did", "milk", "over"]}, {"sent": "who will spill the beans next ?", "words": ["the", "beans", "spill", "who", "will"]}, {"sent": "if you shake them the milk will spill all over the place .", "words": ["the", "all", "if", "will", "them", "shake", "you", "spill", "milk", "over"]}, {"sent": "be careful dont spill .", "words": ["dont", "be", "spill", "careful"]}, {"sent": "better drink it before you spill it .", "words": ["better", "drink", "you", "it", "spill"]}, {"sent": "xxx so you better not spill .", "words": ["better", "so", "you", "spill", "not"]}, {"sent": "youre gonna spill your milk if youre not careful .", "words": ["your", "if", "spill", "careful", "milk", "not"]}, {"sent": "otherwise youll spill it .", "words": ["it", "spill"]}, {"sent": "dont spill it .", "words": ["dont", "it", "spill"]}, {"sent": "youll spill it .", "words": ["it", "spill"]}, {"sent": "thats in the fridge so we wont spill that one .", "words": ["the", "we", "so", "in", "spill", "that"]}, {"sent": "so I wont spill any on my dress ?", "words": ["my", "so", "any", "spill", "on", "dress"]}, {"sent": "we dont wanna spill it on the floor .", "words": ["the", "we", "dont", "it", "spill", "wanna", "on"]}, {"sent": "dont spill it .", "words": ["dont", "it", "spill"]}, {"sent": "how to spill things ?", "words": ["to", "how", "spill"]}, {"sent": "Mommy did Laura spill the milk deliberate or by accident ?", "words": ["the", "by", "spill", "did", "milk"]}, {"sent": "dont spill it .", "words": ["dont", "it", "spill"]}, {"sent": "people do spill tea in tea shops dont they ?", "words": ["do", "dont", "in", "they", "spill"]}, {"sent": "if you tip it up like that its going to spill isnt it ?", "words": ["up", "if", "to", "like", "you", "it", "spill", "that"]}, {"sent": "both spill the beans .", "words": ["the", "beans", "spill"]}, {"sent": "so we dont spill .", "words": ["dont", "so", "spill", "we"]}, {"sent": "I sent this back to you dont spill it .", "words": ["back", "to", "you", "this", "dont", "it", "spill"]}, {"sent": "did xxx spill on his pants when you were there ?", "words": ["you", "there", "on", "pants", "spill", "did", "his", "when", "were"]}, {"sent": "because if it does the juice will spill out .", "words": ["the", "does", "if", "it", "spill", "out", "will", "juice", "because"]}, {"sent": "whoa lets not spill them all out why dont we eat them outta the jar .", "words": ["the", "all", "we", "them", "eat", "dont", "spill", "out", "jar", "not", "why"]}, {"sent": "dont you spill it .", "words": ["dont", "it", "spill", "you"]}, {"sent": "because youre going to spill it .", "words": ["to", "it", "spill", "because"]}, {"sent": "dont spill them .", "words": ["dont", "them", "spill"]}, {"sent": "dont spill your chocolate too .", "words": ["your", "chocolate", "dont", "spill", "too"]}, {"sent": "spill it or pour it ?", "words": ["it", "pour", "spill"]}, {"sent": "watch you dont spill it now .", "words": ["watch", "you", "dont", "it", "spill"]}, {"sent": "yes its breathing the fumes from the oil spill .", "words": ["the", "spill"]}, {"sent": "watch you dont spill it .", "words": ["watch", "you", "dont", "it", "spill"]}, {"sent": "no because it will spill .", "words": ["will", "it", "spill", "because"]}, {"sent": "Peter put your milk on the table its gonna spill hm .", "words": ["the", "put", "your", "table", "on", "spill", "milk"]}, {"sent": "you want me to be the farmer and I dont know about the oil spill ?", "words": ["the", "to", "be", "you", "and", "dont", "spill", "about", "me"]}, {"sent": "Laura thats gonna spill on the floor and I wont like it will I ?", "words": ["the", "will", "like", "and", "it", "spill", "on"]}, {"sent": "theres an oil spill and my cat is in the oil spill .", "words": ["the", "my", "is", "and", "cat", "an", "in", "spill"]}, {"sent": "youre gonna spill it I know it .", "words": ["it", "spill"]}, {"sent": "dont spill it .", "words": ["dont", "it", "spill"]}, {"sent": "did you spill some ?", "words": ["spill", "some", "did", "you"]}, {"sent": "then it wont spill .", "words": ["then", "it", "spill"]}, {"sent": "they stay strong and healthy the brown horse eats all day in the stable or the field he munches on some hay .", "words": ["the", "all", "brown", "and", "they", "in", "on", "he", "horse", "stay", "some"]}, {"sent": "say brown .", "words": ["say", "brown"]}, {"sent": "red brown .", "words": ["red", "brown"]}, {"sent": "and this is a brown cat .", "words": ["brown", "is", "and", "this", "cat", "a"]}, {"sent": "well hafta do something with the brown and black ones .", "words": ["the", "do", "brown", "with", "and", "black"]}, {"sent": "it was about the big brown hare wasnt it ?", "words": ["the", "brown", "was", "it", "about", "big"]}, {"sent": "once upon a time there was a big farm with all kinds of animals on it even a funny brown dog .", "words": ["all", "brown", "of", "was", "with", "there", "it", "on", "big", "dog", "a"]}, {"sent": "brown and yellow .", "words": ["yellow", "and", "brown"]}, {"sent": "to put the brown sugar in .", "words": ["the", "put", "brown", "to", "in"]}, {"sent": "and thats brown .", "words": ["brown", "and"]}, {"sent": "brown .", "words": ["brown"]}, {"sent": "thats brown .", "words": ["brown"]}, {"sent": "you need a brown cat back .", "words": ["need", "back", "brown", "you", "cat", "a"]}, {"sent": "lets see this one is a brown sturgeon and yellow pork fish yep I think we saw these .", "words": ["these", "think", "brown", "is", "yellow", "we", "and", "this", "fish", "see", "a"]}, {"sent": "more brown windows ?", "words": ["more", "brown"]}, {"sent": "does it have a little brown piece ?", "words": ["does", "brown", "little", "have", "it", "a"]}, {"sent": "mister brown can buzz .", "words": ["can", "brown"]}, {"sent": "big brown bear .", "words": ["bear", "big", "brown"]}, {"sent": "the brown .", "words": ["the", "brown"]}, {"sent": "this little piggy had brown bread and butter .", "words": ["bread", "brown", "and", "little", "this", "butter"]}, {"sent": "brown .", "words": ["brown"]}, {"sent": "do you want brown bread or cereal ?", "words": ["do", "bread", "brown", "you", "cereal"]}, {"sent": "mamma brown bear smiled I guess we could do a little more .", "words": ["do", "we", "brown", "bear", "little", "more", "could", "a"]}, {"sent": "heres a teensie piece of brown .", "words": ["of", "brown", "a"]}, {"sent": "oh the wonderful sounds mister brown can do he can sound like a cow .", "words": ["the", "do", "can", "brown", "like", "he", "cow", "a"]}, {"sent": "I was dreaming last night how about how bad it was for me to have brown sugar .", "words": ["how", "for", "brown", "was", "last", "to", "bad", "have", "it", "about", "me"]}, {"sent": "I love you all the way up to your toes said big nut brown hare swinging little nut brown hare up over his head .", "words": ["the", "all", "up", "your", "brown", "to", "head", "you", "little", "his", "love", "over", "big"]}, {"sent": "and both grandmas have got dark brown eyes .", "words": ["have", "brown", "dark", "and"]}, {"sent": "some sort of orangey brown and some dark brown .", "words": ["brown", "of", "and", "dark", "some"]}, {"sent": "Id quite like to go to a cafe where they sell brown biscuits that look like swans .", "words": ["where", "go", "brown", "to", "like", "they", "look", "that", "a"]}, {"sent": "was it like a bench a long brown bench ?", "words": ["brown", "was", "like", "long", "it", "bench", "a"]}, {"sent": "big brown bird .", "words": ["bird", "big", "brown"]}, {"sent": "was it a brown bag ?", "words": ["brown", "was", "it", "a"]}, {"sent": "and brown .", "words": ["brown", "and"]}, {"sent": "its a brown horsie .", "words": ["brown", "a"]}, {"sent": "gee you like that brown crayon .", "words": ["brown", "crayon", "like", "you", "that"]}, {"sent": "mm yeah thats brown this is the black one you can have the black one .", "words": ["the", "can", "is", "brown", "you", "black", "this", "have"]}, {"sent": "and there looks to be a blue bird a pink rabbit a yellow fish and brown horse .", "words": ["brown", "to", "be", "bird", "and", "blue", "there", "fish", "horse", "yellow", "a"]}, {"sent": "dark brown .", "words": ["brown", "dark"]}, {"sent": "thatll have lotsof brown on it .", "words": ["have", "on", "it", "brown"]}, {"sent": "thats a nice brown and white horse isnt it ?", "words": ["brown", "nice", "and", "it", "horse", "white", "a"]}, {"sent": "round marks or brown marks ?", "words": ["brown"]}, {"sent": "brown has gone a bit funny hasnt it ?", "words": ["it", "a", "brown"]}, {"sent": "and with the brown crayon .", "words": ["the", "brown", "crayon", "with", "and"]}, {"sent": "yellow boxes a brown one ?", "words": ["yellow", "brown", "a"]}, {"sent": "two brown pencils .", "words": ["brown"]}, {"sent": "you dont eat brown bread ?", "words": ["bread", "brown", "you", "eat", "dont"]}, {"sent": "this brown cat is standing up .", "words": ["up", "is", "brown", "this", "cat"]}, {"sent": "to stay strong and healthy the brown horse eats all day .", "words": ["the", "all", "brown", "to", "and", "horse", "stay"]}, {"sent": "after weve wrapped them in pretty paper well wrap them again in brown paper .", "words": ["brown", "paper", "them", "in", "pretty"]}, {"sent": "its in the brown one .", "words": ["the", "in", "brown"]}, {"sent": "brown into the box .", "words": ["the", "box", "into", "brown"]}, {"sent": "brown ?", "words": ["brown"]}, {"sent": "and what about this brown bear ?", "words": ["what", "brown", "and", "bear", "this", "about"]}, {"sent": "and I had a brown dress ?", "words": ["brown", "dress", "a", "and"]}, {"sent": "the brown chocolate icecream .", "words": ["the", "chocolate", "brown"]}, {"sent": "this is a brown cat thats standing up .", "words": ["up", "brown", "is", "this", "cat", "a"]}, {"sent": "once all of the animals were out playing on the farm when along came a big brown monkey and he jumped right in to the middle of the farm and said .", "words": ["the", "all", "when", "brown", "of", "to", "monkey", "and", "big", "in", "he", "out", "on", "were", "a"]}, {"sent": "DryadsSaddle has a pale yellow brown cap with darker scales .", "words": ["yellow", "brown", "with", "a"]}, {"sent": "but hes hes has Jim got brown eyes ?", "words": ["but", "brown"]}, {"sent": "brown bricks ?", "words": ["brown"]}, {"sent": "brown .", "words": ["brown"]}, {"sent": "Eleanor whats the brown thing in the icecream ?", "words": ["the", "in", "brown"]}, {"sent": "the squirrels eyes are brown .", "words": ["the", "are", "brown"]}, {"sent": "wow arent there a lot of brown houses ?", "words": ["brown", "of", "a lot", "there", "a"]}, {"sent": "it is brown .", "words": ["brown", "it", "is"]}, {"sent": "it does look a little bit like brown doesnt it ?", "words": ["does", "brown", "like", "little", "it", "look", "a"]}, {"sent": "when I looked at your foot last night I thought your foot was gonna be brown forever .", "words": ["your", "brown", "was", "last", "be", "foot", "at", "when"]}, {"sent": "mm brown .", "words": ["brown"]}, {"sent": "heres a brown .", "words": ["brown", "a"]}, {"sent": "so now we have a red yes we draw them we have a red circle we have a red triangle a brown square are there more crayons ?", "words": ["red", "we", "brown", "so", "draw", "them", "have", "there", "more", "are", "a"]}, {"sent": "are they red or are they brown ?", "words": ["red", "they", "are", "brown"]}, {"sent": "brown and orange .", "words": ["orange", "and", "brown"]}, {"sent": "just the brown ones .", "words": ["the", "brown"]}, {"sent": "a little brown puppy jumped up and grabbed the balloon strings with his teeth .", "words": ["up", "the", "brown", "with", "little", "and", "puppy", "balloon", "his", "a"]}, {"sent": "right Shirley has brown eyes .", "words": ["brown"]}, {"sent": "do you think its brown ?", "words": ["do", "think", "brown", "you"]}, {"sent": "brown .", "words": ["brown"]}, {"sent": "brown .", "words": ["brown"]}, {"sent": "maybe kind of brown or maybe gray .", "words": ["of", "brown"]}, {"sent": "heres brown .", "words": ["brown"]}, {"sent": "how do how do um people get brown skin ?", "words": ["get", "do", "how", "brown"]}, {"sent": "you have brown bear book .", "words": ["brown", "bear", "have", "you", "book"]}, {"sent": "would you like a brown car ?", "words": ["brown", "car", "like", "you", "would", "a"]}, {"sent": "two brown .", "words": ["brown"]}, {"sent": "here is brown .", "words": ["here", "brown", "is"]}, {"sent": "a brown square are there more crayons ?", "words": ["brown", "there", "more", "are", "a"]}, {"sent": "but its brown .", "words": ["but", "brown"]}, {"sent": "brown bear ?", "words": ["bear", "brown"]}, {"sent": "heres one with some brown long hair .", "words": ["brown", "with", "long", "some", "hair"]}, {"sent": "big brown eyes .", "words": ["big", "brown"]}, {"sent": "what little brown boy Thomas ?", "words": ["what", "brown", "little"]}, {"sent": "brown as a .", "words": ["a", "brown"]}, {"sent": "Tobys brown .", "words": ["brown"]}, {"sent": "little little brown baby mouse .", "words": ["brown", "mouse", "little"]}, {"sent": "make brown .", "words": ["make", "brown"]}, {"sent": "use the brown one now .", "words": ["the", "brown"]}, {"sent": "brown .", "words": ["brown"]}, {"sent": "I hope your tooth fall falls out .", "words": ["tooth", "your", "fall", "out"]}, {"sent": "no because the tooth fairy has a big bow on her head .", "words": ["the", "tooth", "head", "her", "on", "big", "because", "a"]}, {"sent": "you have a tooth .", "words": ["have", "tooth", "a", "you"]}, {"sent": "your tooth hurts ?", "words": ["tooth", "your"]}, {"sent": "with tooth brushes ?", "words": ["tooth", "with"]}, {"sent": "you got your tooth .", "words": ["tooth", "your", "you"]}, {"sent": "in your tooth .", "words": ["tooth", "in", "your"]}, {"sent": "look this guy has a buck tooth .", "words": ["tooth", "this", "a", "look"]}, {"sent": "looks like shes getting a tooth .", "words": ["tooth", "like", "a"]}, {"sent": "um and she she caught a top tooth yesterday and so I just think .", "words": ["think", "tooth", "so", "and", "she", "a"]}, {"sent": "do you know why the tooth fairy likes to eat so much ?", "words": ["the", "do", "tooth", "to", "so", "you", "eat", "much", "why"]}, {"sent": "he hes got abscessed tooth on both sides .", "words": ["tooth", "on", "he"]}, {"sent": "you working on that tooth again little girl ?", "words": ["tooth", "little", "you", "on", "that"]}, {"sent": "mm look at this tooth this is the tooth from a prehistoric shark like a dinosaur shark .", "words": ["the", "look", "is", "tooth", "like", "this", "at", "a"]}, {"sent": "do they have a big sharp tooth in there ?", "words": ["do", "tooth", "have", "they", "in", "there", "big", "a"]}, {"sent": "give me your tooth brush .", "words": ["your", "tooth", "brush", "me", "give"]}, {"sent": "are we gonna do uh get a yoga card after we do this tooth brushing and a sticker ?", "words": ["do", "we", "tooth", "and", "get", "this", "are", "a"]}, {"sent": "do you have a tooth coming in ?", "words": ["do", "tooth", "you", "have", "in", "a"]}, {"sent": "youve got a generous tooth fairy .", "words": ["tooth", "a"]}, {"sent": "did you lose a tooth ?", "words": ["tooth", "did", "a", "you"]}, {"sent": "yeah would you get no just get the ah tooth pick well get it right out no big deal .", "words": ["the", "tooth", "pick", "you", "get", "it", "out", "would", "big"]}, {"sent": "show Susan where you had a bloody tooth .", "words": ["where", "show", "tooth", "you", "a"]}, {"sent": "Kathryn was nearly a year before she got a tooth .", "words": ["tooth", "was", "she", "a"]}, {"sent": "no you got a tooth .", "words": ["tooth", "a", "you"]}, {"sent": "okay remember when Opal loses her tooth ?", "words": ["tooth", "her", "when"]}, {"sent": "the tooth fairy works harder than Santa Claus because theres only one night when Santa goes down all those chimneys but the tooth fairy is busy every night .", "words": ["the", "all", "those", "is", "tooth", "every", "but", "down", "when", "because"]}, {"sent": "each year the tooth fairy has a great big party and all the other fairies come to have refreshments and to play games .", "words": ["the", "all", "each", "tooth", "to", "play", "party", "and", "other", "have", "big", "a"]}, {"sent": "oh you got a loose tooth ?", "words": ["tooth", "a", "you"]}, {"sent": "lost tooth ?", "words": ["tooth"]}, {"sent": "we could read a great book that we havent read we could read the tooth fairy book .", "words": ["the", "we", "tooth", "read", "book", "could", "that", "a"]}, {"sent": "because up above you have another tooth growing and it wants to push out the other one xxx this one falls out and you get a big strong tooth coming out .", "words": ["up", "the", "push", "tooth", "to", "you", "have", "another", "and", "it", "other", "this", "out", "get", "above", "big", "because", "a"]}, {"sent": "and the tooth and teeth .", "words": ["the", "tooth", "and"]}, {"sent": "not a tooth in your mouth .", "words": ["your", "tooth", "mouth", "in", "not", "a"]}, {"sent": "toothpaste and a tooth .", "words": ["tooth", "a", "and"]}, {"sent": "you see your tooth ?", "words": ["tooth", "see", "your", "you"]}, {"sent": "they came very quickly but oh you suffered everytime you got a tooth .", "words": ["tooth", "you", "they", "but", "a"]}, {"sent": "this is the tooth fairy .", "words": ["the", "this", "tooth", "is"]}, {"sent": "I think he is liking that icecream cone because he has a tooth coming in so hes like putting the pressure back there .", "words": ["the", "think", "back", "is", "tooth", "so", "like", "there", "in", "he", "that", "because", "a"]}, {"sent": "Megaldone thats a big tooth .", "words": ["tooth", "big", "a"]}, {"sent": "eh like a tooth brush .", "words": ["tooth", "brush", "like", "a"]}, {"sent": "wobbly tooth didnt it ?", "words": ["tooth", "it"]}, {"sent": "you know its pretty silly if youre drawing him and he doesnt have a buck tooth and the cartoon that you draw does have a buck tooth .", "words": ["the", "does", "if", "tooth", "draw", "and", "you", "have", "him", "pretty", "he", "that", "a"]}, {"sent": "a little tooth coming through .", "words": ["tooth", "little", "a"]}, {"sent": "does your tooth hurt ?", "words": ["tooth", "does", "your", "hurt"]}, {"sent": "another tooth is coming through .", "words": ["another", "tooth", "is"]}, {"sent": "because their gerbil when their gerbil broke a tooth xxx .", "words": ["their", "tooth", "when", "because", "a"]}, {"sent": "tell Courtney you broke your front tooth ?", "words": ["tooth", "your", "you"]}, {"sent": "he looks like he needs a tooth brushing too doesnt he ?", "words": ["tooth", "like", "he", "too", "a"]}, {"sent": "she needs them because when you are asleep she comes and takes away the tooth that you have put under your pillow .", "words": ["the", "put", "your", "tooth", "them", "you", "and", "asleep", "that", "are", "away", "have", "under", "when", "she", "because", "pillow"]}, {"sent": "Id rather have a baby than get a tooth out .", "words": ["tooth", "have", "get", "out", "a"]}, {"sent": "fang tooth .", "words": ["tooth"]}, {"sent": "did you get your tooth ?", "words": ["your", "tooth", "you", "get", "did"]}, {"sent": "and if youve got and Im not saying youve got a little hole in your tooth .", "words": ["your", "if", "tooth", "little", "and", "in", "not", "a"]}, {"sent": "tooth teeth no what was he doing .", "words": ["tooth", "what", "was", "he"]}, {"sent": "three tooth .", "words": ["tooth"]}, {"sent": "thats we use the same tooth toothpaste .", "words": ["the", "tooth", "same", "we"]}, {"sent": "yes oh yes I see that rotten tooth right there .", "words": ["tooth", "see", "that", "there"]}, {"sent": "do you think we got all the tooth dirties ?", "words": ["the", "all", "do", "think", "we", "tooth", "you"]}, {"sent": "have you got a tooth that hurts Thomas ?", "words": ["tooth", "you", "have", "that", "a"]}, {"sent": "what is on your tooth there ?", "words": ["what", "your", "is", "tooth", "there", "on"]}, {"sent": "xxx a great big tooth xxx for Christmas .", "words": ["tooth", "for", "big", "a"]}, {"sent": "oh hes got tooth paste .", "words": ["tooth"]}, {"sent": "a tooth ?", "words": ["tooth", "a"]}, {"sent": "youve got a new tooth too yeah .", "words": ["tooth", "too", "new", "a"]}, {"sent": "she hasnt gotten a tooth in gosh I think she got all her teeth my the time she was a year old .", "words": ["the", "all", "think", "my", "old", "tooth", "was", "in", "her", "she", "a"]}, {"sent": "do you see any tooth brushes in the bathroom ?", "words": ["the", "do", "tooth", "any", "you", "in", "bathroom", "see"]}, {"sent": "is your tooth a little bit sore ?", "words": ["your", "is", "tooth", "little", "a"]}, {"sent": "you got a tooth now .", "words": ["tooth", "a", "you"]}, {"sent": "are you another VanIddison with a sweet tooth ?", "words": ["tooth", "with", "you", "another", "are", "a"]}, {"sent": "I broke my tooth .", "words": ["tooth", "my"]}, {"sent": "you tap your tooth .", "words": ["tooth", "your", "you"]}, {"sent": "its like that each tooth is like that big .", "words": ["each", "is", "tooth", "like", "big", "that"]}, {"sent": "tooth trouble .", "words": ["tooth"]}, {"sent": "little tooth coming through .", "words": ["tooth", "little"]}, {"sent": "oh Ive an awful sweet tooth .", "words": ["tooth", "an"]}, {"sent": "when did she chip her tooth ?", "words": ["when", "tooth", "did", "her", "she"]}, {"sent": "thats its tooth .", "words": ["tooth"]}, {"sent": "its tooth time .", "words": ["tooth"]}, {"sent": "cause a dentist is really a tooth doctor .", "words": ["tooth", "is", "a"]}, {"sent": "xxx your tooth brush .", "words": ["tooth", "brush", "your"]}, {"sent": "ach has he got a tooth ?", "words": ["tooth", "he", "a"]}, {"sent": "another yellow tooth .", "words": ["another", "yellow", "tooth"]}, {"sent": "someones waiting to have a tooth checked ?", "words": ["have", "to", "tooth", "a"]}, {"sent": "its a saber tooth tiger .", "words": ["tooth", "tiger", "a"]}, {"sent": "um does she have a sweet tooth ?", "words": ["does", "tooth", "have", "she", "a"]}, {"sent": "but his tooth .", "words": ["his", "tooth", "but"]}, {"sent": "can you brush the bears tooth ?", "words": ["can", "the", "brush", "tooth", "you"]}, {"sent": "how did you lose that tooth ?", "words": ["how", "tooth", "you", "did", "that"]}, {"sent": "she will take your tooth home with her along with all the others she has collected that night .", "words": ["the", "all", "your", "tooth", "with", "her", "that", "will", "take", "she", "home"]}, {"sent": "xxx then he sat the slug in the tooth fairys seat and moved the tooth fairy to another seat .", "words": ["the", "tooth", "to", "and", "another", "in", "he", "then"]}, {"sent": "if my other tooth comes through they can stay there .", "words": ["can", "my", "if", "tooth", "other", "they", "there", "stay"]}, {"sent": "what happened to your tooth ?", "words": ["to", "what", "your", "tooth"]}, {"sent": "no thats a different fairy because the tooth fairy has a big bow on her head .", "words": ["the", "tooth", "head", "her", "on", "big", "because", "a"]}, {"sent": "because the saber tooth tigers and those things can kill the Tyranosaurus Rex .", "words": ["the", "can", "those", "tooth", "and", "because"]}, {"sent": "where we gonna put the saber tooth tiger ?", "words": ["where", "the", "put", "we", "tooth", "tiger"]}, {"sent": "is it stuck on your tooth ?", "words": ["your", "is", "tooth", "it", "stuck", "on"]}, {"sent": "a tooth what ?", "words": ["tooth", "what", "a"]}, {"sent": "if I had a tooth pulled what would that be called ?", "words": ["what", "if", "tooth", "be", "would", "that", "a"]}, {"sent": "one of the best know sabertooth tigers is the Smiladon which means knife tooth in Latin .", "words": ["the", "is", "of", "tooth", "knife", "in", "which"]}, {"sent": "xxx keys .", "words": ["keys"]}, {"sent": "we have real keys at home we could give you .", "words": ["give", "we", "you", "have", "at", "could", "keys", "home"]}, {"sent": "oh they have keys you cant have that .", "words": ["you", "have", "they", "keys", "that"]}, {"sent": "keys .", "words": ["keys"]}, {"sent": "yeah you like the keys .", "words": ["the", "keys", "like", "you"]}, {"sent": "got the keys ?", "words": ["the", "keys"]}, {"sent": "you like the keys ?", "words": ["the", "keys", "like", "you"]}, {"sent": "oh now you have two keys .", "words": ["have", "keys", "you"]}, {"sent": "those keys ?", "words": ["keys", "those"]}, {"sent": "keys or something .", "words": ["keys"]}, {"sent": "why are my keys in your train ?", "words": ["my", "your", "train", "in", "are", "keys", "why"]}, {"sent": "where are the keys going ?", "words": ["where", "the", "keys", "are"]}, {"sent": "he ran back upstairs to get his keys and his wallet and he forgot to shut the gate didnt he ?", "words": ["the", "back", "to", "and", "get", "he", "his", "keys"]}, {"sent": "whatre you gonna do with those keys ?", "words": ["do", "those", "with", "you", "keys"]}, {"sent": "I got some different keys .", "words": ["some", "keys"]}, {"sent": "you wanna go up and get the wallet and the keys and we will go for a drive in the car .", "words": ["up", "the", "for", "go", "we", "car", "and", "you", "get", "drive", "in", "wanna", "will", "keys", "a"]}, {"sent": "can I bring the keys back to Manuela ?", "words": ["can", "the", "back", "to", "bring", "keys"]}, {"sent": "mummie dropped her keys .", "words": ["keys", "her"]}, {"sent": "xxx keys .", "words": ["keys"]}, {"sent": "its just like mommys keys .", "words": ["keys", "like"]}, {"sent": "thats where Mummys keys are anyways .", "words": ["where", "keys", "are"]}, {"sent": "well wheres mothers keys ?", "words": ["keys"]}, {"sent": "you playing with your little ring of keys .", "words": ["your", "of", "with", "little", "you", "keys"]}, {"sent": "Carolines keys arent they look .", "words": ["they", "keys", "look"]}, {"sent": "oh this oh these are different keys .", "words": ["this", "keys", "are", "these"]}, {"sent": "a dinosaur keyring and where are your keys for it ?", "words": ["where", "for", "your", "and", "are", "it", "keys", "a"]}, {"sent": "she liked the keys only after I gave them to her though .", "words": ["the", "to", "them", "keys", "her", "she"]}, {"sent": "we need keys to lock ourselves in and other people out .", "words": ["need", "we", "to", "and", "other", "in", "out", "keys"]}, {"sent": "but I think the keys are about her favorite .", "words": ["the", "think", "are", "about", "keys", "her", "but"]}, {"sent": "just the waffles and the keys .", "words": ["the", "keys", "and"]}, {"sent": "you can chew on the keys .", "words": ["can", "the", "you", "on", "keys"]}, {"sent": "keys yeah you got the keys alright .", "words": ["the", "keys", "you"]}, {"sent": "you like the keys ?", "words": ["the", "keys", "like", "you"]}, {"sent": "xxx the breads more fun than the keys ?", "words": ["the", "more", "keys"]}, {"sent": "well she puts the keys in her mouth because thats what she does for everything all her play keys but .", "words": ["the", "what", "does", "for", "all", "mouth", "play", "in", "but", "keys", "her", "she", "because"]}, {"sent": "dont drop the keys in there .", "words": ["the", "dont", "in", "there", "drop", "keys"]}, {"sent": "do we eat keys ?", "words": ["eat", "do", "keys", "we"]}, {"sent": "keys yes and thats a lollipop .", "words": ["keys", "lollipop", "a", "and"]}, {"sent": "theres keys .", "words": ["keys"]}, {"sent": "spoons heres some keys .", "words": ["some", "keys"]}, {"sent": "you want those keys again ?", "words": ["keys", "those", "you"]}, {"sent": "and there are little hammers hitting the strings sothat when we press the keys down the hammer hits the string and makes a sound .", "words": ["the", "hammer", "we", "down", "little", "and", "there", "are", "keys", "when", "a"]}, {"sent": "I thought he would like the keys more well he I he has a tendency to know like whats real and whats a toy you know ?", "words": ["the", "to", "like", "and", "you", "more", "toy", "he", "would", "keys", "a"]}, {"sent": "you love these keys .", "words": ["these", "keys", "love", "you"]}, {"sent": "look at the keys .", "words": ["the", "keys", "at", "look"]}, {"sent": "um but after the visit we ended up going and buying him some keys .", "words": ["the", "up", "we", "and", "him", "some", "keys", "but"]}, {"sent": "between the spoon and the keys .", "words": ["the", "keys", "spoon", "and"]}, {"sent": "and keys .", "words": ["keys", "and"]}, {"sent": "he liked the keys .", "words": ["the", "keys", "he"]}, {"sent": "keys .", "words": ["keys"]}, {"sent": "heres your keys I fixed them .", "words": ["keys", "your", "them"]}, {"sent": "Ill go get the keys .", "words": ["get", "the", "keys", "go"]}, {"sent": "so she really liked the keys and the spoons because they make noise .", "words": ["the", "so", "and", "they", "keys", "make", "she", "because"]}, {"sent": "heres some keys .", "words": ["some", "keys"]}, {"sent": "keys would go in there pretty good .", "words": ["go", "there", "in", "pretty", "good", "would", "keys"]}, {"sent": "you wanna eat the keys ?", "words": ["the", "you", "eat", "wanna", "keys"]}, {"sent": "I get my car keys .", "words": ["get", "car", "my", "keys"]}, {"sent": "use your own keys .", "words": ["keys", "your"]}, {"sent": "you got the keys ?", "words": ["the", "keys", "you"]}, {"sent": "uhoh xxx to your keys ?", "words": ["to", "keys", "your"]}, {"sent": "your keys .", "words": ["keys", "your"]}, {"sent": "heres your keys .", "words": ["keys", "your"]}, {"sent": "yeah keys .", "words": ["keys"]}, {"sent": "oh back to the keys .", "words": ["the", "to", "keys", "back"]}, {"sent": "those are keys .", "words": ["keys", "are", "those"]}, {"sent": "shes teething right now so anything little shh keys like this anything she chews up but .", "words": ["up", "so", "like", "little", "this", "keys", "she", "but"]}, {"sent": "um you havent seen my full set of keys did you ?", "words": ["my", "full", "of", "you", "did", "keys"]}, {"sent": "you love keys ?", "words": ["keys", "love", "you"]}, {"sent": "Daddys keys yup .", "words": ["keys"]}, {"sent": "she wouldnt put the bottle down for the longest until I gave her the keys .", "words": ["the", "put", "for", "keys", "bottle", "down", "her", "she"]}, {"sent": "well youve got keys .", "words": ["keys"]}, {"sent": "which she loves keys .", "words": ["keys", "which", "she"]}, {"sent": "youre gonna eat the keys .", "words": ["eat", "the", "keys"]}, {"sent": "put the keys on ?", "words": ["the", "put", "keys", "on"]}, {"sent": "keys .", "words": ["keys"]}, {"sent": "keys .", "words": ["keys"]}, {"sent": "well you tell me where the keys are .", "words": ["where", "the", "you", "are", "me", "keys"]}, {"sent": "should we buy those keys ?", "words": ["keys", "those", "buy", "we"]}, {"sent": "three keys .", "words": ["keys"]}, {"sent": "having fun with those keys ?", "words": ["keys", "with", "those"]}, {"sent": "you find some keys ?", "words": ["some", "keys", "find", "you"]}, {"sent": "theres those keys .", "words": ["keys", "those"]}, {"sent": "they are your keys .", "words": ["they", "keys", "are", "your"]}, {"sent": "you like those keys dont ya ?", "words": ["those", "like", "you", "dont", "keys"]}, {"sent": "thats where the keys go .", "words": ["where", "the", "keys", "go"]}, {"sent": "wherere the keys ?", "words": ["the", "keys"]}, {"sent": "has he got everybodys keys so he can get in ?", "words": ["can", "so", "get", "in", "he", "keys"]}, {"sent": "keys .", "words": ["keys"]}, {"sent": "lets put just this arm in and then you can hold Mommys keys .", "words": ["can", "put", "hold", "and", "you", "this", "in", "arm", "then", "keys"]}, {"sent": "like car keys and stuff .", "words": ["car", "keys", "like", "and"]}, {"sent": "shake all your keys inside and then just push down the snap .", "words": ["the", "all", "your", "push", "down", "inside", "shake", "and", "then", "keys"]}, {"sent": "uhhuh its the keys .", "words": ["the", "keys"]}, {"sent": "oh look at the keys .", "words": ["the", "keys", "at", "look"]}, {"sent": "haha you like the keys .", "words": ["the", "keys", "like", "you"]}, {"sent": "theres no keys there .", "words": ["there", "keys"]}, {"sent": "I havent got those keys .", "words": ["keys", "those"]}, {"sent": "what do those keys go to ?", "words": ["what", "do", "those", "go", "to", "keys"]}, {"sent": "weve got two lotsof magnets and none of them are numbers .", "words": ["of", "none", "them", "and", "are"]}, {"sent": "you are not drinking none of it .", "words": ["of", "none", "you", "are", "it", "not"]}, {"sent": "are there none on the floor .", "words": ["the", "none", "there", "are", "on"]}, {"sent": "theres none left .", "words": ["none"]}, {"sent": "none of them really work well do they ?", "words": ["do", "work", "of", "none", "them", "they"]}, {"sent": "hes got none on there has he ?", "words": ["on", "none", "he", "there"]}, {"sent": "none in there ?", "words": ["none", "in", "there"]}, {"sent": "none of the other little children have broken it yet .", "words": ["the", "of", "none", "other", "little", "have", "broken", "it"]}, {"sent": "we needta move the transporter or none of them are going to get home to eat their sausages .", "words": ["the", "their", "we", "of", "none", "to", "them", "get", "eat", "are", "home"]}, {"sent": "is there none there with long trousers on ?", "words": ["is", "none", "with", "long", "there", "on"]}, {"sent": "but none of the particular books that .", "words": ["the", "of", "none", "that", "but"]}, {"sent": "but with the fluoride and all Imean its amazing that none of our boys have fillings yet .", "words": ["the", "all", "of", "none", "with", "and", "our", "have", "that", "but"]}, {"sent": "none .", "words": ["none"]}, {"sent": "I dont want none of your kisses .", "words": ["of", "dont", "your", "none"]}, {"sent": "girls dont wear no no none a these for head .", "words": ["these", "for", "none", "head", "dont", "a"]}, {"sent": "you dont want none ?", "words": ["dont", "none", "you"]}, {"sent": "none ya business .", "words": ["none"]}, {"sent": "I didnt get none .", "words": ["get", "none"]}, {"sent": "thats why there should be none of it left .", "words": ["of", "be", "none", "there", "it", "why"]}, {"sent": "none .", "words": ["none"]}, {"sent": "this little piggy had none .", "words": ["this", "none", "little"]}, {"sent": "we have we have alotof paintings up on the wall and none of yours .", "words": ["the", "up", "we", "of", "none", "and", "have", "on"]}, {"sent": "no none of them cried not one .", "words": ["of", "none", "them", "not"]}, {"sent": "and it didnt work none of it .", "words": ["work", "of", "none", "and", "it"]}, {"sent": "oh theres none so rare as compare with King Cole and his fiddlers three .", "words": ["none", "so", "with", "and", "his"]}, {"sent": "none of the windows was his window .", "words": ["the", "of", "none", "was", "his", "window"]}, {"sent": "daddy dont want none ?", "words": ["dont", "none"]}, {"sent": "this little piggy had none .", "words": ["this", "none", "little"]}, {"sent": "he cant eat none of them Dad .", "words": ["of", "none", "them", "eat", "he"]}, {"sent": "dont gid her none dont give her .", "words": ["dont", "give", "her", "none"]}, {"sent": "I got none of these .", "words": ["of", "none", "these"]}, {"sent": "none .", "words": ["none"]}, {"sent": "you get none .", "words": ["get", "none", "you"]}, {"sent": "none of those yet .", "words": ["of", "none", "those"]}, {"sent": "not nothing like not none of these .", "words": ["these", "of", "none", "like", "not"]}, {"sent": "if you do not shake the bottle none will come out then alot will .", "words": ["the", "do", "if", "none", "then", "shake", "you", "out", "will", "bottle", "not"]}, {"sent": "there is none .", "words": ["there", "none", "is"]}, {"sent": "but this little piggy had none .", "words": ["this", "none", "but", "little"]}, {"sent": "and this little piggie had none .", "words": ["this", "none", "little", "and"]}, {"sent": "and this little piggy had none .", "words": ["this", "none", "little", "and"]}, {"sent": "none of the buttons fell off .", "words": ["of", "none", "off", "the"]}, {"sent": "none of that .", "words": ["of", "none", "that"]}, {"sent": "youre one none .", "words": ["none"]}, {"sent": "Sharleen no this is none of your business .", "words": ["your", "is", "of", "none", "this"]}, {"sent": "well you xxx that none of them were dangerous but they are fighting .", "words": ["of", "none", "them", "you", "they", "are", "but", "that", "were"]}, {"sent": "none of us are deaf .", "words": ["of", "none", "are", "us"]}, {"sent": "none of them are simple are they ?", "words": ["of", "none", "them", "they", "are"]}, {"sent": "like none of her babies came with a bottle so Im thinking maybe thats why she actually liked to play with it .", "words": ["of", "none", "so", "to", "with", "like", "play", "it", "bottle", "her", "she", "why", "a"]}, {"sent": "and this little piggie had none .", "words": ["this", "none", "little", "and"]}, {"sent": "and this piggy had none .", "words": ["this", "none", "and"]}, {"sent": "none of them are very funny .", "words": ["of", "none", "are", "them"]}, {"sent": "none of them ?", "words": ["of", "none", "them"]}, {"sent": "yeah but none of them match so how do you play ?", "words": ["do", "how", "of", "none", "so", "play", "them", "you", "but"]}, {"sent": "thats why theres none in it .", "words": ["none", "in", "it", "why"]}, {"sent": "and I got none xxx ah .", "words": ["none", "and"]}, {"sent": "none of them ?", "words": ["of", "none", "them"]}, {"sent": "youve got two now and Ive got none .", "words": ["none", "and"]}, {"sent": "I know none of these taste very good do they ?", "words": ["these", "do", "of", "none", "taste", "they", "good"]}, {"sent": "so Ill be upstairs and Don will say whats the bread song like there is none make it up .", "words": ["the", "up", "bread", "is", "so", "be", "none", "say", "like", "and", "there", "it", "will", "make"]}, {"sent": "dont give hah her none .", "words": ["dont", "give", "her", "none"]}, {"sent": "well neither of those none of those are Toby are they ?", "words": ["those", "of", "none", "they", "are"]}, {"sent": "and this little piggy ate none .", "words": ["this", "none", "little", "and"]}, {"sent": "I dont have none .", "words": ["have", "dont", "none"]}, {"sent": "but this little piggy had none .", "words": ["this", "none", "but", "little"]}, {"sent": "and this little piggy had none .", "words": ["this", "none", "little", "and"]}, {"sent": "aint no more I didnt get none .", "words": ["get", "none", "more"]}, {"sent": "none of them even remotely like anything we have at home .", "words": ["we", "of", "none", "them", "like", "have", "at", "home"]}, {"sent": "none of these are matching our cards Shem .", "words": ["these", "of", "none", "our", "are"]}, {"sent": "Brett has none stars .", "words": ["none"]}, {"sent": "I aint showing you none a mine .", "words": ["none", "mine", "a", "you"]}, {"sent": "is there none of them going to the station ?", "words": ["the", "is", "of", "none", "to", "them", "there"]}, {"sent": "dont give her none .", "words": ["dont", "give", "her", "none"]}, {"sent": "but you dont say Brett has none stars .", "words": ["none", "say", "you", "dont", "but"]}, {"sent": "it feels like that theres none in .", "words": ["none", "like", "in", "it", "that"]}, {"sent": "well youve got none left .", "words": ["none"]}, {"sent": "none of these will do because none of these are white ones are they ?", "words": ["do", "these", "of", "none", "they", "are", "will", "white", "because"]}, {"sent": "and none of those in particular that I can think of so .", "words": ["can", "think", "those", "of", "none", "so", "and", "in", "that"]}, {"sent": "none of them were into um .", "words": ["into", "of", "none", "them", "were"]}, {"sent": "one little piggy had none .", "words": ["none", "little"]}, {"sent": "none ya business .", "words": ["none"]}, {"sent": "am I a special candies and you dont got none .", "words": ["none", "you", "and", "dont", "am", "a"]}, {"sent": "come on I didnt get none before .", "words": ["on", "none", "get"]}, {"sent": "none ?", "words": ["none"]}, {"sent": "none of the Teletubbies are are brown .", "words": ["the", "brown", "of", "none", "are"]}, {"sent": "but none of the insects they happened upon had any idea where her mother had gone .", "words": ["the", "where", "of", "none", "any", "they", "her", "but"]}, {"sent": "Mama Sally dont give me none .", "words": ["dont", "give", "me", "none"]}, {"sent": "no I dont want none .", "words": ["dont", "none"]}, {"sent": "really none of us have any favorite foods thats so sad .", "words": ["of", "none", "so", "any", "have", "us", "sad"]}, {"sent": "no none at all .", "words": ["all", "none", "at"]}, {"sent": "and this pig had none .", "words": ["this", "pig", "none", "and"]}, {"sent": "so none of the ones that are in there per say .", "words": ["the", "of", "so", "none", "say", "there", "in", "are", "that"]}, {"sent": "none of the kids except for you and Robin and JoAnne .", "words": ["the", "for", "of", "none", "you", "and"]}, {"sent": "but none of them look like that .", "words": ["of", "none", "them", "like", "look", "that", "but"]}, {"sent": "oh I havent got none .", "words": ["none"]}, {"sent": "there isnt none there now .", "words": ["there", "none"]}, {"sent": "none .", "words": ["none"]}, {"sent": "yes I only brought none from you mark .", "words": ["none", "you"]}, {"sent": "oo told Jt none of his business .", "words": ["of", "none", "his"]}, {"sent": "no none for doggies .", "words": ["none", "for"]}, {"sent": "none of them are square .", "words": ["of", "none", "are", "them"]}, {"sent": "hes sad ?", "words": ["sad"]}, {"sent": "oh did you notice they were so sad .", "words": ["so", "you", "they", "did", "sad", "were"]}, {"sent": "he was sad first of all and then he was happy .", "words": ["all", "happy", "of", "was", "first", "and", "he", "sad", "then"]}, {"sent": "you wanna play a sad Batman song ?", "words": ["play", "you", "wanna", "sad", "a"]}, {"sent": "why would it be sad ?", "words": ["be", "it", "would", "sad", "why"]}, {"sent": "hell be sad Nathaniel .", "words": ["be", "sad"]}, {"sent": "should I make him happy or sad ?", "words": ["sad", "him", "make", "happy"]}, {"sent": "oh shes a sad face .", "words": ["sad", "face", "a"]}, {"sent": "oh is the flower feeling sad today ?", "words": ["the", "sad", "flower", "is"]}, {"sent": "Im sad .", "words": ["sad"]}, {"sent": "and Im feeling so sad .", "words": ["so", "sad", "and"]}, {"sent": "are you happy or are you sad Fraser ?", "words": ["sad", "are", "happy", "you"]}, {"sent": "you see some faces that look sad ?", "words": ["sad", "you", "look", "see", "that", "some"]}, {"sent": "why would you be sad ?", "words": ["be", "you", "would", "sad", "why"]}, {"sent": "and hes sad because hes so fat maybe .", "words": ["sad", "so", "because", "and"]}, {"sent": "is it is it sad ?", "words": ["sad", "it", "is"]}, {"sent": "Daddy and I felt very sad last night .", "words": ["sad", "last", "and"]}, {"sent": "Elsa looks so sad what happened ?", "words": ["so", "sad", "what"]}, {"sent": "theyre sad .", "words": ["sad"]}, {"sent": "he looks sad doesnt he ?", "words": ["sad", "he"]}, {"sent": "she was sad because her kitten was stuck high up on a roof .", "words": ["up", "was", "stuck", "roof", "high", "on", "sad", "her", "she", "because", "a"]}, {"sent": "he looks sad .", "words": ["sad", "he"]}, {"sent": "why is Fraser sad ?", "words": ["sad", "why", "is"]}, {"sent": "would you be very sad ?", "words": ["be", "sad", "you", "would"]}, {"sent": "sad why ?", "words": ["sad", "why"]}, {"sent": "was there something maybe sad that happened in the Land Before Time ?", "words": ["the", "was", "there", "in", "sad", "that"]}, {"sent": "is he very sad ?", "words": ["sad", "he", "is"]}, {"sent": "can you show Amanda your sad face ?", "words": ["can", "your", "face", "show", "you", "sad"]}, {"sent": "oh what a sad boy .", "words": ["what", "sad", "a"]}, {"sent": "oh and then we have another tiny bear whos looking a bit sad isnt he ?", "words": ["tiny", "we", "and", "have", "another", "bear", "he", "sad", "then", "a"]}, {"sent": "I dont know why is he sad ?", "words": ["is", "dont", "he", "sad", "why"]}, {"sent": "why is he sad ?", "words": ["sad", "he", "why", "is"]}, {"sent": "the hand is so sad .", "words": ["the", "is", "hand", "so", "sad"]}, {"sent": "it was sad because of his pride Jwww .", "words": ["of", "was", "it", "his", "sad", "because"]}, {"sent": "happy sad .", "words": ["sad", "happy"]}, {"sent": "hes sitting by the window and its raining outside and hes very sad .", "words": ["the", "outside", "and", "by", "sad", "window"]}, {"sent": "and today Lola was sad .", "words": ["was", "sad", "and"]}, {"sent": "I know said Happy maybe hes feeling sad and needs a little cheering up .", "words": ["up", "little", "and", "sad", "a"]}, {"sent": "thats a sad face .", "words": ["sad", "face", "a"]}, {"sent": "no theyre not sad but theyre very dirty .", "words": ["sad", "dirty", "not", "but"]}, {"sent": "that makes me sad .", "words": ["me", "sad", "that"]}, {"sent": "sad mother duck went out one day .", "words": ["duck", "sad", "out"]}, {"sent": "ah looka there shes all sad and gonna cry .", "words": ["all", "cry", "and", "there", "sad"]}, {"sent": "Miffy cried when she saw how cold and sad it was .", "words": ["how", "cold", "was", "and", "it", "sad", "when", "she"]}, {"sent": "Im not surprised hes sad if you threw him on the floor .", "words": ["the", "if", "you", "him", "on", "sad", "not"]}, {"sent": "theyre not sad because they miss their brother ?", "words": ["their", "they", "sad", "not", "because"]}, {"sent": "hes sad because all the other trains have been telling lies about him .", "words": ["the", "all", "other", "have", "him", "about", "sad", "because"]}, {"sent": "why am I sad ?", "words": ["sad", "am", "why"]}, {"sent": "happy dog sad dog .", "words": ["sad", "dog", "happy"]}, {"sent": "and how do you feel sad what do you do when you feel sad sad darling .", "words": ["do", "how", "what", "you", "and", "sad", "when"]}, {"sent": "I dont know why shes so sad today .", "words": ["dont", "sad", "so", "why"]}, {"sent": "theyre sad .", "words": ["sad"]}, {"sent": "whys he sad ?", "words": ["sad", "he"]}, {"sent": "Jwww sad ?", "words": ["sad"]}, {"sent": "Im very sad Aran .", "words": ["sad"]}, {"sent": "you youre not sad about that Roman ?", "words": ["you", "about", "sad", "that", "not"]}, {"sent": "oy hes sad .", "words": ["sad"]}, {"sent": "Milo was sad .", "words": ["was", "sad"]}, {"sent": "Dad is sad .", "words": ["sad", "is"]}, {"sent": "is he sad ?", "words": ["sad", "he", "is"]}, {"sent": "he but did it make him happy or sad ?", "words": ["happy", "him", "it", "he", "did", "sad", "make", "but"]}, {"sent": "is that a happy or a sad ?", "words": ["happy", "is", "sad", "that", "a"]}, {"sent": "sad xxx .", "words": ["sad"]}, {"sent": "shall we draw some faces you tell me Eleanor is this happy or is this sad .", "words": ["sad", "happy", "we", "is", "me", "draw", "you", "this", "some"]}, {"sent": "thats a bit sad .", "words": ["sad", "a"]}, {"sent": "oh poor guy hes sad hohohoho .", "words": ["sad", "poor"]}, {"sent": "youre feeling sad ?", "words": ["sad"]}, {"sent": "happy sad .", "words": ["sad", "happy"]}, {"sent": "is that a sad face ?", "words": ["face", "is", "sad", "that", "a"]}, {"sent": "thats a sad face you start at the bottom like a rainbow you know kind of like drawing a rainbow .", "words": ["the", "bottom", "face", "of", "like", "you", "at", "sad", "a"]}, {"sent": "thats a sad state of affairs .", "words": ["of", "sad", "a"]}, {"sent": "why are you sad ?", "words": ["sad", "are", "why", "you"]}, {"sent": "hes getting married today and I was sad because I wanted to be at his wedding .", "words": ["to", "was", "be", "and", "at", "his", "sad", "because"]}, {"sent": "oh that ones very sad .", "words": ["sad", "that"]}, {"sent": "why is wasp sad darling ?", "words": ["sad", "why", "is"]}, {"sent": "why are they sad do you think ?", "words": ["do", "think", "you", "they", "are", "sad", "why"]}, {"sent": "mhm he is a sad fish .", "words": ["is", "he", "fish", "sad", "a"]}, {"sent": "happy sad .", "words": ["sad", "happy"]}, {"sent": "ones sad .", "words": ["sad"]}, {"sent": "because hes sad about that what are you sad about ?", "words": ["what", "you", "are", "about", "sad", "that", "because"]}, {"sent": "hes sad ?", "words": ["sad"]}, {"sent": "that does make you sad to think about Peanuts .", "words": ["does", "think", "to", "you", "that", "about", "sad", "make"]}, {"sent": "did I make you sad ?", "words": ["sad", "make", "did", "you"]}, {"sent": "its sad .", "words": ["sad"]}, {"sent": "can I see your sad face ?", "words": ["can", "sad", "your", "face", "see"]}, {"sent": "Jesus do you ever feel sad .", "words": ["do", "sad", "you"]}, {"sent": "whos sad ?", "words": ["sad"]}, {"sent": "oh is that is that a happy face or a sad face ?", "words": ["face", "happy", "is", "sad", "that", "a"]}, {"sent": "you think hes sad .", "words": ["sad", "think", "you"]}, {"sent": "and then youre sad because your bellys hungry huh .", "words": ["your", "hungry", "and", "sad", "then", "because"]}, {"sent": "an then we go back to the sad again ?", "words": ["the", "back", "go", "we", "to", "an", "sad", "then"]}, {"sent": "how sad .", "words": ["sad", "how"]}, {"sent": "and that ones sad .", "words": ["sad", "that", "and"]}, {"sent": "she is sad .", "words": ["sad", "she", "is"]}, {"sent": "thats sad isnt it ?", "words": ["sad", "it"]}, {"sent": "oy Ellens going to be sad .", "words": ["to", "sad", "be"]}, {"sent": "youre not a very good driver falling over on your bump .", "words": ["bump", "your", "good", "on", "over", "not", "a"]}, {"sent": "bump .", "words": ["bump"]}, {"sent": "did he bump his head ?", "words": ["bump", "head", "he", "did", "his"]}, {"sent": "you want to go bump ?", "words": ["bump", "go", "to", "you", "want to"]}, {"sent": "ooh those are real bump bump bump see ?", "words": ["bump", "see", "are", "those"]}, {"sent": "bump bump bump bump bump bump bump .", "words": ["bump"]}, {"sent": "oh the things they will bump .", "words": ["the", "they", "bump", "will"]}, {"sent": "bump head .", "words": ["bump", "head"]}, {"sent": "oh well you were holding your head saying bump so I thought perhaps you had .", "words": ["bump", "your", "so", "head", "you", "were"]}, {"sent": "bumpity bump bump .", "words": ["bump"]}, {"sent": "bump is baby Sarah .", "words": ["bump", "is"]}, {"sent": "xxx got your bump .", "words": ["bump", "your"]}, {"sent": "bumpity bump .", "words": ["bump"]}, {"sent": "bump .", "words": ["bump"]}, {"sent": "and as theyre going bumpity bump the babies are going wah wah and the Mummys are saying .", "words": ["bump", "the", "are", "and"]}, {"sent": "mind you dont bump your head .", "words": ["bump", "your", "head", "you", "dont"]}, {"sent": "bumpity bumpity bumpity bump .", "words": ["bump"]}, {"sent": "you have a funny little bump on your back horsie .", "words": ["bump", "your", "back", "little", "have", "you", "on", "a"]}, {"sent": "go bump in the car .", "words": ["bump", "the", "go", "car", "in"]}, {"sent": "you dont wanna fall and bump your head again do you ?", "words": ["bump", "do", "your", "head", "fall", "and", "you", "dont", "wanna"]}, {"sent": "did I bump your elbow then ?", "words": ["bump", "then", "your", "did"]}, {"sent": "a bump ?", "words": ["bump", "a"]}, {"sent": "the riders on the bus go bumpity bump bump bumpity bump bump bumpity bump bump .", "words": ["the", "bump", "go", "on", "bus"]}, {"sent": "oh big bump .", "words": ["bump", "big"]}, {"sent": "oh and you couldnt get Kirsten over the bump ?", "words": ["the", "bump", "you", "get", "and", "over"]}, {"sent": "watch out or youll bump again .", "words": ["bump", "watch", "out"]}, {"sent": "go bump like oh .", "words": ["bump", "like", "go"]}, {"sent": "bump .", "words": ["bump"]}, {"sent": "the riders on the bus go bumpity bump .", "words": ["the", "bump", "go", "on", "bus"]}, {"sent": "with a bump in the middle .", "words": ["bump", "the", "with", "in", "a"]}, {"sent": "bump .", "words": ["bump"]}, {"sent": "I gonna bump .", "words": ["bump"]}, {"sent": "you might bump your nose .", "words": ["bump", "nose", "your", "you"]}, {"sent": "no I didnt mean to bump you .", "words": ["bump", "to", "you"]}, {"sent": "bump her head ?", "words": ["bump", "head", "her"]}, {"sent": "whats the bump bump ?", "words": ["the", "bump"]}, {"sent": "bump .", "words": ["bump"]}, {"sent": "oh did you bump on the chair ?", "words": ["bump", "the", "chair", "you", "did", "on"]}, {"sent": "yes its much better to bump on a pillow would you like to go in the other room and bump your head on a pillow hah its the telephone .", "words": ["the", "go", "like", "bump", "on", "much", "pillow", "to", "head", "you", "and", "would", "a", "better", "your", "other", "in", "room", "telephone"]}, {"sent": "it went bump .", "words": ["bump", "it"]}, {"sent": "you went bump ?", "words": ["bump", "you"]}, {"sent": "bump jump ajump .", "words": ["bump", "jump"]}, {"sent": "where did you bump your head ?", "words": ["where", "bump", "your", "head", "you", "did"]}, {"sent": "xxx do the bump Simba .", "words": ["the", "do", "bump"]}, {"sent": "bumpity bump bump bumpity bump bump .", "words": ["bump"]}, {"sent": "a big bump at the end .", "words": ["bump", "the", "at", "big", "a"]}, {"sent": "bumpity bump bump .", "words": ["bump"]}, {"sent": "you didnt bump your head then .", "words": ["bump", "your", "head", "you", "then"]}, {"sent": "say bump the stairs .", "words": ["bump", "the", "say", "stairs"]}, {"sent": "it is a bump .", "words": ["bump", "it", "a", "is"]}, {"sent": "bump ?", "words": ["bump"]}, {"sent": "did you bump your head ?", "words": ["bump", "your", "head", "you", "did"]}, {"sent": "where did you bump your head I dont really see .", "words": ["where", "bump", "your", "head", "you", "dont", "did", "see"]}, {"sent": "bump .", "words": ["bump"]}, {"sent": "xxx Ill bump bump bump .", "words": ["bump"]}, {"sent": "and even if she did she wouldnt be tall enough to bump her head on it .", "words": ["bump", "if", "be", "to", "head", "and", "it", "did", "on", "her", "she"]}, {"sent": "did he fall down and bump his head ?", "words": ["bump", "head", "fall", "and", "he", "did", "his", "down"]}, {"sent": "youre gonna bump your head .", "words": ["bump", "head", "your"]}, {"sent": "come here sweets youre gonna bump it .", "words": ["here", "bump", "it"]}, {"sent": "wasnt a big bump .", "words": ["bump", "big", "a"]}, {"sent": "well you will bump your head if you dont come out from under the table .", "words": ["bump", "the", "your", "if", "table", "head", "you", "dont", "under", "out", "will"]}, {"sent": "bumpity bump .", "words": ["bump"]}, {"sent": "bumpity bumpity bump bump bump .", "words": ["bump"]}, {"sent": "bump .", "words": ["bump"]}, {"sent": "where did you bump your head I dont really see right now .", "words": ["where", "bump", "your", "head", "you", "dont", "did", "see"]}, {"sent": "bumpity bump .", "words": ["bump"]}, {"sent": "because if they bump into the rocks theyll crash and then theyll sink into the water wont they ?", "words": ["bump", "the", "if", "into", "and", "they", "sink", "water", "then", "because"]}, {"sent": "bump your head .", "words": ["bump", "head", "your"]}, {"sent": "you bump your head ?", "words": ["bump", "head", "your", "you"]}, {"sent": "youre gonna get a big bump on your nose if you blow it ?", "words": ["bump", "your", "if", "nose", "you", "get", "it", "on", "big", "blow", "a"]}, {"sent": "who got a bump ?", "words": ["bump", "a", "who"]}, {"sent": "the riders on the bus go bumpity bump .", "words": ["the", "bump", "go", "on", "bus"]}, {"sent": "bump a bump .", "words": ["bump", "a"]}, {"sent": "bump .", "words": ["bump"]}, {"sent": "a big bump .", "words": ["bump", "big", "a"]}, {"sent": "he went to bed with a bump on his .", "words": ["bump", "to", "with", "bed", "his", "he", "on", "a"]}, {"sent": "did Nathaniel bump his head ?", "words": ["bump", "his", "head", "did"]}, {"sent": "where you drive around and bump into other cars .", "words": ["where", "bump", "into", "and", "drive", "you", "other", "around"]}, {"sent": "I had a bump right away .", "words": ["bump", "away", "a"]}, {"sent": "no there really is seven on the bump of gump .", "words": ["the", "bump", "is", "of", "there", "on"]}, {"sent": "did I bump your head ?", "words": ["bump", "head", "your", "did"]}, {"sent": "bumpity bump bump .", "words": ["bump"]}, {"sent": "hey thats a mean old bumpy bump .", "words": ["old", "bump", "a"]}, {"sent": "bumpity bump bump .", "words": ["bump"]}, {"sent": "your Mummys lovely with her big bump .", "words": ["bump", "your", "her", "with", "big"]}, {"sent": "and over the bump .", "words": ["the", "bump", "over", "and"]}, {"sent": "you bump ?", "words": ["bump", "you"]}, {"sent": "its a bump .", "words": ["bump", "a"]}, {"sent": "the truck hit a bump .", "words": ["the", "bump", "truck", "hit", "a"]}, {"sent": "did he bump himself ?", "words": ["bump", "he", "did"]}, {"sent": "oh did you make a bump in the road ?", "words": ["bump", "the", "you", "in", "did", "make", "a"]}, {"sent": "you did have a big bump .", "words": ["bump", "you", "have", "did", "big", "a"]}, {"sent": "bumpity bump .", "words": ["bump"]}, {"sent": "no because thats a bump sticking .", "words": ["bump", "because", "a"]}, {"sent": "so am I he reached into his pocket and gave the hen the grain which the miller had given him the black hen went to her nest there she sat very still for a long while .", "words": ["the", "for", "into", "so", "to", "her", "long", "and", "black", "there", "him", "he", "am", "hen", "his", "which", "she", "a"]}, {"sent": "what are you the rooster or the hen ?", "words": ["the", "what", "you", "rooster", "are", "hen"]}, {"sent": "theres a hen yes .", "words": ["hen", "a"]}, {"sent": "thats a mother hen with her chick .", "words": ["her", "with", "hen", "a"]}, {"sent": "thats a big hen isnt it ?", "words": ["it", "big", "hen", "a"]}, {"sent": "oh the hen the hens gonna crush it .", "words": ["the", "it", "hen"]}, {"sent": "I had a little hen the prettiest ever seen she washed up the dishes and kept the house clean .", "words": ["the", "up", "clean", "little", "and", "house", "hen", "she", "a"]}, {"sent": "its a hen .", "words": ["hen", "a"]}, {"sent": "who has got into the hen house ?", "words": ["the", "into", "house", "hen", "who"]}, {"sent": "you wanna do the hen ?", "words": ["the", "do", "you", "wanna", "hen"]}, {"sent": "mother hen and the chicks have not seen mother duck .", "words": ["the", "and", "have", "hen", "duck", "not"]}, {"sent": "brown hen .", "words": ["hen", "brown"]}, {"sent": "he looks like he is looking at the chicken and the hen .", "words": ["the", "is", "like", "hen", "and", "he", "at", "chicken"]}, {"sent": "bok bok orange hen .", "words": ["orange", "hen"]}, {"sent": "and the mother hen with her chick .", "words": ["the", "with", "and", "hen", "her"]}, {"sent": "I fed my hen under yonder tree .", "words": ["tree", "under", "my", "hen"]}, {"sent": "theres the hen .", "words": ["the", "hen"]}, {"sent": "and a big white hen standing on one leg and under the hen was a quiet egg .", "words": ["the", "egg", "was", "and", "under", "leg", "quiet", "hen", "on", "white", "big", "a"]}, {"sent": "she has forgotten to shut the gate to the hen house .", "words": ["the", "to", "house", "hen", "she"]}, {"sent": "and then feeling a bit happier he got out his hen that layed golden eggs .", "words": ["and", "he", "out", "hen", "his", "then", "that", "a"]}, {"sent": "a big fat hen ?", "words": ["big", "hen", "a"]}, {"sent": "theyve helped a lost chick to find mother hen .", "words": ["to", "hen", "find", "a"]}, {"sent": "now wheres the hen ?", "words": ["the", "hen"]}, {"sent": "der man gate hen nein .", "words": ["hen"]}, {"sent": "says mother hen .", "words": ["hen"]}, {"sent": "so Danny and the hen and the goose and the goat all galloped along until they met .", "words": ["the", "all", "so", "and", "they", "hen", "goose"]}, {"sent": "wheres his hen hiding ?", "words": ["his", "hen"]}, {"sent": "and under the hen was a quiet egg .", "words": ["the", "egg", "was", "and", "under", "quiet", "hen", "a"]}, {"sent": "der man gate hen nein .", "words": ["hen"]}, {"sent": "Mamas magic hen .", "words": ["hen"]}, {"sent": "and then the hen laid him two eggs .", "words": ["the", "and", "him", "hen", "then"]}, {"sent": "I might just decide that I dont like the look of your hen house .", "words": ["the", "your", "of", "like", "hen", "dont", "house", "look", "that"]}, {"sent": "heres a cooked hen .", "words": ["hen", "a"]}, {"sent": "hen .", "words": ["hen"]}, {"sent": "well actually darling this is the Daddy hen .", "words": ["the", "this", "hen", "is"]}, {"sent": "its a very grand hen house .", "words": ["house", "hen", "a"]}, {"sent": "there was a hen chasing her .", "words": ["was", "there", "hen", "her", "a"]}, {"sent": "hen .", "words": ["hen"]}, {"sent": "I bought me a hen and the hen pleased me .", "words": ["the", "and", "hen", "me", "a"]}, {"sent": "bear bunny duck alligator and hen .", "words": ["bunny", "and", "bear", "alligator", "hen", "duck"]}, {"sent": "a hen says .", "words": ["hen", "a"]}, {"sent": "its not a hen .", "words": ["hen", "not", "a"]}, {"sent": "hen .", "words": ["hen"]}, {"sent": "well we havent decided yet whether this is the best hen house or not have we ?", "words": ["the", "is", "we", "have", "this", "house", "hen", "not"]}, {"sent": "a big fat hen .", "words": ["big", "hen", "a"]}, {"sent": "thats right the hen fell .", "words": ["the", "hen"]}, {"sent": "is it a hen ?", "words": ["it", "hen", "a", "is"]}, {"sent": "is the hen having something to eat ?", "words": ["the", "is", "to", "eat", "hen"]}, {"sent": "because it might make life hard for a hen .", "words": ["for", "it", "hard", "hen", "make", "because", "a"]}, {"sent": "jack went to the farmyard the black hen said tsk tsk but she had no eggs .", "words": ["the", "to", "black", "hen", "she", "but"]}, {"sent": "a hen .", "words": ["hen", "a"]}, {"sent": "Jack snatched the hen and raced down the beanstalk .", "words": ["the", "down", "and", "hen"]}, {"sent": "did you see a hen that you could ride on or something ?", "words": ["ride", "you", "did", "hen", "on", "see", "could", "that", "a"]}, {"sent": "youre making a hen house are you ?", "words": ["you", "are", "house", "hen", "a"]}, {"sent": "the hen has laid golden eggs .", "words": ["the", "hen"]}, {"sent": "you know when I went to my friends hen night ?", "words": ["my", "to", "you", "hen", "when"]}, {"sent": "have you put the cup for hen in there ?", "words": ["the", "cup", "put", "for", "you", "have", "there", "in", "hen"]}, {"sent": "he reached into his pocket and gave the hen the grain which the miller had given him .", "words": ["the", "into", "and", "him", "he", "hen", "his", "which"]}, {"sent": "a hen yeah .", "words": ["hen", "a"]}, {"sent": "I think this is a very funny hen house Aran .", "words": ["think", "is", "this", "house", "hen", "a"]}, {"sent": "theres the hen .", "words": ["the", "hen"]}, {"sent": "because weve a hen night and a stag night .", "words": ["and", "hen", "because", "a"]}, {"sent": "do you wanna start with the dog the horse the pig or the hen ?", "words": ["the", "do", "with", "you", "pig", "wanna", "hen", "horse", "dog"]}, {"sent": "wheres the hen go ?", "words": ["the", "go", "hen"]}, {"sent": "and the hen .", "words": ["the", "hen", "and"]}, {"sent": "a hen says bock bock bock bock bock bock bock bock bogack bogack bock bock bock bogack bogack .", "words": ["hen", "a"]}, {"sent": "and a hen .", "words": ["hen", "a", "and"]}, {"sent": "wheres the hen that laid the eggs ?", "words": ["the", "that", "hen"]}, {"sent": "any xxx any more white pieces of a hen ?", "words": ["of", "any", "more", "hen", "white", "a"]}, {"sent": "shall we put another hen in there then ?", "words": ["put", "we", "another", "there", "in", "hen", "then"]}, {"sent": "you cant have a Daddy hen .", "words": ["have", "hen", "a", "you"]}, {"sent": "and the and the hen I mean the hen and the baby chicks ?", "words": ["the", "hen", "and"]}, {"sent": "yeah not a hen .", "words": ["hen", "not", "a"]}, {"sent": "those are eggs that the the hen has laid .", "words": ["the", "those", "are", "hen", "that"]}, {"sent": "hen in a hat hooray hooray .", "words": ["hat", "in", "a", "hen"]}, {"sent": "my hen house is the best hen house theres ever been .", "words": ["the", "my", "is", "house", "hen"]}, {"sent": "well Ill build a new hen house then if youre not going to .", "words": ["if", "to", "house", "build", "hen", "then", "new", "not", "a"]}, {"sent": "Im sure farmer Pyjama has the most wonderful hen house that ever was .", "words": ["the", "was", "house", "hen", "that"]}, {"sent": "it says Clucky is a hen .", "words": ["it", "hen", "a", "is"]}, {"sent": "take the hen down so it doesnt get drenched .", "words": ["the", "so", "get", "it", "hen", "down", "take"]}, {"sent": "a hen .", "words": ["hen", "a"]}, {"sent": "a hen and her chicks .", "words": ["her", "hen", "and", "a"]}, {"sent": "sometimes nine and sometimes ten hicketypickety my fine hen .", "words": ["my", "fine", "hen", "and"]}, {"sent": "you found the hen .", "words": ["the", "hen", "you"]}, {"sent": "are you the mother hen ?", "words": ["the", "are", "hen", "you"]}, {"sent": "your the hen ?", "words": ["the", "your", "hen"]}, {"sent": "a big oh thats big thats a big hen .", "words": ["big", "hen", "a"]}, {"sent": "the hen lets see .", "words": ["the", "see", "hen"]}, {"sent": "and a big white hen standing on one leg .", "words": ["and", "leg", "hen", "on", "white", "big", "a"]}, {"sent": "and then theres a hen with a line of chicks and the hen that youve got in your hand .", "words": ["the", "your", "hand", "of", "with", "and", "in", "hen", "then", "that", "a"]}, {"sent": "any more pieces with white and hen on them ?", "words": ["any", "with", "them", "and", "more", "hen", "on", "white"]}, {"sent": "heres the hen .", "words": ["the", "hen"]}, {"sent": "hen in a hat .", "words": ["hat", "in", "a", "hen"]}, {"sent": "and the hen .", "words": ["the", "hen", "and"]}, {"sent": "its a bear and a hen together what would you call that ?", "words": ["what", "would", "bear", "and", "you", "hen", "that", "a"]}, {"sent": "well I think thats the best hen house anybody has ever seen .", "words": ["the", "think", "house", "hen"]}, {"sent": "hello pussy cat .", "words": ["cat"]}, {"sent": "the cat ?", "words": ["the", "cat"]}, {"sent": "thats a kitty cat and whats that ?", "words": ["and", "cat", "kitty", "that", "a"]}, {"sent": "you didnt see a cat ?", "words": ["see", "cat", "a", "you"]}, {"sent": "well I showed him the domestic ones the dog and cat .", "words": ["the", "and", "cat", "him", "dog"]}, {"sent": "we already have a cat and two hamsters .", "words": ["we", "and", "have", "cat", "a"]}, {"sent": "I see a purple cat looking at me .", "words": ["me", "cat", "at", "see", "a"]}, {"sent": "wheres he going to whats he going to do is he going to take that poorly cat home .", "words": ["do", "is", "to", "cat", "that", "he", "take", "home"]}, {"sent": "and a kitty cat .", "words": ["cat", "kitty", "a", "and"]}, {"sent": "cat .", "words": ["cat"]}, {"sent": "he is gonna eat the cat .", "words": ["the", "is", "eat", "cat", "he"]}, {"sent": "theres a cat .", "words": ["cat", "a"]}, {"sent": "the cat thats right .", "words": ["the", "cat"]}, {"sent": "this is a sweet little pussy cat .", "words": ["is", "little", "this", "cat", "a"]}, {"sent": "bye bye cat .", "words": ["cat"]}, {"sent": "you have the cat wow .", "words": ["have", "the", "cat", "you"]}, {"sent": "is the cat purring ?", "words": ["the", "cat", "is"]}, {"sent": "a cat and two bats are trying on hats .", "words": ["and", "cat", "are", "on", "a"]}, {"sent": "cat .", "words": ["cat"]}, {"sent": "there is two owls and a pussy cat and a .", "words": ["is", "and", "there", "cat", "a"]}, {"sent": "c as in cat ?", "words": ["cat", "in"]}, {"sent": "hi cat ears .", "words": ["cat"]}, {"sent": "beaming like a Cheshire cat .", "words": ["cat", "like", "a"]}, {"sent": "he has got the tree as well as the cat .", "words": ["the", "cat", "tree", "he"]}, {"sent": "oh that poor pussy cat .", "words": ["cat", "that", "poor"]}, {"sent": "kitty cat ?", "words": ["cat", "kitty"]}, {"sent": "cat .", "words": ["cat"]}, {"sent": "its about a fat cat .", "words": ["cat", "about", "a"]}, {"sent": "the cat jumps outof the carriage .", "words": ["the", "cat"]}, {"sent": "thats a cat there isnt it ?", "words": ["cat", "it", "there", "a"]}, {"sent": "yeah yeah we do have a cat at home too but she doesnt like I think she started like sometimes shell imitate like kitty an approximation of the word for me but puppy shes definitely more interested in the dog than the cat .", "words": ["the", "think", "of", "like", "an", "more", "me", "have", "kitty", "she", "home", "for", "too", "a", "do", "we", "cat", "in", "puppy", "at", "dog", "but"]}, {"sent": "pussy cat yes .", "words": ["cat"]}, {"sent": "not cat house .", "words": ["cat", "house", "not"]}, {"sent": "oh and then they look back and the cat has come out from behind the tree .", "words": ["the", "tree", "back", "behind", "and", "they", "cat", "out", "look", "then"]}, {"sent": "well its sposta be the shape of a pussy cat because weve shaked weve shaken xxx .", "words": ["the", "of", "be", "cat", "because", "a"]}, {"sent": "yeah I thought you were talking about Gus the cat for a minute .", "words": ["the", "for", "you", "cat", "about", "were", "a"]}, {"sent": "theres another cat on your garden now look .", "words": ["your", "another", "cat", "garden", "look", "on"]}, {"sent": "the cat thats being the baby ?", "words": ["the", "cat"]}, {"sent": "shed purr like this toy cat purrs .", "words": ["this", "toy", "like", "cat"]}, {"sent": "the cat ?", "words": ["the", "cat"]}, {"sent": "a cat shelter I think its called .", "words": ["cat", "think", "a"]}, {"sent": "say cat .", "words": ["cat", "say"]}, {"sent": "is this a big kitty cat or a little kitty cat ?", "words": ["is", "little", "this", "cat", "kitty", "big", "a"]}, {"sent": "you be my cat and I be a witch .", "words": ["my", "be", "and", "you", "cat", "a"]}, {"sent": "well not a baby cat but a xxx cat .", "words": ["cat", "not", "but", "a"]}, {"sent": "whats a kitty cat say ?", "words": ["cat", "kitty", "say", "a"]}, {"sent": "the cat is in the bath full of water .", "words": ["the", "full", "is", "of", "cat", "in", "water"]}, {"sent": "whats Sues pussy cat called ?", "words": ["cat"]}, {"sent": "theres a bit of his pussy cat .", "words": ["of", "his", "cat", "a"]}, {"sent": "can I have the cat ?", "words": ["can", "the", "have", "cat"]}, {"sent": "I should have used orange for the pussy cat shouldnt I ?", "words": ["the", "for", "have", "cat", "orange"]}, {"sent": "and whose cat was Jack ?", "words": ["was", "cat", "and"]}, {"sent": "aah theres the blue cat .", "words": ["the", "cat", "blue"]}, {"sent": "hes looking at the cat .", "words": ["the", "cat", "at"]}, {"sent": "and the cat went on top and ran away .", "words": ["the", "and", "cat", "away", "on"]}, {"sent": "kitty cat .", "words": ["cat", "kitty"]}, {"sent": "you cant catch us ginger cat laughs Jane with such delight Sam skates around in circles and ties ginger up in his kite .", "words": ["up", "catch", "with", "you", "and", "us", "cat", "in", "around", "his"]}, {"sent": "you can have a biscuit but you dont have cat biscuits .", "words": ["can", "you", "have", "dont", "cat", "but", "a"]}, {"sent": "and wheres the cat in his bed ?", "words": ["the", "bed", "and", "cat", "in", "his"]}, {"sent": "mother cat asked Mr Fixit how much money it would cost to fix it .", "words": ["money", "how", "to", "fix", "cat", "it", "would", "much"]}, {"sent": "leave the cat alone and finish eating .", "words": ["the", "cat", "finish", "and"]}, {"sent": "and a cat .", "words": ["cat", "a", "and"]}, {"sent": "because the cat had become very frightened and it wouldnt go out the house and it was just very very weak and very very sad .", "words": ["the", "go", "was", "and", "cat", "it", "house", "out", "sad", "because"]}, {"sent": "where is the cat ?", "words": ["where", "the", "cat", "is"]}, {"sent": "look again thats the cat meow can you find the dog ?", "words": ["the", "can", "find", "you", "cat", "look", "dog"]}, {"sent": "have you got a cat and a dog please ?", "words": ["and", "have", "cat", "you", "dog", "a"]}, {"sent": "oh I see oh so its not a real pussy cat ?", "words": ["so", "cat", "see", "not", "a"]}, {"sent": "fish for the cat yep .", "words": ["the", "cat", "for", "fish"]}, {"sent": "talk to the toy pussy cat while youre waiting .", "words": ["the", "to", "cat", "toy", "talk"]}, {"sent": "to keep up the cat .", "words": ["up", "to", "cat", "the"]}, {"sent": "yeah we have a kitty cat .", "words": ["we", "have", "cat", "kitty", "a"]}, {"sent": "hey diddle diddle the cat and the fiddle the cow jumped over the moon .", "words": ["the", "moon", "and", "cat", "over", "cow"]}, {"sent": "look Jean Im trying to get the cat a fish for dinner .", "words": ["the", "for", "to", "fish", "get", "cat", "look", "a"]}, {"sent": "thats Sues pussy cat called Smudge .", "words": ["cat"]}, {"sent": "now nobody has any cheese but the mouse and the cat how about that .", "words": ["the", "how", "any", "and", "cat", "that", "mouse", "about", "cheese", "but"]}, {"sent": "he doesnt know how to get the cat out .", "words": ["the", "how", "to", "get", "cat", "he", "out"]}, {"sent": "the fat cat .", "words": ["the", "cat"]}, {"sent": "see the cat over there ?", "words": ["the", "cat", "there", "see", "over"]}, {"sent": "cat outside .", "words": ["cat", "outside"]}, {"sent": "did you make a valentine for kitty cat ?", "words": ["for", "you", "cat", "kitty", "did", "make", "a"]}, {"sent": "what letter will chase away a cat ?", "words": ["what", "chase", "cat", "away", "will", "a"]}, {"sent": "the other cat can be looking from the barn looking way down at the other cat drinking the milk .", "words": ["the", "can", "be", "other", "cat", "at", "milk", "down"]}, {"sent": "you cant see said the cat .", "words": ["the", "see", "cat", "you"]}, {"sent": "that cat he said just look and see .", "words": ["and", "cat", "he", "look", "see", "that"]}, {"sent": "the cat runs in fear of the dog I can tell .", "words": ["the", "can", "of", "cat", "in", "dog"]}, {"sent": "purple cat purple cat what do you see ?", "words": ["what", "do", "you", "cat", "see"]}, {"sent": "but it was actually fur from the grey and white cat because this naughty ginger cat that were feeding now had attacked him .", "words": ["the", "were", "was", "because", "and", "cat", "this", "it", "him", "white", "that", "naughty", "but"]}, {"sent": "cat in the basket .", "words": ["the", "cat", "in", "basket"]}, {"sent": "and this grey cat is lying down .", "words": ["is", "and", "this", "cat", "down"]}, {"sent": "is that a cat ?", "words": ["cat", "that", "a", "is"]}, {"sent": "PostmanPats cat .", "words": ["cat"]}, {"sent": "and the cat runs right into that into that tube again .", "words": ["the", "into", "and", "cat", "that"]}, {"sent": "a cat .", "words": ["cat", "a"]}, {"sent": "wed better take some cat food as well for Purdie .", "words": ["better", "for", "cat", "food", "some", "take"]}, {"sent": "listen to the cat .", "words": ["the", "to", "cat", "listen"]}, {"sent": "and meanwhile the little boy who was the cowboy caught the cat .", "words": ["the", "was", "little", "and", "cat", "who"]}, {"sent": "hows your pussy cat Jeannine ?", "words": ["cat", "your"]}, {"sent": "oh are you a cat now ?", "words": ["cat", "are", "a", "you"]}, {"sent": "Thomas we needta call the police again because theres a pussy cat stuck at the top of the lift .", "words": ["the", "we", "of", "cat", "stuck", "at", "because", "a"]}, {"sent": "okay Ill read .", "words": ["read"]}, {"sent": "wanna read you a story .", "words": ["you", "read", "wanna", "story", "a"]}, {"sent": "take that and then put it back tonight because Id like to read it .", "words": ["put", "back", "to", "then", "like", "and", "it", "read", "take", "that", "because"]}, {"sent": "should we read this ?", "words": ["this", "read", "we"]}, {"sent": "okay lets read it .", "words": ["it", "read"]}, {"sent": "you have a lot of good books here that you havent read for a long time .", "words": ["for", "of", "a lot", "long", "you", "have", "here", "read", "good", "that", "a"]}, {"sent": "should we read Dora ?", "words": ["read", "we"]}, {"sent": "why do you wanna read a story ?", "words": ["do", "you", "read", "wanna", "story", "why", "a"]}, {"sent": "youve read that quite a few times .", "words": ["that", "read", "a"]}, {"sent": "do you wanna read that one first ?", "words": ["do", "first", "you", "read", "wanna", "that"]}, {"sent": "do you want me to read two books to you ?", "words": ["do", "to", "you", "read", "me"]}, {"sent": "would you like me to read the book to you ?", "words": ["the", "to", "like", "you", "read", "would", "book", "me"]}, {"sent": "what story that we read the other night does that remind you of ?", "words": ["the", "what", "does", "we", "of", "other", "you", "read", "story", "that"]}, {"sent": "should we read shy Charles ?", "words": ["read", "we"]}, {"sent": "read ?", "words": ["read"]}, {"sent": "wanna read Pat the bunny ?", "words": ["the", "bunny", "read", "wanna"]}, {"sent": "hes using his torch to read a book isnt he ?", "words": ["to", "read", "he", "his", "book", "a"]}, {"sent": "lets look at this first page together can you read a couple of sentences to me ?", "words": ["can", "look", "of", "to", "first", "you", "this", "read", "at", "me", "a"]}, {"sent": "what sidit awright read this for me .", "words": ["what", "for", "this", "read", "me"]}, {"sent": "and I can read teddy a story .", "words": ["can", "and", "read", "story", "a"]}, {"sent": "which one shall we read first ?", "words": ["first", "which", "read", "we"]}, {"sent": "Mummy read it ?", "words": ["it", "read"]}, {"sent": "you read me it me .", "words": ["me", "it", "read", "you"]}, {"sent": "gonna read a book ?", "words": ["book", "read", "a"]}, {"sent": "do ya wanna read a book ?", "words": ["do", "read", "wanna", "book", "a"]}, {"sent": "something we read but shh reading wasnt really what she wanted to do it didnt seem like .", "words": ["what", "do", "we", "to", "like", "it", "read", "she", "but"]}, {"sent": "can Eleanor read that book to Olivia Mama and Jwww because wed like to hear it .", "words": ["can", "to", "like", "and", "hear", "it", "read", "book", "that", "because"]}, {"sent": "I xxx said thats the music and you hafta learn how to read the music .", "words": ["the", "how", "to", "you", "and", "read"]}, {"sent": "oh we dont hafta read it .", "words": ["dont", "it", "read", "we"]}, {"sent": "and whats is there a favorite book that you actually tend to read her a lot of the ones at home ?", "words": ["the", "is", "of", "to", "a lot", "her", "and", "you", "there", "read", "at", "book", "home", "that", "a"]}, {"sent": "you wanna read the picture book ?", "words": ["the", "picture", "you", "read", "wanna", "book"]}, {"sent": "do you wanna read a story ?", "words": ["do", "you", "read", "wanna", "story", "a"]}, {"sent": "you want me to read it ?", "words": ["to", "you", "it", "read", "me"]}, {"sent": "wanna read that one ?", "words": ["that", "read", "wanna"]}, {"sent": "you wanna play those things after you read it ?", "words": ["those", "play", "you", "it", "read", "wanna"]}, {"sent": "we read another book ?", "words": ["another", "book", "read", "we"]}, {"sent": "I cant even read this part .", "words": ["this", "read"]}, {"sent": "lets see this one we didnt read this one .", "words": ["see", "read", "this", "we"]}, {"sent": "xxx read the writing not looking at the pictures .", "words": ["the", "read", "not", "at"]}, {"sent": "I couldnt read it .", "words": ["it", "read"]}, {"sent": "do you want me to read you a story ?", "words": ["do", "to", "you", "read", "me", "story", "a"]}, {"sent": "okay Ill read .", "words": ["read"]}, {"sent": "you wanna read the book ?", "words": ["the", "you", "read", "wanna", "book"]}, {"sent": "be good if you could read it really uhn wouldnt it ?", "words": ["if", "be", "you", "it", "read", "good", "could"]}, {"sent": "and Ill read you another book .", "words": ["you", "another", "and", "read", "book"]}, {"sent": "here lets read a book .", "words": ["here", "book", "read", "a"]}, {"sent": "you read it then .", "words": ["then", "it", "read", "you"]}, {"sent": "shall we read it ?", "words": ["it", "read", "we"]}, {"sent": "which books did you read to her ?", "words": ["to", "her", "you", "read", "did", "which"]}, {"sent": "oh babys gonna read it ?", "words": ["it", "read"]}, {"sent": "what else can we read about what other state should we read about ?", "words": ["can", "what", "we", "other", "read", "about"]}, {"sent": "mommy a poem to read ?", "words": ["to", "read", "a"]}, {"sent": "oh read read .", "words": ["read"]}, {"sent": "would you like me to read to you ?", "words": ["to", "like", "you", "read", "would", "me"]}, {"sent": "okay I will read it .", "words": ["will", "it", "read"]}, {"sent": "youre going to read me another story ?", "words": ["to", "another", "read", "me", "story"]}, {"sent": "lets read this caterpillar book .", "words": ["book", "this", "read"]}, {"sent": "Im going to read this book to you .", "words": ["to", "you", "this", "read", "book"]}, {"sent": "next year youll probably be able to read these books .", "words": ["to", "be", "read", "these"]}, {"sent": "I read it wrong .", "words": ["it", "read"]}, {"sent": "you wanna read a book ?", "words": ["you", "read", "wanna", "book", "a"]}, {"sent": "well I thought you wanted to read oh you want me down there with you oh .", "words": ["to", "with", "you", "there", "read", "me", "down"]}, {"sent": "yeah I was just gonna say how about if we read a little bit at a time one story a day .", "words": ["how", "if", "we", "was", "say", "little", "read", "about", "at", "story", "a"]}, {"sent": "you always love to read books .", "words": ["to", "read", "love", "you"]}, {"sent": "and then after Daddy had read it we could say thank you Daddy for sharing your newspaper .", "words": ["for", "your", "we", "say", "and", "you", "it", "read", "then", "could"]}, {"sent": "I havent read that one yet either Becky .", "words": ["that", "read"]}, {"sent": "this is a long one do you wanna read a shorter one ?", "words": ["do", "is", "long", "you", "this", "read", "wanna", "a"]}, {"sent": "you read .", "words": ["read", "you"]}, {"sent": "she is yeah shes going to read her books .", "words": ["is", "to", "read", "her", "she"]}, {"sent": "dont you like to read ?", "words": ["to", "like", "you", "dont", "read"]}, {"sent": "do you want to read it ?", "words": ["do", "to", "you", "it", "read", "want to"]}, {"sent": "what shall we read ?", "words": ["what", "read", "we"]}, {"sent": "did you did mom read any of your Babar stories yet ?", "words": ["your", "of", "any", "you", "read", "did"]}, {"sent": "can you read it out loud so we can read it ?", "words": ["can", "we", "so", "you", "it", "read", "out", "loud"]}, {"sent": "well sit down here and read to Syrup .", "words": ["to", "and", "here", "read", "sit", "down"]}, {"sent": "what page should we read now ?", "words": ["what", "read", "we"]}, {"sent": "I havent read this yet .", "words": ["this", "read"]}, {"sent": "Sweetie did you want to read another book ?", "words": ["to", "you", "another", "read", "did", "want to", "book"]}, {"sent": "should we read Daisy Duck ?", "words": ["read", "we"]}, {"sent": "wanna read a book ?", "words": ["book", "read", "wanna", "a"]}, {"sent": "xxx read to you xxx .", "words": ["to", "read", "you"]}, {"sent": "Eleanor first read it darling .", "words": ["it", "first", "read"]}, {"sent": "you read that page first .", "words": ["that", "first", "read", "you"]}, {"sent": "read this page .", "words": ["this", "read"]}, {"sent": "we dont read and write at the same time .", "words": ["the", "we", "write", "and", "dont", "read", "same", "at"]}, {"sent": "do you think you could learn to read ?", "words": ["do", "think", "to", "you", "read", "could"]}, {"sent": "do you wanna read that book or another book ?", "words": ["do", "you", "another", "read", "wanna", "book", "that"]}, {"sent": "and then well typically read one or two .", "words": ["then", "read", "and"]}, {"sent": "okay remember those books we left in your room that we were gonna read ?", "words": ["your", "those", "we", "in", "room", "read", "that", "were"]}, {"sent": "and read a bit of the paper and watch the news in peace .", "words": ["the", "watch", "of", "paper", "and", "in", "read", "a"]}, {"sent": "weve read this before .", "words": ["this", "read"]}, {"sent": "help teddy read a bedtime story .", "words": ["help", "read", "story", "a"]}, {"sent": "shall we read this book ?", "words": ["book", "this", "read", "we"]}, {"sent": "well you sit and be Farmer Jones and you read it out loud and tell me the story okay ?", "words": ["the", "story", "be", "and", "you", "it", "read", "out", "me", "sit", "loud"]}, {"sent": "you wanna read ?", "words": ["read", "wanna", "you"]}, {"sent": "did you ever read James and The Giant Peach ?", "words": ["read", "did", "and", "you"]}, {"sent": "lets see if we can read the name .", "words": ["can", "the", "if", "we", "read", "see"]}, {"sent": "theres a wasp one here if you wanna w read it .", "words": ["if", "you", "here", "it", "read", "wanna", "a"]}, {"sent": "sit down and Mommy read the story .", "words": ["the", "story", "and", "read", "sit", "down"]}, {"sent": "do you know how the frog goes ?", "words": ["the", "do", "how", "frog", "you"]}, {"sent": "and what did Freddy do with that frog ?", "words": ["what", "do", "with", "and", "frog", "did", "that"]}, {"sent": "the frog .", "words": ["the", "frog"]}, {"sent": "so does she have you said she really liked the frog .", "words": ["the", "does", "so", "you", "have", "frog", "she"]}, {"sent": "theres a frog in the box .", "words": ["the", "box", "frog", "in", "a"]}, {"sent": "this girl made a frog .", "words": ["this", "frog", "a"]}, {"sent": "theres a big frog right there .", "words": ["there", "big", "frog", "a"]}, {"sent": "thats called frog spawn .", "words": ["frog"]}, {"sent": "a frog vitamin ?", "words": ["frog", "a"]}, {"sent": "its your toy frog from the bath .", "words": ["the", "toy", "your", "frog"]}, {"sent": "wheres the frog ?", "words": ["the", "frog"]}, {"sent": "why dont you just close your eyes and Ill hide the frog .", "words": ["the", "your", "hide", "you", "and", "dont", "frog", "close", "why"]}, {"sent": "is that a frog ?", "words": ["that", "frog", "a", "is"]}, {"sent": "frog .", "words": ["frog"]}, {"sent": "a frog ?", "words": ["frog", "a"]}, {"sent": "its a frog .", "words": ["frog", "a"]}, {"sent": "is this what you mean a frog ?", "words": ["what", "is", "you", "frog", "this", "a"]}, {"sent": "theres a frog .", "words": ["frog", "a"]}, {"sent": "you dont know where frog and toad is .", "words": ["where", "is", "frog", "and", "dont", "you"]}, {"sent": "whats the frog say ?", "words": ["the", "say", "frog"]}, {"sent": "heres a frog John .", "words": ["frog", "a"]}, {"sent": "green frog whats the frog say ?", "words": ["the", "say", "green", "frog"]}, {"sent": "tree frog .", "words": ["tree", "frog"]}, {"sent": "where was the green frog ?", "words": ["where", "the", "was", "frog", "green"]}, {"sent": "and a frog .", "words": ["frog", "a", "and"]}, {"sent": "what about the frog ?", "words": ["the", "what", "about", "frog"]}, {"sent": "a big frog .", "words": ["big", "frog", "a"]}, {"sent": "its a frog isnt it ?", "words": ["it", "frog", "a"]}, {"sent": "yeah thats a frog I think .", "words": ["think", "frog", "a"]}, {"sent": "a jumping frog with long green legs .", "words": ["with", "long", "frog", "green", "a"]}, {"sent": "frog .", "words": ["frog"]}, {"sent": "that is a green frog .", "words": ["is", "frog", "green", "that", "a"]}, {"sent": "the dog thinks its a frog .", "words": ["the", "frog", "dog", "a"]}, {"sent": "I think the frog looks funny sitting up here dont you ?", "words": ["the", "up", "think", "frog", "here", "dont", "you"]}, {"sent": "the frog is walking sideways .", "words": ["the", "is", "frog"]}, {"sent": "cant you be a frog ?", "words": ["be", "frog", "a", "you"]}, {"sent": "its a little tree frog .", "words": ["tree", "frog", "little", "a"]}, {"sent": "frog frog frog .", "words": ["frog"]}, {"sent": "poor frog .", "words": ["frog", "poor"]}, {"sent": "a dog wearing a frog mask .", "words": ["frog", "dog", "a"]}, {"sent": "and theres a frog .", "words": ["frog", "a", "and"]}, {"sent": "xxx the frog .", "words": ["the", "frog"]}, {"sent": "carrot what else should I get at the grocery store should I buy a frog to eat ?", "words": ["the", "what", "store", "to", "frog", "get", "eat", "at", "buy", "a"]}, {"sent": "well we have um we have a frog a stuffed a frog that William plays with at home .", "words": ["we", "with", "frog", "have", "at", "home", "that", "a"]}, {"sent": "frog frog frog .", "words": ["frog"]}, {"sent": "that was a very nice kiss you gave that frog .", "words": ["was", "nice", "you", "frog", "that", "kiss", "a"]}, {"sent": "frog jump down here ?", "words": ["here", "jump", "down", "frog"]}, {"sent": "and a pink frog .", "words": ["frog", "a", "and"]}, {"sent": "this is the frog .", "words": ["the", "this", "frog", "is"]}, {"sent": "a frog lets see .", "words": ["see", "frog", "a"]}, {"sent": "she hears frog .", "words": ["she", "frog"]}, {"sent": "you saw a frog ?", "words": ["frog", "a", "you"]}, {"sent": "is the elephant in front of the frog or behind the frog ?", "words": ["the", "is", "of", "behind", "elephant", "frog", "in"]}, {"sent": "my frog .", "words": ["my", "frog"]}, {"sent": "do you wanna wind up the frog ?", "words": ["up", "do", "the", "you", "frog", "wanna", "wind"]}, {"sent": "oh and I showed him the frog but he didnt gear towards it I hadta prompt it .", "words": ["the", "frog", "and", "him", "it", "he", "but"]}, {"sent": "a frog .", "words": ["frog", "a"]}, {"sent": "wheres your frog ?", "words": ["your", "frog"]}, {"sent": "red eyed tree frog .", "words": ["red", "tree", "frog"]}, {"sent": "uh we have a frog .", "words": ["have", "frog", "a", "we"]}, {"sent": "thats a frog .", "words": ["frog", "a"]}, {"sent": "theres a frog .", "words": ["frog", "a"]}, {"sent": "a frog .", "words": ["frog", "a"]}, {"sent": "the silly frog .", "words": ["the", "frog"]}, {"sent": "leaving the frog the sheep .", "words": ["the", "sheep", "frog"]}, {"sent": "a frog ?", "words": ["frog", "a"]}, {"sent": "looks like a frog .", "words": ["frog", "like", "a"]}, {"sent": "I got the frog .", "words": ["the", "frog"]}, {"sent": "that is a daddy frog .", "words": ["that", "frog", "a", "is"]}, {"sent": "xxx theres another frog you can put it over here with the other frog .", "words": ["can", "the", "put", "with", "you", "another", "frog", "here", "it", "other", "over"]}, {"sent": "oh my gosh maybe the little frog fell off .", "words": ["the", "off", "my", "frog", "little"]}, {"sent": "no there wasnt a frog in there with that box of toys .", "words": ["box", "of", "with", "frog", "there", "in", "that", "a"]}, {"sent": "its a frog .", "words": ["frog", "a"]}, {"sent": "its a frog .", "words": ["frog", "a"]}, {"sent": "nail brushes you have one its a frog .", "words": ["nail", "frog", "have", "you", "a"]}, {"sent": "here you go its a frog .", "words": ["go", "frog", "here", "you", "a"]}, {"sent": "who made that frog ?", "words": ["that", "frog", "who"]}, {"sent": "is the frog on your dress ?", "words": ["the", "your", "is", "frog", "on", "dress"]}, {"sent": "poor frog .", "words": ["frog", "poor"]}, {"sent": "the duck and a frog .", "words": ["the", "frog", "and", "duck", "a"]}, {"sent": "a frog huh ?", "words": ["frog", "a"]}, {"sent": "tree frog .", "words": ["tree", "frog"]}, {"sent": "thats not a frog .", "words": ["frog", "not", "a"]}, {"sent": "and one is a frog .", "words": ["a", "frog", "is", "and"]}, {"sent": "I think we hafta do the frog green .", "words": ["the", "do", "think", "we", "frog", "green"]}, {"sent": "its a frog .", "words": ["frog", "a"]}, {"sent": "someones coming to join us frog .", "words": ["to", "frog", "us"]}, {"sent": "so whats happening to this poor frog ?", "words": ["to", "so", "frog", "poor", "this"]}, {"sent": "wheres the frog ?", "words": ["the", "frog"]}, {"sent": "a frog ?", "words": ["frog", "a"]}, {"sent": "shall we say hello to the frog ?", "words": ["the", "we", "to", "say", "frog"]}, {"sent": "how about the frog ?", "words": ["the", "how", "about", "frog"]}, {"sent": "hi frog .", "words": ["frog"]}, {"sent": "why dont you have some springrolls instead frog ?", "words": ["you", "have", "dont", "frog", "some", "why"]}, {"sent": "oh Im a frog .", "words": ["frog", "a"]}, {"sent": "theres a frog in your pocket is there ?", "words": ["your", "is", "frog", "there", "in", "a"]}, {"sent": "frog noises .", "words": ["frog"]}, {"sent": "frog on your head .", "words": ["on", "head", "your", "frog"]}, {"sent": "thats all about um a toad or a frog isnt it ?", "words": ["all", "frog", "it", "about", "a"]}, {"sent": "no that one didnt fit .", "words": ["fit", "that"]}, {"sent": "it doesnt fit on there does it ?", "words": ["does", "fit", "there", "it", "on"]}, {"sent": "it willnt fit that way will it ?", "words": ["will", "fit", "it", "that"]}, {"sent": "I dont think thatll fit on that tree .", "words": ["think", "tree", "fit", "dont", "on", "that"]}, {"sent": "these little wooden ones fit on .", "words": ["on", "these", "fit", "little"]}, {"sent": "that wont fit in there .", "words": ["in", "fit", "that", "there"]}, {"sent": "because I just wouldnt be able to fit into things like that any more .", "words": ["into", "to", "be", "fit", "any", "like", "more", "that", "because"]}, {"sent": "but were gonna be were gonna share this week in art class and have a good time and no ones gonna have a fit okay ?", "words": ["share", "be", "fit", "and", "have", "this", "in", "good", "were", "but", "a"]}, {"sent": "oh I guess it does fit .", "words": ["does", "fit", "it"]}, {"sent": "could a baby fit in there ?", "words": ["fit", "there", "in", "could", "a"]}, {"sent": "Im not fit at all .", "words": ["all", "fit", "not", "at"]}, {"sent": "that looks like it might fit you but it might be mamas too .", "words": ["be", "fit", "like", "you", "it", "too", "that", "but"]}, {"sent": "youre not you dont even fit in there .", "words": ["fit", "you", "dont", "in", "there", "not"]}, {"sent": "I dont think they will fit actually Thomas .", "words": ["think", "fit", "dont", "they", "will"]}, {"sent": "wanna fit through the street ?", "words": ["the", "fit", "street", "wanna"]}, {"sent": "you needta fit things on top of them at the join .", "words": ["the", "of", "fit", "them", "you", "at", "on"]}, {"sent": "it wont fit .", "words": ["fit", "it"]}, {"sent": "oh it wont fit .", "words": ["fit", "it"]}, {"sent": "will it fit there ?", "words": ["will", "there", "fit", "it"]}, {"sent": "can you not fit another one on ?", "words": ["can", "fit", "you", "another", "on", "not"]}, {"sent": "will it fit ?", "words": ["will", "fit", "it"]}, {"sent": "youre sposta fit them into eachother you see .", "words": ["into", "fit", "them", "you", "see"]}, {"sent": "I dont think this frockll fit will it with three sets of clothing on .", "words": ["think", "of", "fit", "with", "dont", "this", "it", "on", "will"]}, {"sent": "oh he wont quite fit in will he ?", "words": ["will", "in", "fit", "he"]}, {"sent": "I bet the pizza would fit in there .", "words": ["the", "fit", "pizza", "there", "in", "would"]}, {"sent": "see if hell fit on the chair .", "words": ["the", "if", "fit", "chair", "on", "see"]}, {"sent": "hell fit .", "words": ["fit"]}, {"sent": "oh you are gonna fit that on that one ?", "words": ["fit", "you", "are", "on", "that"]}, {"sent": "um we took tried to take it the real icecream away from her and she had a fit .", "words": ["the", "we", "to", "fit", "her", "and", "it", "away", "take", "she", "a"]}, {"sent": "does it not fit in ?", "words": ["does", "fit", "in", "it", "not"]}, {"sent": "Ill put the other ones back if they dont fit .", "words": ["the", "put", "if", "back", "fit", "other", "they", "dont"]}, {"sent": "perhaps if you put them as they were then they would all fit in .", "words": ["all", "put", "if", "fit", "them", "you", "they", "in", "would", "then", "were"]}, {"sent": "can you fit that in your mouth ?", "words": ["can", "your", "fit", "mouth", "you", "in", "that"]}, {"sent": "I dont think theyll fit actually .", "words": ["dont", "think", "fit"]}, {"sent": "and that fit down here .", "words": ["fit", "and", "here", "down", "that"]}, {"sent": "um I dont know if it will fit on my head lets see .", "words": ["my", "if", "fit", "head", "dont", "on", "it", "will", "see"]}, {"sent": "does that fit in there ?", "words": ["does", "fit", "there", "in", "that"]}, {"sent": "they dont fit on the big one .", "words": ["the", "fit", "dont", "they", "on", "big"]}, {"sent": "they all gonna fit in that jail ?", "words": ["all", "fit", "they", "in", "that"]}, {"sent": "they willnt all fit in .", "words": ["all", "they", "fit", "in"]}, {"sent": "shall we put in as many bricks as we can fit ?", "words": ["can", "put", "we", "fit", "in"]}, {"sent": "oh do you think that will fit in there ?", "words": ["do", "think", "fit", "you", "there", "in", "will", "that"]}, {"sent": "it doesnt really fit me .", "words": ["me", "fit", "it"]}, {"sent": "no fit .", "words": ["fit"]}, {"sent": "no I wouldnt fit in your bed anyways would I ?", "words": ["your", "fit", "bed", "in", "would"]}, {"sent": "does that baby fit better ?", "words": ["better", "does", "fit", "that"]}, {"sent": "lion fit on .", "words": ["on", "fit", "lion"]}, {"sent": "cant fit in that one its a triangle .", "words": ["in", "fit", "that", "a"]}, {"sent": "only one can fit there .", "words": ["can", "there", "fit"]}, {"sent": "it wont fit sweetheart .", "words": ["fit", "it"]}, {"sent": "Annes gonna fit in there .", "words": ["in", "fit", "there"]}, {"sent": "they fit together .", "words": ["they", "fit"]}, {"sent": "hm I hope they fit you well hafta check and see if they fit .", "words": ["if", "fit", "and", "you", "they", "see"]}, {"sent": "ah does he fit in there does he ?", "words": ["does", "fit", "there", "in", "he"]}, {"sent": "oh how does it fit ?", "words": ["does", "how", "it", "fit"]}, {"sent": "hope those socks will fit you though .", "words": ["will", "fit", "those", "you"]}, {"sent": "oh he dont fit in there .", "words": ["fit", "dont", "in", "there", "he"]}, {"sent": "then itll fit on .", "words": ["on", "then", "fit"]}, {"sent": "would you fit in here ?", "words": ["fit", "you", "here", "in", "would"]}, {"sent": "think it will fit there ?", "words": ["think", "fit", "there", "it", "will"]}, {"sent": "that doesnt fit .", "words": ["fit", "that"]}, {"sent": "does it fit mummy ?", "words": ["does", "fit", "it"]}, {"sent": "you just hafta fit the inside now dont you ?", "words": ["the", "fit", "inside", "you", "dont"]}, {"sent": "someone was too big to fit through .", "words": ["to", "was", "fit", "too", "big"]}, {"sent": "does he fit ?", "words": ["does", "fit", "he"]}, {"sent": "see then make his tail fit with his body .", "words": ["fit", "with", "his", "see", "then", "make"]}, {"sent": "what does fit on it ?", "words": ["what", "does", "fit", "it", "on"]}, {"sent": "I dont think thats going to fit .", "words": ["to", "dont", "think", "fit"]}, {"sent": "put it on there does it fit on ?", "words": ["does", "put", "fit", "there", "it", "on"]}, {"sent": "he wont fit .", "words": ["fit", "he"]}, {"sent": "elephant willnt fit .", "words": ["fit", "elephant"]}, {"sent": "they willnt fit him will they ?", "words": ["will", "they", "fit", "him"]}, {"sent": "they dont fit ?", "words": ["dont", "they", "fit"]}, {"sent": "I wonder if thisll fit ya if these fit you those would be cute .", "words": ["these", "those", "if", "cute", "be", "fit", "you", "would"]}, {"sent": "well will it fit in the bus ?", "words": ["the", "fit", "in", "it", "will", "bus"]}, {"sent": "does fit very well .", "words": ["does", "fit"]}, {"sent": "oh I bet this is gonna fit .", "words": ["this", "fit", "is"]}, {"sent": "do you know what could fit though ?", "words": ["do", "what", "fit", "you", "could"]}, {"sent": "yeah I bet a couple would fit in that .", "words": ["fit", "in", "would", "that", "a"]}, {"sent": "I dont think itll fit in though .", "words": ["dont", "think", "fit", "in"]}, {"sent": "oops he doesnt fit in that .", "words": ["in", "fit", "he", "that"]}, {"sent": "they just took him outof the barroom this afternoon oh around xxx in an epileptic fit .", "words": ["the", "fit", "they", "him", "this", "in", "an", "around"]}, {"sent": "I see does that fit in this block by the way ?", "words": ["the", "does", "block", "fit", "by", "this", "in", "see", "that"]}, {"sent": "will it still not fit ?", "words": ["will", "fit", "it", "not"]}, {"sent": "Lily has shoes on well thats not going to fit Lily its too small .", "words": ["to", "fit", "too", "on", "not"]}, {"sent": "it willnt fit .", "words": ["fit", "it"]}, {"sent": "does it not fit ?", "words": ["does", "fit", "it", "not"]}, {"sent": "have a fit .", "words": ["have", "fit", "a"]}, {"sent": "for it to fit .", "words": ["to", "it", "fit", "for"]}, {"sent": "those are hard to fit arent they ?", "words": ["those", "to", "fit", "they", "are", "hard"]}, {"sent": "probably about as many as you can fit on isnt it four ?", "words": ["can", "fit", "you", "it", "about", "on"]}, {"sent": "see if you can fit him .", "words": ["can", "if", "fit", "you", "him", "see"]}, {"sent": "oh Im surprised they all fit in the car .", "words": ["the", "all", "car", "fit", "they", "in"]}, {"sent": "where does it fit ?", "words": ["where", "does", "fit", "it"]}, {"sent": "do you think we could fit them in the house ?", "words": ["the", "do", "think", "we", "fit", "them", "you", "in", "house", "could"]}, {"sent": "I think its going to fit isnt it ?", "words": ["to", "think", "it", "fit"]}, {"sent": "where does that fit ?", "words": ["where", "does", "fit", "that"]}, {"sent": "can you see her leg anywhere ?", "words": ["can", "you", "leg", "see", "her"]}, {"sent": "look on your other side of your leg .", "words": ["your", "of", "other", "leg", "look", "on"]}, {"sent": "xxx leg stuck in the door .", "words": ["the", "in", "stuck", "leg", "door"]}, {"sent": "you hurting my leg .", "words": ["my", "leg", "you"]}, {"sent": "did you hurt your leg ?", "words": ["your", "hurt", "you", "did", "leg"]}, {"sent": "is it under this leg ?", "words": ["is", "under", "this", "it", "leg"]}, {"sent": "wheres Danielles leg gone ?", "words": ["leg"]}, {"sent": "it can be by your leg .", "words": ["can", "your", "be", "by", "it", "leg"]}, {"sent": "up your leg ?", "words": ["up", "your", "leg"]}, {"sent": "youve got it all on your leg look .", "words": ["all", "your", "it", "leg", "look", "on"]}, {"sent": "with my extra leg .", "words": ["my", "with", "leg"]}, {"sent": "I got your leg leg leg .", "words": ["your", "leg"]}, {"sent": "is your leg still there ?", "words": ["there", "your", "leg", "is"]}, {"sent": "swinging down from a thread she hung on by one leg .", "words": ["by", "leg", "on", "down", "she", "a"]}, {"sent": "chicken leg maybe ?", "words": ["chicken", "leg"]}, {"sent": "when when did you hurt your leg ?", "words": ["your", "hurt", "you", "did", "leg", "when"]}, {"sent": "about the little boy who broke his leg .", "words": ["the", "little", "leg", "about", "who", "his"]}, {"sent": "you drive it up and down my leg .", "words": ["up", "my", "and", "drive", "you", "it", "leg", "down"]}, {"sent": "just move your leg darling .", "words": ["your", "leg"]}, {"sent": "wheres you leg ?", "words": ["leg", "you"]}, {"sent": "hes trapped his leg .", "words": ["his", "leg"]}, {"sent": "gimme that chicken leg .", "words": ["chicken", "that", "leg"]}, {"sent": "well Ive got to take her clothes off so I can find her leg hole and put the leg back in .", "words": ["off", "can", "put", "the", "back", "to", "so", "and", "in", "leg", "find", "take", "her"]}, {"sent": "is that your bad leg ?", "words": ["your", "is", "bad", "leg", "that"]}, {"sent": "I think because while I had my broken leg my husband was shopping so they would go shopping together .", "words": ["my", "think", "go", "was", "so", "broken", "they", "leg", "would", "because"]}, {"sent": "oh it was on the kittys leg ?", "words": ["the", "was", "it", "leg", "on"]}, {"sent": "thats a table leg .", "words": ["table", "leg", "a"]}, {"sent": "lift your leg up then .", "words": ["up", "then", "your", "leg"]}, {"sent": "shes awful bad leg very lame woman .", "words": ["leg", "bad"]}, {"sent": "do your leg exercises .", "words": ["do", "your", "leg"]}, {"sent": "theres my poorly leg .", "words": ["my", "leg"]}, {"sent": "I only have one leg xxx .", "words": ["have", "leg"]}, {"sent": "table leg .", "words": ["table", "leg"]}, {"sent": "is he on top of my leg or under my leg ?", "words": ["my", "is", "of", "under", "he", "leg", "on"]}, {"sent": "are you climbing up Daddys leg ?", "words": ["up", "are", "leg", "you"]}, {"sent": "on my leg ?", "words": ["on", "my", "leg"]}, {"sent": "oh yes Ive got a leg .", "words": ["leg", "a"]}, {"sent": "Im gonna give you horse bite on your leg .", "words": ["bite", "your", "you", "on", "leg", "horse", "give"]}, {"sent": "can you hop on one leg John ?", "words": ["can", "on", "leg", "you"]}, {"sent": "whats that on your leg ?", "words": ["on", "that", "leg", "your"]}, {"sent": "elastic here tie all the ribbon around tie it all around your leg so it wont go off .", "words": ["the", "all", "off", "your", "go", "so", "here", "it", "leg", "around"]}, {"sent": "I saw she was interested in um the bandaid on the leg .", "words": ["the", "was", "in", "leg", "on", "she"]}, {"sent": "has he hurt his leg ?", "words": ["hurt", "his", "he", "leg"]}, {"sent": "and a horse bite on your leg .", "words": ["bite", "your", "and", "on", "leg", "horse", "a"]}, {"sent": "dont kneel on my leg .", "words": ["on", "dont", "my", "leg"]}, {"sent": "no you think thats her leg ?", "words": ["think", "her", "leg", "you"]}, {"sent": "somebody broke his leg didnt they ?", "words": ["his", "they", "leg"]}, {"sent": "yeah he has a blanket on his leg .", "words": ["his", "on", "he", "leg", "blanket", "a"]}, {"sent": "and then we can put the table leg down and we can have more space .", "words": ["can", "the", "put", "table", "we", "and", "have", "more", "leg", "then", "down"]}, {"sent": "does your leg still hurt ?", "words": ["hurt", "does", "your", "leg"]}, {"sent": "hold his leg look .", "words": ["his", "hold", "leg", "look"]}, {"sent": "leg .", "words": ["leg"]}, {"sent": "you just leapt over my leg .", "words": ["my", "over", "leg", "you"]}, {"sent": "leg .", "words": ["leg"]}, {"sent": "whats on your leg ?", "words": ["on", "your", "leg"]}, {"sent": "did you hurt your leg when you fell off the horse ?", "words": ["off", "the", "your", "hurt", "you", "did", "leg", "horse", "when"]}, {"sent": "hell have my leg off .", "words": ["have", "my", "leg", "off"]}, {"sent": "he would have the peg leg probably .", "words": ["the", "have", "he", "leg", "would"]}, {"sent": "can I put this other leg in your trousers please .", "words": ["can", "put", "your", "other", "this", "in", "leg"]}, {"sent": "shall we try and balance the roof on your leg ?", "words": ["the", "your", "we", "and", "roof", "leg", "on", "try"]}, {"sent": "and your back of your leg .", "words": ["your", "back", "of", "and", "leg"]}, {"sent": "your whole pant leg ?", "words": ["your", "leg"]}, {"sent": "you giving me her leg ?", "words": ["me", "her", "leg", "you"]}, {"sent": "its a chicken leg .", "words": ["chicken", "leg", "a"]}, {"sent": "yeah but look at this this birdie only showing one leg on this .", "words": ["look", "this", "leg", "at", "on", "but"]}, {"sent": "near your leg .", "words": ["your", "leg"]}, {"sent": "did you hurt your leg on Jwww ?", "words": ["your", "hurt", "you", "did", "leg", "on"]}, {"sent": "put your leg down .", "words": ["put", "down", "your", "leg"]}, {"sent": "got wobbly leg ?", "words": ["leg"]}, {"sent": "your leg is attached to your foot .", "words": ["your", "is", "to", "foot", "leg"]}, {"sent": "my leg .", "words": ["my", "leg"]}, {"sent": "a leg goes here and a leg goes here .", "words": ["here", "leg", "and", "a"]}, {"sent": "you want a bandaid on your leg ?", "words": ["your", "you", "leg", "on", "a"]}, {"sent": "sure you have your xxx on your leg dont you ?", "words": ["your", "you", "have", "dont", "leg", "on"]}, {"sent": "put one leg up .", "words": ["up", "put", "leg"]}, {"sent": "pull his leg along .", "words": ["his", "leg", "pull"]}, {"sent": "can I have this leg please ?", "words": ["can", "this", "have", "leg"]}, {"sent": "on the leg .", "words": ["on", "the", "leg"]}, {"sent": "is that Frasers leg ?", "words": ["that", "leg", "is"]}, {"sent": "hes got a poorly leg ?", "words": ["leg", "a"]}, {"sent": "its stuck on my leg .", "words": ["on", "my", "stuck", "leg"]}, {"sent": "right there by your leg .", "words": ["there", "your", "leg", "by"]}, {"sent": "the roosters pecking your leg .", "words": ["the", "your", "leg"]}, {"sent": "where is your leg .", "words": ["where", "your", "leg", "is"]}, {"sent": "what did you hurt your leg on ?", "words": ["what", "your", "hurt", "you", "did", "leg", "on"]}, {"sent": "on your other leg .", "words": ["on", "your", "leg", "other"]}, {"sent": "other leg .", "words": ["leg", "other"]}, {"sent": "a sharks going to bite her leg off is it ?", "words": ["off", "is", "to", "it", "leg", "bite", "her", "a"]}, {"sent": "lets fix the leg .", "words": ["the", "fix", "leg"]}, {"sent": "thats I think thats his leg Liz .", "words": ["his", "think", "leg"]}, {"sent": "well wait fifteen minutes for my leg .", "words": ["wait", "for", "leg", "my"]}, {"sent": "lift her leg up ?", "words": ["up", "her", "leg"]}, {"sent": "can I have your other leg please .", "words": ["can", "your", "other", "have", "leg"]}, {"sent": "put him on your leg so you can see him .", "words": ["can", "put", "your", "so", "you", "him", "leg", "on", "see"]}, {"sent": "my gligs leg doesnt fall off does it ?", "words": ["off", "does", "my", "fall", "it", "leg"]}, {"sent": "put him on your leg again .", "words": ["put", "your", "him", "leg", "on"]}, {"sent": "Daddy has got a poorly leg .", "words": ["leg", "a"]}, {"sent": "its a leg .", "words": ["leg", "a"]}, {"sent": "youre buying a lot .", "words": ["a lot", "a"]}, {"sent": "youre running round a lot arent you ?", "words": ["a lot", "you", "a"]}, {"sent": "yeah a lot of kids have favorite things .", "words": ["of", "have", "a lot", "a"]}, {"sent": "okay we got a lot of stuff for you .", "words": ["for", "we", "of", "a lot", "you", "a"]}, {"sent": "a lot of glue everywhere .", "words": ["of", "glue", "a lot", "a"]}, {"sent": "I think we also I think we get a lot of vroom .", "words": ["think", "we", "of", "a lot", "get", "a"]}, {"sent": "an octopus has a lot of legs .", "words": ["of", "an", "a lot", "a"]}, {"sent": "that boy comes came up here a lot another resident said .", "words": ["up", "a lot", "here", "another", "that", "a"]}, {"sent": "and hes a lot more active now than he was at seven or eight months .", "words": ["a lot", "was", "and", "more", "he", "at", "a"]}, {"sent": "at the um store she likes finding like all the little animal figures a lot so .", "words": ["the", "all", "a lot", "store", "so", "like", "little", "animal", "at", "she", "a"]}, {"sent": "Isabelle had a lot of cards didnt she ?", "words": ["of", "a lot", "she", "a"]}, {"sent": "we have a lot in common honey I would bring cheese too I think and bread too .", "words": ["think", "bread", "we", "a lot", "bring", "and", "have", "in", "would", "too", "cheese", "a"]}, {"sent": "a lot of the pieces look the same this is going to be a tricky puzzle .", "words": ["the", "is", "of", "to", "be", "a lot", "this", "same", "puzzle", "look", "a"]}, {"sent": "bought a lot uhn havent they ?", "words": ["a lot", "they", "a"]}, {"sent": "wow you can do a lot of .", "words": ["can", "do", "of", "a lot", "you", "a"]}, {"sent": "theres going to be a lot of people here .", "words": ["of", "to", "be", "a lot", "here", "a"]}, {"sent": "you have a lot to say .", "words": ["a lot", "to", "say", "you", "have", "a"]}, {"sent": "yeah that was a lot within the past month shes been doing so .", "words": ["the", "a lot", "was", "so", "that", "a"]}, {"sent": "I bought a lot of stuff .", "words": ["of", "a lot", "a"]}, {"sent": "but um the past few years youve had a lot of um Teletubby cards .", "words": ["the", "of", "a lot", "but", "a"]}, {"sent": "but theres a lot of dirt round his feet .", "words": ["of", "a lot", "his", "but", "a"]}, {"sent": "weve got a lot of drinking chocolate .", "words": ["of", "chocolate", "a lot", "a"]}, {"sent": "um I havent really sat down with him lately and and had a lot of one on one time with him .", "words": ["of", "a lot", "with", "and", "him", "on", "down", "a"]}, {"sent": "so are you going to use your toothbrush a lot .", "words": ["toothbrush", "your", "a lot", "to", "so", "you", "are", "a"]}, {"sent": "what a lot .", "words": ["a lot", "what", "a"]}, {"sent": "oh thats a lot of wheels .", "words": ["of", "a lot", "a"]}, {"sent": "a lot of envelopes we get are rectangles rather than squares arent they ?", "words": ["we", "of", "a lot", "get", "they", "are", "a"]}, {"sent": "but I think Purdies is because shes old and she sleeps a lot .", "words": ["old", "think", "is", "a lot", "and", "but", "she", "because", "a"]}, {"sent": "there will be a lot of meowing wont there ?", "words": ["of", "be", "a lot", "there", "will", "a"]}, {"sent": "thats a lot of wheels huh ?", "words": ["of", "a lot", "a"]}, {"sent": "a lot of people have made that over the years .", "words": ["the", "of", "a lot", "have", "that", "over", "a"]}, {"sent": "we use those quite a lot dont we ?", "words": ["those", "we", "a lot", "dont", "a"]}, {"sent": "theyre a lot like worms .", "words": ["a lot", "like", "a"]}, {"sent": "yeah theres a lot of that .", "words": ["of", "that", "a lot", "a"]}, {"sent": "thats a lot of farm animals .", "words": ["of", "a lot", "a"]}, {"sent": "not a lot .", "words": ["a lot", "not", "a"]}, {"sent": "and Providence has a lot of red bud trees which .", "words": ["red", "of", "a lot", "and", "which", "a"]}, {"sent": "it did have a lot of wheels .", "words": ["of", "a lot", "have", "it", "did", "a"]}, {"sent": "theres a lot of customers in the morning and .", "words": ["the", "of", "a lot", "and", "in", "a"]}, {"sent": "so youve had quite a lot to eat today Thomas .", "words": ["a lot", "to", "so", "eat", "a"]}, {"sent": "a lot of candy you are right .", "words": ["of", "a lot", "you", "are", "candy", "a"]}, {"sent": "so um so a lot of times when those things are are novel she goes for them .", "words": ["for", "those", "of", "so", "a lot", "them", "are", "when", "she", "a"]}, {"sent": "its a lot of bees on a mobile .", "words": ["of", "on", "a lot", "a"]}, {"sent": "if he gets a lot of water in his boat its going to sink .", "words": ["if", "of", "to", "a lot", "in", "he", "sink", "water", "his", "boat", "a"]}, {"sent": "he ate a lot of food .", "words": ["of", "a lot", "food", "he", "a"]}, {"sent": "we hafta do a lot of tapes about breakfast time .", "words": ["do", "we", "of", "a lot", "about", "a"]}, {"sent": "looks like there might be a little drop more inside but not a lot .", "words": ["a lot", "be", "inside", "like", "little", "there", "more", "drop", "not", "but", "a"]}, {"sent": "um not a lot .", "words": ["a lot", "not", "a"]}, {"sent": "when you were a lot younger thats all you useta say .", "words": ["all", "a lot", "say", "you", "when", "were", "a"]}, {"sent": "weve not been able to do a lot this morning .", "words": ["do", "a lot", "to", "this", "not", "a"]}, {"sent": "does he does he cry a lot does he ?", "words": ["does", "a lot", "cry", "he", "a"]}, {"sent": "a lot smaller than our dog .", "words": ["our", "a lot", "dog", "a"]}, {"sent": "she plays with a lot of cars and trains .", "words": ["of", "a lot", "with", "and", "she", "a"]}, {"sent": "nurse shark and you get a lot in a cup .", "words": ["cup", "a lot", "you", "get", "and", "in", "a"]}, {"sent": "we eat like pizza a lot so .", "words": ["we", "a lot", "so", "like", "pizza", "eat", "a"]}, {"sent": "theres gonna be a lot of unleaded gas in that car Nicholas .", "words": ["of", "be", "car", "a lot", "in", "that", "a"]}, {"sent": "oh because if she had the flu this week she better drink a lot .", "words": ["the", "better", "if", "drink", "a lot", "this", "she", "because", "a"]}, {"sent": "two and a lot .", "words": ["a lot", "a", "and"]}, {"sent": "think youve got a lot of clothes havent you ?", "words": ["think", "of", "a lot", "you", "a"]}, {"sent": "he talks a lot more at home .", "words": ["a lot", "more", "he", "at", "a", "home"]}, {"sent": "Eleanor youre making a lot of noise .", "words": ["of", "a lot", "a"]}, {"sent": "I mean but he liked this a lot .", "words": ["a lot", "this", "he", "but", "a"]}, {"sent": "a lot ?", "words": ["a lot", "a"]}, {"sent": "they do like to spend a lot of time in trees .", "words": ["do", "of", "to", "a lot", "like", "they", "in", "a"]}, {"sent": "the baby went pushing three times thats a lot three times .", "words": ["the", "a lot", "a"]}, {"sent": "there are a lot of cows on the farm arent there ?", "words": ["the", "of", "a lot", "there", "are", "on", "a"]}, {"sent": "yes arent there a lot of clothes .", "words": ["of", "there", "a lot", "a"]}, {"sent": "I noticed a lot of my friends like like xxx the house he just happened to see it like xxx .", "words": ["the", "my", "of", "to", "a lot", "like", "it", "house", "he", "see", "a"]}, {"sent": "I wanna get a lot of your voice on the tape recorder though .", "words": ["the", "your", "of", "a lot", "tape", "get", "wanna", "on", "a"]}, {"sent": "he told you to drink a lot .", "words": ["drink", "a lot", "to", "you", "he", "a"]}, {"sent": "I think a lot of people must have brought buses dont you ?", "words": ["think", "of", "a lot", "you", "have", "dont", "a"]}, {"sent": "oh we had a lot more huh ?", "words": ["a lot", "more", "a", "we"]}, {"sent": "I think a lot of xxx daycare theres .", "words": ["of", "think", "a lot", "a"]}, {"sent": "you got a lot of bathing suits girl .", "words": ["of", "a lot", "a", "you"]}, {"sent": "what a lot of arms weve got .", "words": ["of", "what", "a lot", "a"]}, {"sent": "so they do a lot of baby doll stuff .", "words": ["do", "of", "so", "a lot", "doll", "they", "a"]}, {"sent": "something is making Mummy sneeze a lot .", "words": ["a lot", "a", "is"]}, {"sent": "theres a lot going on here huh ?", "words": ["on", "here", "a lot", "a"]}, {"sent": "well you like that toy a lot better than I do honey .", "words": ["better", "do", "a lot", "like", "you", "toy", "that", "a"]}, {"sent": "we fed the baby a lot of pizza .", "words": ["the", "we", "of", "a lot", "pizza", "a"]}, {"sent": "he was really interested in a lot of different things .", "words": ["of", "was", "a lot", "in", "he", "a"]}, {"sent": "daddy is right you did get a lot of hair .", "words": ["is", "of", "a lot", "you", "get", "did", "hair", "a"]}, {"sent": "you seem to be having a lot of fun with it dont you ?", "words": ["of", "to", "be", "a lot", "with", "you", "dont", "it", "a"]}, {"sent": "he eats and drinks a lot that penguin doesnt he ?", "words": ["a lot", "penguin", "and", "he", "that", "a"]}, {"sent": "a lot of loud noises .", "words": ["of", "a lot", "loud", "a"]}, {"sent": "the only problem was I wouldve loved you to have slept a bit more during the day because I had a lot of jobs to do .", "words": ["the", "do", "of", "was", "to", "a lot", "you", "have", "more", "because", "a"]}, {"sent": "it would be a lot easier .", "words": ["a lot", "be", "it", "would", "a"]}, {"sent": "she rides horses a lot in shows doesnt she ?", "words": ["a lot", "in", "she", "a"]}, {"sent": "a lot of sneezing .", "words": ["of", "a lot", "a"]}, {"sent": "got a lot of books and files in his hands hasnt he ?", "words": ["of", "a lot", "and", "in", "he", "his", "a"]}, {"sent": "they do a lot of singing and dancing dont they ?", "words": ["do", "of", "a lot", "and", "they", "dont", "a"]}, {"sent": "this is a lot going on here .", "words": ["is", "a lot", "here", "this", "on", "a"]}, {"sent": "you gotta chew those a lot though .", "words": ["a lot", "those", "a", "you"]}, {"sent": "um we dont have a lot of babies .", "words": ["we", "of", "a lot", "have", "dont", "a"]}, {"sent": "so weve got a lot of things havent we ?", "words": ["we", "of", "so", "a lot", "a"]}, {"sent": "a lot of the like big interlocking block type things .", "words": ["the", "block", "of", "a lot", "like", "big", "a"]}, {"sent": "I bet it is a lot for all that meat .", "words": ["all", "for", "is", "a lot", "meat", "it", "that", "a"]}, {"sent": "you know the one that we dont really play with a lot now but you useta say it was xxx .", "words": ["the", "we", "a lot", "was", "play", "with", "say", "you", "dont", "it", "that", "but", "a"]}, {"sent": "there was a lot xxx .", "words": ["was", "there", "a lot", "a"]}, {"sent": "a lot ?", "words": ["a lot", "a"]}, {"sent": "she also thinks that um I might have told you when we were in Ireland we were looking at the moon and the cloud moved in front of it so we kept saying so we kept saying the moons hiding behind the cloud so a lot of times now shell point to clouds and say moon because she thinks the moons hiding if she cant find it .", "words": ["the", "of", "when", "have", "it", "that", "she", "were", "moon", "if", "a lot", "so", "to", "behind", "you", "and", "cloud", "find", "because", "a", "we", "say", "in", "at"]}, {"sent": "oh when she was younger she used to say moon for the moon .", "words": ["the", "for", "moon", "to", "was", "say", "when", "she"]}, {"sent": "heres a little moon .", "words": ["moon", "little", "a"]}, {"sent": "good night moon .", "words": ["moon", "good"]}, {"sent": "I dont think you see the moon honey .", "words": ["the", "think", "moon", "you", "dont", "see"]}, {"sent": "the moon .", "words": ["the", "moon"]}, {"sent": "moon yeah .", "words": ["moon"]}, {"sent": "we need a m for moon .", "words": ["for", "need", "moon", "we", "a"]}, {"sent": "the cow jumping over the moon .", "words": ["the", "over", "moon", "cow"]}, {"sent": "yeah when you flip it over its the moon .", "words": ["the", "when", "moon", "you", "it", "over"]}, {"sent": "moon .", "words": ["moon"]}, {"sent": "um and hes got the regular goodnight moon not this one .", "words": ["the", "moon", "and", "this", "not"]}, {"sent": "good night moon one two three .", "words": ["moon", "good"]}, {"sent": "youre not the man in the moon he cried staring at the smiling face .", "words": ["the", "moon", "face", "in", "he", "at", "not"]}, {"sent": "cows jumping over the moon here isnt he ?", "words": ["the", "moon", "here", "he", "over"]}, {"sent": "flying towards the moon arent they ?", "words": ["the", "they", "moon"]}, {"sent": "its the moon .", "words": ["the", "moon"]}, {"sent": "you want goodnight moon .", "words": ["moon", "you"]}, {"sent": "the moon ?", "words": ["the", "moon"]}, {"sent": "theyre holes in the moon .", "words": ["the", "in", "moon"]}, {"sent": "when it goes dark the moon comes out doesnt it ?", "words": ["the", "moon", "it", "out", "dark", "when"]}, {"sent": "so it became dark because the moon was covering the sun and the longnecks thought they hadta what did they think they were sposta do to push the sun back up again ?", "words": ["the", "what", "think", "it", "were", "up", "push", "moon", "so", "was", "sun", "to", "and", "because", "do", "back", "they", "did", "dark"]}, {"sent": "theyre going towards the moon are they ?", "words": ["the", "they", "are", "moon"]}, {"sent": "what kinda moon is that ?", "words": ["what", "that", "moon", "is"]}, {"sent": "look at the moon in the sky .", "words": ["the", "look", "moon", "sky", "in", "at"]}, {"sent": "wheres the moon ?", "words": ["the", "moon"]}, {"sent": "are you hiding the moon ?", "words": ["the", "are", "moon", "you"]}, {"sent": "m for moon .", "words": ["for", "moon"]}, {"sent": "no thats not a moon .", "words": ["moon", "not", "a"]}, {"sent": "its like um Luna the moon isnt it ?", "words": ["the", "it", "moon", "like"]}, {"sent": "he says moon ?", "words": ["he", "moon"]}, {"sent": "this rocket is on top of the moon so this right here is the moon .", "words": ["the", "moon", "is", "of", "so", "here", "this", "on"]}, {"sent": "theres the moon and the star .", "words": ["the", "star", "moon", "and"]}, {"sent": "a bite out of the moon .", "words": ["the", "moon", "of", "out", "bite", "a"]}, {"sent": "are they going up to the moon ?", "words": ["up", "the", "moon", "to", "they", "are"]}, {"sent": "oh thats the moon up in the sky you know sometimes the moon is like a circle and then sometimes the moon looks like a banana ?", "words": ["the", "up", "moon", "sky", "is", "like", "you", "and", "banana", "in", "then", "a"]}, {"sent": "half Earth half moon youre right .", "words": ["moon"]}, {"sent": "did you have goodnight moon out at all ?", "words": ["all", "moon", "you", "have", "did", "out", "at"]}, {"sent": "then I can read goodnight moon .", "words": ["can", "then", "read", "moon"]}, {"sent": "moon is bright .", "words": ["moon", "is"]}, {"sent": "the moon is cold .", "words": ["the", "moon", "cold", "is"]}, {"sent": "um looking at good night moon ?", "words": ["moon", "good", "at"]}, {"sent": "now the moon is coming out .", "words": ["the", "moon", "out", "is"]}, {"sent": "uh there was an old woman tossed up in a blanket ninety times as high as the moon .", "words": ["up", "old", "the", "moon", "was", "there", "an", "in", "high", "blanket", "a"]}, {"sent": "thats not the moon .", "words": ["the", "moon", "not"]}, {"sent": "we can make a moon .", "words": ["can", "moon", "we", "make", "a"]}, {"sent": "goodnight moon .", "words": ["moon"]}, {"sent": "where do you find the moon ?", "words": ["where", "do", "the", "moon", "you", "find"]}, {"sent": "thats the cow that jumped over the moon .", "words": ["the", "moon", "that", "over", "cow"]}, {"sent": "are you holding your feet up high into the moon ?", "words": ["up", "the", "your", "into", "moon", "you", "are", "high"]}, {"sent": "ooh look its a cow and a moon .", "words": ["moon", "and", "look", "cow", "a"]}, {"sent": "and the moon was up in the dark night sky .", "words": ["the", "up", "moon", "sky", "was", "and", "in", "dark"]}, {"sent": "carry moon beams home in a jar ?", "words": ["moon", "carry", "in", "home", "jar", "a"]}, {"sent": "big moon .", "words": ["big", "moon"]}, {"sent": "goodnight moon one two three .", "words": ["moon"]}, {"sent": "it looked like two Smarties the sun and the moon didnt it ?", "words": ["the", "moon", "sun", "like", "and", "it"]}, {"sent": "moon .", "words": ["moon"]}, {"sent": "good night moon ?", "words": ["moon", "good"]}, {"sent": "and see the moon .", "words": ["the", "see", "moon", "and"]}, {"sent": "theres your moon .", "words": ["your", "moon"]}, {"sent": "theres no moon uhn is there ?", "words": ["there", "moon", "is"]}, {"sent": "jumped over the moon .", "words": ["the", "over", "moon"]}, {"sent": "four cows jumping over the moon .", "words": ["the", "over", "moon"]}, {"sent": "thats round like the moon and the sun .", "words": ["the", "moon", "sun", "like", "and"]}, {"sent": "towards the moon .", "words": ["the", "moon"]}, {"sent": "let us turn let us here you hold on to this moon .", "words": ["hold", "moon", "to", "you", "here", "us", "this", "on"]}, {"sent": "and then she started saying its you hafta tell by the inflection the difference between moon which sounds like moo and whats the cow saying moo because they sound different but not in terms of like the consonants they only sound different in terms of her intonation .", "words": ["the", "moon", "of", "not", "her", "like", "because", "by", "you", "and", "they", "in", "but", "then", "which", "she", "cow"]}, {"sent": "yeah you like goodnight moon .", "words": ["moon", "like", "you"]}, {"sent": "look at the moon ?", "words": ["the", "at", "moon", "look"]}, {"sent": "does Laura like going to the moon ?", "words": ["the", "does", "moon", "to", "like"]}, {"sent": "hes really into the moon .", "words": ["the", "moon", "into"]}, {"sent": "the f oh the stars and the moon .", "words": ["the", "moon", "and"]}, {"sent": "but because youd made a sun and could make some a moon .", "words": ["moon", "sun", "and", "but", "some", "could", "make", "because", "a"]}, {"sent": "its the moon .", "words": ["the", "moon"]}, {"sent": "teddy smiled and nodded and the more he looked at the man in the moon the nearer his smily face seemed to get .", "words": ["the", "moon", "face", "to", "and", "get", "more", "in", "he", "at", "his"]}, {"sent": "crescent shaped moon .", "words": ["moon"]}, {"sent": "wheres the moon ?", "words": ["the", "moon"]}, {"sent": "night night moon again .", "words": ["moon"]}, {"sent": "four cows jumping over the moon .", "words": ["the", "over", "moon"]}, {"sent": "did she fly to the moon ?", "words": ["the", "moon", "to", "did", "she"]}, {"sent": "we saw the sun and the moon this morning didnt we ?", "words": ["the", "moon", "we", "sun", "and", "this"]}, {"sent": "do you think cows usually jump over the moon ?", "words": ["the", "do", "think", "moon", "you", "jump", "over"]}, {"sent": "you know a nurseryrhyme about the moon dont you ?", "words": ["the", "moon", "you", "dont", "about", "a"]}, {"sent": "uh the moon ?", "words": ["the", "moon"]}, {"sent": "wheres that moon ?", "words": ["that", "moon"]}, {"sent": "good night cow jumping over the moon .", "words": ["the", "moon", "good", "over", "cow"]}, {"sent": "and a heart and the moon and some stars .", "words": ["the", "moon", "and", "some", "a"]}, {"sent": "it must be away to the man in the moon .", "words": ["the", "moon", "to", "be", "in", "it", "away"]}, {"sent": "have you made a moon before ?", "words": ["have", "moon", "a", "you"]}, {"sent": "that it does look kinda like the moon bear .", "words": ["the", "does", "moon", "like", "bear", "it", "look", "that"]}, {"sent": "the moon .", "words": ["the", "moon"]}, {"sent": "is the moon in the sky ?", "words": ["the", "moon", "sky", "is", "in"]}, {"sent": "oh now youre gonna color the other moon .", "words": ["the", "moon", "other"]}, {"sent": "moon .", "words": ["moon"]}, {"sent": "mm the moon huh .", "words": ["the", "moon"]}, {"sent": "the moon .", "words": ["the", "moon"]}, {"sent": "thats not the moon .", "words": ["the", "moon", "not"]}, {"sent": "the moon is up in the sky ?", "words": ["the", "up", "moon", "is", "sky", "in"]}, {"sent": "and the moon went over the sun it went dark .", "words": ["the", "moon", "sun", "and", "it", "dark", "over"]}, {"sent": "funny bunny ?", "words": ["bunny"]}, {"sent": "see the bunny .", "words": ["the", "see", "bunny"]}, {"sent": "I bet most of them have got a bunny in .", "words": ["bunny", "of", "them", "have", "in", "a"]}, {"sent": "um we have we have a bunny we have dogs we have .", "words": ["have", "bunny", "a", "we"]}, {"sent": "whats the bunny do ?", "words": ["the", "do", "bunny"]}, {"sent": "can you say bunny ?", "words": ["can", "bunny", "say", "you"]}, {"sent": "what else does a bunny have ?", "words": ["what", "does", "bunny", "have", "a"]}, {"sent": "incase the easter bunny decides to visit you then hell know exactly where to come .", "words": ["the", "where", "bunny", "to", "you", "then"]}, {"sent": "yeah youre smart bunny .", "words": ["bunny"]}, {"sent": "bunny and a deer .", "words": ["deer", "bunny", "a", "and"]}, {"sent": "look at that cute bunny .", "words": ["bunny", "look", "cute", "at", "that"]}, {"sent": "hes a funny bunny isnt he ?", "words": ["bunny", "he", "a"]}, {"sent": "wheres the other bunny ?", "words": ["the", "bunny", "other"]}, {"sent": "the farmer says hes too old to be the Easter bunny said Moo .", "words": ["the", "old", "bunny", "to", "be", "too"]}, {"sent": "the EasterBunny went back to the bunny patch .", "words": ["the", "to", "bunny", "back"]}, {"sent": "whats that the the bunny in the hat .", "words": ["the", "bunny", "in", "hat", "that"]}, {"sent": "can you give the baba to the bunny ?", "words": ["can", "the", "bunny", "to", "you", "give"]}, {"sent": "are you gonna color in the bunny ?", "words": ["the", "bunny", "you", "in", "are"]}, {"sent": "lets not play with the bunny now honey .", "words": ["the", "bunny", "play", "with", "not"]}, {"sent": "Judy can pat the bunny .", "words": ["can", "the", "bunny"]}, {"sent": "I chase the butterflies and the butterflies chase me thats what the bunny says .", "words": ["the", "what", "bunny", "and", "chase", "me"]}, {"sent": "should we give the bunny a carrot ?", "words": ["the", "bunny", "we", "give", "a"]}, {"sent": "pat the bunny .", "words": ["the", "bunny"]}, {"sent": "and Mr bunny rabbit that goes for you too .", "words": ["bunny", "for", "you", "and", "too", "that"]}, {"sent": "that is the bunny .", "words": ["the", "bunny", "that", "is"]}, {"sent": "you give bunny a hug ?", "words": ["bunny", "you", "hug", "give", "a"]}, {"sent": "a bunny rabbit ?", "words": ["bunny", "a"]}, {"sent": "does bunny know which ones are red ?", "words": ["red", "does", "bunny", "are", "which"]}, {"sent": "and thats a bunny .", "words": ["bunny", "a", "and"]}, {"sent": "pj funny bunny was very sad he did not like being a bunny his mother made him eat cooked carrots every day he had far too many brothers and sisters .", "words": ["bunny", "was", "like", "and", "eat", "him", "he", "did", "carrots", "every", "his", "sad", "too", "not", "a"]}, {"sent": "can we sing to bunny ?", "words": ["can", "bunny", "we", "to", "sing"]}, {"sent": "now the fishys touching bunny .", "words": ["the", "bunny"]}, {"sent": "playing with your bunny .", "words": ["your", "bunny", "with"]}, {"sent": "did you tell bunny rabbit that were gonna go see Donald ?", "words": ["bunny", "go", "you", "did", "see", "that", "were"]}, {"sent": "see the bunny ?", "words": ["the", "see", "bunny"]}, {"sent": "no Easter bunny ?", "words": ["bunny"]}, {"sent": "the bunny goes hophop hophop .", "words": ["the", "bunny"]}, {"sent": "did you see the bunny ?", "words": ["the", "bunny", "you", "did", "see"]}, {"sent": "why dont you make make a crocodile and a a bunny .", "words": ["bunny", "you", "and", "dont", "make", "why", "a"]}, {"sent": "see bunny ?", "words": ["see", "bunny"]}, {"sent": "thats the the bunny in the crocus in the hidden garden .", "words": ["the", "garden", "in", "bunny"]}, {"sent": "bunny is sleeping .", "words": ["bunny", "is"]}, {"sent": "and you dont come out of your room till bunny wakes up .", "words": ["up", "bunny", "your", "of", "you", "and", "dont", "room", "out"]}, {"sent": "poor bunny .", "words": ["bunny", "poor"]}, {"sent": "I dont wanna be a bunny any more said pj .", "words": ["bunny", "be", "any", "dont", "more", "wanna", "a"]}, {"sent": "call the bunny .", "words": ["the", "bunny"]}, {"sent": "lets make the bunny come out again .", "words": ["the", "bunny", "make", "out"]}, {"sent": "okay well help bunny rabbit hold hold the squirt gun .", "words": ["the", "help", "bunny", "hold"]}, {"sent": "and that made pj funny bunny very unhappy .", "words": ["bunny", "that", "and"]}, {"sent": "theres a bunny .", "words": ["bunny", "a"]}, {"sent": "here bunny .", "words": ["here", "bunny"]}, {"sent": "I just see your little bunny .", "words": ["see", "bunny", "your", "little"]}, {"sent": "bunny xxx .", "words": ["bunny"]}, {"sent": "what a funny bunny .", "words": ["what", "bunny", "a"]}, {"sent": "a bunny .", "words": ["bunny", "a"]}, {"sent": "a bunny .", "words": ["bunny", "a"]}, {"sent": "yeah bunnys awake then Eleanor wakes up when bunny wakes up .", "words": ["up", "bunny", "awake", "then", "when"]}, {"sent": "like this bunny rabbit here still in the bathroom .", "words": ["the", "bunny", "like", "here", "this", "in", "bathroom"]}, {"sent": "bunny .", "words": ["bunny"]}, {"sent": "bunny well that is very close .", "words": ["close", "bunny", "that", "is"]}, {"sent": "and does the bunny the bunny tail go back and forth like that ?", "words": ["the", "does", "bunny", "back", "go", "like", "and", "that"]}, {"sent": "ooh bunny likes the squishy .", "words": ["the", "bunny"]}, {"sent": "did he find a bunny rabbit in your ear ?", "words": ["bunny", "your", "in", "he", "did", "find", "ear", "a"]}, {"sent": "bunny bunny .", "words": ["bunny"]}, {"sent": "can Cindy pat the bunny ?", "words": ["can", "the", "bunny"]}, {"sent": "did you see the bunny rabbits ?", "words": ["the", "bunny", "you", "did", "see"]}, {"sent": "bunny .", "words": ["bunny"]}, {"sent": "how big is bunny ?", "words": ["how", "bunny", "big", "is"]}, {"sent": "what color is that bunny ?", "words": ["what", "bunny", "that", "is"]}, {"sent": "what does the bunny say ?", "words": ["the", "what", "does", "bunny", "say"]}, {"sent": "shall we put the bunny in the barn ?", "words": ["the", "put", "bunny", "we", "in"]}, {"sent": "a bunny .", "words": ["bunny", "a"]}, {"sent": "now bunny .", "words": ["bunny"]}, {"sent": "your baby bunny .", "words": ["bunny", "your"]}, {"sent": "remember that little bunny ?", "words": ["bunny", "that", "little"]}, {"sent": "the bunny .", "words": ["the", "bunny"]}, {"sent": "bunny .", "words": ["bunny"]}, {"sent": "you putting money in your bunny ?", "words": ["money", "bunny", "your", "you", "in"]}, {"sent": "and the birds and the bunny rabbit and the fox .", "words": ["the", "bunny", "and"]}, {"sent": "is this bunny friends with that bunny ?", "words": ["bunny", "is", "with", "this", "that"]}, {"sent": "your bionic bunny spy glasses said Arthur okay great trade .", "words": ["bunny", "your", "glasses"]}, {"sent": "bunny goes by hoppity hop .", "words": ["bunny", "by"]}, {"sent": "and theyve got bunny .", "words": ["bunny", "and"]}, {"sent": "Judy can pat the bunny now you pat the bunny ooh hes so soft .", "words": ["can", "the", "bunny", "soft", "so", "you"]}, {"sent": "whered bunny go ?", "words": ["bunny", "go"]}, {"sent": "you trying to eat your chocolate bunny ?", "words": ["bunny", "your", "to", "chocolate", "you", "eat"]}, {"sent": "how does the bunny feel .", "words": ["the", "does", "how", "bunny"]}, {"sent": "thats the bunny book .", "words": ["the", "book", "bunny"]}, {"sent": "a momma bunny and a baby bunny .", "words": ["bunny", "and", "a"]}, {"sent": "uh peekaboo bunny .", "words": ["bunny"]}, {"sent": "okay Ill read you that book bunny .", "words": ["bunny", "you", "read", "book", "that"]}, {"sent": "thats a bunny rabbit .", "words": ["bunny", "a"]}, {"sent": "bunny rabbit .", "words": ["bunny"]}, {"sent": "wheres the bunny rabbit ?", "words": ["the", "bunny"]}, {"sent": "does the bunny eat the carrots ?", "words": ["the", "does", "bunny", "eat", "carrots"]}, {"sent": "hes a funny bunny isnt he ?", "words": ["bunny", "he", "a"]}, {"sent": "ah a bunny .", "words": ["bunny", "a"]}, {"sent": "its very dark .", "words": ["dark"]}, {"sent": "they are dark green .", "words": ["they", "are", "green", "dark"]}, {"sent": "in the dark dark s house .", "words": ["the", "in", "house", "dark"]}, {"sent": "it is dark .", "words": ["dark", "it", "is"]}, {"sent": "theyre beautiful dark purple .", "words": ["dark"]}, {"sent": "the night of March twenty fourth nineteen eighty nine was dark and cold .", "words": ["the", "of", "was", "and", "dark", "cold"]}, {"sent": "its looking a bit dark isnt it ?", "words": ["it", "dark", "a"]}, {"sent": "isnt it dark ?", "words": ["it", "dark"]}, {"sent": "is it dark yet ?", "words": ["dark", "it", "is"]}, {"sent": "not far from the ceremony in a cave at the back side of Pride rock a scraggly lion with a dark mane grumbled .", "words": ["the", "back", "of", "dark", "with", "in", "at", "rock", "lion", "not", "a"]}, {"sent": "when its dark outside daddy will be home .", "words": ["outside", "be", "dark", "will", "when", "home"]}, {"sent": "it went dark didnt it ?", "words": ["it", "dark"]}, {"sent": "he has blonde hair you have dark hair .", "words": ["you", "have", "he", "dark", "hair"]}, {"sent": "that ones a little dark but thats him .", "words": ["little", "him", "dark", "that", "but", "a"]}, {"sent": "quickly TomCat ran out into the dark down to the house of the FiddlerCat .", "words": ["the", "into", "of", "to", "house", "out", "dark", "down"]}, {"sent": "it was dark when we went wasnt it ?", "words": ["we", "was", "it", "dark", "when"]}, {"sent": "now its dark outside .", "words": ["outside", "dark"]}, {"sent": "is it is it dark outside ?", "words": ["outside", "dark", "it", "is"]}, {"sent": "one night when it was dark I saw the moon shining in the sky .", "words": ["the", "moon", "sky", "was", "in", "it", "dark", "when"]}, {"sent": "theres a dark dark cellar .", "words": ["dark", "a"]}, {"sent": "dark said little bear .", "words": ["bear", "little", "dark"]}, {"sent": "its not dark brown is the color of wood .", "words": ["the", "brown", "is", "of", "dark", "not"]}, {"sent": "but its not dark now is it ?", "words": ["is", "it", "dark", "not", "but"]}, {"sent": "but I really like your hair its so its so shiny and dark and smooth .", "words": ["your", "so", "like", "and", "dark", "hair", "but"]}, {"sent": "are they dark in color ?", "words": ["they", "in", "are", "dark"]}, {"sent": "well we still got a while before it gets dark .", "words": ["dark", "it", "a", "we"]}, {"sent": "it is dark isnt it ?", "words": ["dark", "it", "is"]}, {"sent": "oh it was dark outside ?", "words": ["was", "it", "outside", "dark"]}, {"sent": "Hildas having such a good time she doesnt notice its gotten dark .", "words": ["dark", "she", "good", "a"]}, {"sent": "in the dark ?", "words": ["the", "in", "dark"]}, {"sent": "oh does that glow in the dark ?", "words": ["the", "does", "in", "dark", "that"]}, {"sent": "youre going over to the dark side .", "words": ["the", "to", "over", "dark"]}, {"sent": "dark blue ?", "words": ["blue", "dark"]}, {"sent": "well this is when Snow White is ah lost in the dark forest .", "words": ["the", "is", "this", "in", "dark", "when"]}, {"sent": "well its a navy blue bag and so its dark inside there .", "words": ["so", "inside", "blue", "and", "there", "dark", "a"]}, {"sent": "little bear likes the dark .", "words": ["bear", "the", "dark", "little"]}, {"sent": "its getting dark .", "words": ["dark"]}, {"sent": "I need to turn the light on its a little dark in here today huh ?", "words": ["the", "need", "light", "to", "need to", "little", "here", "in", "dark", "on", "a"]}, {"sent": "its dark blue and na dont look dont color it on dere like your boy ?", "words": ["your", "dark", "like", "blue", "and", "dont", "it", "look", "on"]}, {"sent": "alright well turn it off because if I turn the other lights off it might be a little bit dark in here .", "words": ["off", "the", "if", "be", "other", "little", "here", "in", "it", "dark", "because", "a"]}, {"sent": "because its dark ?", "words": ["because", "dark"]}, {"sent": "outside the dark is getting darker and the cold is getting colder .", "words": ["the", "is", "outside", "and", "dark", "cold"]}, {"sent": "show me where the dark is .", "words": ["where", "the", "is", "show", "dark", "me"]}, {"sent": "is it dark outside ?", "words": ["outside", "dark", "it", "is"]}, {"sent": "its not dark .", "words": ["not", "dark"]}, {"sent": "dark red .", "words": ["red", "dark"]}, {"sent": "oh so the sky has dark parts and blue parts ?", "words": ["the", "sky", "so", "and", "blue", "dark"]}, {"sent": "thats another dark green piece isnt it ?", "words": ["another", "it", "green", "dark"]}, {"sent": "MrPlod follows the goblins out of the dark wood and over the bridge .", "words": ["the", "of", "and", "out", "dark", "over"]}, {"sent": "its so dark .", "words": ["so", "dark"]}, {"sent": "I told xxx to save the dark blue .", "words": ["the", "to", "blue", "dark"]}, {"sent": "that was to make it dark last night so you two could get to sleep .", "words": ["sleep", "to", "was", "last", "so", "you", "get", "that", "it", "dark", "could", "make"]}, {"sent": "it isnt dark .", "words": ["it", "dark"]}, {"sent": "the light one or the dark one ?", "words": ["the", "dark", "light"]}, {"sent": "is it dark where the keys going into the keyhole ?", "words": ["where", "the", "into", "is", "it", "dark", "keys"]}, {"sent": "do you like do you like the dark ?", "words": ["the", "do", "like", "you", "dark"]}, {"sent": "its dark well .", "words": ["dark"]}, {"sent": "well its through the dark tunnel really isnt it ?", "words": ["the", "it", "dark"]}, {"sent": "ah its dark in here .", "words": ["here", "in", "dark"]}, {"sent": "I suppose you can see them in the dark .", "words": ["can", "the", "them", "you", "in", "dark", "see"]}, {"sent": "but its deep blue its dark blue .", "words": ["dark", "but", "blue"]}, {"sent": "he likes the dark does he ?", "words": ["the", "does", "he", "dark"]}, {"sent": "it is dark is it ?", "words": ["dark", "it", "is"]}, {"sent": "what light blue or dark blue ?", "words": ["light", "what", "dark", "blue"]}, {"sent": "you have very dark and long hair .", "words": ["long", "and", "have", "you", "dark", "hair"]}, {"sent": "people who are often very dark .", "words": ["dark", "are", "who"]}, {"sent": "is it dark ?", "words": ["dark", "it", "is"]}, {"sent": "it useta be really blonde but now youre going dark I think .", "words": ["think", "be", "it", "dark", "but"]}, {"sent": "a dark brown .", "words": ["brown", "dark", "a"]}, {"sent": "shines in the dark .", "words": ["the", "in", "dark"]}, {"sent": "uh sure dark purple ?", "words": ["dark"]}, {"sent": "well Smudge is a dark cat .", "words": ["cat", "dark", "a", "is"]}, {"sent": "those silly guys for being at the playground when its getting dark dark dark huh .", "words": ["the", "for", "those", "dark", "at", "when"]}, {"sent": "sometimes we can make it dark by going over and over it .", "words": ["can", "we", "by", "and", "it", "dark", "over", "make"]}, {"sent": "its not real dark .", "words": ["not", "dark"]}, {"sent": "and in the dark dark street theres a dark dark house .", "words": ["the", "and", "in", "house", "dark", "street", "a"]}, {"sent": "is that one in the dark and that ones in the light ?", "words": ["the", "is", "light", "and", "in", "dark", "that"]}, {"sent": "dark brown .", "words": ["brown", "dark"]}, {"sent": "it was a thing people see in the dark and it was sitting on a flying saucer .", "words": ["the", "was", "and", "in", "it", "dark", "on", "see", "a"]}, {"sent": "and ran as fast as he could in the dark .", "words": ["the", "fast", "and", "in", "he", "dark", "could"]}, {"sent": "and then theres some very dark clouds particularly over Sues house house .", "words": ["and", "house", "dark", "some", "then", "over"]}, {"sent": "are you going to make another dark tunnel ?", "words": ["to", "you", "another", "are", "dark", "make"]}, {"sent": "see that dark green ?", "words": ["see", "that", "green", "dark"]}, {"sent": "but sure look how dark your side of the family is .", "words": ["the", "how", "look", "your", "is", "of", "dark", "but"]}, {"sent": "this is dark green not light green .", "words": ["is", "light", "this", "green", "dark", "not"]}, {"sent": "its very dark brown .", "words": ["brown", "dark"]}, {"sent": "it is a bit dark in there .", "words": ["is", "there", "in", "it", "dark", "a"]}, {"sent": "a dark grayish blue .", "words": ["blue", "dark", "a"]}, {"sent": "well then move it up til it gets dark .", "words": ["up", "then", "it", "dark"]}, {"sent": "Puddle tramped through dark woods until a starry sky opened before him .", "words": ["him", "sky", "a", "dark"]}, {"sent": "the moon will come out when its dark .", "words": ["the", "moon", "out", "dark", "will", "when"]}, {"sent": "you have a light green and Ill have a dark green .", "words": ["light", "you", "have", "and", "green", "dark", "a"]}, {"sent": "and orange car and a dark blue car .", "words": ["car", "and", "blue", "orange", "dark", "a"]}, {"sent": "will it be dark when we go home ?", "words": ["we", "go", "be", "it", "dark", "will", "when", "home"]}, {"sent": "if a great big purple dragon comes to see me in the dark .", "words": ["the", "if", "to", "me", "in", "dark", "see", "big", "a"]}, {"sent": "dark blue .", "words": ["blue", "dark"]}, {"sent": "you hit the ball and the panda bear throws it to Sleepy Bear ?", "words": ["the", "to", "and", "ball", "you", "bear", "it", "hit"]}, {"sent": "ball .", "words": ["ball"]}, {"sent": "put the ball in the box .", "words": ["the", "put", "box", "ball", "in"]}, {"sent": "can you throw it the ball to Mommy ?", "words": ["can", "the", "throw", "to", "you", "ball", "it"]}, {"sent": "a ball .", "words": ["ball", "a"]}, {"sent": "wheres Olivias ball ?", "words": ["ball"]}, {"sent": "ball yes .", "words": ["ball"]}, {"sent": "catch the ball .", "words": ["the", "ball", "catch"]}, {"sent": "where your ball at ?", "words": ["where", "ball", "your", "at"]}, {"sent": "let us see if we can find a ball in here .", "words": ["can", "if", "we", "ball", "us", "in", "here", "find", "see", "a"]}, {"sent": "the pink ball .", "words": ["the", "ball"]}, {"sent": "get that ball outta da hall way .", "words": ["get", "ball", "that"]}, {"sent": "the ball you want the hippo to catch the ball ?", "words": ["the", "to", "catch", "you", "ball"]}, {"sent": "okay bring the ball here .", "words": ["the", "ball", "here", "bring"]}, {"sent": "you want the ball ?", "words": ["the", "ball", "you"]}, {"sent": "instead of a golf ball .", "words": ["of", "ball", "a"]}, {"sent": "you see a ball on the tv ?", "words": ["the", "you", "ball", "tv", "on", "see", "a"]}, {"sent": "what kind of ball ?", "words": ["of", "what", "ball"]}, {"sent": "did you ever see a square ball ?", "words": ["you", "ball", "did", "see", "a"]}, {"sent": "it is your ball .", "words": ["ball", "it", "your", "is"]}, {"sent": "a ball .", "words": ["ball", "a"]}, {"sent": "you rolled a big ball but you wanna bounce this one .", "words": ["you", "ball", "this", "wanna", "big", "but", "a"]}, {"sent": "so its a whole ball .", "words": ["ball", "so", "a"]}, {"sent": "Bobby stay over here with the ball please .", "words": ["the", "with", "here", "ball", "stay", "over"]}, {"sent": "ball yeah .", "words": ["ball"]}, {"sent": "can I have the ball ?", "words": ["can", "the", "have", "ball"]}, {"sent": "oh do you want this ball ?", "words": ["ball", "do", "this", "you"]}, {"sent": "you want bouncy ball ?", "words": ["ball", "you"]}, {"sent": "balls this is one ball .", "words": ["ball", "this", "is"]}, {"sent": "Yuc look at the ball .", "words": ["the", "ball", "at", "look"]}, {"sent": "theres your ball .", "words": ["ball", "your"]}, {"sent": "youre gonna hafta stand up if you wanna reach that ball .", "words": ["up", "if", "stand", "you", "ball", "wanna", "that"]}, {"sent": "your ball .", "words": ["ball", "your"]}, {"sent": "you want the ball ?", "words": ["the", "ball", "you"]}, {"sent": "wheres that ball ?", "words": ["ball", "that"]}, {"sent": "ball oh you got the ball .", "words": ["ball", "the", "you"]}, {"sent": "ball .", "words": ["ball"]}, {"sent": "you wanna play with the ball ?", "words": ["the", "play", "with", "you", "ball", "wanna"]}, {"sent": "you get the blue ball too okay ?", "words": ["the", "blue", "get", "ball", "you", "too"]}, {"sent": "wanna throw the ball to me ?", "words": ["the", "throw", "to", "ball", "wanna", "me"]}, {"sent": "peek a boo patty cake this little piggie roll the ball ride a horse jiggety jiggy .", "words": ["the", "cake", "little", "ball", "this", "horse", "ride", "a"]}, {"sent": "oh you cant hold the ball .", "words": ["the", "ball", "hold", "you"]}, {"sent": "ball youre right thats a ball .", "words": ["ball", "a"]}, {"sent": "do you see a soccer ball here ?", "words": ["do", "you", "ball", "here", "see", "a"]}, {"sent": "that in football you dont get the ball .", "words": ["the", "you", "get", "dont", "in", "ball", "that"]}, {"sent": "get the ball .", "words": ["get", "the", "ball"]}, {"sent": "can you throw me the ball ?", "words": ["can", "the", "throw", "you", "ball", "me"]}, {"sent": "do you think the ball goes in the swimming pool ?", "words": ["the", "do", "think", "pool", "you", "ball", "in"]}, {"sent": "oh didja hit the ball ?", "words": ["the", "ball", "hit"]}, {"sent": "I remember we had a small striped ball right ?", "words": ["ball", "a", "we"]}, {"sent": "are you behind the ball ?", "words": ["the", "behind", "you", "ball", "are"]}, {"sent": "a ball .", "words": ["ball", "a"]}, {"sent": "okay take the ball .", "words": ["the", "ball", "take"]}, {"sent": "wheres that big ball that I bought you ?", "words": ["ball", "big", "that", "you"]}, {"sent": "you step over in that and you hit the ball .", "words": ["the", "and", "you", "ball", "in", "over", "hit", "that"]}, {"sent": "its already a ball .", "words": ["ball", "a"]}, {"sent": "uhoh whered the ball go ?", "words": ["the", "ball", "go"]}, {"sent": "you get the ball and come over here and throw the ball to mummy .", "words": ["the", "throw", "to", "and", "get", "ball", "you", "here", "over"]}, {"sent": "ball ?", "words": ["ball"]}, {"sent": "are you behind the ball ?", "words": ["the", "behind", "you", "ball", "are"]}, {"sent": "thats a ball thats right .", "words": ["ball", "a"]}, {"sent": "the big ball .", "words": ["the", "ball", "big"]}, {"sent": "ball .", "words": ["ball"]}, {"sent": "hes in a ball .", "words": ["ball", "in", "a"]}, {"sent": "gimme the ball Alex .", "words": ["the", "ball"]}, {"sent": "catch the ball .", "words": ["the", "ball", "catch"]}, {"sent": "give me the ball .", "words": ["the", "me", "give", "ball"]}, {"sent": "heres the little ball .", "words": ["the", "ball", "little"]}, {"sent": "yeah the ball ball ball is stuck stuck stuck .", "words": ["the", "ball", "stuck", "is"]}, {"sent": "look here is another ball here is another ball .", "words": ["is", "here", "ball", "another", "look"]}, {"sent": "the girl with the ball .", "words": ["the", "ball", "with"]}, {"sent": "its a saucy ball isnt it ?", "words": ["ball", "it", "a"]}, {"sent": "send me the big ball can you can you kick the big ball M this one xxx this ball the big ball .", "words": ["the", "can", "you", "ball", "this", "kick", "me", "big"]}, {"sent": "ball in the box .", "words": ["ball", "the", "in", "box"]}, {"sent": "is she hiding the ball ?", "words": ["the", "ball", "she", "is"]}, {"sent": "you like to push that ball ?", "words": ["push", "to", "like", "you", "ball", "that"]}, {"sent": "throw mommy the ball Sul .", "words": ["the", "ball", "throw"]}, {"sent": "make a ball .", "words": ["ball", "make", "a"]}, {"sent": "good thing it is a soft ball huh ?", "words": ["soft", "is", "ball", "it", "good", "a"]}, {"sent": "you got another ball .", "words": ["another", "ball", "you"]}, {"sent": "*SI1: pink ball .", "words": ["ball"]}, {"sent": "we can make the biggest ball right ?", "words": ["can", "the", "we", "ball", "make"]}, {"sent": "shes playing with the ball .", "words": ["the", "ball", "with"]}, {"sent": "we dont throw the ball over here .", "words": ["the", "throw", "we", "ball", "dont", "here", "over"]}, {"sent": "ball are you gone ?", "words": ["ball", "are", "you"]}, {"sent": "one ball two balls .", "words": ["ball"]}, {"sent": "huh theres a ball .", "words": ["ball", "a"]}, {"sent": "okay no you just catch the ball .", "words": ["the", "ball", "catch", "you"]}, {"sent": "hes playing bat ball .", "words": ["ball", "bat"]}, {"sent": "is the ball out there ?", "words": ["the", "is", "ball", "there", "out"]}, {"sent": "ball ball .", "words": ["ball"]}, {"sent": "LaLas ball .", "words": ["ball"]}, {"sent": "getting rougher and rougher with the ball .", "words": ["the", "ball", "with", "and"]}, {"sent": "he came down with a bump from up there on the ball and Sally and I we saw all the things .", "words": ["bump", "up", "the", "all", "we", "with", "and", "ball", "there", "he", "on", "down", "a"]}, {"sent": "see if you can go find the ball .", "words": ["can", "the", "if", "go", "you", "ball", "find", "see"]}, {"sent": "*SI1: I put it down when I was looking for the ball .", "words": ["the", "put", "for", "was", "ball", "it", "down", "when"]}, {"sent": "wanna throw me the ball ?", "words": ["the", "throw", "ball", "wanna", "me"]}, {"sent": "if we take the one big ball out .", "words": ["the", "if", "we", "ball", "out", "take", "big"]}, {"sent": "orange jacket .", "words": ["orange", "jacket"]}, {"sent": "you can make a big blue jacket for the snowman .", "words": ["can", "the", "for", "you", "blue", "make", "snowman", "big", "jacket", "a"]}, {"sent": "jacket .", "words": ["jacket"]}, {"sent": "lets take your jacket off .", "words": ["your", "take", "jacket", "off"]}, {"sent": "looks like shes in a straight jacket today .", "words": ["in", "jacket", "like", "a"]}, {"sent": "well I mean Im happy to make you a bed jacket if you want .", "words": ["if", "happy", "to", "bed", "you", "make", "jacket", "a"]}, {"sent": "and jacket .", "words": ["jacket", "and"]}, {"sent": "he is all dressed in a rain a rainhat and jacket .", "words": ["all", "rain", "is", "and", "in", "he", "jacket", "a"]}, {"sent": "Ill take my jacket off .", "words": ["off", "my", "take", "jacket"]}, {"sent": "you just leave the jacket on for a little while more and the were going to take it off .", "words": ["the", "off", "for", "to", "you", "little", "and", "more", "it", "on", "take", "were", "jacket", "a"]}, {"sent": "a cowboy jacket or a zebra robe ?", "words": ["zebra", "jacket", "a"]}, {"sent": "put your jacket on .", "words": ["on", "put", "your", "jacket"]}, {"sent": "you bet I like my jacket .", "words": ["my", "jacket", "like", "you"]}, {"sent": "hey j j why dont you put your jacket on and go outside for a little while okay ?", "words": ["put", "for", "your", "go", "outside", "you", "and", "dont", "little", "on", "jacket", "why", "a"]}, {"sent": "its actually going to make a lot of noise with your jacket on .", "words": ["your", "of", "to", "a lot", "with", "on", "make", "jacket", "a"]}, {"sent": "Eleanor wheres your jacket that Jeannine gave you ?", "words": ["that", "your", "jacket", "you"]}, {"sent": "you got the jacket on his shoes .", "words": ["the", "you", "on", "his", "jacket"]}, {"sent": "this jacket is the very newest style said Theodore the tailor .", "words": ["the", "this", "jacket", "is"]}, {"sent": "the driver has got a green jacket on yours I think he has got a blue jacket .", "words": ["the", "think", "blue", "he", "green", "on", "jacket", "a"]}, {"sent": "do you wanna put a jacket on oh youre gonna take his pants off .", "words": ["off", "do", "put", "you", "his", "pants", "wanna", "on", "take", "jacket", "a"]}, {"sent": "have I got a jacket like that ?", "words": ["like", "have", "that", "jacket", "a"]}, {"sent": "ah you can sit there and listen to the story with the jacket on .", "words": ["can", "the", "story", "listen", "to", "with", "and", "you", "there", "on", "sit", "jacket"]}, {"sent": "shes got her denim jacket on .", "words": ["on", "her", "jacket"]}, {"sent": "and xxx when he came in he was wearing a new orange jacket .", "words": ["was", "and", "in", "orange", "he", "new", "when", "jacket", "a"]}, {"sent": "wheres the jacket ?", "words": ["the", "jacket"]}, {"sent": "yeah g get your jacket .", "words": ["get", "your", "jacket"]}, {"sent": "is that your BobtheBuilder jacket ?", "words": ["your", "that", "jacket", "is"]}, {"sent": "do you like Daddys jacket ?", "words": ["do", "jacket", "like", "you"]}, {"sent": "thats Awwws jacket .", "words": ["jacket"]}, {"sent": "do I like my jacket ?", "words": ["do", "my", "jacket", "like"]}, {"sent": "look at this quilted jacket .", "words": ["this", "at", "jacket", "look"]}, {"sent": "shorts and a jacket .", "words": ["shorts", "jacket", "a", "and"]}, {"sent": "its a little scarecrow with a blue jacket and green pants .", "words": ["with", "blue", "little", "and", "pants", "green", "jacket", "a"]}, {"sent": "her jacket .", "words": ["her", "jacket"]}, {"sent": "put your jacket on isnt there ?", "words": ["put", "your", "there", "on", "jacket"]}, {"sent": "jacket .", "words": ["jacket"]}, {"sent": "maybe hes getting his money or his jacket or something .", "words": ["his", "jacket", "money"]}, {"sent": "Im showing off my new jacket I made .", "words": ["off", "my", "new", "jacket"]}, {"sent": "do you wanna put that jacket on ?", "words": ["do", "put", "you", "wanna", "on", "that", "jacket"]}, {"sent": "thats why I was asking you if it was possible to put a vest over or a jacket or something over .", "words": ["put", "if", "was", "to", "you", "it", "over", "jacket", "why", "a"]}, {"sent": "more jacket and a belt .", "words": ["belt", "and", "more", "jacket", "a"]}, {"sent": "this is a handkerchief in a mans jacket .", "words": ["is", "this", "in", "jacket", "a"]}, {"sent": "a dear little jacket .", "words": ["jacket", "little", "a"]}, {"sent": "Peters jacket got caught on a gooseberry net .", "words": ["on", "jacket", "a"]}, {"sent": "well I dont think Flopsie wore a little a little jacket did she ?", "words": ["think", "little", "dont", "did", "she", "jacket", "a"]}, {"sent": "is that your jacket Fraser ?", "words": ["your", "that", "jacket", "is"]}, {"sent": "jacket right .", "words": ["jacket"]}, {"sent": "do you wanna put that jacket on ?", "words": ["do", "put", "you", "wanna", "on", "that", "jacket"]}, {"sent": "fleecy jacket .", "words": ["jacket"]}, {"sent": "Thomas dont because thats Daddys suit jacket .", "words": ["dont", "jacket", "because"]}, {"sent": "if you wanna go out put your winter jacket on .", "words": ["put", "your", "if", "go", "you", "out", "wanna", "on", "jacket"]}, {"sent": "oh shes taken the jacket off the stick now .", "words": ["the", "stick", "jacket", "off"]}, {"sent": "see if kept my jacket on I got this Jackie .", "words": ["my", "if", "this", "on", "see", "jacket"]}, {"sent": "xxx I see a yellow jacket nest right there .", "words": ["yellow", "there", "see", "jacket", "a"]}, {"sent": "you want me to put your jacket on ?", "words": ["put", "your", "to", "you", "on", "me", "jacket"]}, {"sent": "what somebodys jacket ?", "words": ["what", "jacket"]}, {"sent": "I didnt put a jacket on her because youll wear xxx .", "words": ["put", "on", "her", "jacket", "because", "a"]}, {"sent": "jacket on .", "words": ["on", "jacket"]}, {"sent": "jacket .", "words": ["jacket"]}, {"sent": "check my jacket my little jacket .", "words": ["my", "jacket", "little"]}, {"sent": "wheres your jacket ?", "words": ["your", "jacket"]}, {"sent": "and on your jacket .", "words": ["on", "your", "jacket", "and"]}, {"sent": "is that his jacket ?", "words": ["his", "that", "jacket", "is"]}, {"sent": "put your jacket on .", "words": ["on", "put", "your", "jacket"]}, {"sent": "thats Dads jacket .", "words": ["jacket"]}, {"sent": "I put my jacket on my bed and went to wash my face and hands .", "words": ["put", "my", "face", "to", "bed", "and", "on", "wash", "jacket"]}, {"sent": "thats a fluffy jacket for you .", "words": ["for", "jacket", "you", "a"]}, {"sent": "green jacket .", "words": ["jacket", "green"]}, {"sent": "thats a jacket .", "words": ["jacket", "a"]}, {"sent": "thats just his jacket .", "words": ["his", "jacket"]}, {"sent": "just think if you had your jacket then you could have a costume then couldnt you ?", "words": ["think", "your", "if", "you", "have", "then", "jacket", "a", "could"]}, {"sent": "its a little bit too warm for the weather at the moment but you wear the jacket part of it .", "words": ["the", "for", "of", "little", "you", "at", "it", "too", "jacket", "but", "a"]}, {"sent": "Im going to wear a jacket .", "words": ["to", "jacket", "a"]}, {"sent": "Eleanor put your jacket on please .", "words": ["on", "put", "your", "jacket"]}, {"sent": "cant get the jacket on .", "words": ["get", "the", "on", "jacket"]}, {"sent": "he has on a blue jacket red tie .", "words": ["red", "blue", "he", "on", "jacket", "a"]}, {"sent": "because we do have your jacket if we need it .", "words": ["do", "your", "if", "need", "we", "have", "it", "jacket", "because"]}, {"sent": "xxx are you putting more jacket on ?", "words": ["you", "more", "are", "on", "jacket"]}, {"sent": "how does the jacket go ?", "words": ["the", "does", "how", "go", "jacket"]}, {"sent": "and put this jacket on .", "words": ["put", "and", "this", "on", "jacket"]}, {"sent": "your life jacket bumblebee ?", "words": ["your", "jacket"]}, {"sent": "do you like my jacket ?", "words": ["do", "my", "like", "you", "jacket"]}, {"sent": "thats Dwwws jacket .", "words": ["jacket"]}, {"sent": "xxx take your jacket off .", "words": ["your", "take", "jacket", "off"]}, {"sent": "little Maggie jacket .", "words": ["jacket", "little"]}, {"sent": "do you wanna get your jacket gahb ?", "words": ["do", "your", "you", "get", "wanna", "jacket"]}, {"sent": "you want your jacket ?", "words": ["your", "jacket", "you"]}, {"sent": "thats a jacket .", "words": ["jacket", "a"]}, {"sent": "if you would take of your jacket Jason and Kevin if you would take off your jacket .", "words": ["off", "your", "if", "of", "you", "and", "would", "take", "jacket"]}, {"sent": "no you dont write on your jacket .", "words": ["your", "write", "you", "dont", "on", "jacket"]}, {"sent": "youve had your denim jacket on today havent you ?", "words": ["on", "your", "jacket", "you"]}, {"sent": "put the jacket on there .", "words": ["the", "put", "there", "on", "jacket"]}, {"sent": "what a which what jacket do you want ?", "words": ["what", "do", "you", "which", "jacket", "a"]}, {"sent": "jacket mm .", "words": ["jacket"]}, {"sent": "jacket .", "words": ["jacket"]}, {"sent": "thats your jacket .", "words": ["your", "jacket"]}, {"sent": "if you wanna wear your blue jacket then you can go out okay ?", "words": ["can", "your", "if", "go", "blue", "you", "wanna", "out", "then", "jacket"]}, {"sent": "she had a red blouse on underneath a black jacket ?", "words": ["red", "black", "jacket", "on", "she", "a"]}, {"sent": "jacket .", "words": ["jacket"]}, {"sent": "yeah her jacket .", "words": ["her", "jacket"]}, {"sent": "no thats a white sheep .", "words": ["white", "sheep", "a"]}, {"sent": "thats what the sheep says .", "words": ["the", "what", "sheep"]}, {"sent": "slide with the sheep .", "words": ["the", "sheep", "slide", "with"]}, {"sent": "sheep .", "words": ["sheep"]}, {"sent": "what I I know why youre saying sheep .", "words": ["what", "sheep", "why"]}, {"sent": "heres a sheep .", "words": ["sheep", "a"]}, {"sent": "six sleepy sheep .", "words": ["sleepy", "sheep"]}, {"sent": "does the sheep like it ?", "words": ["the", "does", "like", "it", "sheep"]}, {"sent": "farmers raise sheep for their wool .", "words": ["for", "sheep", "their"]}, {"sent": "yep the sheep .", "words": ["the", "sheep"]}, {"sent": "see the sheep ?", "words": ["the", "see", "sheep"]}, {"sent": "apart from sheep .", "words": ["sheep"]}, {"sent": "sheep .", "words": ["sheep"]}, {"sent": "look at the sheep .", "words": ["the", "at", "sheep", "look"]}, {"sent": "thats right xxx how about the sheep ?", "words": ["the", "how", "sheep", "about"]}, {"sent": "should we take the sheep for a ride ?", "words": ["the", "for", "we", "ride", "sheep", "take", "a"]}, {"sent": "no sheep .", "words": ["sheep"]}, {"sent": "sheep .", "words": ["sheep"]}, {"sent": "baaa said the sheep .", "words": ["the", "sheep"]}, {"sent": "I dont think you should be pulling the wool off the sheep .", "words": ["the", "off", "think", "be", "you", "dont", "sheep"]}, {"sent": "sheep .", "words": ["sheep"]}, {"sent": "maybe youd like to sing baa baa black sheep .", "words": ["to", "like", "sing", "black", "sheep"]}, {"sent": "the sheep .", "words": ["the", "sheep"]}, {"sent": "I can find room for a little sheep .", "words": ["can", "for", "little", "room", "sheep", "find", "a"]}, {"sent": "and the other thing was a a sheep wasnt it ?", "words": ["the", "was", "other", "and", "it", "sheep", "a"]}, {"sent": "thats a sheep .", "words": ["sheep", "a"]}, {"sent": "sheep .", "words": ["sheep"]}, {"sent": "right okay which is big sheep turning over ?", "words": ["is", "which", "big", "sheep", "over"]}, {"sent": "the sheepdogs gonna go in the field with the sheep is he ?", "words": ["the", "go", "is", "with", "in", "sheep", "he"]}, {"sent": "yeah and sheep .", "words": ["sheep", "and"]}, {"sent": "sheep .", "words": ["sheep"]}, {"sent": "sheep .", "words": ["sheep"]}, {"sent": "mommy sheep is called a ewe .", "words": ["sheep", "a", "is"]}, {"sent": "thats a sheep .", "words": ["sheep", "a"]}, {"sent": "its a sheep .", "words": ["sheep", "a"]}, {"sent": "is that a sheep ?", "words": ["sheep", "that", "a", "is"]}, {"sent": "why dont you get your sheep ?", "words": ["your", "you", "get", "dont", "sheep", "why"]}, {"sent": "what do sheep say ?", "words": ["what", "do", "sheep", "say"]}, {"sent": "do you know what happens to sheep Thomas ?", "words": ["do", "what", "to", "you", "sheep"]}, {"sent": "a baby sheep ?", "words": ["sheep", "a"]}, {"sent": "wheres the sheep ?", "words": ["the", "sheep"]}, {"sent": "its a sheep .", "words": ["sheep", "a"]}, {"sent": "just playing baba black sheep .", "words": ["black", "sheep"]}, {"sent": "thats the sheep again .", "words": ["the", "sheep"]}, {"sent": "puppies quickly learn how to round up sheep .", "words": ["up", "to", "how", "sheep"]}, {"sent": "maybe the sheep ?", "words": ["the", "sheep"]}, {"sent": "thats some real sheep there .", "words": ["some", "sheep", "there"]}, {"sent": "a sheep .", "words": ["sheep", "a"]}, {"sent": "thats a baa baa black sheep .", "words": ["black", "sheep", "a"]}, {"sent": "sheep .", "words": ["sheep"]}, {"sent": "sheep .", "words": ["sheep"]}, {"sent": "I think its a baby sheep .", "words": ["think", "sheep", "a"]}, {"sent": "the farmer xxx his sheep right ?", "words": ["the", "his", "sheep"]}, {"sent": "wherere they going to live these sheep ?", "words": ["to", "they", "these", "sheep"]}, {"sent": "can you point to the sheep ?", "words": ["can", "the", "to", "you", "sheep"]}, {"sent": "theres the sheep there .", "words": ["the", "there", "sheep"]}, {"sent": "sheep .", "words": ["sheep"]}, {"sent": "you want ta put the sheep in the bag ?", "words": ["the", "put", "you", "in", "sheep"]}, {"sent": "thats a sheep .", "words": ["sheep", "a"]}, {"sent": "wheres that black sheep ?", "words": ["black", "sheep", "that"]}, {"sent": "and theres lotsof sheep .", "words": ["sheep", "and"]}, {"sent": "what about sheep ?", "words": ["what", "sheep", "about"]}, {"sent": "sheep .", "words": ["sheep"]}, {"sent": "pet the sheep .", "words": ["the", "sheep"]}, {"sent": "a bighorn sheep standing big and tall .", "words": ["big", "sheep", "and", "a"]}, {"sent": "yeah thats a sheep feel it .", "words": ["it", "sheep", "a"]}, {"sent": "pretty much all of them xxx dogs are here favorite I think she likes um the sheep she was the petting zoo she can feed the sheep .", "words": ["the", "all", "think", "zoo", "can", "of", "was", "them", "here", "are", "pretty", "sheep", "feed", "much", "she"]}, {"sent": "says sheep .", "words": ["sheep"]}, {"sent": "shall we put him with the sheep ?", "words": ["the", "put", "we", "with", "him", "sheep"]}, {"sent": "maybe the sheep lives there .", "words": ["the", "there", "sheep"]}, {"sent": "whats the sheep say ?", "words": ["the", "sheep", "say"]}, {"sent": "and the sheep .", "words": ["the", "sheep", "and"]}, {"sent": "are the sheep running ?", "words": ["the", "are", "sheep"]}, {"sent": "mhm is it a black sheep ?", "words": ["is", "black", "it", "sheep", "a"]}, {"sent": "this sheep youve given me Thomas is it off one of your fire engines ?", "words": ["off", "your", "is", "of", "this", "it", "sheep", "me"]}, {"sent": "the sheep can creep in .", "words": ["the", "in", "can", "sheep"]}, {"sent": "sheep shove sheep grunt sheep dont think to look up front .", "words": ["up", "think", "to", "dont", "sheep", "look"]}, {"sent": "he didnt wanna go to sleep thats why he didnt wanna think of sheep .", "words": ["think", "go", "sleep", "to", "of", "he", "sheep", "wanna", "why"]}, {"sent": "sheep is soft .", "words": ["soft", "sheep", "is"]}, {"sent": "is that the sheep ?", "words": ["the", "sheep", "that", "is"]}, {"sent": "doesnt matter whether its one sheep one sweet one grape whatever it is if you take something from a shop without paying that is stealing .", "words": ["if", "is", "you", "that", "it", "sheep", "take", "a"]}, {"sent": "wheres the baby sheep ?", "words": ["the", "sheep"]}, {"sent": "the sheep .", "words": ["the", "sheep"]}, {"sent": "what does a sheep that is a sheep .", "words": ["what", "does", "is", "sheep", "that", "a"]}, {"sent": "a sheep says .", "words": ["sheep", "a"]}, {"sent": "nice little white sheep standing in a row .", "words": ["nice", "little", "in", "sheep", "white", "a"]}, {"sent": "look at this a sheep .", "words": ["look", "this", "sheep", "at", "a"]}, {"sent": "no where does a sheep live ?", "words": ["where", "does", "sheep", "a"]}, {"sent": "hes the boss said a sheep .", "words": ["the", "sheep", "a"]}, {"sent": "all those sheep .", "words": ["sheep", "all", "those"]}, {"sent": "the sheep goes there doesnt it ?", "words": ["the", "there", "it", "sheep"]}, {"sent": "is that sheep still alive ?", "words": ["sheep", "that", "is"]}, {"sent": "Littlebopeep found a sheep .", "words": ["sheep", "a"]}, {"sent": "a cow a donkey and a sheep ?", "words": ["and", "sheep", "donkey", "cow", "a"]}, {"sent": "see sheep instead .", "words": ["see", "sheep"]}, {"sent": "we want some sheep .", "words": ["some", "sheep", "we"]}, {"sent": "theres mama sheep .", "words": ["sheep"]}, {"sent": "but its too windy today .", "words": ["windy", "but", "too"]}, {"sent": "its bright and windy .", "words": ["windy", "and"]}, {"sent": "its windy up there very very windy .", "words": ["up", "there", "windy"]}, {"sent": "its the windmills turning round quickly because its windy and its raining .", "words": ["the", "windy", "because", "and"]}, {"sent": "you dont want a windy day ?", "words": ["dont", "windy", "a", "you"]}, {"sent": "because when its very wet or very cold and windy she doesnt like to be outside .", "words": ["wet", "to", "be", "outside", "windy", "like", "and", "she", "when", "cold", "because"]}, {"sent": "windy isnt it ?", "words": ["it", "windy"]}, {"sent": "he still gets very windy .", "words": ["he", "windy"]}, {"sent": "and if you go out today and its windy or its sunny the cream will protect your face again from getting even more sore wont it ?", "words": ["the", "your", "if", "face", "go", "windy", "you", "and", "more", "it", "out", "will"]}, {"sent": "it was quite windy this morning wasnt it ?", "words": ["was", "it", "windy", "this"]}, {"sent": "do you remember a while ago we went with Daddy on that paddle steamer and it was really windy ?", "words": ["do", "we", "was", "with", "windy", "you", "and", "it", "on", "that", "a"]}, {"sent": "its quite windy outside isnt it ?", "words": ["outside", "it", "windy"]}, {"sent": "is it windy to you ?", "words": ["is", "to", "windy", "you", "it"]}, {"sent": "very windy day today .", "words": ["windy"]}, {"sent": "and it was windy .", "words": ["was", "it", "windy", "and"]}, {"sent": "its windy ?", "words": ["windy"]}, {"sent": "it is quite windy .", "words": ["it", "windy", "is"]}, {"sent": "on a windy day ?", "words": ["on", "windy", "a"]}, {"sent": "windy wasnt it ?", "words": ["it", "windy"]}, {"sent": "it was so cold and wet and windy and horrible .", "words": ["wet", "was", "so", "windy", "and", "it", "cold"]}, {"sent": "its not as windy as yesterday .", "words": ["windy", "not"]}, {"sent": "its warm and windy outside .", "words": ["outside", "windy", "and"]}, {"sent": "well its windy .", "words": ["windy"]}, {"sent": "its not particularly windy is it ?", "words": ["it", "windy", "not", "is"]}, {"sent": "yeah thats right sweetie windy .", "words": ["windy"]}, {"sent": "well you hafta leave something on because its windy .", "words": ["on", "windy", "because", "you"]}, {"sent": "is it windy today look outside ?", "words": ["is", "outside", "windy", "it", "look"]}, {"sent": "is it very windy today ?", "words": ["it", "windy", "is"]}, {"sent": "its quite windy again isnt it today ?", "words": ["it", "windy"]}, {"sent": "Rosies very windy did you say ?", "words": ["did", "say", "windy", "you"]}, {"sent": "windy .", "words": ["windy"]}, {"sent": "and then when we went out it was all windy and overcast and .", "words": ["all", "we", "was", "windy", "and", "it", "out", "then", "when"]}, {"sent": "it is real windy .", "words": ["it", "windy", "is"]}, {"sent": "mean its its windy too .", "words": ["windy", "too"]}, {"sent": "well I think what happened Thomas before Christmas it was very windy .", "words": ["what", "think", "was", "windy", "it"]}, {"sent": "because you can do that when its windy cant you ?", "words": ["can", "do", "when", "windy", "you", "that", "because"]}, {"sent": "its getting very windy .", "words": ["windy"]}, {"sent": "it is so stormy and windy and pretty ?", "words": ["is", "so", "windy", "and", "it", "pretty"]}, {"sent": "its getting windy isnt it ?", "words": ["it", "windy"]}, {"sent": "it is windy today boy .", "words": ["it", "windy", "is"]}, {"sent": "do you think its windy or not ?", "words": ["do", "think", "windy", "you", "not"]}, {"sent": "youre windy .", "words": ["windy"]}, {"sent": "too windy .", "words": ["windy", "too"]}, {"sent": "its getting windy out .", "words": ["windy", "out"]}, {"sent": "its windy outside isnt it ?", "words": ["outside", "it", "windy"]}, {"sent": "and its very windy .", "words": ["windy", "and"]}, {"sent": "its windy huh ?", "words": ["windy"]}, {"sent": "and because it was windy he was probably playing with his kite .", "words": ["was", "with", "windy", "and", "it", "he", "his", "because"]}, {"sent": "a windy day .", "words": ["windy", "a"]}, {"sent": "we could read Spots windy day .", "words": ["could", "read", "windy", "we"]}, {"sent": "I thought it felt a bit cool and windy but it has just started to rain .", "words": ["rain", "to", "windy", "and", "it", "but", "a"]}, {"sent": "very windy isnt it ?", "words": ["it", "windy"]}, {"sent": "windy ?", "words": ["windy"]}, {"sent": "is it bit windy ?", "words": ["it", "windy", "is"]}, {"sent": "because its very windy today isnt it ?", "words": ["it", "windy", "because"]}, {"sent": "oh you are a bit windy yeah .", "words": ["are", "windy", "a", "you"]}, {"sent": "it wasnt too windy thats right .", "words": ["it", "windy", "too"]}, {"sent": "but its getting quite windy now .", "words": ["windy", "but"]}, {"sent": "is it a very windy day then is it ?", "words": ["is", "windy", "it", "then", "a"]}, {"sent": "and wasnt it windy outside ?", "words": ["outside", "it", "windy", "and"]}, {"sent": "because its too windy out and his boat cant go out see he only has a little boat and when its too when the wind is too strong sometimes it could make it could blow away little boats it just you know it could blow them right down its very dangerous .", "words": ["the", "go", "could", "when", "it", "away", "down", "boat", "is", "windy", "and", "little", "you", "too", "see", "make", "blow", "because", "wind", "a", "them", "he", "out", "his"]}, {"sent": "it was very windy .", "words": ["was", "it", "windy"]}, {"sent": "on cold wet and windy afternoons when OldMacdonald lets his animals shelter in the warm barn there .", "words": ["the", "wet", "windy", "and", "his", "there", "in", "on", "when", "cold"]}, {"sent": "its windy and its wet and its not very nice is it ?", "words": ["wet", "is", "nice", "windy", "and", "it", "not"]}, {"sent": "its more windy .", "words": ["more", "windy"]}, {"sent": "thats what I mean it must have been a storm or something for them to blow down was it very windy ?", "words": ["what", "for", "to", "was", "them", "windy", "have", "it", "down", "blow", "a"]}, {"sent": "here oh its windy out theyll get dry soon .", "words": ["dry", "windy", "here", "get", "out"]}, {"sent": "is it a very windy day ?", "words": ["it", "windy", "a", "is"]}, {"sent": "too windy to fly the kite .", "words": ["the", "to", "windy", "too"]}, {"sent": "and three friends on a windy day .", "words": ["on", "windy", "a", "and"]}, {"sent": "sunny rainy or windy dont you ?", "words": ["dont", "windy", "you"]}, {"sent": "come on windy .", "words": ["on", "windy"]}, {"sent": "it is so windy .", "words": ["so", "it", "windy", "is"]}, {"sent": "because sometimes it can be hot and windy .", "words": ["can", "be", "windy", "and", "hot", "it", "because"]}, {"sent": "bit windy ?", "words": ["windy"]}, {"sent": "its too windy .", "words": ["windy", "too"]}, {"sent": "it is a bit windy though .", "words": ["it", "windy", "a", "is"]}, {"sent": "must be windy .", "words": ["be", "windy"]}, {"sent": "but it was too windy in the end wasnt it ?", "words": ["the", "was", "windy", "in", "it", "too", "but"]}, {"sent": "its very windy out there this morning youre gonna hafta wear xxx .", "words": ["there", "windy", "out", "this"]}, {"sent": "windy .", "words": ["windy"]}, {"sent": "it was a windy windy day .", "words": ["was", "it", "windy", "a"]}, {"sent": "yeah its really windy out .", "words": ["windy", "out"]}, {"sent": "wasnt it cold and windy ?", "words": ["it", "windy", "cold", "and"]}, {"sent": "its windy out today .", "words": ["windy", "out"]}, {"sent": "because she was colicy and windy and had like xxx .", "words": ["was", "windy", "like", "and", "she", "because"]}, {"sent": "come on windy .", "words": ["on", "windy"]}, {"sent": "it was a windy windy morning .", "words": ["was", "it", "windy", "a"]}, {"sent": "is it windy outside ?", "words": ["outside", "it", "windy", "is"]}, {"sent": "its very windy today .", "words": ["windy"]}, {"sent": "nope too windy .", "words": ["windy", "too"]}, {"sent": "its very windy out .", "words": ["windy", "out"]}, {"sent": "thats how we know its windy .", "words": ["how", "windy", "we"]}, {"sent": "very very windy .", "words": ["windy"]}, {"sent": "its a bit windy round here isnt it ?", "words": ["here", "it", "windy", "a"]}, {"sent": "what a windy road .", "words": ["what", "windy", "a"]}, {"sent": "take the windy off .", "words": ["the", "take", "windy", "off"]}, {"sent": "xxx on a windy bright march day xxx .", "words": ["on", "windy", "a"]}, {"sent": "windy day .", "words": ["windy"]}, {"sent": "and but the peas and the spaghetti seem to be his favorite .", "words": ["the", "spaghetti", "to", "be", "peas", "and", "his", "but"]}, {"sent": "Pooh Bear meet peas .", "words": ["peas"]}, {"sent": "do you wanna cook the peas next ?", "words": ["the", "do", "peas", "you", "cook", "wanna"]}, {"sent": "you like the peas dont you ?", "words": ["the", "peas", "like", "you", "dont"]}, {"sent": "and putting the he was playing with the spaghetti and the peas .", "words": ["the", "spaghetti", "was", "peas", "with", "and", "he"]}, {"sent": "five little peas in a pea pod press .", "words": ["in", "peas", "a", "little"]}, {"sent": "the peas .", "words": ["the", "peas"]}, {"sent": "for supper so I dont know if its the peas and the spaghetti are the only ones that shes eat actually no actually no she probably she liked the bread to and shes had that .", "words": ["the", "bread", "for", "if", "spaghetti", "so", "to", "peas", "and", "eat", "dont", "are", "that", "she"]}, {"sent": "you want some more peas ?", "words": ["some", "more", "peas", "you"]}, {"sent": "peas .", "words": ["peas"]}, {"sent": "I prefer my peas green than red .", "words": ["red", "my", "peas", "green"]}, {"sent": "um he eats peas but he calls them balls .", "words": ["peas", "he", "but", "them"]}, {"sent": "does Eleanor like peas ?", "words": ["does", "peas", "like"]}, {"sent": "mummies dropped the peas .", "words": ["the", "peas"]}, {"sent": "peas yeah ?", "words": ["peas"]}, {"sent": "peas .", "words": ["peas"]}, {"sent": "you can cook some peas .", "words": ["can", "peas", "you", "cook", "some"]}, {"sent": "peas .", "words": ["peas"]}, {"sent": "peas in each hand .", "words": ["in", "peas", "each", "hand"]}, {"sent": "you cant eat those peas .", "words": ["eat", "peas", "those", "you"]}, {"sent": "we had peas last night didnt we ?", "words": ["last", "peas", "we"]}, {"sent": "but I dont think shes ever seen peas qui .", "words": ["dont", "think", "peas", "but"]}, {"sent": "those peas are silly huh ?", "words": ["peas", "those", "are"]}, {"sent": "hey look peas .", "words": ["peas", "look"]}, {"sent": "and um peas and green beans .", "words": ["peas", "green beans", "and", "beans", "green"]}, {"sent": "Peter who was quite lost saw a little old mouse carrying peas to her family .", "words": ["old", "was", "to", "peas", "little", "who", "her", "mouse", "a"]}, {"sent": "wheres the peas ?", "words": ["the", "peas"]}, {"sent": "you had peas for lunch .", "words": ["peas", "for", "you"]}, {"sent": "you just want the peas .", "words": ["the", "peas", "you"]}, {"sent": "peas .", "words": ["peas"]}, {"sent": "and Grandad had peas .", "words": ["peas", "and"]}, {"sent": "you gonna eat your peas ?", "words": ["eat", "peas", "your", "you"]}, {"sent": "rice and peas .", "words": ["peas", "and"]}, {"sent": "how about peas ?", "words": ["how", "peas", "about"]}, {"sent": "peas he probably I mean he knows what peas are I .", "words": ["what", "peas", "he", "are"]}, {"sent": "peas .", "words": ["peas"]}, {"sent": "oh are there more peas ?", "words": ["there", "more", "are", "peas"]}, {"sent": "peas .", "words": ["peas"]}, {"sent": "um she loves cheese so she would recognize the cheese the pizza the peas the bread .", "words": ["the", "bread", "so", "peas", "pizza", "would", "cheese", "she"]}, {"sent": "peas and juice and fruit .", "words": ["peas", "juice", "and"]}, {"sent": "yeah we definitely have youknow the standard peas and corn .", "words": ["the", "we", "peas", "and", "have", "corn"]}, {"sent": "are you gonna give me some peas ?", "words": ["peas", "you", "are", "me", "give", "some"]}, {"sent": "what do you think of peas ?", "words": ["what", "do", "think", "of", "peas", "you"]}, {"sent": "some peas ?", "words": ["some", "peas"]}, {"sent": "peas .", "words": ["peas"]}, {"sent": "and your peas ?", "words": ["peas", "your", "and"]}, {"sent": "go get the peas .", "words": ["get", "the", "peas", "go"]}, {"sent": "you want some waffles with your peas ?", "words": ["your", "peas", "with", "you", "some"]}, {"sent": "its a petal from the sweet peas on the mantelpiece .", "words": ["the", "on", "peas", "a"]}, {"sent": "I got peas too .", "words": ["peas", "too"]}, {"sent": "peas .", "words": ["peas"]}, {"sent": "um last time he I remember he really liked the peas .", "words": ["the", "last", "peas", "he"]}, {"sent": "actual peas he likes to squish them .", "words": ["to", "peas", "he", "them"]}, {"sent": "hafta eat them peas .", "words": ["eat", "peas", "them"]}, {"sent": "peas .", "words": ["peas"]}, {"sent": "you always say that and I know you didnt have peas because I know what you had for your lunch and it wasnt peas .", "words": ["what", "for", "your", "peas", "say", "and", "have", "you", "it", "that", "because"]}, {"sent": "I wish youd eat the peas but you wont .", "words": ["the", "peas", "wish", "eat", "you", "but"]}, {"sent": "peas and beans and chicken .", "words": ["chicken", "peas", "beans", "and"]}, {"sent": "oh its peas .", "words": ["peas"]}, {"sent": "peas .", "words": ["peas"]}, {"sent": "go for the peas yummy yum .", "words": ["the", "peas", "for", "go"]}, {"sent": "smash up the peas and not that .", "words": ["up", "the", "peas", "and", "that", "not"]}, {"sent": "I like peas .", "words": ["peas", "like"]}, {"sent": "you cant eat the pasta and the peas you cant eat them both .", "words": ["the", "peas", "them", "and", "eat", "you"]}, {"sent": "and you always say red peas .", "words": ["red", "peas", "say", "you", "and"]}, {"sent": "she eat peas or spaghetti or anything like that ?", "words": ["spaghetti", "peas", "like", "eat", "that", "she"]}, {"sent": "peas and worms ?", "words": ["peas", "and"]}, {"sent": "you like peas ?", "words": ["peas", "like", "you"]}, {"sent": "yes theyre peas .", "words": ["peas"]}, {"sent": "you like peas .", "words": ["peas", "like", "you"]}, {"sent": "shes only had peas out of all of this .", "words": ["all", "of", "peas", "this", "out"]}, {"sent": "like is that peas ?", "words": ["peas", "that", "like", "is"]}, {"sent": "you love peas .", "words": ["peas", "love", "you"]}, {"sent": "what are peas ?", "words": ["what", "peas", "are"]}, {"sent": "oh oh you got the peas .", "words": ["the", "peas", "you"]}, {"sent": "more peas please look you have peas in both hands .", "words": ["peas", "you", "have", "more", "in", "look"]}, {"sent": "peas .", "words": ["peas"]}, {"sent": "so many peas .", "words": ["so", "peas"]}, {"sent": "peas .", "words": ["peas"]}, {"sent": "want some peas ?", "words": ["some", "peas"]}, {"sent": "we got peas .", "words": ["peas", "we"]}, {"sent": "and you ate the chips and peas and and one piece of bread but left the burger .", "words": ["the", "bread", "of", "peas", "you", "and", "but"]}, {"sent": "and peas .", "words": ["peas", "and"]}, {"sent": "oh theres the peas .", "words": ["the", "peas"]}, {"sent": "you really like those peas huh ?", "words": ["peas", "those", "like", "you"]}, {"sent": "will you put the peas in the bowl ?", "words": ["the", "bowl", "put", "peas", "you", "in", "will"]}, {"sent": "and peas .", "words": ["peas", "and"]}, {"sent": "youre gonna share the peas ?", "words": ["the", "peas", "share"]}, {"sent": "I think Grandad has peas .", "words": ["think", "peas"]}, {"sent": "she fed the fox her peas .", "words": ["the", "peas", "her", "she"]}, {"sent": "peas ?", "words": ["peas"]}, {"sent": "mash potatoes and peas .", "words": ["peas", "and"]}, {"sent": "okay Dan say thankyou for your peas .", "words": ["your", "peas", "for", "say"]}, {"sent": "peas on mommys head .", "words": ["on", "head", "peas"]}, {"sent": "some peas ?", "words": ["some", "peas"]}, {"sent": "give the baby some peas .", "words": ["the", "some", "give", "peas"]}, {"sent": "and lets see this has carrots and potatoes and peas that sounds yumyum .", "words": ["peas", "and", "this", "carrots", "see", "that"]}, {"sent": "you can put spaghetti on a plate and we can put peas on a plate .", "words": ["can", "put", "plate", "spaghetti", "we", "peas", "you", "and", "on", "a"]}, {"sent": "you want peas with it ?", "words": ["it", "peas", "with", "you"]}, {"sent": "its very hot .", "words": ["hot"]}, {"sent": "who said you might get a hot bum ?", "words": ["you", "get", "hot", "who", "a"]}, {"sent": "hot hot hot .", "words": ["hot"]}, {"sent": "it is hot isnt it ?", "words": ["hot", "it", "is"]}, {"sent": "how can that water be hot ?", "words": ["can", "how", "be", "hot", "water", "that"]}, {"sent": "we didnt actually go to very much because it was so hot wasnt it Fraser ?", "words": ["go", "we", "to", "was", "so", "hot", "it", "much", "because"]}, {"sent": "theyre too hot for her ?", "words": ["hot", "for", "her", "too"]}, {"sent": "hot cross buns hot cross buns .", "words": ["hot"]}, {"sent": "hot fire .", "words": ["hot"]}, {"sent": "can you say hot ?", "words": ["can", "hot", "say", "you"]}, {"sent": "hot and .", "words": ["hot", "and"]}, {"sent": "are you hot Laura ?", "words": ["hot", "are", "you"]}, {"sent": "yeah is the table hot ?", "words": ["the", "hot", "table", "is"]}, {"sent": "this time of the year in Arizona in Tucson its hot .", "words": ["the", "of", "this", "in", "hot"]}, {"sent": "getting it good and hot ?", "words": ["hot", "it", "good", "and"]}, {"sent": "the toaster gets hot doesnt it ?", "words": ["the", "hot", "it"]}, {"sent": "its hot still ?", "words": ["hot"]}, {"sent": "theres hot tea on the table .", "words": ["on", "the", "hot", "table"]}, {"sent": "very hot .", "words": ["hot"]}, {"sent": "its hot .", "words": ["hot"]}, {"sent": "it is hot .", "words": ["hot", "it", "is"]}, {"sent": "on such a hot day ?", "words": ["on", "hot", "a"]}, {"sent": "a bit hot .", "words": ["hot", "a"]}, {"sent": "its hot so sip it carefully .", "words": ["so", "hot", "it"]}, {"sent": "its too hot .", "words": ["hot", "too"]}, {"sent": "hot I know .", "words": ["hot"]}, {"sent": "I feel hot and sick all over .", "words": ["all", "and", "hot", "sick", "over"]}, {"sent": "and said this is too hot .", "words": ["is", "and", "this", "hot", "too"]}, {"sent": "because the waters very hot .", "words": ["the", "hot", "because"]}, {"sent": "and theyll be hot so well be careful once you put them in .", "words": ["put", "be", "so", "them", "and", "you", "hot", "in", "careful"]}, {"sent": "oh theres lots of hot air balloons in this one isnt there ?", "words": ["of", "this", "in", "hot", "there"]}, {"sent": "because its hot like hotchocolate .", "words": ["hot", "like", "because"]}, {"sent": "I dont think its too hot .", "words": ["dont", "think", "hot", "too"]}, {"sent": "it is hot your mommy drinks coffee doesnt she ?", "words": ["your", "is", "hot", "it", "coffee", "she"]}, {"sent": "you feel very hot .", "words": ["hot", "you"]}, {"sent": "heres the stove right here its very hot you hafta be careful okay ?", "words": ["the", "be", "you", "here", "hot", "stove", "careful"]}, {"sent": "ooo teas hot .", "words": ["hot"]}, {"sent": "xxx ow hot .", "words": ["hot"]}, {"sent": "how about cooking up some hot cereal ?", "words": ["up", "how", "hot", "about", "some", "cereal"]}, {"sent": "whats hot ?", "words": ["hot"]}, {"sent": "a mixture of stones and hot tar is tipped from a truck in front of this machine .", "words": ["is", "of", "truck", "and", "this", "hot", "in", "a"]}, {"sent": "this piece is for Victor but its so hot I really dont want it on your plate .", "words": ["for", "your", "plate", "is", "so", "this", "hot", "dont", "it", "on", "but"]}, {"sent": "its burning hot .", "words": ["hot"]}, {"sent": "a real hot sweet .", "words": ["hot", "a"]}, {"sent": "very hot actually .", "words": ["hot"]}, {"sent": "two hot peppers .", "words": ["hot"]}, {"sent": "its hot .", "words": ["hot"]}, {"sent": "because its too hot to drink .", "words": ["drink", "to", "hot", "too", "because"]}, {"sent": "and theyre hot .", "words": ["hot", "and"]}, {"sent": "because it was very hot .", "words": ["was", "hot", "it", "because"]}, {"sent": "make it hot hot hot hot .", "words": ["hot", "it", "make"]}, {"sent": "and when its very hot you wear shorts and a tshirt .", "words": ["shorts", "you", "and", "hot", "when", "a"]}, {"sent": "is it nice and hot ?", "words": ["is", "nice", "and", "hot", "it"]}, {"sent": "eat your hot dog please .", "words": ["eat", "hot", "your", "dog"]}, {"sent": "xxx its a hot day .", "words": ["hot", "a"]}, {"sent": "I know its very hot .", "words": ["hot"]}, {"sent": "thats not hot anymore .", "words": ["hot", "not"]}, {"sent": "uh hey this oven is hot .", "words": ["this", "hot", "oven", "is"]}, {"sent": "do you mean that tomorrow morning you want hot chocolate for breakfast ?", "words": ["do", "for", "chocolate", "you", "hot", "that"]}, {"sent": "you can smell hot .", "words": ["can", "hot", "you"]}, {"sent": "oh that is hot .", "words": ["hot", "that", "is"]}, {"sent": "it is a little hot .", "words": ["is", "little", "hot", "it", "a"]}, {"sent": "is it hot under the towel ?", "words": ["the", "is", "under", "hot", "it", "towel"]}, {"sent": "is it not hot ?", "words": ["hot", "it", "not", "is"]}, {"sent": "it was a lovely hot sunny day and we were outside .", "words": ["we", "was", "outside", "and", "hot", "it", "were", "a"]}, {"sent": "ahhah hot milk .", "words": ["milk", "hot"]}, {"sent": "theyre hot .", "words": ["hot"]}, {"sent": "I dont want it too hot but I want it hot .", "words": ["dont", "hot", "it", "too", "but"]}, {"sent": "oh its a little bit hot .", "words": ["hot", "little", "a"]}, {"sent": "hot huh ?", "words": ["hot"]}, {"sent": "phew its a change from being hot when youre pinnie was on fire .", "words": ["was", "hot", "on", "when", "a"]}, {"sent": "oh hot and cold did you do that ?", "words": ["do", "you", "and", "hot", "did", "that", "cold"]}, {"sent": "is it a hot day today ?", "words": ["hot", "it", "a", "is"]}, {"sent": "dont because the panll be hot .", "words": ["the", "be", "dont", "hot", "because"]}, {"sent": "wheres your hot ?", "words": ["hot", "your"]}, {"sent": "youll just hafta wait a minute because Im going to get you some hot milk and get SleepingBeauty .", "words": ["wait", "to", "you", "get", "and", "hot", "milk", "some", "because", "a"]}, {"sent": "its not hot .", "words": ["hot", "not"]}, {"sent": "empty hot bottle .", "words": ["hot", "bottle", "empty"]}, {"sent": "it was a hot drink .", "words": ["drink", "was", "hot", "it", "a"]}, {"sent": "oh you dont like hot drinks ?", "words": ["dont", "hot", "like", "you"]}, {"sent": "hot ttttt can you say hot ?", "words": ["can", "hot", "say", "you"]}, {"sent": "hot hot hot .", "words": ["hot"]}, {"sent": "its hot is it ?", "words": ["hot", "it", "is"]}, {"sent": "its hot water .", "words": ["hot", "water"]}, {"sent": "five hot chillys .", "words": ["hot"]}, {"sent": "stay here because its very hot .", "words": ["here", "stay", "hot", "because"]}, {"sent": "yeah were under the covers its hot .", "words": ["the", "under", "hot", "were"]}, {"sent": "its not so hot any more .", "words": ["so", "any", "hot", "more", "not"]}, {"sent": "thats hot too ?", "words": ["hot", "too"]}, {"sent": "its hot ?", "words": ["hot"]}, {"sent": "get get some of this carrot because its not too hot .", "words": ["of", "get", "this", "hot", "too", "some", "not", "because"]}, {"sent": "its not too hot .", "words": ["hot", "not", "too"]}, {"sent": "I thought youd give me a hot dog and Id give you some money .", "words": ["money", "and", "you", "hot", "me", "give", "dog", "some", "a"]}, {"sent": "and this man is hot .", "words": ["this", "hot", "is", "and"]}, {"sent": "are they hot or cold ?", "words": ["they", "hot", "are", "cold"]}, {"sent": "oh was it too hot ?", "words": ["was", "hot", "it", "too"]}, {"sent": "everytime she says hot she goes hot hot hot .", "words": ["hot", "she"]}, {"sent": "are you hot ?", "words": ["hot", "are", "you"]}, {"sent": "dont touch it yet kutz its really much too hot .", "words": ["touch", "dont", "hot", "it", "too", "much"]}, {"sent": "wheres the dog ?", "words": ["the", "dog"]}, {"sent": "what is this dog holding ?", "words": ["what", "dog", "this", "is"]}, {"sent": "you were a man dog and you got squished by a car and youre dead .", "words": ["car", "and", "you", "by", "were", "dog", "a"]}, {"sent": "thats a nice dog .", "words": ["nice", "dog", "a"]}, {"sent": "and her little dog Oliver he sits in the window doesnt he ?", "words": ["the", "little", "and", "in", "he", "window", "her", "dog"]}, {"sent": "dog a wow wow .", "words": ["dog", "a"]}, {"sent": "see if you can write dog .", "words": ["can", "if", "write", "you", "see", "dog"]}, {"sent": "your dog .", "words": ["your", "dog"]}, {"sent": "wheres the dog ?", "words": ["the", "dog"]}, {"sent": "have you got a dog ?", "words": ["have", "dog", "a", "you"]}, {"sent": "do you see the dog ?", "words": ["the", "do", "you", "see", "dog"]}, {"sent": "oh looks like a dog .", "words": ["dog", "like", "a"]}, {"sent": "a dog named Colonel .", "words": ["dog", "a"]}, {"sent": "hes running after the dog with a fork in his hand and hes jumping down the stairs which I hope you never do .", "words": ["the", "do", "hand", "with", "and", "you", "in", "fork", "his", "stairs", "down", "which", "dog", "a"]}, {"sent": "thats called xxx dog .", "words": ["dog"]}, {"sent": "thats a nice dog .", "words": ["nice", "dog", "a"]}, {"sent": "um dog .", "words": ["dog"]}, {"sent": "have you ever been bitten by a dog ?", "words": ["by", "have", "you", "dog", "a"]}, {"sent": "this is a special dog that looks after sheep .", "words": ["is", "this", "sheep", "that", "dog", "a"]}, {"sent": "the dog said Minnie .", "words": ["the", "dog"]}, {"sent": "oh heres a little dog .", "words": ["dog", "little", "a"]}, {"sent": "for the dog ?", "words": ["the", "for", "dog"]}, {"sent": "heres a big dog whos jumping up .", "words": ["up", "big", "dog", "a"]}, {"sent": "thats a dog there .", "words": ["there", "dog", "a"]}, {"sent": "Jos dog Sooty .", "words": ["dog"]}, {"sent": "and has the dog had some babys ?", "words": ["the", "some", "dog", "and"]}, {"sent": "have you got a dog ?", "words": ["have", "dog", "a", "you"]}, {"sent": "a cat and a dog ?", "words": ["cat", "dog", "and", "a"]}, {"sent": "I think theyve got a dog on the back of there .", "words": ["the", "think", "back", "of", "there", "on", "dog", "a"]}, {"sent": "you think the dog wants to eat some food too ?", "words": ["the", "think", "to", "you", "eat", "food", "too", "some", "dog"]}, {"sent": "Mama a high chair for a dog .", "words": ["for", "chair", "high chair", "high", "dog", "a"]}, {"sent": "dog .", "words": ["dog"]}, {"sent": "dog and this is a seal .", "words": ["is", "and", "this", "dog", "a"]}, {"sent": "I gave our dog two tins of stewed steak .", "words": ["of", "dog", "our"]}, {"sent": "youre making the dog bark .", "words": ["the", "dog"]}, {"sent": "probably the dog is outside somewhere .", "words": ["the", "outside", "dog", "is"]}, {"sent": "look hes like a dog with a bone .", "words": ["with", "like", "look", "dog", "a"]}, {"sent": "the message went from dog to dog .", "words": ["the", "to", "dog"]}, {"sent": "the missing bone its found its found with hound dog paw prints all around oh oh .", "words": ["the", "all", "with", "around", "dog"]}, {"sent": "shes got a dog .", "words": ["dog", "a"]}, {"sent": "its its not a bottom dog .", "words": ["dog", "bottom", "not", "a"]}, {"sent": "is it the dog house ?", "words": ["the", "is", "it", "house", "dog"]}, {"sent": "after Ryan um after Ryan pays for his dog were gonna get going okay ?", "words": ["for", "get", "his", "were", "dog"]}, {"sent": "dog .", "words": ["dog"]}, {"sent": "he says dog a lot .", "words": ["a lot", "he", "dog", "a"]}, {"sent": "do you wanna go see the dog ?", "words": ["the", "do", "go", "you", "wanna", "see", "dog"]}, {"sent": "a spotty dog .", "words": ["dog", "a"]}, {"sent": "the big fierce dog sprang at him and chased him down the street .", "words": ["the", "street", "and", "him", "at", "down", "big", "dog"]}, {"sent": "is that a puppy dog ?", "words": ["is", "puppy", "that", "dog", "a"]}, {"sent": "dog .", "words": ["dog"]}, {"sent": "wheres wheres dog gone ?", "words": ["dog"]}, {"sent": "Gruffs the dog .", "words": ["the", "dog"]}, {"sent": "gonna put the dog away ?", "words": ["the", "put", "away", "dog"]}, {"sent": "aw youre giving the dog some loving ?", "words": ["the", "some", "dog"]}, {"sent": "grandma and grandad have got a dog as well .", "words": ["have", "dog", "a", "and"]}, {"sent": "the dog is stuck .", "words": ["the", "stuck", "dog", "is"]}, {"sent": "is that the dog ?", "words": ["the", "that", "dog", "is"]}, {"sent": "the dog is pretty funny .", "words": ["the", "pretty", "dog", "is"]}, {"sent": "the dog ran away in fear of the pig .", "words": ["the", "of", "pig", "in", "away", "dog"]}, {"sent": "what are you doing with that dog ?", "words": ["what", "with", "you", "are", "that", "dog"]}, {"sent": "look the fancy dog .", "words": ["the", "dog", "look"]}, {"sent": "its a dog .", "words": ["dog", "a"]}, {"sent": "hes probably the only dog youll ever have .", "words": ["the", "have", "dog"]}, {"sent": "his Mommy is the dog ?", "words": ["his", "the", "dog", "is"]}, {"sent": "the dog is downstairs sweetie .", "words": ["the", "dog", "is"]}, {"sent": "another dog .", "words": ["another", "dog"]}, {"sent": "I can ki there Im doing downward dog now .", "words": ["can", "there", "dog"]}, {"sent": "about a funny dog .", "words": ["dog", "about", "a"]}, {"sent": "whats the dog doing ?", "words": ["the", "dog"]}, {"sent": "hm looks like a dog peed in here .", "words": ["like", "here", "in", "dog", "a"]}, {"sent": "oh you want a dog for him as well ?", "words": ["for", "you", "him", "dog", "a"]}, {"sent": "he knows cow pig horse dog cat .", "words": ["pig", "cat", "he", "horse", "dog", "cow"]}, {"sent": "you putting the man on the dog on the horses head ?", "words": ["the", "head", "you", "on", "dog"]}, {"sent": "do you think we should get him some dog bones ?", "words": ["do", "think", "we", "you", "get", "him", "some", "dog"]}, {"sent": "this dog is lying on the floor .", "words": ["the", "is", "this", "on", "dog"]}, {"sent": "dog do you want a dog ?", "words": ["do", "dog", "a", "you"]}, {"sent": "does that that looks like downward dog a little bit .", "words": ["does", "like", "little", "that", "dog", "a"]}, {"sent": "look look where is the dog .", "words": ["where", "the", "is", "look", "dog"]}, {"sent": "he went away wheres the dog ?", "words": ["the", "away", "he", "dog"]}, {"sent": "heres the dog .", "words": ["the", "dog"]}, {"sent": "the the dog is shaking off the water .", "words": ["the", "off", "is", "water", "dog"]}, {"sent": "whose dog is that ?", "words": ["that", "dog", "is"]}, {"sent": "I thought you called Michael a dog .", "words": ["dog", "a", "you"]}, {"sent": "dog food .", "words": ["food", "dog"]}, {"sent": "yeah thats a dog .", "words": ["dog", "a"]}, {"sent": "were only singing you daft dog .", "words": ["were", "dog", "you"]}, {"sent": "thats the dog .", "words": ["the", "dog"]}, {"sent": "xxx dog ?", "words": ["dog"]}, {"sent": "whats happened to your dog ?", "words": ["to", "your", "dog"]}, {"sent": "the black dog called the cat .", "words": ["the", "cat", "black", "dog"]}, {"sent": "aw give the dog a stroke .", "words": ["the", "give", "dog", "a"]}, {"sent": "dog not go outside .", "words": ["outside", "not", "dog", "go"]}, {"sent": "what about the dog ?", "words": ["the", "what", "dog", "about"]}, {"sent": "the turtle is hers the budgie is his and the dog is .", "words": ["the", "is", "hers", "and", "his", "turtle", "dog"]}, {"sent": "whats the dog do ?", "words": ["the", "do", "dog"]}, {"sent": "gonna go in the bedroom and visit the dog ?", "words": ["the", "go", "and", "in", "dog", "bedroom"]}, {"sent": "wheres the dog now ?", "words": ["the", "dog"]}, {"sent": "dog .", "words": ["dog"]}, {"sent": "another dog .", "words": ["another", "dog"]}, {"sent": "you dont want any cereal ?", "words": ["dont", "cereal", "any", "you"]}, {"sent": "I know the pieces of cereal fell down when you .", "words": ["the", "when", "of", "you", "down", "cereal"]}, {"sent": "more cereal ?", "words": ["more", "cereal"]}, {"sent": "do you want a cut in your cereal ?", "words": ["do", "your", "cut", "you", "in", "cereal", "a"]}, {"sent": "its cereal .", "words": ["cereal"]}, {"sent": "when did you have cereal ?", "words": ["when", "you", "have", "did", "cereal"]}, {"sent": "and what cereal will they have ?", "words": ["what", "and", "have", "they", "will", "cereal"]}, {"sent": "would you like one of these instead of cereal ?", "words": ["these", "of", "like", "you", "would", "cereal"]}, {"sent": "Mommy hasta mix the milk in the cereal .", "words": ["the", "milk", "in", "cereal"]}, {"sent": "thats a lot of cereal .", "words": ["of", "cereal", "a lot", "a"]}, {"sent": "oh good grief you could of had cereal and coffee and toast or something if I had realized that .", "words": ["if", "toast", "of", "you", "and", "that", "coffee", "good", "could", "cereal"]}, {"sent": "here comes a big bite of cereal .", "words": ["of", "here", "cereal", "bite", "big", "a"]}, {"sent": "its going round your bowl of cereal is it ?", "words": ["bowl", "your", "is", "of", "it", "cereal"]}, {"sent": "cereal bar ?", "words": ["cereal"]}, {"sent": "what kind of cereal do you want tomorrow Morgan ?", "words": ["what", "do", "of", "you", "cereal"]}, {"sent": "do you like cereal ?", "words": ["do", "cereal", "like", "you"]}, {"sent": "what sort of cereal is that ?", "words": ["what", "is", "of", "that", "cereal"]}, {"sent": "its a cereal thats got all sorts of little bits in .", "words": ["all", "of", "little", "in", "cereal", "a"]}, {"sent": "my cereal .", "words": ["my", "cereal"]}, {"sent": "it looks like a bowl of cereal there .", "words": ["bowl", "of", "like", "there", "it", "cereal", "a"]}, {"sent": "its a cereal box a book truck and a cake .", "words": ["box", "cake", "truck", "and", "book", "cereal", "a"]}, {"sent": "cereal and strawberries .", "words": ["cereal", "and"]}, {"sent": "and you want some cereal ?", "words": ["some", "cereal", "you", "and"]}, {"sent": "I had enough milk and cereal .", "words": ["milk", "cereal", "and"]}, {"sent": "you will have some cereal wont you ?", "words": ["you", "have", "will", "some", "cereal"]}, {"sent": "cereal ?", "words": ["cereal"]}, {"sent": "Julians going to have some cereal and some toast for breakfast .", "words": ["for", "toast", "to", "and", "have", "some", "cereal"]}, {"sent": "you useta have a beautiful teletubbie cereal bowl .", "words": ["bowl", "you", "have", "cereal", "a"]}, {"sent": "dont you have enough cereal ?", "words": ["have", "dont", "cereal", "you"]}, {"sent": "cereal .", "words": ["cereal"]}, {"sent": "is he in the little pot with the cereal ?", "words": ["the", "is", "with", "little", "in", "he", "cereal"]}, {"sent": "you want some cereal ?", "words": ["some", "cereal", "you"]}, {"sent": "give give me back the cereal .", "words": ["the", "back", "me", "give", "cereal"]}, {"sent": "would you like RiceKrispies or Cornflakes or cereal bar ?", "words": ["cereal", "like", "you", "would"]}, {"sent": "what kind of cereal ?", "words": ["of", "what", "cereal"]}, {"sent": "want milk in the cereal ?", "words": ["milk", "the", "in", "cereal"]}, {"sent": "the cereal is just on the floor making a mess .", "words": ["the", "is", "on", "cereal", "a"]}, {"sent": "cereal ?", "words": ["cereal"]}, {"sent": "Mommy needs some more cereal .", "words": ["some", "more", "cereal"]}, {"sent": "and we have the cereal .", "words": ["the", "we", "and", "have", "cereal"]}, {"sent": "thats a little spoon just like you use for your cereal huh ?", "words": ["for", "your", "like", "you", "little", "spoon", "cereal", "a"]}, {"sent": "thats the sound of the cereal going on the cereal .", "words": ["the", "of", "on", "cereal"]}, {"sent": "what kind of cereal ?", "words": ["of", "what", "cereal"]}, {"sent": "Alex would you like some cereal ?", "words": ["like", "you", "would", "some", "cereal"]}, {"sent": "you want some of your rice cereal ?", "words": ["your", "of", "you", "some", "cereal"]}, {"sent": "like some cereal ?", "words": ["some", "cereal", "like"]}, {"sent": "AppleJack cereal .", "words": ["cereal"]}, {"sent": "Mommys gonna make you some cereal .", "words": ["some", "cereal", "make", "you"]}, {"sent": "thats for cereal ?", "words": ["for", "cereal"]}, {"sent": "how about some cereal ?", "words": ["some", "how", "cereal", "about"]}, {"sent": "thats a type of hot cereal .", "words": ["of", "hot", "cereal", "a"]}, {"sent": "good cereal .", "words": ["cereal", "good"]}, {"sent": "cereal ?", "words": ["cereal"]}, {"sent": "you can go down and have your cereal .", "words": ["can", "your", "go", "and", "you", "have", "down", "cereal"]}, {"sent": "instead of putting cereal in your bowl and putting milk on pouring milk on .", "words": ["bowl", "your", "of", "and", "in", "on", "milk", "cereal"]}, {"sent": "you want a bite of cereal ?", "words": ["of", "you", "bite", "cereal", "a"]}, {"sent": "who helped you get the cereal ?", "words": ["the", "you", "get", "who", "cereal"]}, {"sent": "cereal in the bowl ?", "words": ["the", "in", "cereal", "bowl"]}, {"sent": "what sort of cereal ?", "words": ["of", "what", "cereal"]}, {"sent": "you want yoghurt drink in your cereal ?", "words": ["your", "drink", "you", "in", "cereal"]}, {"sent": "by bottle by breast with cups and with spoons milk and then cereal carrots and prunes .", "words": ["with", "then", "and", "by", "carrots", "milk", "bottle", "cereal"]}, {"sent": "where is the bowl of cereal ?", "words": ["where", "the", "bowl", "is", "of", "cereal"]}, {"sent": "mind the cereal .", "words": ["the", "cereal"]}, {"sent": "do you want your cereal in this bowl ?", "words": ["bowl", "do", "your", "you", "this", "in", "cereal"]}, {"sent": "why dont you come and eat your cereal with Naima so that I can do a few things that I wanna do ?", "words": ["can", "do", "your", "so", "with", "and", "eat", "dont", "you", "that", "wanna", "cereal", "why", "a"]}, {"sent": "and I need some cereal Im all out .", "words": ["all", "need", "and", "out", "some", "cereal"]}, {"sent": "those are cookies made outta RiceKrispies cereal .", "words": ["are", "those", "cereal"]}, {"sent": "eat your cereal .", "words": ["eat", "your", "cereal"]}, {"sent": "you ate all your yummy cereal .", "words": ["all", "cereal", "your", "you"]}, {"sent": "cereal yumyum .", "words": ["cereal"]}, {"sent": "you can have a little taste of cereal right there .", "words": ["can", "of", "taste", "little", "have", "you", "there", "cereal", "a"]}, {"sent": "Daddy puts two puts out two bowls and Caillou gets the cereal .", "words": ["the", "cereal", "out", "and"]}, {"sent": "what kind of cereal do you want ?", "words": ["what", "do", "of", "you", "cereal"]}, {"sent": "no we dont have butter on cereal .", "words": ["we", "have", "dont", "butter", "on", "cereal"]}, {"sent": "xxx this xxx awful lot of cereal xxx .", "words": ["of", "this", "cereal"]}, {"sent": "how about having uh this new kind of cereal .", "words": ["how", "of", "this", "about", "new", "cereal"]}, {"sent": "granola is cereal .", "words": ["cereal", "is"]}, {"sent": "thats the bird at the back of the cereal box .", "words": ["the", "box", "back", "of", "bird", "at", "cereal"]}, {"sent": "mm I think Ill put this cereal in some yogurt .", "words": ["think", "put", "yogurt", "this", "in", "some", "cereal"]}, {"sent": "Im making cereal and toast .", "words": ["cereal", "toast", "and"]}, {"sent": "youve got cereal in your eye ?", "words": ["your", "in", "cereal", "eye"]}, {"sent": "well I dont think I need cereal but Im thinking I might like to have some cereal .", "words": ["think", "need", "to", "like", "have", "dont", "some", "cereal", "but"]}, {"sent": "would you like RiceKrispies Cornflakes or a cereal bar ?", "words": ["like", "you", "would", "cereal", "a"]}, {"sent": "and cereal ?", "words": ["cereal", "and"]}, {"sent": "did you say my cereal has bones in it ?", "words": ["my", "say", "you", "in", "it", "did", "cereal"]}, {"sent": "just think if anybody invites you round for breakfast Ill hafta say Im sorry Thomas cant come because he doesnt know how to eat cereal politely .", "words": ["think", "for", "how", "if", "to", "say", "you", "eat", "he", "cereal", "because"]}, {"sent": "do you think she might like some cereal ?", "words": ["do", "think", "like", "you", "some", "cereal", "she"]}, {"sent": "youve your juice and Ive got my tea so what are we going to have some toast or some cereal ?", "words": ["what", "my", "your", "toast", "we", "so", "to", "and", "have", "are", "some", "juice", "cereal"]}, {"sent": "no xxx its the cereal give it to Mommy .", "words": ["the", "to", "it", "give", "cereal"]}, {"sent": "some cereal for .", "words": ["some", "for", "cereal"]}, {"sent": "cereal xxx .", "words": ["cereal"]}, {"sent": "two bowls of cereal for the cow and the pig and Ethan who just made a mess .", "words": ["the", "for", "of", "and", "pig", "who", "cereal", "cow", "a"]}, {"sent": "cereal xxx kids .", "words": ["cereal"]}, {"sent": "thats a car that carries cereal .", "words": ["car", "cereal", "that", "a"]}, {"sent": "I dont want my cereal now .", "words": ["dont", "my", "cereal"]}, {"sent": "what are you going to put on the cereal ?", "words": ["the", "what", "put", "to", "you", "are", "on", "cereal"]}, {"sent": "mind you youve eaten some of your cereal from breakfast havent you ?", "words": ["your", "of", "you", "some", "cereal"]}, {"sent": "guess who drinks diet coke ?", "words": ["coke", "who"]}, {"sent": "you could go get some coke and you could also get some of those peanutbutter cookies .", "words": ["those", "go", "of", "you", "get", "coke", "and", "some", "could"]}, {"sent": "coke goes in the fridge .", "words": ["coke", "the", "in"]}, {"sent": "but when do you have coke ?", "words": ["do", "you", "have", "coke", "when", "but"]}, {"sent": "you want some of Daddys coke ?", "words": ["of", "some", "coke", "you"]}, {"sent": "Daddys gonna get some coke .", "words": ["get", "some", "coke"]}, {"sent": "would you like a bottle of coke or a bottle of milk ?", "words": ["of", "like", "you", "coke", "would", "milk", "bottle", "a"]}, {"sent": "I dont know if there is coke .", "words": ["if", "is", "coke", "dont", "there"]}, {"sent": "can I have some coke ?", "words": ["can", "some", "have", "coke"]}, {"sent": "do you not mean coke ?", "words": ["coke", "do", "not", "you"]}, {"sent": "coke or pop .", "words": ["coke", "pop"]}, {"sent": "its coke .", "words": ["coke"]}, {"sent": "itll be alotof coke .", "words": ["coke", "be"]}, {"sent": "and you drink what do you drink coke outof ?", "words": ["what", "do", "drink", "you", "and", "coke"]}, {"sent": "so thats Thomass drink over there because thats coke .", "words": ["drink", "so", "coke", "there", "over", "because"]}, {"sent": "who would like some coke ?", "words": ["who", "like", "coke", "would", "some"]}, {"sent": "lemonade does that and coke .", "words": ["coke", "does", "that", "and"]}, {"sent": "oh that is Daddys coke can .", "words": ["coke", "can", "that", "is"]}, {"sent": "they dont ask for diet coke they ask for cola light .", "words": ["for", "light", "coke", "dont", "they"]}, {"sent": "theyre carrying coke or coal ?", "words": ["coke"]}, {"sent": "you dont want the coke ?", "words": ["the", "dont", "coke", "you"]}, {"sent": "who do you know that drinks brandy and coke ?", "words": ["do", "and", "you", "coke", "who", "that"]}, {"sent": "a drink please of coke .", "words": ["of", "coke", "drink", "a"]}, {"sent": "coke will be fine .", "words": ["coke", "be", "will", "fine"]}, {"sent": "daddy took his coke .", "words": ["his", "coke"]}, {"sent": "coke .", "words": ["coke"]}, {"sent": "is it coke ?", "words": ["coke", "it", "is"]}, {"sent": "has Daddy got some diet coke ?", "words": ["coke", "some"]}, {"sent": "and you had chicken and chips and coke did you ?", "words": ["you", "coke", "and", "did", "chicken"]}, {"sent": "hello diet coke man .", "words": ["coke"]}, {"sent": "do you like coke .", "words": ["coke", "do", "like", "you"]}, {"sent": "dont spill your coke okay ?", "words": ["coke", "dont", "your", "spill"]}, {"sent": "coke .", "words": ["coke"]}, {"sent": "do you drink coke in a baby cup ?", "words": ["cup", "do", "drink", "you", "coke", "in", "a"]}, {"sent": "it is coke .", "words": ["coke", "it", "is"]}, {"sent": "you want a drink of coke out of Daddys glass ?", "words": ["drink", "of", "you", "coke", "glass", "out", "a"]}, {"sent": "maybe coke .", "words": ["coke"]}, {"sent": "rice pudding with coke and tomato sauce .", "words": ["sauce", "with", "and", "coke", "pudding"]}, {"sent": "let Daddy put some coke in here .", "words": ["put", "coke", "here", "in", "some"]}, {"sent": "oh I didnt get that coke did I or did I ?", "words": ["get", "did", "that", "coke"]}, {"sent": "coke .", "words": ["coke"]}, {"sent": "icecream and coke and that with their meals .", "words": ["their", "with", "and", "coke", "that"]}, {"sent": "oh youre bringing me some cans of diet coke and ginger beer ?", "words": ["of", "and", "coke", "me", "some"]}, {"sent": "lets wait until we finish our juice and then we can have a little bit of coke .", "words": ["can", "wait", "we", "of", "finish", "our", "and", "have", "little", "coke", "then", "juice", "a"]}, {"sent": "coke .", "words": ["coke"]}, {"sent": "Carl you want a coke a ginnie ale uh cherry ?", "words": ["coke", "a", "you"]}, {"sent": "shall I pour the milk away and put the coke in the cup ?", "words": ["the", "cup", "put", "pour", "and", "coke", "in", "away", "milk"]}, {"sent": "whats coke ?", "words": ["coke"]}, {"sent": "a coke bottle .", "words": ["coke", "bottle", "a"]}, {"sent": "I just got a coke for daddy when he wakes up .", "words": ["up", "for", "coke", "he", "when", "a"]}, {"sent": "Daddys thirsty for coke .", "words": ["coke", "for", "thirsty"]}, {"sent": "trains dont drink coke Dominic .", "words": ["coke", "dont", "drink"]}, {"sent": "xxx who likes coke ?", "words": ["coke", "who"]}, {"sent": "Daddy is gonna have a coke .", "words": ["have", "coke", "a", "is"]}, {"sent": "you dont have diet coke at breakfast time .", "words": ["you", "have", "dont", "coke", "at"]}, {"sent": "please mister carrot could I have some diet coke please ?", "words": ["have", "some", "could", "coke"]}, {"sent": "two two coke and two hi two Hic ?", "words": ["coke", "and"]}, {"sent": "a wee glass of coke .", "words": ["of", "coke", "glass", "a"]}, {"sent": "I dont hafta work tomorrow and I got some rum at home and Im gonna get some coke and Im gonna have me a couple of rum and cokes and Im gonna knock this cold right out ta me .", "words": ["work", "of", "me", "and", "get", "dont", "coke", "have", "knock", "this", "out", "at", "some", "cold", "a", "home"]}, {"sent": "right get me my coke .", "words": ["get", "me", "my", "coke"]}, {"sent": "coke what else ?", "words": ["coke", "what"]}, {"sent": "and that theyre alcoholic drinks but things like orange juice and coke and water and lemonade are nonalcoholic or soft drinks .", "words": ["soft", "like", "and", "coke", "orange", "are", "water", "juice", "that", "but"]}, {"sent": "coke ?", "words": ["coke"]}, {"sent": "you dont like coke do you ?", "words": ["do", "like", "you", "coke", "dont"]}, {"sent": "before that you pointed to the m of Macdonalds and said I wanna buy some chips and diet coke .", "words": ["the", "of", "to", "you", "and", "coke", "wanna", "some", "that", "buy"]}, {"sent": "who was the coke for ?", "words": ["the", "for", "was", "coke", "who"]}, {"sent": "whatre you doing with the coke Sarah ?", "words": ["the", "with", "coke", "you"]}, {"sent": "and they form a sort of ramp so that the men can load up all the trolleys with all the boxes of cans and beer and coke and whatever else they sell .", "words": ["the", "can", "up", "all", "of", "so", "with", "and", "coke", "they", "that", "a"]}, {"sent": "about getting two Hic and two coke .", "words": ["coke", "about", "and"]}, {"sent": "you dont want any uh coke ?", "words": ["coke", "dont", "any", "you"]}, {"sent": "its drinking coke Nina .", "words": ["coke"]}, {"sent": "I dont think coke can sing .", "words": ["can", "think", "coke", "dont", "sing"]}, {"sent": "spilt a bit of coke .", "words": ["of", "coke", "a"]}, {"sent": "no I dont want anymore coke .", "words": ["coke", "dont"]}, {"sent": "is that coke too ?", "words": ["coke", "too", "that", "is"]}, {"sent": "you want that coke ?", "words": ["coke", "that", "you"]}, {"sent": "thats what Mommy puts in her coke .", "words": ["coke", "what", "in", "her"]}, {"sent": "spell coke .", "words": ["coke"]}, {"sent": "some diet coke .", "words": ["coke", "some"]}, {"sent": "coke .", "words": ["coke"]}, {"sent": "would you like some coke ?", "words": ["like", "you", "coke", "would", "some"]}, {"sent": "would you like some coke ?", "words": ["like", "you", "coke", "would", "some"]}, {"sent": "youre not having coke .", "words": ["coke", "not"]}, {"sent": "coke ?", "words": ["coke"]}, {"sent": "coke .", "words": ["coke"]}, {"sent": "you want a coke ?", "words": ["coke", "a", "you"]}, {"sent": "Daddy does he likes coke .", "words": ["coke", "does", "he"]}, {"sent": "I drink vodka and coke yousee .", "words": ["coke", "drink", "and"]}, {"sent": "and he wants a drink of coke please .", "words": ["drink", "of", "and", "coke", "he", "a"]}, {"sent": "the fish is swimming in diet coke ?", "words": ["the", "is", "coke", "in", "fish"]}, {"sent": "do you drink coke and throw it away ?", "words": ["do", "throw", "drink", "and", "coke", "you", "it", "away"]}, {"sent": "uh Carl what you we have um coke and orange left .", "words": ["what", "we", "you", "have", "coke", "and", "orange"]}, {"sent": "the xxx and the coke .", "words": ["the", "coke", "and"]}, {"sent": "some diet coke .", "words": ["coke", "some"]}, {"sent": "coke is daddys juice .", "words": ["coke", "juice", "is"]}, {"sent": "do you like coke ?", "words": ["coke", "do", "like", "you"]}, {"sent": "did you have a little sip of Daddys diet coke ?", "words": ["of", "little", "have", "you", "coke", "did", "a"]}, {"sent": "do you like coke ?", "words": ["coke", "do", "like", "you"]}, {"sent": "then he ate some French beans and finally he ate a nice radish .", "words": ["nice", "and", "beans", "he", "some", "then", "a"]}, {"sent": "you like beans ?", "words": ["beans", "like", "you"]}, {"sent": "you stirring the beans ?", "words": ["the", "beans", "you"]}, {"sent": "where are the beans ?", "words": ["where", "the", "are", "beans"]}, {"sent": "what are jammy beans ?", "words": ["what", "are", "beans"]}, {"sent": "what do we have for lunch today we have beans .", "words": ["what", "do", "for", "we", "have", "beans"]}, {"sent": "kidney beans .", "words": ["beans"]}, {"sent": "get some more beans .", "words": ["get", "some", "more", "beans"]}, {"sent": "here you gonna get some beans ?", "words": ["you", "here", "get", "beans", "some"]}, {"sent": "have some green beans with your pizza ?", "words": ["your", "with", "pizza", "green beans", "have", "beans", "green", "some"]}, {"sent": "chips and beans ?", "words": ["beans", "and"]}, {"sent": "those are great new beans .", "words": ["new", "are", "those", "beans"]}, {"sent": "Ill give you five beans for that cow .", "words": ["for", "you", "beans", "give", "that", "cow"]}, {"sent": "where did all the beans go ?", "words": ["where", "all", "the", "go", "beans", "did"]}, {"sent": "but we havent got any sausages and beans here at the moment have we ?", "words": ["the", "we", "any", "and", "here", "have", "beans", "at", "but"]}, {"sent": "oh green beans .", "words": ["green beans", "beans", "green"]}, {"sent": "do you wanna eat your beans ?", "words": ["do", "your", "you", "eat", "beans", "wanna"]}, {"sent": "all the coffee beans do you keep some and sell them as coffee beans ?", "words": ["the", "all", "do", "them", "you", "and", "coffee", "beans", "some"]}, {"sent": "what color is green beans ?", "words": ["what", "is", "green beans", "beans", "green"]}, {"sent": "you dont like beans ?", "words": ["dont", "beans", "like", "you"]}, {"sent": "green beans wants a sip ?", "words": ["green beans", "beans", "green", "a"]}, {"sent": "these are beans for you sweetheart .", "words": ["these", "for", "you", "are", "beans"]}, {"sent": "jelly beans .", "words": ["jelly", "beans"]}, {"sent": "can you pour beans and rice ?", "words": ["can", "pour", "and", "you", "beans"]}, {"sent": "with the baked beans on his beak .", "words": ["the", "with", "on", "beans", "his"]}, {"sent": "no youre not having jelly beans for breakfast silly .", "words": ["jelly", "beans", "not", "for"]}, {"sent": "couscous and beans ?", "words": ["beans", "and"]}, {"sent": "thats a pretend tin of beans .", "words": ["of", "pretend", "beans", "a"]}, {"sent": "lookit lookit all these beans that are in your chair .", "words": ["all", "these", "your", "chair", "in", "beans", "are", "that"]}, {"sent": "heres some beans .", "words": ["some", "beans"]}, {"sent": "do you like beans ?", "words": ["do", "beans", "like", "you"]}, {"sent": "chips and baked beans ?", "words": ["beans", "and"]}, {"sent": "you like the beans that she made ?", "words": ["the", "like", "you", "beans", "that", "she"]}, {"sent": "I think its baked beans with beef burgers .", "words": ["think", "beans", "with"]}, {"sent": "making some beans ?", "words": ["some", "beans"]}, {"sent": "would you like a tin of beans ?", "words": ["of", "like", "you", "beans", "would", "a"]}, {"sent": "hih green beans green beans green beans .", "words": ["green beans", "beans", "green"]}, {"sent": "beans .", "words": ["beans"]}, {"sent": "and you have one piece with your beans .", "words": ["your", "with", "you", "have", "and", "beans"]}, {"sent": "let me see if I need to put salt in the beans oh I do .", "words": ["the", "do", "put", "need", "if", "need to", "to", "me", "salt", "in", "beans", "see"]}, {"sent": "can I have some baked beans Eleanor ?", "words": ["can", "some", "have", "beans"]}, {"sent": "perhaps not the beef burgers but you like the beans .", "words": ["the", "like", "you", "beans", "not", "but"]}, {"sent": "and some string beans .", "words": ["some", "beans", "and"]}, {"sent": "how about some vegetables some beans can you give the mommy some beans ?", "words": ["can", "the", "how", "you", "beans", "about", "some", "give"]}, {"sent": "some green beans ?", "words": ["some", "green beans", "beans", "green"]}, {"sent": "beans .", "words": ["beans"]}, {"sent": "but anyway I had decided since I have these beans and franks I hafta eat them and um because Id let four or five sit for a couple a weeks .", "words": ["these", "for", "them", "because", "and", "have", "eat", "beans", "sit", "but", "a"]}, {"sent": "well how are you going to eat the rest of the beans ?", "words": ["the", "how", "of", "to", "you", "eat", "are", "beans"]}, {"sent": "well whenever youre done Ill clean you up and you can either pour practice pouring the beans or you could wash one of your friends .", "words": ["clean", "can", "up", "the", "your", "of", "pour", "and", "you", "beans", "could", "wash"]}, {"sent": "you tell me while I put the beans away please .", "words": ["the", "put", "you", "beans", "away", "me"]}, {"sent": "five beans for our cow cried his mother .", "words": ["for", "our", "beans", "his", "cow"]}, {"sent": "um yeah and when shes she knows green beans .", "words": ["green beans", "and", "beans", "green", "when", "she"]}, {"sent": "baked beans and beef burgers .", "words": ["beans", "and"]}, {"sent": "its bean you have beans and TVP in there .", "words": ["and", "have", "you", "in", "beans", "there"]}, {"sent": "oh youre gonna take all the beans .", "words": ["the", "all", "beans", "take"]}, {"sent": "I like baked beans .", "words": ["beans", "like"]}, {"sent": "we have a donut and a piece of pizza and green beans .", "words": ["donut", "we", "of", "pizza", "and", "have", "green beans", "beans", "green", "a"]}, {"sent": "baked beans .", "words": ["beans"]}, {"sent": "and green beans .", "words": ["green beans", "beans", "green", "and"]}, {"sent": "chips and beans ?", "words": ["beans", "and"]}, {"sent": "beans and soups and .", "words": ["beans", "and"]}, {"sent": "you can have some baked beans .", "words": ["can", "you", "have", "beans", "some"]}, {"sent": "so youve grown all your coffee beans ?", "words": ["all", "your", "so", "coffee", "beans"]}, {"sent": "do you see the beans ?", "words": ["the", "do", "you", "beans", "see"]}, {"sent": "would you like some beans and rice ?", "words": ["like", "you", "and", "beans", "would", "some"]}, {"sent": "think there might be a tin of beans in that basket .", "words": ["think", "of", "be", "there", "in", "beans", "basket", "that", "a"]}, {"sent": "well weve got a lot of beans to pick up .", "words": ["up", "of", "to", "pick", "a lot", "beans", "a"]}, {"sent": "these green beans are not looking so good .", "words": ["these", "so", "green beans", "beans", "are", "green", "good", "not"]}, {"sent": "try putting some beans up there and see what happens .", "words": ["up", "what", "see", "and", "there", "beans", "some", "try"]}, {"sent": "some more beans .", "words": ["some", "more", "beans"]}, {"sent": "beans ?", "words": ["beans"]}, {"sent": "green beans ?", "words": ["green beans", "beans", "green"]}, {"sent": "thats green beans .", "words": ["green beans", "beans", "green"]}, {"sent": "milk on beans doesnt sound very nice Thomas .", "words": ["milk", "on", "beans", "nice"]}, {"sent": "okay heres some more green beans .", "words": ["green beans", "more", "beans", "green", "some"]}, {"sent": "we havent got any beans .", "words": ["beans", "any", "we"]}, {"sent": "I dont like the string beans .", "words": ["the", "dont", "beans", "like"]}, {"sent": "baked beans .", "words": ["beans"]}, {"sent": "well I hope you learnt your lesson and you wont throw blue coffee beans at next doors windows .", "words": ["your", "throw", "blue", "and", "you", "coffee", "beans", "at"]}, {"sent": "I bet you didnt have sausage and beans either .", "words": ["have", "beans", "and", "you"]}, {"sent": "green beans taste good .", "words": ["taste", "green beans", "beans", "green", "good"]}, {"sent": "and Ill have a tin of baked beans .", "words": ["of", "and", "have", "beans", "a"]}, {"sent": "could I have some baked beans please ?", "words": ["have", "some", "could", "beans"]}, {"sent": "hes still got baked beans on his beak John .", "words": ["on", "his", "beans"]}, {"sent": "she knows green beans but she eats them pureed she wont eat them cut up for some reason .", "words": ["up", "for", "cut", "them", "green beans", "eat", "beans", "green", "some", "she", "but"]}, {"sent": "your baby likes baked beans .", "words": ["beans", "your"]}, {"sent": "we have them with beans dont we .", "words": ["we", "them", "with", "have", "dont", "beans"]}, {"sent": "baked beans .", "words": ["beans"]}, {"sent": "beans .", "words": ["beans"]}, {"sent": "the rest of the beans that were not eating .", "words": ["the", "of", "beans", "that", "not", "were"]}, {"sent": "let me eat the rice and beans first and Ill let you know .", "words": ["the", "first", "and", "eat", "you", "beans", "me"]}, {"sent": "do you wanna put beans with the melon ?", "words": ["the", "do", "put", "with", "you", "beans", "wanna", "melon"]}, {"sent": "I spilled the beans .", "words": ["the", "beans"]}, {"sent": "Mommy and Naima can make spicy beans thats right .", "words": ["can", "beans", "make", "and"]}, {"sent": "like oats and beans and barley .", "words": ["beans", "like", "and"]}, {"sent": "you like baked beans .", "words": ["beans", "like", "you"]}, {"sent": "and some beans .", "words": ["some", "beans", "and"]}, {"sent": "tell Courtney about your pool .", "words": ["your", "pool", "about"]}, {"sent": "and is that your swimming pool ?", "words": ["your", "is", "pool", "and", "that"]}, {"sent": "the paddling pool has a hole in it .", "words": ["the", "pool", "in", "it", "a"]}, {"sent": "in the swimming pool ?", "words": ["the", "in", "pool"]}, {"sent": "oh yeah theres the paddling pool .", "words": ["the", "pool"]}, {"sent": "or in the swimming pool .", "words": ["the", "in", "pool"]}, {"sent": "there were some big frogs in the swimming pool werent there ?", "words": ["the", "pool", "there", "in", "some", "big", "were"]}, {"sent": "did you like swimming in the big pool ?", "words": ["the", "pool", "like", "you", "in", "did", "big"]}, {"sent": "and lets see if she can dive into the pool of water .", "words": ["can", "the", "if", "into", "of", "pool", "and", "water", "see", "she"]}, {"sent": "have you been playing in the paddling pool Fraser ?", "words": ["the", "pool", "you", "have", "in"]}, {"sent": "theyre not playing pool ah haha ha .", "words": ["pool", "not"]}, {"sent": "well it was a big swimming pool .", "words": ["was", "pool", "it", "big", "a"]}, {"sent": "and do you remember being in the pool ?", "words": ["the", "do", "pool", "you", "and", "in"]}, {"sent": "you cry when you go to the swimming pool .", "words": ["the", "go", "to", "cry", "pool", "you", "when"]}, {"sent": "in your big swimming pool ?", "words": ["big", "in", "your", "pool"]}, {"sent": "is that a swimming pool ?", "words": ["that", "pool", "a", "is"]}, {"sent": "shall we buy a swimming pool in the summer for Nina ?", "words": ["the", "for", "we", "pool", "in", "buy", "a"]}, {"sent": "I dont think youll be going out in your paddling pool today .", "words": ["think", "your", "be", "pool", "dont", "in", "out"]}, {"sent": "pool .", "words": ["pool"]}, {"sent": "ya you jumped in the pool and you kicked in the bath and you blew bubbles didnt you ?", "words": ["the", "pool", "and", "you", "in", "bubbles"]}, {"sent": "its a pool table .", "words": ["pool", "table", "a"]}, {"sent": "its about time you stood in the pool and jumped in the pool isnt it ?", "words": ["the", "pool", "you", "and", "in", "it", "about"]}, {"sent": "uhhuh they in a pool .", "words": ["they", "in", "pool", "a"]}, {"sent": "youre alright in the paddling pool .", "words": ["the", "in", "pool"]}, {"sent": "is there a little pool for little kids ?", "words": ["for", "is", "pool", "little", "there", "a"]}, {"sent": "thats Jwww and Cwww in the paddling pool outside .", "words": ["the", "outside", "pool", "and", "in"]}, {"sent": "or is it a swimming pool ?", "words": ["it", "pool", "a", "is"]}, {"sent": "the cheese biscuit has been taken out of the pool Thomas .", "words": ["the", "of", "pool", "out", "cheese"]}, {"sent": "you can slide in right into the pool .", "words": ["can", "the", "slide", "into", "pool", "you", "in"]}, {"sent": "fifty times up and down the pool .", "words": ["up", "the", "pool", "and", "down"]}, {"sent": "in the pool thats right we spent all day in the pool .", "words": ["the", "all", "we", "pool", "in"]}, {"sent": "shes in her kiddie pool .", "words": ["in", "her", "pool"]}, {"sent": "I think her Mummys filling the paddling pool .", "words": ["the", "think", "her", "pool"]}, {"sent": "they are grouped around the swimming pool .", "words": ["the", "pool", "they", "are", "around"]}, {"sent": "know what he did after he got out of the pool ?", "words": ["the", "what", "of", "pool", "he", "did", "out"]}, {"sent": "when you get all sandy in the sandbox you can go wash yourself off in the swimming pool .", "words": ["the", "all", "sandbox", "can", "yourself", "off", "go", "pool", "wash", "you", "get", "in", "when"]}, {"sent": "and did you swim in the wee pool ?", "words": ["the", "swim", "pool", "you", "and", "in", "did"]}, {"sent": "this is me swimming in my pool .", "words": ["my", "is", "pool", "this", "in", "me"]}, {"sent": "what would you do in the swimming pool ?", "words": ["the", "what", "do", "pool", "you", "in", "would"]}, {"sent": "she was never outof the pool .", "words": ["the", "was", "pool", "she"]}, {"sent": "we could make him swimming in a pool .", "words": ["we", "pool", "in", "him", "could", "make", "a"]}, {"sent": "Ive been in the ball pool .", "words": ["the", "in", "pool", "ball"]}, {"sent": "right you like to go in the pool ?", "words": ["the", "go", "to", "pool", "like", "you", "in"]}, {"sent": "there is a paddling pool isnt there ?", "words": ["there", "pool", "a", "is"]}, {"sent": "thats the swimming pool .", "words": ["the", "pool"]}, {"sent": "theres he trying to get in the ducky in the pool .", "words": ["the", "to", "pool", "get", "in", "he"]}, {"sent": "wheres a pool ?", "words": ["pool", "a"]}, {"sent": "swimming pool shapes ?", "words": ["pool"]}, {"sent": "in the swimming pool .", "words": ["the", "in", "pool"]}, {"sent": "now the dragon fell in the pool and what happened next ?", "words": ["the", "what", "pool", "and", "in"]}, {"sent": "were going to make a pool for him .", "words": ["for", "to", "pool", "him", "make", "were", "a"]}, {"sent": "not in the swimming pool .", "words": ["the", "in", "pool", "not"]}, {"sent": "you can you are at the swimming pool .", "words": ["can", "the", "pool", "you", "are", "at"]}, {"sent": "why did the dragon fall in the pool ?", "words": ["the", "pool", "fall", "in", "did", "why"]}, {"sent": "itll be fun to go in the pool .", "words": ["the", "go", "to", "be", "pool", "in"]}, {"sent": "paddling pool ?", "words": ["pool"]}, {"sent": "Mandy you get in the pool .", "words": ["the", "pool", "you", "get", "in"]}, {"sent": "what did you do in the pool ?", "words": ["the", "what", "do", "pool", "you", "in", "did"]}, {"sent": "and this is a swimming pool so what do you suppose these people should do ?", "words": ["what", "do", "these", "is", "so", "pool", "and", "you", "this", "a"]}, {"sent": "what did you do in the swimming pool ?", "words": ["the", "what", "do", "pool", "you", "in", "did"]}, {"sent": "two elephants are by a pool of water .", "words": ["of", "pool", "by", "are", "water", "a"]}, {"sent": "uh what happened when the dragon fell in the pool ?", "words": ["the", "what", "pool", "in", "when"]}, {"sent": "is she in her paddling pool ?", "words": ["is", "pool", "in", "her", "she"]}, {"sent": "xxx with its pool .", "words": ["with", "pool"]}, {"sent": "swimming pool I think .", "words": ["think", "pool"]}, {"sent": "now the dragon got out of the pool .", "words": ["the", "of", "pool", "out"]}, {"sent": "at half past seven this morning there was a man cleaning out the swimming pool and there were some pussy cats at the side of the pool .", "words": ["the", "of", "was", "pool", "and", "this", "there", "out", "at", "some", "were", "a"]}, {"sent": "and then tomorrow when we go in the pool theyll be lovely and clean again for you .", "words": ["the", "clean", "for", "go", "we", "be", "pool", "and", "you", "in", "then", "when"]}, {"sent": "well hey maybe I can slide into the pool .", "words": ["can", "the", "slide", "into", "pool"]}, {"sent": "Tinkerbell after lunch maybe we could go for a swim in the pool what do you think ?", "words": ["the", "what", "do", "for", "think", "go", "we", "swim", "pool", "you", "in", "could", "a"]}, {"sent": "like at the swimming pool .", "words": ["the", "pool", "like", "at"]}, {"sent": "thatll be lovely having a paddling pool .", "words": ["be", "pool", "a"]}, {"sent": "because most childrens pool .", "words": ["pool", "because"]}, {"sent": "and what do you do in the swimming pool ?", "words": ["the", "what", "do", "pool", "you", "and", "in"]}, {"sent": "can get your ball pool out downstairs couldnt we ?", "words": ["can", "your", "we", "pool", "ball", "get", "out"]}, {"sent": "alright you wont go to the pool tomorrow .", "words": ["the", "go", "to", "pool", "you"]}, {"sent": "into the ball pool .", "words": ["the", "ball", "pool", "into"]}, {"sent": "how come you fell in the swimming pool ?", "words": ["the", "how", "pool", "you", "in"]}, {"sent": "because we wanted to stay at a hotel with an indoor pool but they didnt have the pool so what did we do we put on our bathing suits and we all went in the bath right ?", "words": ["the", "what", "put", "our", "an", "with", "have", "on", "stay", "all", "to", "so", "and", "because", "a", "do", "we", "pool", "they", "in", "did", "at", "but"]}, {"sent": "oh no our paddling pool .", "words": ["pool", "our"]}, {"sent": "a blue pool ?", "words": ["blue", "pool", "a"]}, {"sent": "or was it near the pool ?", "words": ["the", "was", "it", "pool"]}, {"sent": "and you can put a swimming pool on it .", "words": ["can", "put", "pool", "you", "and", "it", "on", "a"]}, {"sent": "and this man was was eating den he went in an pool almost got drown .", "words": ["was", "pool", "and", "this", "in", "an", "he"]}, {"sent": "in pool ?", "words": ["in", "pool"]}, {"sent": "no and I think her mother had told her not to go into the pool .", "words": ["the", "think", "into", "go", "to", "pool", "and", "her", "not"]}, {"sent": "a swimming pool .", "words": ["pool", "a"]}, {"sent": "can you jump into the pool ?", "words": ["can", "the", "into", "pool", "you", "jump"]}, {"sent": "what did we see when we went to the swimming pool ?", "words": ["the", "what", "we", "to", "pool", "did", "see", "when"]}, {"sent": "Maisie paddles in the paddling pool .", "words": ["the", "in", "pool"]}, {"sent": "you swim at the Y at the pool at the Y ?", "words": ["the", "swim", "pool", "you", "at"]}, {"sent": "Ill put Pooh Bear in the pool .", "words": ["the", "in", "put", "pool"]}, {"sent": "lets build this pool here .", "words": ["here", "this", "build", "pool"]}, {"sent": "but the big pool that Mummy and Daddy went in was absolutely freezing .", "words": ["the", "was", "pool", "and", "big", "in", "that", "but"]}, {"sent": "did you have the paddling pool ?", "words": ["the", "pool", "you", "have", "did"]}, {"sent": "its too cold for the paddling pool .", "words": ["the", "for", "pool", "too", "cold"]}, {"sent": "were going to swimming pool and what ?", "words": ["what", "to", "pool", "and", "were"]}, {"sent": "when the dragon fell in the pool what happened next ?", "words": ["the", "what", "pool", "in", "when"]}, {"sent": "or the swimming pool .", "words": ["the", "pool"]}, {"sent": "bang your eye .", "words": ["your", "eye"]}, {"sent": "should we give him an eye ?", "words": ["we", "eye", "him", "an", "give"]}, {"sent": "did it get in your eye ?", "words": ["your", "eye", "get", "in", "it", "did"]}, {"sent": "I saw some something in your eye .", "words": ["some", "in", "your", "eye"]}, {"sent": "its all cock eye .", "words": ["all", "eye"]}, {"sent": "the eye doctor .", "words": ["the", "eye"]}, {"sent": "why dont you try the other eye ?", "words": ["the", "eye", "you", "other", "dont", "try", "why"]}, {"sent": "whats the matter with your eye ?", "words": ["the", "with", "eye", "your"]}, {"sent": "mind your eye .", "words": ["your", "eye"]}, {"sent": "now keep an eye on him .", "words": ["on", "an", "eye", "him"]}, {"sent": "and they treated the eye with drops .", "words": ["the", "with", "eye", "and", "they"]}, {"sent": "one eye on one piece and one eye on another piece .", "words": ["on", "eye", "another", "and"]}, {"sent": "no thats my eye .", "words": ["my", "eye"]}, {"sent": "we read the GoodNightMoon every night before bed so that caught my eye .", "words": ["the", "my", "we", "so", "bed", "eye", "read", "every", "that"]}, {"sent": "one eye open one eye close .", "words": ["open", "close", "eye"]}, {"sent": "dont poke his eye .", "words": ["his", "dont", "eye"]}, {"sent": "blinks its red eye .", "words": ["red", "eye"]}, {"sent": "that hit you in the eye .", "words": ["the", "eye", "you", "in", "hit", "that"]}, {"sent": "heres the eye .", "words": ["the", "eye"]}, {"sent": "oh dont put it in your eye .", "words": ["put", "your", "eye", "dont", "in", "it"]}, {"sent": "you look at your eye ?", "words": ["your", "eye", "you", "at", "look"]}, {"sent": "eye brows .", "words": ["eye"]}, {"sent": "one eye on there .", "words": ["on", "there", "eye"]}, {"sent": "who drew on Loftys eye ?", "words": ["on", "eye", "who"]}, {"sent": "her eye ?", "words": ["her", "eye"]}, {"sent": "I see your eye .", "words": ["see", "your", "eye"]}, {"sent": "they have a special eye .", "words": ["have", "they", "eye", "a"]}, {"sent": "theres your eye patch .", "words": ["your", "eye"]}, {"sent": "yes I dont wanna have a black eye like daddy did huh .", "words": ["eye", "like", "have", "dont", "black", "did", "wanna", "a"]}, {"sent": "what kind of eye is this .", "words": ["what", "is", "of", "eye", "this"]}, {"sent": "cats eye thats right .", "words": ["eye"]}, {"sent": "eye .", "words": ["eye"]}, {"sent": "keep your eye on the ball .", "words": ["the", "your", "eye", "ball", "on"]}, {"sent": "and these are the eye colors .", "words": ["the", "these", "eye", "and", "are"]}, {"sent": "he made you get it in his eye ?", "words": ["eye", "you", "get", "in", "it", "he", "his"]}, {"sent": "now how about an eye and another eye ?", "words": ["how", "eye", "and", "another", "an", "about"]}, {"sent": "ope got something in my eye comere .", "words": ["in", "eye", "my"]}, {"sent": "one eye .", "words": ["eye"]}, {"sent": "wheres his eye ?", "words": ["his", "eye"]}, {"sent": "I dont want him on my eye .", "words": ["my", "eye", "dont", "him", "on"]}, {"sent": "heres one eye .", "words": ["eye"]}, {"sent": "can you see the pussycats eye on there ?", "words": ["can", "the", "eye", "you", "there", "on", "see"]}, {"sent": "is your eye alright now Poppet ?", "words": ["your", "eye", "is"]}, {"sent": "he looked at me with his beady eye .", "words": ["with", "eye", "he", "at", "his", "me"]}, {"sent": "dont poke Eleanor in her eye .", "words": ["dont", "in", "her", "eye"]}, {"sent": "oh youve bumped your eye on the corner of the box .", "words": ["the", "your", "box", "of", "eye", "on"]}, {"sent": "I spy with my little eye .", "words": ["my", "with", "eye", "little"]}, {"sent": "look what was in my eye .", "words": ["what", "my", "was", "eye", "in", "look"]}, {"sent": "that almost me in the eye again .", "words": ["the", "eye", "in", "me", "that"]}, {"sent": "why does why does he have one eye ?", "words": ["does", "eye", "have", "he", "why"]}, {"sent": "is your eye not sore now Nathan ?", "words": ["not", "your", "eye", "is"]}, {"sent": "dont touch my eye .", "words": ["dont", "my", "eye", "touch"]}, {"sent": "wheres Naimas other eye ?", "words": ["eye", "other"]}, {"sent": "my eye .", "words": ["my", "eye"]}, {"sent": "close one eye .", "words": ["close", "eye"]}, {"sent": "and thats your eye .", "words": ["your", "eye", "and"]}, {"sent": "so does does Mikes friend have just one eye ?", "words": ["have", "so", "does", "eye"]}, {"sent": "clowns eye .", "words": ["eye"]}, {"sent": "Im sorry you fell off the bed and bruised your eye up a little bit .", "words": ["off", "the", "up", "your", "bed", "eye", "and", "you", "little", "a"]}, {"sent": "you have the diamond eye .", "words": ["have", "the", "eye", "you"]}, {"sent": "Thomass right eye .", "words": ["eye"]}, {"sent": "I dont think something in your eye Becky .", "words": ["think", "your", "eye", "dont", "in"]}, {"sent": "I can just see something Im keeping an eye on in my chair .", "words": ["can", "my", "eye", "chair", "an", "in", "on", "see"]}, {"sent": "oh you poked your eye ouchie .", "words": ["your", "eye", "you"]}, {"sent": "he has got a bandage on his eye hasnt he ?", "words": ["eye", "his", "he", "on", "a"]}, {"sent": "the camera has one big round eye .", "words": ["the", "big", "camera", "eye"]}, {"sent": "whats the matter with your eye Carl ?", "words": ["the", "with", "eye", "your"]}, {"sent": "because youve got something in the corner of your eye .", "words": ["the", "your", "of", "eye", "in", "because"]}, {"sent": "theres one with an eye on it look .", "words": ["with", "eye", "an", "it", "look", "on"]}, {"sent": "oh they can see with the other eye ?", "words": ["can", "the", "with", "eye", "other", "they", "see"]}, {"sent": "well if you spit cereal I guess sometimes it lands in your eye .", "words": ["your", "if", "eye", "you", "in", "it", "cereal"]}, {"sent": "rubbing my eye .", "words": ["my", "eye"]}, {"sent": "yes in your eye didnt it .", "words": ["in", "it", "your", "eye"]}, {"sent": "get your finger out of your eye .", "words": ["your", "of", "eye", "get", "finger", "out"]}, {"sent": "eye eye eye eye eye .", "words": ["eye"]}, {"sent": "just think if its in that nest its keeping an eye on you all the time not just when Dimitra brings it .", "words": ["the", "all", "think", "if", "when", "eye", "you", "in", "an", "it", "on", "that", "not"]}, {"sent": "you didnt poke the other baby in the eye .", "words": ["the", "eye", "other", "you", "in"]}, {"sent": "well the other eye should be over here somewhere .", "words": ["the", "be", "eye", "other", "here", "over"]}, {"sent": "one eye .", "words": ["eye"]}, {"sent": "and his other eye and his mouth .", "words": ["mouth", "eye", "other", "and", "his"]}, {"sent": "eye .", "words": ["eye"]}, {"sent": "oh you made an eye .", "words": ["an", "eye", "you"]}, {"sent": "his eye wants to go here .", "words": ["go", "to", "eye", "here", "his"]}, {"sent": "with a wink of his eye and a twist of his head the party can start I made it Norm said .", "words": ["the", "can", "of", "head", "with", "eye", "and", "party", "it", "his", "a"]}, {"sent": "youve drawn the the most beautiful eye .", "words": ["the", "eye"]}, {"sent": "who got this this bad eye ?", "words": ["this", "eye", "bad", "who"]}, {"sent": "use your eye .", "words": ["your", "eye"]}, {"sent": "eye .", "words": ["eye"]}, {"sent": "if you hit that little eye it should make a sound .", "words": ["if", "eye", "little", "you", "it", "make", "hit", "that", "a"]}, {"sent": "why are you giving him the milk in his eye ?", "words": ["the", "eye", "you", "his", "him", "are", "in", "milk", "why"]}, {"sent": "I nearly poked your eye out .", "words": ["your", "eye", "out"]}, {"sent": "what did you do get blood on your eye ?", "words": ["what", "do", "your", "eye", "you", "get", "did", "on"]}, {"sent": "youve got paint on your eye .", "words": ["on", "paint", "eye", "your"]}, {"sent": "you be the eye doctor .", "words": ["the", "be", "eye", "you"]}, {"sent": "eye spy with my little eye something beginning with p .", "words": ["my", "with", "eye", "little"]}, {"sent": "heres an eye .", "words": ["an", "eye"]}, {"sent": "you seem to have a little bruise xxx right by your eye .", "words": ["your", "to", "eye", "little", "have", "you", "by", "a"]}, {"sent": "thats the eye .", "words": ["the", "eye"]}, {"sent": "if I had one eye would I be a monster ?", "words": ["if", "be", "eye", "would", "a"]}, {"sent": "can you carry it ?", "words": ["can", "it", "carry", "you"]}, {"sent": "what does this train carry Thomas ?", "words": ["what", "does", "train", "carry", "this"]}, {"sent": "oh hes gonna help her carry it right ?", "words": ["carry", "help", "it", "her"]}, {"sent": "its a bit too heavy to carry round isnt it ?", "words": ["to", "carry", "it", "too", "heavy", "a"]}, {"sent": "some cars carry scrap .", "words": ["carry", "some"]}, {"sent": "its not crying but it will do if you carry on doing nasty things to it .", "words": ["do", "if", "to", "you", "carry", "on", "it", "will", "not", "but"]}, {"sent": "do you want me to carry to carry little Actionman and you carry big Actionman ?", "words": ["do", "to", "you", "carry", "little", "and", "me", "big"]}, {"sent": "let Mummy carry it darling .", "words": ["carry", "it"]}, {"sent": "you must be strong if you can carry her .", "words": ["can", "if", "be", "you", "carry", "her"]}, {"sent": "for him to carry the tray like that .", "words": ["the", "for", "to", "like", "carry", "him", "that", "tray"]}, {"sent": "right you you carry on let me go and get it .", "words": ["go", "you", "carry", "and", "get", "it", "on", "me"]}, {"sent": "are you going to carry that ?", "words": ["to", "you", "carry", "are", "that"]}, {"sent": "you wanna carry the moose book you bring the moose book here Naima .", "words": ["the", "bring", "moose", "carry", "you", "here", "wanna", "book"]}, {"sent": "its not very heavy to carry him .", "words": ["to", "carry", "him", "heavy", "not"]}, {"sent": "and you cant carry him can you now ?", "words": ["can", "you", "carry", "and", "him"]}, {"sent": "lady carry a pushchair ?", "words": ["carry", "a"]}, {"sent": "well you cant carry her .", "words": ["carry", "her", "you"]}, {"sent": "and mother drives us to the store to buy food for supper the grocery man and his helper carry out the packages .", "words": ["the", "for", "to", "store", "and", "carry", "us", "food", "out", "his", "buy"]}, {"sent": "shall I carry on making my big Yorkshire pudding .", "words": ["my", "carry", "on", "pudding", "big"]}, {"sent": "I think its very possible that it does carry newspapers .", "words": ["does", "think", "carry", "it", "that"]}, {"sent": "carry it around .", "words": ["carry", "it", "around"]}, {"sent": "do you carry faking bacon ?", "words": ["carry", "do", "you"]}, {"sent": "you think its big enough to carry you ?", "words": ["think", "to", "you", "carry", "big"]}, {"sent": "I couldnt carry it .", "words": ["carry", "it"]}, {"sent": "Nomi dont carry them in a pile like that .", "words": ["them", "like", "carry", "dont", "in", "that", "a"]}, {"sent": "well you carry on being a fireman .", "words": ["carry", "on", "a", "you"]}, {"sent": "how can he carry all those things ?", "words": ["can", "all", "how", "those", "carry", "he"]}, {"sent": "oh well Ill carry on watching it then .", "words": ["carry", "on", "it", "then"]}, {"sent": "carry me back to .", "words": ["carry", "me", "to", "back"]}, {"sent": "xxx me carry .", "words": ["carry", "me"]}, {"sent": "youve got to carry her .", "words": ["carry", "to", "her"]}, {"sent": "I should carry you everywhere .", "words": ["carry", "you"]}, {"sent": "can you carry them all at once ?", "words": ["can", "all", "them", "you", "carry", "at"]}, {"sent": "what does he carry in his lorry ?", "words": ["what", "does", "carry", "in", "he", "his"]}, {"sent": "that she can carry look at that .", "words": ["can", "carry", "at", "look", "that", "she"]}, {"sent": "can you carry it ?", "words": ["can", "it", "carry", "you"]}, {"sent": "okay then we can ride the horsie for the baby do you think you can carry the taperecorder ?", "words": ["can", "the", "do", "for", "think", "we", "you", "carry", "then", "ride"]}, {"sent": "Ill carry the picnic basket .", "words": ["carry", "the", "basket"]}, {"sent": "DW helped Arthur carry boxes of junk outside .", "words": ["carry", "of", "outside"]}, {"sent": "do you want Mummy to carry you ?", "words": ["to", "do", "carry", "you"]}, {"sent": "Im putting the sling on you so you can carry Sleepy Bear .", "words": ["the", "can", "so", "you", "carry", "on"]}, {"sent": "so he can actually carry a hosepipe on the fire engine .", "words": ["can", "the", "so", "carry", "he", "on", "a"]}, {"sent": "you help me carry all these things to the stairs .", "words": ["the", "all", "help", "these", "to", "you", "carry", "me", "stairs"]}, {"sent": "you trying to carry it in to her ?", "words": ["to", "you", "carry", "in", "it", "her"]}, {"sent": "carry you ?", "words": ["carry", "you"]}, {"sent": "instead of this stupid carry on .", "words": ["of", "this", "on", "carry"]}, {"sent": "you carry baby ghost .", "words": ["carry", "you"]}, {"sent": "where do you want me carry you Thomas ?", "words": ["where", "do", "you", "carry", "me"]}, {"sent": "can Mummy carry on sleeping ?", "words": ["can", "on", "carry"]}, {"sent": "but usually I mean at home shell like carry the doll put the doll to bed do all that stuff so .", "words": ["the", "do", "put", "all", "to", "so", "like", "doll", "bed", "carry", "at", "that", "but", "home"]}, {"sent": "hes got a lot to carry on his head hasnt he ?", "words": ["a lot", "to", "head", "carry", "his", "he", "on", "a"]}, {"sent": "what would you like to carry in your bag ?", "words": ["what", "your", "to", "like", "you", "carry", "in", "would"]}, {"sent": "Im going to carry on with this story now .", "words": ["to", "with", "carry", "this", "on", "story"]}, {"sent": "she hasta carry it .", "words": ["carry", "it", "she"]}, {"sent": "now I cant carry both of you .", "words": ["carry", "of", "you"]}, {"sent": "oh PostmanPat must have found that one heavy mustnt he to carry around ?", "words": ["to", "have", "carry", "he", "around", "heavy", "that"]}, {"sent": "and then you can carry them round .", "words": ["can", "them", "you", "carry", "and", "then"]}, {"sent": "she aint gonna carry nothing .", "words": ["carry", "she"]}, {"sent": "you carry this ?", "words": ["carry", "this", "you"]}, {"sent": "oh youre gonna carry them home in your hands ?", "words": ["your", "them", "carry", "in", "home"]}, {"sent": "okay you carry it to the kitchen .", "words": ["the", "to", "you", "carry", "it", "kitchen"]}, {"sent": "I bet theyre quite heavy to carry those canisters as well arent they ?", "words": ["those", "to", "carry", "they", "heavy"]}, {"sent": "you wanna carry me ?", "words": ["carry", "me", "wanna", "you"]}, {"sent": "what do tanker cars carry ?", "words": ["carry", "what", "do"]}, {"sent": "you could carry drinks in it if you wanted .", "words": ["if", "you", "carry", "in", "it", "could"]}, {"sent": "and you carry on rolling him up .", "words": ["up", "you", "carry", "and", "him", "on"]}, {"sent": "thats not the way to carry that can I show you the way to carry that honey ?", "words": ["the", "can", "show", "to", "you", "carry", "that", "not"]}, {"sent": "carry it with him in ?", "words": ["with", "carry", "in", "him", "it"]}, {"sent": "oh would you like to carry this ?", "words": ["to", "like", "you", "carry", "this", "would"]}, {"sent": "do you wanna carry on talking to me ?", "words": ["do", "to", "you", "carry", "wanna", "on", "me"]}, {"sent": "daddy carry them over .", "words": ["carry", "over", "them"]}, {"sent": "itll carry the blocks for ya .", "words": ["carry", "the", "for"]}, {"sent": "so you dont hafta carry it ?", "words": ["so", "you", "carry", "dont", "it"]}, {"sent": "you dont wanna carry him in the sling ?", "words": ["the", "you", "carry", "dont", "him", "in", "wanna"]}, {"sent": "want ta carry this ?", "words": ["carry", "this"]}, {"sent": "you use this so you can carry your doll and make it easier huh ?", "words": ["can", "your", "so", "doll", "you", "carry", "this", "and", "it", "make"]}, {"sent": "thats called carry me back to old Virginia .", "words": ["old", "back", "to", "carry", "me"]}, {"sent": "this is a little tiny box that I carry in my purse for little tiny things .", "words": ["tiny", "my", "for", "box", "purse", "is", "little", "carry", "this", "in", "that", "a"]}, {"sent": "you carry on drawing now .", "words": ["carry", "on", "you"]}, {"sent": "it was going to carry him away look .", "words": ["to", "was", "carry", "him", "it", "away", "look"]}, {"sent": "because it wants to run away and make sure the people dont catch it so it will carry away a chick or an egg .", "words": ["the", "egg", "to", "so", "catch", "and", "carry", "run", "dont", "away", "it", "an", "will", "make", "because", "a"]}, {"sent": "do you want me to carry you ?", "words": ["do", "to", "you", "carry", "me"]}, {"sent": "or do you want Mommy to carry you ?", "words": ["to", "do", "carry", "you"]}, {"sent": "well boys who are very old carry their lunch buckets .", "words": ["old", "their", "carry", "are", "who"]}, {"sent": "so I can carry it home .", "words": ["can", "so", "carry", "it", "home"]}, {"sent": "you carry your juice .", "words": ["carry", "juice", "your", "you"]}, {"sent": "thats it carry on .", "words": ["carry", "on", "it"]}, {"sent": "you cant carry them downstairs .", "words": ["carry", "them", "you"]}, {"sent": "now you carry that one and Mummy take this .", "words": ["and", "carry", "you", "that", "this", "take"]}, {"sent": "what would she carry them for ?", "words": ["what", "for", "them", "carry", "would", "she"]}, {"sent": "well I shall carry on doing this .", "words": ["carry", "this", "on"]}, {"sent": "well you carry on .", "words": ["carry", "on", "you"]}, {"sent": "well carry it to the table .", "words": ["the", "table", "to", "carry", "it"]}, {"sent": "Ill carry one you carry the other .", "words": ["carry", "the", "other", "you"]}, {"sent": "theyll be very heavy to carry so the men will hafta put them on trolleys and wheel them up and down the ramp .", "words": ["the", "up", "put", "to", "be", "so", "them", "and", "carry", "on", "will", "heavy", "down"]}, {"sent": "and Ill carry the sock you took off and Ill carry the legos .", "words": ["the", "off", "you", "carry", "and", "sock"]}, {"sent": "do you think youll carry some flowers ?", "words": ["do", "think", "you", "carry", "some"]}, {"sent": "thats a lot to carry home .", "words": ["a lot", "to", "carry", "a", "home"]}, {"sent": "can you carry him ?", "words": ["can", "him", "carry", "you"]}, {"sent": "only one tree ?", "words": ["tree"]}, {"sent": "soft tree .", "words": ["tree", "soft"]}, {"sent": "okay put the tree in .", "words": ["the", "tree", "put", "in"]}, {"sent": "Christmas tree .", "words": ["tree"]}, {"sent": "I dont like a theme tree .", "words": ["dont", "tree", "like", "a"]}, {"sent": "this is a tree .", "words": ["this", "tree", "a", "is"]}, {"sent": "theyre up a tree uhn are they ?", "words": ["up", "tree", "they", "are", "a"]}, {"sent": "Kippers popping out of that tree trunk isnt he ?", "words": ["tree", "of", "he", "out", "that"]}, {"sent": "tree .", "words": ["tree"]}, {"sent": "you cant have a tree inside .", "words": ["tree", "inside", "you", "have", "a"]}, {"sent": "did he see the Christmas tree ?", "words": ["the", "tree", "he", "did", "see"]}, {"sent": "let him sit under the tree and have a rest .", "words": ["the", "tree", "and", "have", "under", "him", "sit", "a"]}, {"sent": "what grows on a snake tree ?", "words": ["on", "what", "tree", "a"]}, {"sent": "is it a tree tower ?", "words": ["tree", "it", "a", "is"]}, {"sent": "thats a good tree huh ?", "words": ["tree", "good", "a"]}, {"sent": "James climbed a tree and ate a peanutbutter sandwich .", "words": ["sandwich", "tree", "and", "a"]}, {"sent": "you wanna set the tree up ?", "words": ["the", "up", "tree", "you", "wanna"]}, {"sent": "a Christmas tree .", "words": ["tree", "a"]}, {"sent": "can you see the blossom on that tree and the buds on this one .", "words": ["can", "the", "tree", "you", "and", "this", "on", "see", "that"]}, {"sent": "feel the tree trunk ?", "words": ["the", "tree"]}, {"sent": "thats a Christmas tree isnt it ?", "words": ["tree", "it", "a"]}, {"sent": "but then when we got up this morning she came out of a tree .", "words": ["up", "tree", "we", "of", "this", "out", "then", "when", "she", "but", "a"]}, {"sent": "tree where are we going to put the tree ?", "words": ["where", "the", "tree", "put", "we", "to", "are"]}, {"sent": "I dont think you should climb that tree .", "words": ["think", "tree", "you", "dont", "climb", "that"]}, {"sent": "can you see hes stuck in a tree ?", "words": ["can", "tree", "you", "in", "stuck", "see", "a"]}, {"sent": "behind the tree ?", "words": ["the", "tree", "behind"]}, {"sent": "so this is an apple tree farm .", "words": ["tree", "is", "so", "this", "an", "apple"]}, {"sent": "everybodys running to the coconut tree .", "words": ["the", "to", "tree"]}, {"sent": "wanna put them under the tree ?", "words": ["the", "put", "tree", "them", "under", "wanna"]}, {"sent": "that has fallen off the tree .", "words": ["off", "the", "tree", "that"]}, {"sent": "weve not got a tree .", "words": ["tree", "not", "a"]}, {"sent": "here is another tree and I .", "words": ["tree", "is", "and", "here", "another"]}, {"sent": "wheres the pine tree ?", "words": ["the", "tree"]}, {"sent": "she ate the fruit from Gods special tree .", "words": ["the", "tree", "she"]}, {"sent": "in about twenty years or so it will grow into a tall oak tree that has many acorns .", "words": ["tree", "into", "so", "in", "it", "about", "will", "that", "a"]}, {"sent": "what kind of a tree does kookaburra sit on ?", "words": ["what", "tree", "does", "of", "on", "sit", "a"]}, {"sent": "oh look at this the wind blew this whole tree down .", "words": ["the", "tree", "look", "this", "at", "down", "wind"]}, {"sent": "it does look like a tree .", "words": ["does", "tree", "like", "it", "look", "a"]}, {"sent": "yes theres a kitty on that tree .", "words": ["tree", "kitty", "on", "that", "a"]}, {"sent": "a tree very good .", "words": ["tree", "good", "a"]}, {"sent": "do you see this pink Christmas tree bauble ?", "words": ["do", "tree", "you", "this", "see"]}, {"sent": "thats a smaller tree .", "words": ["tree", "a"]}, {"sent": "and then we could perhaps draw a tree .", "words": ["tree", "we", "draw", "and", "then", "a", "could"]}, {"sent": "are you a raccoon in a hole in the tree ?", "words": ["the", "tree", "you", "in", "are", "a"]}, {"sent": "is it a Christmas tree ?", "words": ["tree", "it", "a", "is"]}, {"sent": "what color are you gonna draw your tree ?", "words": ["what", "tree", "your", "draw", "you", "are"]}, {"sent": "what kind of tree is this .", "words": ["what", "tree", "is", "of", "this"]}, {"sent": "Will gonna put the tree together ?", "words": ["the", "tree", "put"]}, {"sent": "your Christmas tree is over now .", "words": ["over", "tree", "your", "is"]}, {"sent": "christmas tree ?", "words": ["tree"]}, {"sent": "you can get the stickers and you can decorate the tree just like all those people did in the story .", "words": ["can", "the", "tree", "all", "those", "like", "and", "get", "you", "in", "did", "story"]}, {"sent": "now weve got a tree here .", "words": ["here", "tree", "a"]}, {"sent": "woodpeckers grip tree trunks with their toes .", "words": ["tree", "with", "their"]}, {"sent": "shes in the tree house but shes jumping out of the tree house thats a big tree house .", "words": ["the", "tree", "of", "in", "house", "out", "big", "but", "a"]}, {"sent": "Owls house is in a tree .", "words": ["tree", "is", "in", "house", "a"]}, {"sent": "well hed only need his ladders if Purdie got stuck up at tree .", "words": ["up", "tree", "need", "if", "stuck", "at", "his"]}, {"sent": "Santa and his christmas tree ?", "words": ["his", "tree", "and"]}, {"sent": "and hes sitting by the tree now isnt he ?", "words": ["the", "tree", "by", "and", "he"]}, {"sent": "Daddy decorated the tree while Mummy was at work and Ive not paid a lot of attention to it .", "words": ["the", "tree", "work", "of", "was", "to", "a lot", "and", "it", "at", "not", "a"]}, {"sent": "I and tag along k all on their way up the coconut tree .", "words": ["up", "all", "the", "tree", "their", "and", "on"]}, {"sent": "is that a tree ?", "words": ["tree", "that", "a", "is"]}, {"sent": "and he goes up a tree .", "words": ["up", "tree", "and", "he", "a"]}, {"sent": "as they clambered onto the front of the roof the burglars leapt off the back into the branches of a tall tree in the garden .", "words": ["the", "off", "tree", "into", "back", "of", "they", "in", "garden", "roof", "a"]}, {"sent": "were hung on a tree to dry .", "words": ["tree", "to", "dry", "on", "were", "a"]}, {"sent": "Ill take you in the tree .", "words": ["the", "tree", "you", "in", "take"]}, {"sent": "so the two cheeky monkeys swinging in the tree .", "words": ["the", "so", "in", "tree"]}, {"sent": "okay how about the alphabet tree ?", "words": ["the", "tree", "how", "about"]}, {"sent": "he is gonna water the tree .", "words": ["the", "tree", "is", "he", "water"]}, {"sent": "youre in the way of the tree .", "words": ["the", "of", "in", "tree"]}, {"sent": "theres the Christmas tree .", "words": ["the", "tree"]}, {"sent": "a tree apple ?", "words": ["apple", "tree", "a"]}, {"sent": "thats kinda like what falls from our tree .", "words": ["what", "tree", "like", "our"]}, {"sent": "well maybe he rided on a bicycle and then hided behind a tree .", "words": ["tree", "behind", "bicycle", "and", "he", "on", "then", "a"]}, {"sent": "are you gonna chop the tree down ?", "words": ["the", "tree", "you", "are", "down"]}, {"sent": "a squirrel remember when we saw the squirrel in the tree ?", "words": ["the", "tree", "we", "in", "when", "squirrel", "a"]}, {"sent": "why did they spread their blanket under the tree ?", "words": ["the", "tree", "their", "they", "under", "did", "blanket", "why"]}, {"sent": "say tree Edw .", "words": ["tree", "say"]}, {"sent": "it would save me a lot of money if we had a pear tree in the garden .", "words": ["money", "the", "tree", "if", "we", "of", "a lot", "in", "it", "garden", "would", "me", "a"]}, {"sent": "and there is a tree .", "words": ["tree", "is", "and", "there", "a"]}, {"sent": "a tree .", "words": ["tree", "a"]}, {"sent": "he climbs up the tree and he jumps over to this branch .", "words": ["up", "the", "tree", "to", "and", "this", "he", "over"]}, {"sent": "its wrapping round the tree isnt it ?", "words": ["the", "tree", "it"]}, {"sent": "maybe when we go home well find a little tree to climb up .", "words": ["up", "tree", "go", "we", "to", "little", "find", "climb", "when", "a", "home"]}, {"sent": "and I see a tree with red leaves right over there .", "words": ["red", "tree", "with", "and", "there", "see", "over", "a"]}, {"sent": "now the prince the princess and the dragon finished decorating the tree .", "words": ["the", "tree", "and"]}, {"sent": "its not very nice to be stuck in a tree is it ?", "words": ["tree", "is", "to", "be", "nice", "in", "it", "stuck", "not", "a"]}, {"sent": "and in a car in a tree .", "words": ["tree", "car", "and", "in", "a"]}, {"sent": "oh look whos got a Christmas tree up .", "words": ["up", "tree", "a", "look"]}, {"sent": "look theres your tree .", "words": ["tree", "your", "look"]}, {"sent": "youve got a little christmas tree decoration on your head .", "words": ["tree", "your", "head", "little", "on", "a"]}, {"sent": "and what happened to the tree and all the presents .", "words": ["the", "what", "tree", "all", "to", "and"]}, {"sent": "shall we let him play on this tree ?", "words": ["tree", "we", "play", "this", "him", "on"]}, {"sent": "you could make another tree .", "words": ["tree", "you", "another", "could", "make"]}, {"sent": "and a flower and a tree and a watering can and a spade .", "words": ["can", "tree", "flower", "and", "a"]}, {"sent": "a tree .", "words": ["tree", "a"]}, {"sent": "it is another tree of the same kind .", "words": ["the", "tree", "is", "of", "another", "it", "same"]}, {"sent": "a tree for the monkeys and some water and some fish for the penguins .", "words": ["the", "tree", "for", "fish", "and", "water", "some", "a"]}, {"sent": "in a tree .", "words": ["tree", "in", "a"]}, {"sent": "what are you doing down at the beach ?", "words": ["the", "what", "you", "beach", "are", "at", "down"]}, {"sent": "you saw the beach from the plane ?", "words": ["the", "beach", "you"]}, {"sent": "what did we do at the rock beach that was pretty cool .", "words": ["the", "what", "do", "we", "was", "beach", "pretty", "did", "at", "rock", "that"]}, {"sent": "and shes sitting on the beach .", "words": ["on", "the", "beach", "and"]}, {"sent": "is it like a little beach house or something ?", "words": ["is", "like", "little", "beach", "it", "house", "a"]}, {"sent": "oh beach ball .", "words": ["ball", "beach"]}, {"sent": "stones on the beach .", "words": ["on", "the", "beach"]}, {"sent": "Kennys down at the beach .", "words": ["the", "beach", "down", "at"]}, {"sent": "at the beach .", "words": ["the", "beach", "at"]}, {"sent": "well you keep pushing what da ya wanna do do you wanna look at this at the beach ?", "words": ["the", "what", "do", "you", "this", "beach", "at", "wanna", "look"]}, {"sent": "did you go to the beach with Nola ?", "words": ["the", "go", "to", "with", "you", "beach", "did"]}, {"sent": "one day we went to the beach do you remember ?", "words": ["the", "do", "we", "to", "you", "beach"]}, {"sent": "were gonna go to the beach .", "words": ["the", "go", "to", "beach", "were"]}, {"sent": "at the beach ?", "words": ["the", "beach", "at"]}, {"sent": "oh theres a beach .", "words": ["beach", "a"]}, {"sent": "how would you like to get shipped back to the beach ?", "words": ["the", "how", "back", "to", "like", "you", "get", "beach", "would"]}, {"sent": "oh I dont wanna take my clothes now off because its too cold here at this beach .", "words": ["off", "my", "here", "dont", "this", "at", "beach", "wanna", "too", "take", "cold", "because"]}, {"sent": "yeah theyre on the beach in that picture .", "words": ["the", "picture", "beach", "in", "on", "that"]}, {"sent": "beach .", "words": ["beach"]}, {"sent": "is the horse gonna come to the beach too ?", "words": ["the", "is", "to", "beach", "too", "horse"]}, {"sent": "so when you go to the beach you can see some ?", "words": ["the", "can", "go", "so", "to", "you", "beach", "see", "when", "some"]}, {"sent": "there were many horseshoe crabs werent there on that beach ?", "words": ["there", "beach", "on", "that", "were"]}, {"sent": "you going down to the beach this summer ?", "words": ["the", "to", "you", "this", "beach", "down"]}, {"sent": "do we collect shells on the beach ?", "words": ["the", "do", "we", "beach", "on"]}, {"sent": "at the beach .", "words": ["the", "beach", "at"]}, {"sent": "xxx is the beach .", "words": ["the", "beach", "is"]}, {"sent": "down the beach .", "words": ["the", "beach", "down"]}, {"sent": "so that you could go to the beach .", "words": ["the", "go", "so", "to", "you", "beach", "could", "that"]}, {"sent": "you dont remember when we were at the beach in Connecticut ?", "words": ["the", "we", "you", "dont", "beach", "in", "at", "when", "were"]}, {"sent": "and a wee tiny path and there was the beach just there .", "words": ["tiny", "the", "was", "and", "there", "beach", "a"]}, {"sent": "do you remember seeing the beach towels Thomas ?", "words": ["the", "do", "beach", "you"]}, {"sent": "those are her beach shoes .", "words": ["her", "are", "those", "beach"]}, {"sent": "do you wanna go to the beach ?", "words": ["the", "do", "go", "to", "you", "beach", "wanna"]}, {"sent": "you going back to the beach tonight ?", "words": ["the", "back", "to", "you", "beach"]}, {"sent": "right at the far end of the beach we came at last to a little spring running down through the sun behind a large wall .", "words": ["the", "we", "of", "to", "last", "sun", "behind", "little", "beach", "at", "down", "a"]}, {"sent": "we cant go to the beach when its raining .", "words": ["the", "go", "we", "to", "beach", "when"]}, {"sent": "whats been happening down at the beach ?", "words": ["the", "beach", "down", "at"]}, {"sent": "what is on the beach ?", "words": ["the", "what", "is", "beach", "on"]}, {"sent": "beach .", "words": ["beach"]}, {"sent": "you went to the beach ?", "words": ["the", "to", "beach", "you"]}, {"sent": "like at the beach .", "words": ["the", "beach", "like", "at"]}, {"sent": "did we go to the beach ?", "words": ["the", "go", "we", "to", "beach", "did"]}, {"sent": "and we needed something just to wedge them together so Grandma and Grandad went and picked lotsof pebbles from the beach .", "words": ["the", "we", "to", "so", "them", "and", "beach"]}, {"sent": "how about the beach ?", "words": ["the", "how", "beach", "about"]}, {"sent": "what did we do at the beach ?", "words": ["the", "what", "do", "we", "beach", "did", "at"]}, {"sent": "they probably have a tricycle but they dont have it right on the beach .", "words": ["the", "have", "they", "dont", "it", "beach", "tricycle", "on", "but", "a"]}, {"sent": "in the summer it is fun to go to the beach .", "words": ["the", "is", "go", "to", "in", "it", "beach"]}, {"sent": "no shes not going to the beach .", "words": ["the", "to", "beach", "not"]}, {"sent": "and what what did we find when we went on the beach ?", "words": ["the", "what", "we", "and", "beach", "did", "find", "on", "when"]}, {"sent": "the beach was so freezing .", "words": ["the", "was", "beach", "so"]}, {"sent": "and the beach on saturday .", "words": ["the", "beach", "on", "and"]}, {"sent": "and what is all this area called the beach isnt it ?", "words": ["the", "all", "what", "is", "and", "this", "beach", "it"]}, {"sent": "to the to the b to the beach .", "words": ["the", "to", "beach"]}, {"sent": "I think shes collecting shells on the beach .", "words": ["on", "the", "think", "beach"]}, {"sent": "do you wanna go to the beach soon ?", "words": ["the", "do", "go", "to", "you", "beach", "wanna"]}, {"sent": "what are they gonna see at the beach ?", "words": ["the", "what", "they", "beach", "are", "at", "see"]}, {"sent": "on the beach .", "words": ["on", "the", "beach"]}, {"sent": "well were gonna put em back cause were gonna you said you wanted to get out you said Im done and we need to get ready to go to beach right ?", "words": ["put", "need", "back", "we", "go", "to", "need to", "you", "get", "and", "beach", "out", "were"]}, {"sent": "and what did you do on the beach ?", "words": ["the", "what", "do", "you", "and", "beach", "did", "on"]}, {"sent": "a beach .", "words": ["beach", "a"]}, {"sent": "something from the beach in Spain .", "words": ["the", "in", "beach"]}, {"sent": "but by the time we gotta the beach there was um there was pools of water and leftover spaces and stuff like that and you could look for sanddollars that just washed up .", "words": ["the", "up", "look", "for", "we", "of", "was", "like", "by", "and", "there", "beach", "you", "water", "could", "that", "but"]}, {"sent": "one day Corky was at the beach chasing a beach ball .", "words": ["the", "was", "ball", "beach", "at", "a"]}, {"sent": "when you go down to the beach right ?", "words": ["the", "go", "to", "you", "beach", "down", "when"]}, {"sent": "ya cant go ta the beach any more until next summer .", "words": ["the", "go", "any", "beach", "more"]}, {"sent": "down the beach .", "words": ["the", "beach", "down"]}, {"sent": "my daddy could take me to beach with xxx .", "words": ["my", "to", "with", "beach", "me", "could", "take"]}, {"sent": "Armie oh look at this one you can wear that one to the beach .", "words": ["can", "the", "look", "to", "you", "this", "beach", "at", "that"]}, {"sent": "she loves the beach .", "words": ["the", "beach", "she"]}, {"sent": "what did you build on the beach ?", "words": ["the", "what", "you", "beach", "build", "did", "on"]}, {"sent": "you went to the beach ?", "words": ["the", "to", "beach", "you"]}, {"sent": "what is this on the beach ?", "words": ["the", "what", "is", "this", "beach", "on"]}, {"sent": "and your sandals on the beach .", "words": ["the", "your", "and", "beach", "on"]}, {"sent": "go to the beach and xxx youknow ?", "words": ["the", "go", "to", "and", "beach"]}, {"sent": "xxx when you take your pail and you go down go the beach ?", "words": ["the", "your", "when", "go", "and", "you", "beach", "down", "take"]}, {"sent": "down at the beach ?", "words": ["the", "beach", "down", "at"]}, {"sent": "at the beach at the beach .", "words": ["the", "beach", "at"]}, {"sent": "did you go on the beach ?", "words": ["the", "go", "you", "beach", "did", "on"]}, {"sent": "oh Thomas theyre pebbles from the beach .", "words": ["the", "beach"]}, {"sent": "Sarah you going down the beach this summer and go swimming in the water ?", "words": ["the", "go", "you", "and", "this", "beach", "in", "water", "down"]}, {"sent": "and then we walked along the beach .", "words": ["the", "we", "and", "beach", "then"]}, {"sent": "the sandy beach reminded Harold of .", "words": ["the", "of", "beach"]}, {"sent": "I left all my ribbon down the beach too .", "words": ["the", "all", "my", "beach", "too", "down"]}, {"sent": "look at all the kids on the beach .", "words": ["the", "all", "look", "beach", "at", "on"]}, {"sent": "were back at the beach right ?", "words": ["the", "back", "beach", "at", "were"]}, {"sent": "look at all those sea shells on the beach .", "words": ["the", "all", "look", "those", "beach", "at", "on"]}, {"sent": "back to the beach .", "words": ["the", "to", "beach", "back"]}, {"sent": "youve got a red beach there .", "words": ["red", "there", "beach", "a"]}, {"sent": "do you think he wants to go to the beach now ?", "words": ["the", "do", "think", "go", "to", "you", "beach", "he"]}, {"sent": "what did we throw in the water at the beach last time ?", "words": ["the", "what", "throw", "we", "last", "in", "at", "beach", "did", "water"]}, {"sent": "and we danced on the beach .", "words": ["the", "we", "and", "beach", "on"]}, {"sent": "did you go to the beach in the summer ?", "words": ["the", "go", "to", "you", "beach", "in", "did"]}, {"sent": "and Mummy and Daddy walked to the beach and we sat on some chairs .", "words": ["the", "we", "to", "and", "beach", "on", "some"]}, {"sent": "I dont think you could take it to the beach no .", "words": ["the", "think", "to", "you", "dont", "beach", "it", "could", "take"]}, {"sent": "go to the beach .", "words": ["the", "to", "beach", "go"]}, {"sent": "Thomas you look like youre dressed for the beach .", "words": ["the", "for", "like", "you", "beach", "look"]}, {"sent": "yanna go to the beach ?", "words": ["the", "to", "beach", "go"]}, {"sent": "shes such a water bug that I mean you get her at a beach and she just .", "words": ["bug", "her", "you", "get", "and", "beach", "at", "water", "that", "she", "a"]}, {"sent": "we were on the beach all day .", "words": ["the", "all", "we", "beach", "on", "were"]}, {"sent": "not a helicopter .", "words": ["helicopter", "not", "a"]}, {"sent": "flatten you with my helicopter now .", "words": ["helicopter", "my", "with", "you"]}, {"sent": "mind your helicopter .", "words": ["helicopter", "your"]}, {"sent": "thats your man for your helicopter .", "words": ["helicopter", "for", "your"]}, {"sent": "what about a helicopter ?", "words": ["what", "helicopter", "about", "a"]}, {"sent": "and that helicopter .", "words": ["helicopter", "that", "and"]}, {"sent": "that is a nice helicopter .", "words": ["helicopter", "is", "nice", "that", "a"]}, {"sent": "no Kalie he was doing the airplane you get the helicopter .", "words": ["the", "helicopter", "airplane", "was", "you", "get", "he"]}, {"sent": "see a helicopter has a thing up here that goes like this .", "words": ["up", "helicopter", "like", "here", "this", "see", "that", "a"]}, {"sent": "helicopter .", "words": ["helicopter"]}, {"sent": "a helicopter .", "words": ["helicopter", "a"]}, {"sent": "has the helicopter broken down ?", "words": ["the", "helicopter", "down", "broken"]}, {"sent": "helicopter .", "words": ["helicopter"]}, {"sent": "helicopter .", "words": ["helicopter"]}, {"sent": "are you telling me the colors in the helicopter ?", "words": ["the", "helicopter", "you", "in", "are", "me"]}, {"sent": "well its just a little helicopter .", "words": ["helicopter", "little", "a"]}, {"sent": "a helicopter .", "words": ["helicopter", "a"]}, {"sent": "a helicopter .", "words": ["helicopter", "a"]}, {"sent": "its a helicopter .", "words": ["helicopter", "a"]}, {"sent": "here is the helicopter .", "words": ["here", "the", "helicopter", "is"]}, {"sent": "blue baby helicopter ?", "words": ["helicopter", "blue"]}, {"sent": "you get that helicopter and bring it over here .", "words": ["helicopter", "bring", "and", "get", "you", "here", "it", "over", "that"]}, {"sent": "is he going in the helicopter now ?", "words": ["the", "helicopter", "is", "in", "he"]}, {"sent": "this is a special helicopter that lands on ice okay .", "words": ["helicopter", "is", "ice", "this", "on", "that", "a"]}, {"sent": "helicopter .", "words": ["helicopter"]}, {"sent": "Mummys going to wind the helicopter up with a key .", "words": ["the", "up", "helicopter", "to", "with", "a", "wind"]}, {"sent": "I dont think youll be able to see that helicopter .", "words": ["helicopter", "think", "to", "be", "dont", "see", "that"]}, {"sent": "and it was a helicopter that came .", "words": ["helicopter", "was", "and", "it", "that", "a"]}, {"sent": "in the lorry or in the helicopter ?", "words": ["the", "helicopter", "in"]}, {"sent": "shes come by helicopter .", "words": ["helicopter", "by"]}, {"sent": "because if its not a funny shape it doesnt look like a helicopter .", "words": ["helicopter", "if", "like", "it", "look", "not", "because", "a"]}, {"sent": "that is a helicopter .", "words": ["helicopter", "that", "a", "is"]}, {"sent": "the helicopter has gone into the engine shed to sleep as well has he ?", "words": ["the", "helicopter", "into", "sleep", "to", "he"]}, {"sent": "helicopter .", "words": ["helicopter"]}, {"sent": "what noise does a helicopter make ?", "words": ["what", "does", "helicopter", "make", "a"]}, {"sent": "the police helicopter ?", "words": ["the", "helicopter"]}, {"sent": "hes going to jump into this helicopter quickly so it doesnt get him .", "words": ["helicopter", "into", "to", "so", "get", "jump", "this", "it", "him"]}, {"sent": "I bet hed like to ride on a helicopter .", "words": ["helicopter", "to", "like", "on", "ride", "a"]}, {"sent": "the helicopter whirls in the sky .", "words": ["the", "helicopter", "in", "sky"]}, {"sent": "yeah a big helicopter .", "words": ["helicopter", "big", "a"]}, {"sent": "wow a helicopter .", "words": ["helicopter", "a"]}, {"sent": "its the news helicopter .", "words": ["the", "helicopter"]}, {"sent": "are you drawing a helicopter .", "words": ["helicopter", "are", "a", "you"]}, {"sent": "it is a helicopter .", "words": ["helicopter", "it", "a", "is"]}, {"sent": "that is a helicopter .", "words": ["helicopter", "that", "a", "is"]}, {"sent": "and what does the helicopter say ?", "words": ["the", "what", "does", "helicopter", "say", "and"]}, {"sent": "helicopter right .", "words": ["helicopter"]}, {"sent": "do you have a baby helicopter ?", "words": ["do", "helicopter", "you", "have", "a"]}, {"sent": "Im gonna ride in my motorcycle no helicopter .", "words": ["motorcycle", "helicopter", "my", "in", "ride"]}, {"sent": "did they xxx helicopter ?", "words": ["they", "did", "helicopter"]}, {"sent": "its a helicopter isnt it ?", "words": ["helicopter", "it", "a"]}, {"sent": "the the helicopter I should say .", "words": ["the", "helicopter", "say"]}, {"sent": "helicopter is right there .", "words": ["helicopter", "there", "is"]}, {"sent": "a helicopter ?", "words": ["helicopter", "a"]}, {"sent": "youve found the propeller for the helicopter .", "words": ["the", "helicopter", "for"]}, {"sent": "this is a helicopter .", "words": ["this", "helicopter", "a", "is"]}, {"sent": "a helicopter .", "words": ["helicopter", "a"]}, {"sent": "dont think it was a helicopter .", "words": ["helicopter", "think", "was", "dont", "it", "a"]}, {"sent": "I thought it looked like an army helicopter .", "words": ["helicopter", "an", "it", "like"]}, {"sent": "helicopter .", "words": ["helicopter"]}, {"sent": "all right now Ive got the zooming helicopter .", "words": ["the", "all", "helicopter"]}, {"sent": "can you make a helicopter ?", "words": ["can", "helicopter", "you", "make", "a"]}, {"sent": "two police stations the helicopter launch pad .", "words": ["the", "helicopter"]}, {"sent": "I think the helicopter was a Christmas present .", "words": ["the", "helicopter", "think", "present", "was", "a"]}, {"sent": "okay this ones called an airplane and this ones called a helicopter .", "words": ["helicopter", "airplane", "and", "this", "an", "a"]}, {"sent": "helicopter .", "words": ["helicopter"]}, {"sent": "but its a funny helicopter .", "words": ["helicopter", "but", "a"]}, {"sent": "a blue propeller on the helicopter .", "words": ["the", "helicopter", "blue", "on", "a"]}, {"sent": "um do you see theres a helicopter on here ?", "words": ["do", "helicopter", "you", "here", "on", "see", "a"]}, {"sent": "and a helicopter .", "words": ["helicopter", "a", "and"]}, {"sent": "maybe he wants to ride in a helicopter .", "words": ["helicopter", "to", "in", "he", "ride", "a"]}, {"sent": "that is a bear in a helicopter .", "words": ["helicopter", "is", "bear", "in", "that", "a"]}, {"sent": "the helicopter is upsidedown .", "words": ["the", "helicopter", "is"]}, {"sent": "there goes the helicopter .", "words": ["the", "there", "helicopter"]}, {"sent": "oh no Laura well never find your helicopter there .", "words": ["helicopter", "your", "there", "find"]}, {"sent": "okay you do the helicopter and Ill do the airplane .", "words": ["the", "do", "helicopter", "airplane", "and", "you"]}, {"sent": "thats from the helicopter see .", "words": ["the", "helicopter", "see"]}, {"sent": "thats right shes going to hon the helicopter .", "words": ["the", "to", "helicopter"]}, {"sent": "if youre putting all these toys in the wheelbarrow why dont you go and get the helicopter ?", "words": ["the", "all", "these", "helicopter", "if", "go", "you", "and", "dont", "in", "get", "why"]}, {"sent": "and a helicopter there .", "words": ["helicopter", "there", "a", "and"]}, {"sent": "thats a helicopter .", "words": ["helicopter", "a"]}, {"sent": "helicopter .", "words": ["helicopter"]}, {"sent": "you think thats a helicopter ?", "words": ["helicopter", "think", "a", "you"]}, {"sent": "helicopter .", "words": ["helicopter"]}, {"sent": "you wanna get the plane and the helicopter ?", "words": ["the", "helicopter", "and", "get", "you", "wanna"]}, {"sent": "a combination between ambulance and helicopter I think .", "words": ["helicopter", "think", "and", "a"]}, {"sent": "do you wanna play with the helicopter and airplane ?", "words": ["the", "do", "helicopter", "airplane", "play", "with", "you", "and", "wanna"]}, {"sent": "make her hon the helicopter again .", "words": ["the", "helicopter", "her", "make"]}, {"sent": "Percy wants to shout hello to the helicopter .", "words": ["the", "to", "helicopter"]}, {"sent": "theres a helicopter on the television as well .", "words": ["on", "helicopter", "the", "a"]}, {"sent": "this is a helicopter jigsaw puzzle .", "words": ["helicopter", "is", "this", "puzzle", "a"]}, {"sent": "wheres the helicopter ?", "words": ["the", "helicopter"]}, {"sent": "from the helicopter .", "words": ["the", "helicopter"]}, {"sent": "we could have a helicopter as well .", "words": ["helicopter", "we", "have", "could", "a"]}, {"sent": "thats a helicopter .", "words": ["helicopter", "a"]}, {"sent": "helicopter .", "words": ["helicopter"]}, {"sent": "a helicopter .", "words": ["helicopter", "a"]}, {"sent": "the helicopter ?", "words": ["the", "helicopter"]}, {"sent": "you dance .", "words": ["dance", "you"]}, {"sent": "everybody dance now lets dance touch the right side of your body .", "words": ["the", "your", "of", "touch", "dance"]}, {"sent": "are they gonna dance now ?", "words": ["they", "are", "dance"]}, {"sent": "you wanna dance ?", "words": ["dance", "wanna", "you"]}, {"sent": "lets dance with momma .", "words": ["with", "dance"]}, {"sent": "do you want me to teach you the Tweenies dance ?", "words": ["the", "do", "to", "dance", "you", "me"]}, {"sent": "oh xxx the clown dance .", "words": ["the", "dance"]}, {"sent": "dance .", "words": ["dance"]}, {"sent": "the people started to dance Griswald started to dance too .", "words": ["the", "to", "dance", "too"]}, {"sent": "sing song or dance or what ?", "words": ["sing", "what", "dance"]}, {"sent": "my pants that dance .", "words": ["that", "my", "pants", "dance"]}, {"sent": "look at that dance Eleanor .", "words": ["at", "that", "dance", "look"]}, {"sent": "can you dance ?", "words": ["can", "dance", "you"]}, {"sent": "you can dance .", "words": ["can", "dance", "you"]}, {"sent": "they dance ?", "words": ["they", "dance"]}, {"sent": "is that the dance the Tweenies do ?", "words": ["the", "do", "is", "dance", "that"]}, {"sent": "and so like one of the book pages you flip and it starts doing the A B C song so she sits there and looks like shes trying to dance and trying to sing to it so you get these sounds that sound like A B C out of her .", "words": ["the", "these", "of", "so", "to", "her", "like", "dance", "and", "you", "there", "sing", "it", "get", "out", "book", "that", "she"]}, {"sent": "dance dance dance .", "words": ["dance"]}, {"sent": "hes doing a dance .", "words": ["dance", "a"]}, {"sent": "dance to your Daddy my bonnie lad .", "words": ["to", "my", "your", "dance"]}, {"sent": "you can dance about later .", "words": ["can", "about", "dance", "you"]}, {"sent": "and theres people people are all underneath here doing the dance .", "words": ["the", "all", "dance", "and", "here", "are"]}, {"sent": "oh I saw a fox and a wolf dance together .", "words": ["dance", "wolf", "and", "a"]}, {"sent": "lets dance .", "words": ["dance"]}, {"sent": "we all can dance he said when we find the music that we love .", "words": ["can", "all", "the", "we", "dance", "that", "he", "love", "find", "when"]}, {"sent": "with an oink and a moo and a quack quack quack the dance is done but well be back .", "words": ["the", "back", "is", "be", "with", "dance", "and", "an", "but", "a"]}, {"sent": "is that the Teletubbie dance ?", "words": ["the", "that", "dance", "is"]}, {"sent": "you make dolly dance .", "words": ["make", "dance", "you"]}, {"sent": "youre going to dance with Cromer ?", "words": ["to", "with", "dance"]}, {"sent": "can you dance ?", "words": ["can", "dance", "you"]}, {"sent": "when theres a Chi Chinese dragon dance they have lotsof very loud drums Thomas .", "words": ["dance", "have", "they", "when", "loud", "a"]}, {"sent": "you useta do the dance afterwards .", "words": ["the", "do", "dance", "you"]}, {"sent": "now lets dance now lets dance .", "words": ["dance"]}, {"sent": "when I say sit they dance .", "words": ["say", "dance", "they", "sit", "when"]}, {"sent": "when the Tweenies come on we dance .", "words": ["the", "we", "dance", "on", "when"]}, {"sent": "you dont wanna go and dance ?", "words": ["go", "dance", "and", "you", "dont", "wanna"]}, {"sent": "do you wanna have a dance ?", "words": ["do", "dance", "you", "have", "wanna", "a"]}, {"sent": "suddenly Cinderella stopped dreaming oh no she said Ive never been to a ball I dont even know how to dance .", "words": ["how", "to", "dance", "ball", "dont", "she", "a"]}, {"sent": "dance along dance for the birds .", "words": ["the", "for", "dance"]}, {"sent": "thats a very unusual dance .", "words": ["dance", "a"]}, {"sent": "she she um will like kind of dance every time music starts .", "words": ["of", "like", "dance", "every", "will", "she"]}, {"sent": "you always did your dance didnt you ?", "words": ["your", "did", "dance", "you"]}, {"sent": "were going to a dance recital tonight .", "words": ["to", "were", "dance", "a"]}, {"sent": "do you think Paddington would like to dance ?", "words": ["do", "think", "to", "like", "dance", "you", "would"]}, {"sent": "where is the dance ?", "words": ["where", "the", "dance", "is"]}, {"sent": "dance along dance for the .", "words": ["the", "for", "dance"]}, {"sent": "now lets dance now lets dance now lets dance .", "words": ["dance"]}, {"sent": "and shes been doing her dance all year long .", "words": ["all", "long", "dance", "and", "her"]}, {"sent": "yeah we clap or we do like a little dance for her .", "words": ["do", "for", "clap", "we", "like", "dance", "little", "her", "a"]}, {"sent": "oh dance baby .", "words": ["dance"]}, {"sent": "dance dance dance dance .", "words": ["dance"]}, {"sent": "did you go to the big teenage dance with little Jamie ?", "words": ["the", "go", "to", "with", "dance", "you", "little", "did", "big"]}, {"sent": "you gonna dance ?", "words": ["dance", "you"]}, {"sent": "say dance dance dance .", "words": ["say", "dance"]}, {"sent": "youre going to dance when we play the record again ?", "words": ["the", "we", "to", "play", "dance", "when"]}, {"sent": "the questionaire asked um what her favorite video and its absolutely without a doubt Sesame Street dance along .", "words": ["the", "what", "dance", "and", "her", "a"]}, {"sent": "now lets dance now lets dance .", "words": ["dance"]}, {"sent": "show Mama how to do the gaggle dance .", "words": ["the", "do", "how", "show", "to", "dance"]}, {"sent": "you gonna dance ?", "words": ["dance", "you"]}, {"sent": "a whole body dance .", "words": ["dance", "a"]}, {"sent": "and hell dance and clap when deck the halls comes on and so .", "words": ["the", "clap", "so", "dance", "and", "on", "when"]}, {"sent": "did Tigger dance ?", "words": ["did", "dance"]}, {"sent": "but see the other hell dance and dance and wont do it .", "words": ["the", "do", "dance", "other", "and", "it", "see", "but"]}, {"sent": "then one by one each animal whod been there at the dance arrived with Gerald xxx and watched him quite entranced .", "words": ["the", "each", "with", "dance", "by", "and", "there", "him", "animal", "at", "then"]}, {"sent": "dance dance dance .", "words": ["dance"]}, {"sent": "touch the front of your body now everybody dance forward .", "words": ["the", "your", "of", "dance", "touch"]}, {"sent": "dance .", "words": ["dance"]}, {"sent": "ahp okay feet out we gotta do a feet dance .", "words": ["do", "we", "dance", "out", "a"]}, {"sent": "youre not gonna do your dance ?", "words": ["do", "your", "not", "dance"]}, {"sent": "and you can dance with Mommy .", "words": ["can", "with", "dance", "you", "and"]}, {"sent": "dance a dance .", "words": ["dance", "a"]}, {"sent": "and she did the little Kwanza dance for us which Elmo does in the video .", "words": ["the", "does", "for", "dance", "little", "and", "us", "in", "did", "which", "she"]}, {"sent": "shake it dancy dance .", "words": ["it", "shake", "dance"]}, {"sent": "you wanna dance ?", "words": ["dance", "wanna", "you"]}, {"sent": "I dance .", "words": ["dance"]}, {"sent": "whos doing a dance for you ?", "words": ["you", "for", "dance", "a"]}, {"sent": "in dance group they was corny .", "words": ["was", "they", "in", "dance"]}, {"sent": "and you dance to it ?", "words": ["to", "dance", "you", "and", "it"]}, {"sent": "can you dance now ?", "words": ["can", "dance", "you"]}, {"sent": "Ive not seen your dancing before do you do do you dance well ?", "words": ["do", "your", "dance", "you", "not"]}, {"sent": "sing and dance .", "words": ["sing", "dance", "and"]}, {"sent": "he can dance cant he ?", "words": ["can", "he", "dance"]}, {"sent": "then swirl up the dust and make the leaves dance .", "words": ["up", "the", "dance", "and", "then", "make"]}, {"sent": "where do I dance ?", "words": ["where", "do", "dance"]}, {"sent": "the wiggle wiggle dance .", "words": ["the", "dance"]}, {"sent": "you dance ?", "words": ["dance", "you"]}, {"sent": "dance for us then .", "words": ["us", "then", "for", "dance"]}, {"sent": "Billy dance ?", "words": ["dance"]}, {"sent": "does it dance ?", "words": ["does", "it", "dance"]}, {"sent": "how did you learn how to dance like that please Gerald tell us how ?", "words": ["how", "to", "like", "dance", "you", "us", "did", "that"]}, {"sent": "now lets dance now lets dance everybody dance .", "words": ["dance"]}, {"sent": "just as Ariel and flounder approached the surface Ariel saw beautiful lights dance across the water .", "words": ["the", "water", "dance", "and"]}, {"sent": "but he does thoroughly enjoy it and he will dance .", "words": ["does", "dance", "and", "it", "he", "will", "but"]}, {"sent": "wanna come down and dance later ?", "words": ["dance", "down", "wanna", "and"]}, {"sent": "Im gonna teach you a dance Thomas .", "words": ["dance", "a", "you"]}, {"sent": "dance dance dance dance .", "words": ["dance"]}, {"sent": "are you gonna dance ?", "words": ["are", "dance", "you"]}, {"sent": "dance baby dance .", "words": ["dance"]}, {"sent": "were gonna dance to the Beatles .", "words": ["the", "to", "were", "dance"]}, {"sent": "now are you going to stand still while I take your shorts off ?", "words": ["off", "shorts", "your", "to", "stand", "you", "are", "take"]}, {"sent": "okay well take the shorts right mkay heres the shorts .", "words": ["the", "shorts", "take"]}, {"sent": "its outof your shorts .", "words": ["shorts", "your"]}, {"sent": "lets get some shorts that fit ya .", "words": ["shorts", "fit", "get", "some", "that"]}, {"sent": "shorts .", "words": ["shorts"]}, {"sent": "wanna pull up those shorts ?", "words": ["up", "shorts", "those", "pull", "wanna"]}, {"sent": "shall we put these green shorts back on ?", "words": ["these", "put", "shorts", "back", "we", "green", "on"]}, {"sent": "he has on shorts .", "words": ["on", "shorts", "he"]}, {"sent": "Daddys wearing his new shorts .", "words": ["his", "shorts", "new"]}, {"sent": "you wear shorts and trousers dont you ?", "words": ["dont", "shorts", "and", "you"]}, {"sent": "shorts on and then we go out .", "words": ["shorts", "we", "go", "and", "out", "on", "then"]}, {"sent": "and your shorts .", "words": ["shorts", "your", "and"]}, {"sent": "Thomas do the little boys have their tshirt tucked in their shorts or not ?", "words": ["the", "do", "shorts", "their", "little", "have", "in", "not"]}, {"sent": "that is his shorts or his underwear .", "words": ["his", "shorts", "that", "is"]}, {"sent": "lets put these shorts on you .", "words": ["these", "put", "shorts", "you", "on"]}, {"sent": "and then his shorts and his jumper .", "words": ["his", "then", "shorts", "and"]}, {"sent": "you got your shoes and socks and your shorts wet .", "words": ["wet", "shorts", "your", "and", "you"]}, {"sent": "no theyre Frasers shorts .", "words": ["shorts"]}, {"sent": "lets pull your shorts up .", "words": ["up", "shorts", "your", "pull"]}, {"sent": "youve got blue shorts on .", "words": ["on", "shorts", "blue"]}, {"sent": "before he knew it Corduroy was being tossed together with all the sheets and shirts and shorts and slacks .", "words": ["the", "all", "shorts", "was", "with", "and", "it", "he"]}, {"sent": "can you just come here while I pull your shorts up please ?", "words": ["can", "up", "shorts", "your", "pull", "here", "you"]}, {"sent": "under shorts okay .", "words": ["under", "shorts"]}, {"sent": "can I pull your shorts up ?", "words": ["can", "up", "shorts", "your", "pull"]}, {"sent": "those shorts look so nice on him .", "words": ["shorts", "those", "so", "nice", "him", "look", "on"]}, {"sent": "orange shorts hasnt he ?", "words": ["shorts", "orange", "he"]}, {"sent": "these shorts would be cute .", "words": ["these", "shorts", "cute", "be", "would"]}, {"sent": "oh theyre shorts arent they ?", "words": ["they", "shorts"]}, {"sent": "your shorts are falling down Thomas .", "words": ["down", "shorts", "your", "are"]}, {"sent": "Timmy has some shorts .", "words": ["some", "shorts"]}, {"sent": "I dont think well dress you in shorts today .", "words": ["think", "shorts", "you", "dont", "in", "dress"]}, {"sent": "xxx let me fix your shorts .", "words": ["me", "shorts", "your", "fix"]}, {"sent": "your shorts are ready but I think we just needta run an iron over the shirt .", "words": ["the", "think", "shorts", "your", "we", "shirt", "run", "an", "are", "over", "but"]}, {"sent": "and he was wearing shorts because its quite a nice day today isnt it ?", "words": ["shorts", "was", "nice", "and", "it", "he", "because", "a"]}, {"sent": "nice red shorts .", "words": ["red", "shorts", "nice"]}, {"sent": "Im ironing blue shorts now .", "words": ["shorts", "blue"]}, {"sent": "Peter want ta put your shorts on ?", "words": ["on", "put", "shorts", "your"]}, {"sent": "lets put your legs through your shorts legs through your shorts .", "words": ["put", "shorts", "your"]}, {"sent": "shorts or trousers .", "words": ["shorts"]}, {"sent": "thats your shorts .", "words": ["shorts", "your"]}, {"sent": "his shorts is dirty .", "words": ["his", "shorts", "dirty", "is"]}, {"sent": "Im going to pull these shorts up while Ive got the chance .", "words": ["up", "these", "the", "shorts", "to", "pull"]}, {"sent": "those are shorts .", "words": ["shorts", "are", "those"]}, {"sent": "you see blue like your shorts are blue .", "words": ["shorts", "your", "like", "blue", "you", "are", "see"]}, {"sent": "you know when you wear shorts they go up to your knees .", "words": ["up", "shorts", "your", "go", "to", "you", "they", "when"]}, {"sent": "I think its a bit too cold to put shorts on today isnt it ?", "words": ["think", "put", "shorts", "to", "it", "too", "on", "cold", "a"]}, {"sent": "I dont think you need shorts on today Thomas .", "words": ["think", "shorts", "need", "you", "dont", "on"]}, {"sent": "what uh you mean pull up your shorts ?", "words": ["up", "what", "shorts", "your", "pull", "you"]}, {"sent": "you hafta come here then if you want your shorts pulling up .", "words": ["up", "shorts", "your", "if", "you", "here", "then"]}, {"sent": "when you wear shorts xxx knees .", "words": ["shorts", "when", "you"]}, {"sent": "have you got shorts on ?", "words": ["have", "on", "shorts", "you"]}, {"sent": "and you were wearing shorts werent you ?", "words": ["shorts", "were", "you", "and"]}, {"sent": "and the blue shorts .", "words": ["the", "shorts", "blue", "and"]}, {"sent": "oh youve had shorts on all day .", "words": ["on", "all", "shorts"]}, {"sent": "here are your shorts .", "words": ["here", "shorts", "are", "your"]}, {"sent": "blue shorts and a lime green top .", "words": ["shorts", "and", "blue", "green", "a"]}, {"sent": "he wanna wear those shorts ?", "words": ["shorts", "he", "those", "wanna"]}, {"sent": "oh her shorts off .", "words": ["off", "shorts", "her"]}, {"sent": "thats Mommys shorts .", "words": ["shorts"]}, {"sent": "I think your shorts are really nice Thomas .", "words": ["think", "shorts", "your", "nice", "are"]}, {"sent": "because I when I play football in shorts when I fall over in grass wont get me muddy .", "words": ["shorts", "play", "fall", "get", "in", "grass", "over", "me", "when", "because"]}, {"sent": "lets just put your shorts on .", "words": ["on", "put", "shorts", "your"]}, {"sent": "Mama bought you some shorts boy .", "words": ["some", "shorts", "you"]}, {"sent": "and the second thing is shorts and a shirt .", "words": ["the", "shorts", "is", "shirt", "and", "a"]}, {"sent": "hes only got his shorts on hasnt he ?", "words": ["his", "on", "shorts", "he"]}, {"sent": "shorts and tee shirt .", "words": ["shirt", "shorts", "and"]}, {"sent": "I dont know what youre doing with those shorts today .", "words": ["what", "shorts", "those", "with", "dont"]}, {"sent": "oh would you wear shorts if there was snow on the ground ?", "words": ["the", "shorts", "if", "was", "snow", "you", "there", "would", "on"]}, {"sent": "that is a pair of shorts for the boy to wear to school .", "words": ["the", "shorts", "for", "is", "of", "to", "school", "that", "a"]}, {"sent": "youre spilling it all over your shorts .", "words": ["all", "shorts", "your", "it", "over"]}, {"sent": "oh like the cold air blowing up your shorts ?", "words": ["the", "up", "shorts", "your", "like", "cold"]}, {"sent": "theres a pair of shorts here of yours .", "words": ["of", "shorts", "here", "a"]}, {"sent": "xxx here you wanna put your shorts on ?", "words": ["put", "shorts", "your", "you", "here", "wanna", "on"]}, {"sent": "shorts .", "words": ["shorts"]}, {"sent": "how do you know his shorts are dirty ?", "words": ["do", "how", "shorts", "you", "are", "dirty", "his"]}, {"sent": "ah you see this its a pair of shorts ?", "words": ["shorts", "of", "you", "this", "see", "a"]}, {"sent": "are you going to pull your shorts up ?", "words": ["up", "shorts", "your", "to", "pull", "you", "are"]}, {"sent": "time for some shorts and a diaper change .", "words": ["shorts", "for", "diaper", "and", "some", "a"]}, {"sent": "white shorts ?", "words": ["shorts", "white"]}, {"sent": "shorts .", "words": ["shorts"]}, {"sent": "hes got yellow shorts on there hasnt he ?", "words": ["shorts", "there", "he", "on", "yellow"]}, {"sent": "do you just wanna slip your legs out of the shorts ?", "words": ["the", "do", "shorts", "your", "of", "you", "out", "wanna"]}, {"sent": "put the shorts on .", "words": ["the", "put", "shorts", "on"]}, {"sent": "polka dot shorts .", "words": ["shorts"]}, {"sent": "alright think Thomas can wear shorts tomorrow ?", "words": ["can", "think", "shorts"]}, {"sent": "and some red shorts for Noddy .", "words": ["red", "shorts", "for", "and", "some"]}, {"sent": "thats a pair of shorts .", "words": ["of", "shorts", "a"]}, {"sent": "those are your shorts .", "words": ["your", "shorts", "are", "those"]}, {"sent": "think it was a little bit cool for shorts Thomas as well .", "words": ["think", "for", "shorts", "was", "little", "it", "a"]}, {"sent": "hed only got a pair of shorts hadnt he ?", "words": ["of", "shorts", "he", "a"]}, {"sent": "shall we put these trousers on instead of the shorts .", "words": ["the", "these", "put", "shorts", "we", "of", "on"]}, {"sent": "shorts .", "words": ["shorts"]}, {"sent": "which is quite funny because when the men came the other day you were wearing green shorts .", "words": ["the", "were", "shorts", "when", "is", "other", "you", "green", "which", "because"]}, {"sent": "well why have you got your shorts on then ?", "words": ["shorts", "your", "you", "have", "on", "then", "why"]}, {"sent": "they are shorts .", "words": ["shorts", "they", "are"]}, {"sent": "here are your blue shorts .", "words": ["shorts", "your", "blue", "here", "are"]}, {"sent": "wheres Griselda the pig ?", "words": ["the", "pig"]}, {"sent": "theres a pig in the house too ?", "words": ["the", "pig", "in", "house", "too", "a"]}, {"sent": "the pig book is going to go over there .", "words": ["the", "is", "go", "to", "pig", "there", "book", "over"]}, {"sent": "done a pig now .", "words": ["pig", "a"]}, {"sent": "thats a pig .", "words": ["pig", "a"]}, {"sent": "this is the pig .", "words": ["the", "this", "pig", "is"]}, {"sent": "I dont think there is another pig .", "words": ["think", "is", "pig", "another", "dont", "there"]}, {"sent": "put that with the baby pig .", "words": ["the", "put", "with", "pig", "that"]}, {"sent": "and theyre handing the salt to the pig so that he can give Lizzy or Issy .", "words": ["the", "can", "to", "so", "pig", "and", "salt", "he", "give", "that"]}, {"sent": "Mummy pig .", "words": ["pig"]}, {"sent": "theres a pig on that book isnt there ?", "words": ["pig", "there", "on", "book", "that", "a"]}, {"sent": "the pig says youre a grump .", "words": ["the", "a", "pig"]}, {"sent": "poo like the pig .", "words": ["the", "like", "pig"]}, {"sent": "its a pig .", "words": ["pig", "a"]}, {"sent": "hes hitting the pig .", "words": ["the", "pig"]}, {"sent": "a pig .", "words": ["pig", "a"]}, {"sent": "Im a little baby pig .", "words": ["pig", "little", "a"]}, {"sent": "then the little pig makes some soup and its hot and the wolf comes down splashes into the hot soup .", "words": ["the", "wolf", "into", "pig", "little", "and", "hot", "soup", "some", "then", "down"]}, {"sent": "and heres a guinea pig and a baby .", "words": ["pig", "a", "and"]}, {"sent": "here two pig came out .", "words": ["here", "out", "pig"]}, {"sent": "thats it theres one pig .", "words": ["it", "pig"]}, {"sent": "the pig said the pig said Im not outof my head .", "words": ["the", "my", "head", "pig", "not"]}, {"sent": "you want a big pig ?", "words": ["pig", "big", "a", "you"]}, {"sent": "baby pig ?", "words": ["pig"]}, {"sent": "put pig back in the pocket .", "words": ["the", "put", "back", "pig", "in"]}, {"sent": "what does a pig say ?", "words": ["what", "does", "say", "pig", "a"]}, {"sent": "pig one and pig two .", "words": ["and", "pig"]}, {"sent": "now why should a pig bite a dog .", "words": ["pig", "bite", "dog", "why", "a"]}, {"sent": "uhhuh you put another pig in .", "words": ["put", "pig", "another", "you", "in"]}, {"sent": "youre squealing like a little pig arent you ?", "words": ["like", "you", "pig", "little", "a"]}, {"sent": "it is the pig indeed .", "words": ["the", "it", "pig", "is"]}, {"sent": "two identical pigs so theyre called pig twins .", "words": ["so", "pig"]}, {"sent": "pig nose .", "words": ["nose", "pig"]}, {"sent": "how bout the pig ?", "words": ["the", "how", "pig"]}, {"sent": "a pig ?", "words": ["pig", "a"]}, {"sent": "no slurping little pig .", "words": ["pig", "little"]}, {"sent": "where do you think the pig goes ?", "words": ["where", "do", "think", "the", "you", "pig"]}, {"sent": "you put pig pennys in it ?", "words": ["put", "pig", "you", "in", "it"]}, {"sent": "that pig is eating that pig is drinking that pig is singing and dancing that pig is smiling that pig is talking .", "words": ["and", "that", "is", "pig"]}, {"sent": "pig .", "words": ["pig"]}, {"sent": "pig .", "words": ["pig"]}, {"sent": "you looking for the pig ?", "words": ["the", "for", "pig", "you"]}, {"sent": "hes called scruffy pig .", "words": ["pig"]}, {"sent": "Squeeker was the youngest pig but he was the cleverest and he decided to build his house of bricks and a fine strong house it was .", "words": ["the", "fine", "of", "was", "to", "pig", "and", "it", "he", "build", "house", "his", "but", "a"]}, {"sent": "what does a pig say Joel ?", "words": ["what", "does", "say", "pig", "a"]}, {"sent": "and a pig on the sailboat .", "words": ["the", "pig", "and", "on", "a"]}, {"sent": "a baby pig that .", "words": ["pig", "that", "a"]}, {"sent": "youve got a pig in your hand havent you ?", "words": ["your", "hand", "pig", "you", "in", "a"]}, {"sent": "twirl with a pig if you know how .", "words": ["how", "if", "with", "pig", "you", "a"]}, {"sent": "thats another pig .", "words": ["another", "pig"]}, {"sent": "pig .", "words": ["pig"]}, {"sent": "she can have some of your pig ?", "words": ["can", "your", "of", "pig", "have", "some", "she"]}, {"sent": "its a pig .", "words": ["pig", "a"]}, {"sent": "can you find another little bit of the pig ?", "words": ["can", "the", "of", "little", "another", "you", "pig", "find"]}, {"sent": "a little pig .", "words": ["pig", "little", "a"]}, {"sent": "and theres a pig .", "words": ["pig", "a", "and"]}, {"sent": "about the pig .", "words": ["the", "about", "pig"]}, {"sent": "well draw a pig in a minute okay ?", "words": ["pig", "in", "draw", "a"]}, {"sent": "pig and big .", "words": ["big", "and", "pig"]}, {"sent": "is he a pig ?", "words": ["pig", "he", "a", "is"]}, {"sent": "and he said little pig little pig let me come in the first the little pig answered .", "words": ["the", "first", "pig", "little", "and", "in", "he", "me"]}, {"sent": "and soon the third little pig had built himself a solid little house of brick .", "words": ["the", "of", "pig", "little", "and", "house", "a"]}, {"sent": "theres a pig .", "words": ["pig", "a"]}, {"sent": "theres the farmer and his pig .", "words": ["the", "his", "pig", "and"]}, {"sent": "pig .", "words": ["pig"]}, {"sent": "or the pig .", "words": ["the", "pig"]}, {"sent": "another pig .", "words": ["another", "pig"]}, {"sent": "but the third little pig built a real strong house out of what ?", "words": ["the", "what", "of", "pig", "little", "house", "out", "but", "a"]}, {"sent": "a crocodile a cow a butterfly or a snake or a pig .", "words": ["butterfly", "pig", "cow", "a"]}, {"sent": "and to live in our own houses said the third little pig .", "words": ["the", "to", "our", "and", "little", "in", "pig"]}, {"sent": "the pig has fallen to bits now .", "words": ["the", "to", "pig"]}, {"sent": "slow feet quick feet up feet down feet here come clown feet small feet big feet here come pig feet in the house and on the street how many different feet you meet .", "words": ["up", "the", "how", "slow", "pig", "here", "and", "in", "you", "house", "street", "on", "down", "big"]}, {"sent": "what does the pig do ?", "words": ["the", "what", "do", "does", "pig"]}, {"sent": "hey I see a pig .", "words": ["see", "pig", "a"]}, {"sent": "what letter do you think that pig begins with ?", "words": ["what", "do", "think", "with", "you", "pig", "that"]}, {"sent": "here they are Ma pig and pa pig .", "words": ["and", "here", "they", "pig", "are"]}, {"sent": "thats a pig noise John .", "words": ["pig", "a"]}, {"sent": "banged on the door little pig .", "words": ["the", "pig", "little", "on", "door"]}, {"sent": "ah the pig .", "words": ["the", "pig"]}, {"sent": "pig .", "words": ["pig"]}, {"sent": "oh a pig .", "words": ["pig", "a"]}, {"sent": "do not pick on the pig you two .", "words": ["the", "do", "pick", "pig", "you", "on", "not"]}, {"sent": "what noise does a pig make ?", "words": ["what", "does", "pig", "make", "a"]}, {"sent": "he knocked the pig off and ran over the goat .", "words": ["the", "off", "and", "pig", "he", "over"]}, {"sent": "pig gets in there .", "words": ["there", "in", "pig"]}, {"sent": "and whats the sheep gonna say to the pig ?", "words": ["the", "to", "say", "and", "pig", "sheep"]}, {"sent": "one pig ?", "words": ["pig"]}, {"sent": "is it a muddy pig ?", "words": ["pig", "it", "a", "is"]}, {"sent": "smallest pig ?", "words": ["pig"]}, {"sent": "theres the pig .", "words": ["the", "pig"]}, {"sent": "thats a baby lamb having some milk from Mummy pig .", "words": ["pig", "milk", "some", "a", "lamb"]}, {"sent": "how about the little pig who went to market ?", "words": ["the", "how", "to", "pig", "little", "about", "who"]}, {"sent": "pig .", "words": ["pig"]}, {"sent": "and the pig .", "words": ["the", "pig", "and"]}, {"sent": "this is a pig .", "words": ["this", "pig", "a", "is"]}, {"sent": "the little pig squealed .", "words": ["the", "pig", "little"]}, {"sent": "pig .", "words": ["pig"]}, {"sent": "what kind of pig is it ?", "words": ["what", "is", "of", "pig", "it"]}, {"sent": "this is a pig .", "words": ["this", "pig", "a", "is"]}, {"sent": "oh no thats soap dont eat soap .", "words": ["eat", "dont", "soap"]}, {"sent": "made up of old soap powder boxes .", "words": ["up", "of", "old", "soap"]}, {"sent": "well you needta put soap and water on it not a Bandaid .", "words": ["put", "soap", "and", "you", "it", "water", "on", "not", "a"]}, {"sent": "Dav where do you think I found all of these things shampoo and soap and a toothbrush and toothpaste and a towel ?", "words": ["where", "do", "think", "all", "these", "toothbrush", "of", "soap", "you", "and", "towel", "a"]}, {"sent": "I put in the soap and the bleach and stuff I let it soak for hours .", "words": ["the", "put", "for", "soap", "and", "in", "it"]}, {"sent": "okay heres the soap .", "words": ["the", "soap"]}, {"sent": "how do you make soap ?", "words": ["do", "how", "soap", "you", "make"]}, {"sent": "well how many times you washed yourself already with the soap ?", "words": ["the", "how", "yourself", "with", "soap", "you"]}, {"sent": "do you smoke tobacco or soap bubbles ?", "words": ["bubbles", "do", "soap", "you"]}, {"sent": "Daddy washed your hair with your hand soap .", "words": ["your", "hand", "with", "soap", "hair"]}, {"sent": "oh soap ?", "words": ["soap"]}, {"sent": "how did that get soap on it ?", "words": ["how", "soap", "get", "it", "did", "on", "that"]}, {"sent": "oh where has the soap gone ?", "words": ["where", "the", "soap"]}, {"sent": "what do you do thats with thats with soap and washcloth .", "words": ["what", "do", "with", "soap", "you", "and"]}, {"sent": "no a soap .", "words": ["soap", "a"]}, {"sent": "is that where is that where where soap that is the soap container isnt it ?", "words": ["where", "the", "is", "soap", "it", "that"]}, {"sent": "and wash your hands with the cookie monster soap ?", "words": ["the", "cookie", "your", "with", "soap", "and", "wash"]}, {"sent": "soap is in there ?", "words": ["there", "in", "soap", "is"]}, {"sent": "soap stars or something .", "words": ["soap"]}, {"sent": "wash the soap off .", "words": ["the", "wash", "off", "soap"]}, {"sent": "well do soap on there .", "words": ["on", "do", "soap", "there"]}, {"sent": "the whale wash starts thumping and the soap suds start pumping .", "words": ["the", "wash", "soap", "and"]}, {"sent": "Barbie soap ?", "words": ["soap"]}, {"sent": "well we still hafta put a little soap on you .", "words": ["put", "we", "soap", "little", "you", "on", "a"]}, {"sent": "we need water without soap in it now .", "words": ["need", "we", "soap", "in", "it", "water"]}, {"sent": "now dont forget that youve used some of these letters in these two words bath and soap that youve made .", "words": ["these", "of", "soap", "and", "dont", "in", "some", "that"]}, {"sent": "heres a pink soap .", "words": ["soap", "a"]}, {"sent": "are you washing the dollys tummy with some soap ?", "words": ["the", "with", "soap", "you", "are", "tummy", "some"]}, {"sent": "thats a little bit of soap to clean it out right ?", "words": ["clean", "of", "to", "soap", "little", "it", "out", "a"]}, {"sent": "okay lets put soap on your hands .", "words": ["on", "put", "your", "soap"]}, {"sent": "wheres the soap ?", "words": ["the", "soap"]}, {"sent": "the soap is all gone .", "words": ["the", "all", "all gone", "is", "soap"]}, {"sent": "soap .", "words": ["soap"]}, {"sent": "oh is there soap in your ears ?", "words": ["your", "is", "soap", "there", "in"]}, {"sent": "you need soap ?", "words": ["need", "soap", "you"]}, {"sent": "when you were a baby you didnt soap and you dont use use a baby too a soap too much now do you ?", "words": ["do", "soap", "you", "and", "dont", "much", "too", "when", "were", "a"]}, {"sent": "I think you have soap suds on your head .", "words": ["think", "your", "head", "soap", "you", "have", "on"]}, {"sent": "soap .", "words": ["soap"]}, {"sent": "Mummy soap .", "words": ["soap"]}, {"sent": "now you hold it over your eyes and Baaee will put some soap in your hair .", "words": ["put", "your", "hold", "soap", "you", "and", "in", "it", "will", "some", "hair", "over"]}, {"sent": "soap .", "words": ["soap"]}, {"sent": "well this is a way of erasing things with soap and water .", "words": ["is", "of", "with", "soap", "and", "this", "water", "a"]}, {"sent": "soap ?", "words": ["soap"]}, {"sent": "bye soap .", "words": ["soap"]}, {"sent": "oh have you got soap on your hands ?", "words": ["your", "soap", "you", "have", "on"]}, {"sent": "have you got the soap again ?", "words": ["have", "the", "soap", "you"]}, {"sent": "we dont need soap now .", "words": ["dont", "need", "soap", "we"]}, {"sent": "do you wanna eat soap ?", "words": ["do", "soap", "you", "eat", "wanna"]}, {"sent": "Ma can you think of a word with soap ?", "words": ["can", "think", "of", "with", "soap", "you", "a"]}, {"sent": "its its the dish that Mummy keeps the liquid soap in and the sponge .", "words": ["the", "soap", "and", "dish", "in", "that"]}, {"sent": "is it when theyre making soap is it the episode where theyre making soap and she says when its all mixed together in a big long brick its called a billet ?", "words": ["the", "where", "all", "is", "soap", "and", "long", "in", "it", "big", "when", "she", "a"]}, {"sent": "for making the mess in the bathroom with all the soap suds ?", "words": ["the", "all", "for", "with", "soap", "in", "bathroom"]}, {"sent": "now youre busy in the bathroom blowing bubbles with the soap .", "words": ["the", "with", "soap", "in", "bathroom", "bubbles"]}, {"sent": "but you dont get soap in your eyes ?", "words": ["your", "soap", "you", "get", "dont", "in", "but"]}, {"sent": "thats your baby soap .", "words": ["your", "soap"]}, {"sent": "today today we washed our things with soap .", "words": ["our", "with", "soap", "we"]}, {"sent": "soap ?", "words": ["soap"]}, {"sent": "so she hunted for that runaway soap but instead of the soap she found .", "words": ["the", "for", "of", "so", "soap", "that", "she", "but"]}, {"sent": "they have a frog in the bathroom holding soap in his mouth a night light of a frog and all different sorts of posters of animals .", "words": ["the", "all", "light", "of", "mouth", "soap", "frog", "have", "they", "in", "and", "his", "bathroom", "a"]}, {"sent": "he gets soap on the towel and he blows bubbles .", "words": ["the", "soap", "and", "towel", "he", "on", "bubbles"]}, {"sent": "Elmo wants his soap that floats .", "words": ["his", "that", "soap"]}, {"sent": "these are all soap bubbles in this picture .", "words": ["all", "these", "picture", "soap", "this", "in", "are", "bubbles"]}, {"sent": "put soap on it .", "words": ["on", "put", "it", "soap"]}, {"sent": "just going to put some soap on her feet look .", "words": ["put", "to", "soap", "look", "on", "some", "her"]}, {"sent": "can I have soap please and maybe a toy ?", "words": ["can", "soap", "and", "have", "toy", "a"]}, {"sent": "you kept all the soap outof your eyes .", "words": ["the", "all", "your", "soap", "you"]}, {"sent": "wheres soap ?", "words": ["soap"]}, {"sent": "have you got the soap ?", "words": ["have", "the", "soap", "you"]}, {"sent": "wow soap ?", "words": ["soap"]}, {"sent": "soap .", "words": ["soap"]}, {"sent": "soap ?", "words": ["soap"]}, {"sent": "some soap .", "words": ["some", "soap"]}, {"sent": "and this is special bath soap for the bath for you .", "words": ["the", "for", "is", "soap", "and", "you", "this"]}, {"sent": "smell the powdered soap .", "words": ["the", "soap"]}, {"sent": "you dont drink things like soap .", "words": ["drink", "like", "soap", "you", "dont"]}, {"sent": "the next ones soap soap .", "words": ["the", "soap"]}, {"sent": "soap .", "words": ["soap"]}, {"sent": "soap powder .", "words": ["soap"]}, {"sent": "lemme wipe that soap .", "words": ["wipe", "that", "soap"]}, {"sent": "is it soap ?", "words": ["it", "soap", "is"]}, {"sent": "what sort of soap do you have in your hair ?", "words": ["what", "do", "your", "of", "soap", "you", "have", "in", "hair"]}, {"sent": "oh you dont need any soap come on wheres Patsy lets go get Patsy come on lets go .", "words": ["need", "go", "any", "soap", "you", "get", "dont", "on"]}, {"sent": "because he put soap on his eyes .", "words": ["put", "soap", "his", "he", "on", "because"]}, {"sent": "Im making a little soap up and .", "words": ["up", "soap", "and", "little", "a"]}, {"sent": "thats wheres the soap .", "words": ["the", "soap"]}, {"sent": "you got soap in your eyes ?", "words": ["in", "your", "soap", "you"]}, {"sent": "oh theyre all inside a big soap bubble ?", "words": ["all", "inside", "soap", "big", "a"]}, {"sent": "itll come off with a bit of soap wont it ?", "words": ["off", "of", "with", "soap", "it", "a"]}, {"sent": "soap soap soap .", "words": ["soap"]}, {"sent": "do you have soap on your hands too ?", "words": ["do", "your", "soap", "you", "have", "too", "on"]}, {"sent": "bye soap .", "words": ["soap"]}, {"sent": "it could be soap .", "words": ["be", "could", "it", "soap"]}, {"sent": "soap suds .", "words": ["soap"]}, {"sent": "use some soap .", "words": ["some", "soap"]}, {"sent": "a lot of soap .", "words": ["of", "a lot", "soap", "a"]}, {"sent": "splish splosh splash .", "words": ["splash"]}, {"sent": "go splash splash splash .", "words": ["splash", "go"]}, {"sent": "her tail goes splash .", "words": ["splash", "her"]}, {"sent": "he went splash didnt he ?", "words": ["splash", "he"]}, {"sent": "lets see who can make the biggest splash said Vik .", "words": ["can", "the", "splash", "who", "see", "make"]}, {"sent": "dont splash .", "words": ["dont", "splash"]}, {"sent": "following their mommy duck splish splash quack .", "words": ["duck", "splash", "their"]}, {"sent": "splish splash quack .", "words": ["splash"]}, {"sent": "a big splash ?", "words": ["splash", "big", "a"]}, {"sent": "we had a big splash this morning didnt we ?", "words": ["we", "this", "splash", "big", "a"]}, {"sent": "it was an even bigger splash wasnt it ?", "words": ["was", "an", "it", "splash"]}, {"sent": "splash in puddles .", "words": ["in", "splash"]}, {"sent": "Elmo likes to splash glubglub .", "words": ["to", "splash"]}, {"sent": "splash splash splash splash splash splash splash .", "words": ["splash"]}, {"sent": "*OP1: can you say splash ?", "words": ["can", "splash", "say", "you"]}, {"sent": "splash splash splash splash splash splash splash .", "words": ["splash"]}, {"sent": "buz buz pop pop pop ho ho clip clip pop pop cock a doodle do grum grum choo choo choo boom boom splash splash tick tock tick tock ssss knock knock knock a slurp and a whisper and a fish kiss too .", "words": ["do", "pop", "and", "knock", "splash", "fish", "too", "kiss", "a"]}, {"sent": "splash splash splash .", "words": ["splash"]}, {"sent": "big splash quackquack .", "words": ["splash", "big"]}, {"sent": "thats when the sly fox opened up the sack and into the boiling water crashed the stones with a big splash .", "words": ["the", "up", "into", "with", "and", "big", "splash", "water", "when", "a"]}, {"sent": "oh splash .", "words": ["splash"]}, {"sent": "Kim loves to jump in puddles splash .", "words": ["to", "in", "jump", "splash"]}, {"sent": "did you splash him with cold water ?", "words": ["with", "you", "him", "splash", "did", "water", "cold"]}, {"sent": "oh cat likes to splash in the bath .", "words": ["the", "to", "cat", "in", "splash"]}, {"sent": "splish splash splish splash .", "words": ["splash"]}, {"sent": "they splash one another in the water ?", "words": ["the", "another", "they", "in", "splash", "water"]}, {"sent": "try not to splash outof the tub .", "words": ["the", "to", "splash", "try", "not"]}, {"sent": "did it splash out then ?", "words": ["it", "splash", "did", "out", "then"]}, {"sent": "big splash .", "words": ["splash", "big"]}, {"sent": "splash splash splash .", "words": ["splash"]}, {"sent": "big splash ?", "words": ["splash", "big"]}, {"sent": "is that baby big splash ?", "words": ["big", "that", "splash", "is"]}, {"sent": "see the doggies in the water going splash splash splash ?", "words": ["the", "in", "splash", "water", "see"]}, {"sent": "down the chimney came the wolf and splash into the kettle he went .", "words": ["the", "wolf", "into", "and", "splash", "he", "down"]}, {"sent": "splash splash splash .", "words": ["splash"]}, {"sent": "splash splash splash .", "words": ["splash"]}, {"sent": "splash .", "words": ["splash"]}, {"sent": "yes big splash .", "words": ["splash", "big"]}, {"sent": "splash .", "words": ["splash"]}, {"sent": "and shes going splash splash splash in the puddles .", "words": ["the", "in", "splash", "and"]}, {"sent": "splash splash splash .", "words": ["splash"]}, {"sent": "gonna splash in the potty ?", "words": ["the", "in", "splash", "potty"]}, {"sent": "following their mommy duck splish splash quack .", "words": ["duck", "splash", "their"]}, {"sent": "you can make splish splish splash splosh in the puddles cant you ?", "words": ["can", "the", "you", "in", "splash", "make"]}, {"sent": "well it isnt really a big splash because you call the bath a big splash .", "words": ["the", "you", "it", "splash", "big", "because", "a"]}, {"sent": "splash splash .", "words": ["splash"]}, {"sent": "splish splash splish splash splish splash .", "words": ["splash"]}, {"sent": "what a big splash .", "words": ["what", "splash", "big", "a"]}, {"sent": "*OP1: the little girl or boy whatever is going splash splash splash .", "words": ["the", "splash", "is", "little"]}, {"sent": "see you big splash .", "words": ["see", "splash", "big", "you"]}, {"sent": "splash splash splash .", "words": ["splash"]}, {"sent": "splish splash xxx .", "words": ["splash"]}, {"sent": "splash splash quack .", "words": ["splash"]}, {"sent": "dont splash too much .", "words": ["dont", "splash", "much", "too"]}, {"sent": "splash .", "words": ["splash"]}, {"sent": "and then theres not quite as much for you to splash around in .", "words": ["for", "to", "and", "you", "in", "splash", "around", "then", "much", "not"]}, {"sent": "like to get in the water and splash ?", "words": ["the", "to", "like", "and", "get", "in", "splash", "water"]}, {"sent": "splash .", "words": ["splash"]}, {"sent": "splish splash quack .", "words": ["splash"]}, {"sent": "splash him .", "words": ["him", "splash"]}, {"sent": "mama goose honks as she leads the way into the pond its wet to splash and play .", "words": ["the", "wet", "into", "to", "play", "and", "splash", "goose", "she"]}, {"sent": "splash splash .", "words": ["splash"]}, {"sent": "splash .", "words": ["splash"]}, {"sent": "theres a big splash isnt there ?", "words": ["there", "splash", "big", "a"]}, {"sent": "you mustnt splash too much otherwise youll get water everywhere wont you ?", "words": ["water", "you", "get", "splash", "too", "much"]}, {"sent": "and splash in the water .", "words": ["the", "and", "in", "splash", "water"]}, {"sent": "he can hear splish splash in the bath .", "words": ["can", "the", "hear", "in", "splash", "he"]}, {"sent": "splash in the potty .", "words": ["the", "in", "splash", "potty"]}, {"sent": "splash splash splash .", "words": ["splash"]}, {"sent": "splish splash splish splash splish splash .", "words": ["splash"]}, {"sent": "splash splash splash .", "words": ["splash"]}, {"sent": "you splash ?", "words": ["splash", "you"]}, {"sent": "splash .", "words": ["splash"]}, {"sent": "big splash ?", "words": ["splash", "big"]}, {"sent": "you splash and splash ?", "words": ["splash", "and", "you"]}, {"sent": "splash .", "words": ["splash"]}, {"sent": "splash .", "words": ["splash"]}, {"sent": "as soon as they arrive they go for a splash in the sea .", "words": ["the", "for", "go", "they", "in", "splash", "a"]}, {"sent": "splash right ?", "words": ["splash"]}, {"sent": "splash splash .", "words": ["splash"]}, {"sent": "splash water everywhere ?", "words": ["splash", "water"]}, {"sent": "and do you splash too ?", "words": ["do", "you", "and", "splash", "too"]}, {"sent": "would you swim and splash ?", "words": ["swim", "and", "you", "splash", "would"]}, {"sent": "yeah duckies go splish splash quack quack quack quack quack .", "words": ["splash", "go"]}, {"sent": "splash splash .", "words": ["splash"]}, {"sent": "splash .", "words": ["splash"]}, {"sent": "splash .", "words": ["splash"]}, {"sent": "splash splash splash .", "words": ["splash"]}, {"sent": "following their Mommy duck splish splash quack .", "words": ["duck", "splash", "their"]}, {"sent": "splish splash quack .", "words": ["splash"]}, {"sent": "splash .", "words": ["splash"]}, {"sent": "Im not gonna let you get in and splash because it is late .", "words": ["is", "and", "get", "you", "in", "it", "splash", "not", "because"]}, {"sent": "lets turn that page crash splash oh no I had an accident .", "words": ["splash", "that", "an"]}, {"sent": "its all wet .", "words": ["all", "wet"]}, {"sent": "what are you going to do about dollys wet socks ?", "words": ["what", "do", "wet", "to", "you", "are", "about"]}, {"sent": "it isnt that wet .", "words": ["wet", "it", "that"]}, {"sent": "you cant leave her wet or shell get cold .", "words": ["wet", "you", "get", "her", "cold"]}, {"sent": "its wet .", "words": ["wet"]}, {"sent": "it is wet .", "words": ["wet", "it", "is"]}, {"sent": "well have some explaining to do if Bens toy gets wet .", "words": ["do", "wet", "if", "to", "have", "toy", "some"]}, {"sent": "but its wet .", "words": ["wet", "but"]}, {"sent": "do you not mind getting your hair wet in the pool ?", "words": ["the", "do", "wet", "your", "pool", "you", "in", "hair", "not"]}, {"sent": "I dont want wet dribble and slobber all over it .", "words": ["all", "wet", "and", "dont", "it", "over"]}, {"sent": "is the ball wet ?", "words": ["the", "wet", "ball", "is"]}, {"sent": "wet .", "words": ["wet"]}, {"sent": "without hesitating he climbed inside the bag which was filled with pieces of wet laundry .", "words": ["the", "wet", "of", "was", "inside", "with", "he", "which"]}, {"sent": "probably too wet .", "words": ["wet", "too"]}, {"sent": "Matty I dont wanna get wet .", "words": ["get", "dont", "wanna", "wet"]}, {"sent": "its wet .", "words": ["wet"]}, {"sent": "uh why did they all get wet ?", "words": ["all", "wet", "get", "they", "did", "why"]}, {"sent": "is the lady getting wet aswell ?", "words": ["the", "wet", "is"]}, {"sent": "oh this is wet .", "words": ["wet", "this", "is"]}, {"sent": "the other one got wet ?", "words": ["the", "wet", "other"]}, {"sent": "but her parents loved him they kissed his wet pink nose and admired his small black eyes .", "words": ["wet", "and", "black", "they", "him", "his", "her", "but", "nose"]}, {"sent": "well hafta put a little sign round it saying wet floor .", "words": ["wet", "put", "little", "it", "a"]}, {"sent": "I did wet it .", "words": ["wet", "it", "did"]}, {"sent": "does it feel wet ?", "words": ["does", "it", "wet"]}, {"sent": "why did they all get wet ?", "words": ["all", "wet", "get", "they", "did", "why"]}, {"sent": "wet is it ?", "words": ["wet", "it", "is"]}, {"sent": "shes soaking wet .", "words": ["wet"]}, {"sent": "xxx get them wet anyway .", "words": ["get", "wet", "them"]}, {"sent": "are your feet getting wet ?", "words": ["wet", "are", "your"]}, {"sent": "I think its a very wet slobbery kiss .", "words": ["wet", "think", "kiss", "a"]}, {"sent": "and tell her why theyre wet .", "words": ["wet", "her", "why", "and"]}, {"sent": "theyll get ever so wet wont they ?", "words": ["get", "so", "they", "wet"]}, {"sent": "but poor Mummy got really wet .", "words": ["wet", "but", "poor"]}, {"sent": "oh he got all wet .", "words": ["all", "he", "wet"]}, {"sent": "I wont get wet .", "words": ["get", "wet"]}, {"sent": "I think I may have gotten that napkin too wet .", "words": ["wet", "think", "have", "too", "that", "napkin"]}, {"sent": "it sprays the car all wet .", "words": ["the", "all", "wet", "car", "it"]}, {"sent": "that wave has just crashed on him and got him wet .", "words": ["wet", "and", "him", "on", "that"]}, {"sent": "and youd be really really poorly if you go and get wet .", "words": ["wet", "if", "go", "be", "you", "and", "get"]}, {"sent": "are your pants wet ?", "words": ["wet", "are", "your", "pants"]}, {"sent": "I dont wanna get it wet .", "words": ["wet", "get", "dont", "it", "wanna"]}, {"sent": "wet it wet it wet it .", "words": ["wet", "it"]}, {"sent": "wet .", "words": ["wet"]}, {"sent": "wet foot .", "words": ["wet", "foot"]}, {"sent": "is that wet sand ?", "words": ["wet", "that", "is"]}, {"sent": "and the grass feels wet doesnt it ?", "words": ["the", "wet", "and", "it", "grass"]}, {"sent": "do you like getting wet ?", "words": ["wet", "do", "like", "you"]}, {"sent": "give me a wet one .", "words": ["wet", "me", "give", "a"]}, {"sent": "it was too wet to play .", "words": ["wet", "was", "to", "play", "it", "too"]}, {"sent": "whyre your socks wet ?", "words": ["wet", "your"]}, {"sent": "youre getting pretty wet arent you ?", "words": ["wet", "pretty", "you"]}, {"sent": "we got wet didnt we ?", "words": ["wet", "we"]}, {"sent": "wet dry .", "words": ["wet", "dry"]}, {"sent": "is he all wet now that rabbit in the bucket ?", "words": ["the", "wet", "all", "is", "bucket", "in", "he", "that"]}, {"sent": "mines wet .", "words": ["wet"]}, {"sent": "thats wet isnt it ?", "words": ["wet", "it"]}, {"sent": "do you want me to wet it ?", "words": ["do", "wet", "to", "you", "it", "me"]}, {"sent": "put your er thingy on so you dont get wet .", "words": ["wet", "put", "your", "so", "you", "get", "dont", "on"]}, {"sent": "come on thats too wet there .", "words": ["on", "wet", "there", "too"]}, {"sent": "that diaper is so wet ?", "words": ["wet", "is", "diaper", "so", "that"]}, {"sent": "because sometimes you know when things get wet they get really slippery .", "words": ["wet", "you", "get", "they", "when", "because"]}, {"sent": "they will give you some fun on this wet wet wet day .", "words": ["wet", "you", "they", "on", "this", "will", "some", "give"]}, {"sent": "you can see how wet they are .", "words": ["can", "wet", "how", "you", "they", "are", "see"]}, {"sent": "that other Mummy mustve wet it with her watering can .", "words": ["can", "wet", "with", "her", "other", "it", "that"]}, {"sent": "is it still wet ?", "words": ["wet", "it", "is"]}, {"sent": "Jenny turn the other way or else youll get all wet .", "words": ["the", "all", "wet", "other", "get"]}, {"sent": "is it a bit wet a bit wet now ?", "words": ["wet", "it", "a", "is"]}, {"sent": "its all wet .", "words": ["all", "wet"]}, {"sent": "do you think that the hat is getting wet ?", "words": ["the", "do", "think", "wet", "is", "you", "hat", "that"]}, {"sent": "shall Mummy just get it a little bit wet on the back ?", "words": ["the", "wet", "back", "little", "get", "it", "on", "a"]}, {"sent": "oh hes a bit wet isnt he ?", "words": ["wet", "he", "a"]}, {"sent": "but the crocodile wants to get wet doesnt he ?", "words": ["the", "wet", "to", "get", "he", "but"]}, {"sent": "how did the towel get wet ?", "words": ["the", "wet", "how", "get", "towel", "did"]}, {"sent": "Fraser wet .", "words": ["wet"]}, {"sent": "wet .", "words": ["wet"]}, {"sent": "no Matty I dont wanna get wet .", "words": ["get", "dont", "wanna", "wet"]}, {"sent": "I dont think theyre wet .", "words": ["dont", "think", "wet"]}, {"sent": "theyll get wet on the boat wont they ?", "words": ["the", "wet", "get", "they", "on", "boat"]}, {"sent": "two dogs get wet .", "words": ["get", "wet"]}, {"sent": "on thats a wet blow .", "words": ["on", "wet", "blow", "a"]}, {"sent": "air moves over your wet skin and cools you .", "words": ["wet", "your", "you", "and", "over"]}, {"sent": "is Donald Duck gonna get all wet ?", "words": ["get", "all", "wet", "is"]}, {"sent": "are they wet ?", "words": ["they", "are", "wet"]}, {"sent": "because shes still wet look because she leaks I think .", "words": ["wet", "think", "look", "she", "because"]}, {"sent": "I think you might need a clean vest because you feel a little bit wet .", "words": ["clean", "wet", "think", "need", "you", "little", "because", "a"]}, {"sent": "they are all gonna get wet .", "words": ["all", "wet", "get", "they", "are"]}, {"sent": "yes it is wet .", "words": ["wet", "it", "is"]}, {"sent": "a little bit wet Mommys gonna give the plant this water okay ?", "words": ["the", "wet", "plant", "little", "this", "water", "give", "a"]}, {"sent": "your pants are all wet .", "words": ["all", "wet", "your", "are", "pants"]}, {"sent": "its good and wet huh .", "words": ["wet", "good", "and"]}, {"sent": "come over here where its not wet look .", "words": ["where", "wet", "here", "look", "over", "not"]}, {"sent": "sand at the seaside looks wet with water from the sea .", "words": ["the", "wet", "with", "at", "water"]}, {"sent": "its wet .", "words": ["wet"]}, {"sent": "is that wet ?", "words": ["wet", "that", "is"]}, {"sent": "and as the glue got wet it made them both as the glue hit the paper it made the paper wet and the red dye came out didnt it ?", "words": ["the", "wet", "red", "paper", "them", "and", "it", "out", "glue", "hit"]}, {"sent": "new wet hat ?", "words": ["hat", "wet", "new"]}, {"sent": "you want it wet ?", "words": ["wet", "it", "you"]}, {"sent": "I dont want you doing those horrible wet blows .", "words": ["wet", "dont", "those", "you"]}, {"sent": "yeah did you get all wet ?", "words": ["all", "wet", "you", "get", "did"]}, {"sent": "he get the cat all wet ?", "words": ["the", "all", "wet", "get", "cat", "he"]}, {"sent": "or it can be the towel .", "words": ["can", "the", "be", "towel", "it"]}, {"sent": "xxx get a towel .", "words": ["get", "towel", "a"]}, {"sent": "gonna getcha a paper towel .", "words": ["paper", "towel", "a"]}, {"sent": "I wanna switch your towel over here .", "words": ["your", "here", "towel", "wanna", "over"]}, {"sent": "Daddys wet towel is on the floor .", "words": ["the", "wet", "is", "towel", "on"]}, {"sent": "and our towel we will hang our towel up right here .", "words": ["up", "we", "our", "and", "here", "towel", "will"]}, {"sent": "Mommys gonna go get a paper towel .", "words": ["go", "paper", "get", "towel", "a"]}, {"sent": "he doesnt get a towel over him .", "words": ["get", "towel", "him", "he", "over", "a"]}, {"sent": "bye towel .", "words": ["towel"]}, {"sent": "got the towel ?", "words": ["the", "towel"]}, {"sent": "Ill get you a paper towel ?", "words": ["paper", "you", "get", "towel", "a"]}, {"sent": "lets dry your hands on the towel .", "words": ["the", "your", "dry", "towel", "on"]}, {"sent": "you want Daddy get a towel ?", "words": ["get", "towel", "a", "you"]}, {"sent": "Awww thats not a thats not a towel thats my top .", "words": ["towel", "not", "my", "a"]}, {"sent": "get the rest of it and put the paper towel in the wastebasket please .", "words": ["the", "put", "of", "paper", "and", "get", "towel", "it", "in"]}, {"sent": "youre picking up the towel and going to dry up the water on the floor again ?", "words": ["up", "the", "to", "dry", "and", "towel", "water", "on"]}, {"sent": "your hands are dry with the towel .", "words": ["the", "your", "dry", "with", "towel", "are"]}, {"sent": "my towel .", "words": ["my", "towel"]}, {"sent": "youre a little dish towel thief .", "words": ["dish", "towel", "little", "a"]}, {"sent": "the idea is to see how many colors you can get on one paper towel .", "words": ["the", "can", "how", "is", "to", "paper", "you", "get", "towel", "on", "see"]}, {"sent": "towel .", "words": ["towel"]}, {"sent": "its because youve got the towel over your head .", "words": ["the", "your", "head", "towel", "over", "because"]}, {"sent": "wheres the towel ?", "words": ["the", "towel"]}, {"sent": "who is under the towel ?", "words": ["the", "is", "under", "towel", "who"]}, {"sent": "why have you got a towel round you .", "words": ["you", "have", "towel", "why", "a"]}, {"sent": "a towel .", "words": ["towel", "a"]}, {"sent": "yeah a towel .", "words": ["towel", "a"]}, {"sent": "if you have a look under the towel theres a couple under the towel .", "words": ["the", "if", "you", "have", "under", "towel", "look", "a"]}, {"sent": "wrapping Thomas in a big pink towel .", "words": ["in", "towel", "big", "a"]}, {"sent": "okay Lucille lets go in the towel .", "words": ["the", "in", "towel", "go"]}, {"sent": "with a towel .", "words": ["towel", "with", "a"]}, {"sent": "lets play with the towel now .", "words": ["the", "towel", "play", "with"]}, {"sent": "thats another pink towel .", "words": ["another", "towel"]}, {"sent": "let me get towel .", "words": ["get", "me", "towel"]}, {"sent": "you can sit on a towel and have juice .", "words": ["can", "you", "and", "have", "towel", "on", "sit", "juice", "a"]}, {"sent": "paper towel .", "words": ["paper", "towel"]}, {"sent": "what are you doing with that towel ?", "words": ["what", "with", "you", "towel", "are", "that"]}, {"sent": "oh pull the towel away from him .", "words": ["the", "pull", "towel", "away", "him"]}, {"sent": "oh you get a towel and a bench at the preschool ?", "words": ["the", "and", "get", "you", "towel", "at", "bench", "a"]}, {"sent": "hand towel .", "words": ["towel", "hand"]}, {"sent": "a towel .", "words": ["towel", "a"]}, {"sent": "wipe your hand on the towel please .", "words": ["the", "your", "hand", "towel", "on", "wipe"]}, {"sent": "you got your towel ?", "words": ["towel", "your", "you"]}, {"sent": "its funny because they had a towel with Po on and one with Tinky .", "words": ["with", "and", "they", "towel", "on", "because", "a"]}, {"sent": "you get your towel and dry xxx .", "words": ["your", "dry", "and", "get", "you", "towel"]}, {"sent": "put her on her towel look .", "words": ["put", "towel", "look", "on", "her"]}, {"sent": "did we already take the towel off ?", "words": ["the", "off", "we", "towel", "did", "take"]}, {"sent": "theres the towel .", "words": ["the", "towel"]}, {"sent": "a towel .", "words": ["towel", "a"]}, {"sent": "okay I gave you the towel to dry them off with .", "words": ["the", "off", "to", "dry", "them", "with", "you", "towel"]}, {"sent": "you told me what he did with the towel .", "words": ["the", "what", "with", "you", "towel", "he", "did", "me"]}, {"sent": "are we all done with the towel ?", "words": ["the", "all", "we", "with", "towel", "are"]}, {"sent": "oh heres a towel .", "words": ["towel", "a"]}, {"sent": "can you find me a towel ?", "words": ["can", "you", "towel", "find", "me", "a"]}, {"sent": "thats the towel yeah the babys drying on the towel .", "words": ["the", "towel", "on"]}, {"sent": "do you wanna stand on this towel and then you can dry your feet ?", "words": ["can", "do", "your", "stand", "dry", "you", "and", "this", "towel", "wanna", "on", "then"]}, {"sent": "okay let me go get a paper towel for you .", "words": ["for", "go", "paper", "you", "get", "towel", "me", "a"]}, {"sent": "this is a towel .", "words": ["this", "towel", "a", "is"]}, {"sent": "thats Vik and he has a towel around his shoulders .", "words": ["and", "towel", "he", "around", "his", "a"]}, {"sent": "a towel .", "words": ["towel", "a"]}, {"sent": "towel .", "words": ["towel"]}, {"sent": "put the towel away and dump the water .", "words": ["the", "put", "and", "towel", "away", "water", "dump"]}, {"sent": "thats why I have the towel .", "words": ["have", "the", "towel", "why"]}, {"sent": "thats the towel .", "words": ["the", "towel"]}, {"sent": "we dont have a towel .", "words": ["we", "have", "dont", "towel", "a"]}, {"sent": "let Mommy get a paper towel Sweets .", "words": ["get", "paper", "towel", "a"]}, {"sent": "but youve got to put your towel on first .", "words": ["put", "your", "to", "first", "towel", "on", "but"]}, {"sent": "can you find a towel ?", "words": ["can", "you", "towel", "find", "a"]}, {"sent": "its a towel to cover up the other rolls sothat they can rise in peace and comfort .", "words": ["up", "the", "can", "to", "other", "and", "they", "towel", "in", "cover", "a"]}, {"sent": "how did the towel get wet ?", "words": ["the", "wet", "how", "get", "towel", "did"]}, {"sent": "the towel ?", "words": ["the", "towel"]}, {"sent": "the towel has dropped .", "words": ["the", "towel"]}, {"sent": "the towel the washcloth the soap .", "words": ["the", "towel", "soap"]}, {"sent": "like the box better than the towel huh ?", "words": ["the", "better", "box", "like", "towel"]}, {"sent": "go and get your towel .", "words": ["your", "go", "and", "get", "towel"]}, {"sent": "oops we need a towel in here .", "words": ["need", "we", "here", "in", "towel", "a"]}, {"sent": "Mummy has put the towel on the on the ba on the bicycle hasnt she ?", "words": ["the", "put", "bicycle", "towel", "on", "she"]}, {"sent": "towel yeah .", "words": ["towel"]}, {"sent": "towel .", "words": ["towel"]}, {"sent": "how did the towel get wet ?", "words": ["the", "wet", "how", "get", "towel", "did"]}, {"sent": "get the towel .", "words": ["get", "the", "towel"]}, {"sent": "and wipe them on the towel .", "words": ["the", "them", "and", "towel", "on", "wipe"]}, {"sent": "oh he is taking my towel away .", "words": ["my", "is", "towel", "away", "he"]}, {"sent": "he want my towel .", "words": ["my", "towel", "he"]}, {"sent": "with that same dish towel .", "words": ["with", "dish", "towel", "same", "that"]}, {"sent": "you clean your face with your towel .", "words": ["clean", "your", "face", "with", "you", "towel"]}, {"sent": "you wrap yourself all up in the towel .", "words": ["up", "all", "the", "yourself", "you", "in", "towel"]}, {"sent": "put on the paper towel .", "words": ["the", "put", "paper", "towel", "on"]}, {"sent": "will this do as a towel ?", "words": ["do", "this", "towel", "will", "a"]}, {"sent": "when I was playing the towel .", "words": ["the", "was", "towel", "when"]}, {"sent": "get a towel now .", "words": ["get", "towel", "a"]}, {"sent": "towel .", "words": ["towel"]}, {"sent": "uh how did the towel get wet ?", "words": ["the", "wet", "how", "get", "towel", "did"]}, {"sent": "want Mommy have paper towel ?", "words": ["have", "paper", "towel"]}, {"sent": "swim xxx .", "words": ["swim"]}, {"sent": "little yellow ducklings swim across the pond and back .", "words": ["the", "back", "swim", "and", "little", "yellow"]}, {"sent": "little yellow ducklings swim across the pond and back .", "words": ["the", "back", "swim", "and", "little", "yellow"]}, {"sent": "swim swim .", "words": ["swim"]}, {"sent": "Sugars goin to the swim club .", "words": ["the", "to", "swim"]}, {"sent": "did you love to swim ?", "words": ["to", "swim", "you", "did", "love"]}, {"sent": "no I dont wanna swim .", "words": ["dont", "swim", "wanna"]}, {"sent": "some dogs like to swim .", "words": ["to", "some", "swim", "like"]}, {"sent": "swim swim swim swim .", "words": ["swim"]}, {"sent": "come on Paddington called Judy lets go for a swim .", "words": ["for", "go", "swim", "on", "a"]}, {"sent": "he needed to go for a swim .", "words": ["for", "go", "to", "swim", "he", "a"]}, {"sent": "do you like to swim ?", "words": ["do", "to", "swim", "like", "you"]}, {"sent": "youre learning xxx how to swim now .", "words": ["to", "how", "swim"]}, {"sent": "where are you gonna swim ?", "words": ["where", "are", "swim", "you"]}, {"sent": "can you swim ?", "words": ["can", "swim", "you"]}, {"sent": "I swim under water and have slippery shiny skin .", "words": ["swim", "and", "have", "under", "water"]}, {"sent": "is it swim ?", "words": ["it", "swim", "is"]}, {"sent": "youre gonna swim there ?", "words": ["there", "swim"]}, {"sent": "yeah does she know how to swim ?", "words": ["does", "how", "to", "swim", "she"]}, {"sent": "swim across the pond and back .", "words": ["the", "swim", "back", "and"]}, {"sent": "do you like to swim in the water ?", "words": ["the", "do", "to", "swim", "like", "you", "in", "water"]}, {"sent": "they swim and they stick their trunks up and all that sticks up are a little bit of their heads and their trunks .", "words": ["up", "all", "stick", "their", "of", "swim", "and", "little", "they", "are", "that", "a"]}, {"sent": "can he go in the pond and swim in the water ?", "words": ["can", "the", "go", "swim", "and", "in", "he", "water"]}, {"sent": "can he swim ?", "words": ["can", "he", "swim"]}, {"sent": "did you swim ?", "words": ["swim", "did", "you"]}, {"sent": "I know but I can swim more than she can swim because because Im bigger and I do it right .", "words": ["can", "do", "swim", "because", "and", "more", "it", "she", "but"]}, {"sent": "does a jelly fish swim like this Luna ?", "words": ["jelly", "does", "swim", "like", "this", "fish", "a"]}, {"sent": "xxx little ducklings swim across the pond and back .", "words": ["the", "back", "swim", "and", "little"]}, {"sent": "on the water you swim ?", "words": ["the", "swim", "you", "water", "on"]}, {"sent": "we did swim at the bay didnt we ?", "words": ["the", "we", "swim", "did", "at"]}, {"sent": "little yellow ducks swim across the pond and back .", "words": ["the", "back", "swim", "and", "little", "yellow"]}, {"sent": "do you like to swim Grant ?", "words": ["do", "to", "swim", "like", "you"]}, {"sent": "pigs dont know how to swim .", "words": ["to", "dont", "how", "swim"]}, {"sent": "they they cant swim in the tea .", "words": ["the", "they", "in", "swim"]}, {"sent": "have a nice swim .", "words": ["have", "nice", "swim", "a"]}, {"sent": "Dwww likes to swim with Jwww .", "words": ["to", "swim", "with"]}, {"sent": "swim .", "words": ["swim"]}, {"sent": "he likes to swim .", "words": ["to", "he", "swim"]}, {"sent": "swim .", "words": ["swim"]}, {"sent": "swim swim swim .", "words": ["swim"]}, {"sent": "swim swim swim swim swim swim swim swim swim .", "words": ["swim"]}, {"sent": "theyll help you swim willnt they ?", "words": ["they", "help", "swim", "you"]}, {"sent": "and could you swim on your own with one of those lifejackets on ?", "words": ["your", "those", "of", "swim", "with", "you", "and", "on", "could"]}, {"sent": "you wanna swim ?", "words": ["swim", "wanna", "you"]}, {"sent": "you might not wanna swim .", "words": ["swim", "not", "wanna", "you"]}, {"sent": "did you go swim yesterday ?", "words": ["go", "swim", "did", "you"]}, {"sent": "do you want a swim .", "words": ["do", "swim", "a", "you"]}, {"sent": "he can go for a swim that fish cant he ?", "words": ["can", "for", "go", "swim", "he", "fish", "that", "a"]}, {"sent": "swim .", "words": ["swim"]}, {"sent": "try and swim back to me .", "words": ["back", "to", "swim", "and", "me", "try"]}, {"sent": "they can swim around .", "words": ["can", "they", "swim", "around"]}, {"sent": "a mother duck swim .", "words": ["duck", "swim", "a"]}, {"sent": "swim swim swim .", "words": ["swim"]}, {"sent": "swim faster Flounder she cried glancing back at him .", "words": ["back", "swim", "him", "at", "she"]}, {"sent": "little yellow ducklings swim across the pond and back .", "words": ["the", "back", "swim", "and", "little", "yellow"]}, {"sent": "daddy come watch me swim .", "words": ["me", "watch", "swim"]}, {"sent": "have a swim .", "words": ["have", "swim", "a"]}, {"sent": "swim pufferfish swim .", "words": ["swim"]}, {"sent": "swim ?", "words": ["swim"]}, {"sent": "see how you swim .", "words": ["see", "how", "swim", "you"]}, {"sent": "have you do you like to swim ?", "words": ["do", "to", "swim", "like", "you", "have"]}, {"sent": "Henry likes to swim and play with all his underwater friends .", "words": ["all", "to", "play", "swim", "with", "and", "his"]}, {"sent": "can he swim ?", "words": ["can", "he", "swim"]}, {"sent": "well swim in the .", "words": ["the", "in", "swim"]}, {"sent": "and they swim in search of food .", "words": ["of", "swim", "and", "they", "in", "food"]}, {"sent": "swim .", "words": ["swim"]}, {"sent": "they looked in LewisburgSquare but there was no water to swim in .", "words": ["was", "to", "swim", "they", "in", "there", "water", "but"]}, {"sent": "swim .", "words": ["swim"]}, {"sent": "do you wanna come for a swim .", "words": ["do", "for", "swim", "you", "wanna", "a"]}, {"sent": "yes they do swim dont they .", "words": ["dont", "they", "do", "swim"]}, {"sent": "little yellow ducklings swim across the pond and back .", "words": ["the", "back", "swim", "and", "little", "yellow"]}, {"sent": "thats what turtles swim around with .", "words": ["what", "swim", "with", "around"]}, {"sent": "you know I dont think shes in the swim class this time around .", "words": ["the", "think", "swim", "you", "dont", "in", "this", "around"]}, {"sent": "you want the baby to swim .", "words": ["the", "to", "swim", "you"]}, {"sent": "so thats what they swim with .", "words": ["what", "so", "swim", "with", "they"]}, {"sent": "she xxx the proper attire to swim .", "words": ["the", "to", "swim", "she"]}, {"sent": "can you swim ?", "words": ["can", "swim", "you"]}, {"sent": "does he swim ?", "words": ["does", "he", "swim"]}, {"sent": "are you going to swim ?", "words": ["to", "are", "swim", "you"]}, {"sent": "you swim ?", "words": ["swim", "you"]}, {"sent": "thats how you swim huh .", "words": ["how", "swim", "you"]}, {"sent": "are we gonna swim or we gonna drown .", "words": ["are", "swim", "we"]}, {"sent": "the Gentoo penguin can swim under water at twenty two miles per hour .", "words": ["the", "can", "penguin", "swim", "under", "at", "water"]}, {"sent": "now youve had a good swim in the river are you going to come and play ?", "words": ["the", "to", "play", "swim", "you", "and", "in", "are", "good", "a"]}, {"sent": "swim .", "words": ["swim"]}, {"sent": "do you like to swim ?", "words": ["do", "to", "swim", "like", "you"]}, {"sent": "little yellow ducklings swim across the pond and back .", "words": ["the", "back", "swim", "and", "little", "yellow"]}, {"sent": "swim swim swim .", "words": ["swim"]}, {"sent": "where did you learn how to swim ?", "words": ["where", "how", "to", "swim", "you", "did"]}, {"sent": "tell Grammy that youre gonna come and swim in her lake .", "words": ["swim", "her", "and", "in", "that"]}, {"sent": "little yellow ducklings swim across the pond and back following their mama .", "words": ["the", "their", "back", "swim", "and", "little", "yellow"]}, {"sent": "which one do you actually swim with ?", "words": ["do", "swim", "with", "you", "which"]}, {"sent": "little yellow ducklings swim across the pond and back .", "words": ["the", "back", "swim", "and", "little", "yellow"]}, {"sent": "and what what do you do on swim team ?", "words": ["what", "do", "swim", "you", "and", "on"]}, {"sent": "well you cant swim without your armbands on can you ?", "words": ["can", "your", "swim", "you", "on"]}, {"sent": "where do you put your shoes while you swim ?", "words": ["where", "do", "put", "your", "swim", "you"]}, {"sent": "to swim with your arms ?", "words": ["your", "to", "swim", "with"]}, {"sent": "I swim .", "words": ["swim"]}, {"sent": "swim swim swim .", "words": ["swim"]}, {"sent": "I just needta pick it up .", "words": ["pick", "it", "up"]}, {"sent": "pick it up sweetheart .", "words": ["pick", "it", "up"]}, {"sent": "pick it up .", "words": ["pick", "it", "up"]}, {"sent": "maybe well pick up the book so it doesnt get broken .", "words": ["the", "up", "pick", "so", "broken", "get", "it", "book"]}, {"sent": "okay well well hafta pick the car up rrrrrr .", "words": ["pick", "the", "up", "car"]}, {"sent": "oh pick them up .", "words": ["pick", "up", "them"]}, {"sent": "did NinNin come and pick you up and give you a cuddle and a kiss .", "words": ["up", "pick", "you", "and", "did", "give", "kiss", "a"]}, {"sent": "Ill try and pick it up .", "words": ["up", "pick", "and", "it", "try"]}, {"sent": "can you pick it up from the floor please .", "words": ["can", "up", "the", "pick", "you", "it"]}, {"sent": "can you pick it up ?", "words": ["can", "up", "pick", "you", "it"]}, {"sent": "you pick a card and then you hafta make the face on the card .", "words": ["the", "face", "pick", "and", "you", "on", "then", "make", "a"]}, {"sent": "pick it up with your fingers its easier .", "words": ["up", "your", "pick", "with", "it"]}, {"sent": "pick it up .", "words": ["pick", "it", "up"]}, {"sent": "tomorrow when we go to the coop you can pick out a pomegranate if you wanna .", "words": ["the", "can", "if", "go", "we", "to", "pick", "you", "out", "wanna", "when", "a"]}, {"sent": "can you pick another book ?", "words": ["can", "pick", "you", "another", "book"]}, {"sent": "you pick that up ?", "words": ["pick", "up", "that", "you"]}, {"sent": "shall we pick them up ?", "words": ["pick", "up", "them", "we"]}, {"sent": "pick out the purple ones .", "words": ["pick", "the", "out"]}, {"sent": "go pick out the book you want .", "words": ["the", "go", "pick", "you", "out", "book"]}, {"sent": "and pick up my kids .", "words": ["pick", "my", "up", "and"]}, {"sent": "pick another book .", "words": ["pick", "book", "another"]}, {"sent": "why dont ya pick one .", "words": ["pick", "dont", "why"]}, {"sent": "you pick it up .", "words": ["pick", "it", "up", "you"]}, {"sent": "do little chicks pick up sticks or build with bricks ?", "words": ["up", "do", "pick", "with", "little", "build"]}, {"sent": "youknow you could pick .", "words": ["pick", "could", "you"]}, {"sent": "Ill pick it up .", "words": ["pick", "it", "up"]}, {"sent": "right can you come and pick some of these things up please .", "words": ["can", "these", "up", "pick", "of", "and", "you", "some"]}, {"sent": "and then we went to pick Jwww up and we took Nwww with us .", "words": ["up", "we", "pick", "to", "with", "and", "us", "then"]}, {"sent": "pick it up sweetheart .", "words": ["pick", "it", "up"]}, {"sent": "yes it is very easy to pick up .", "words": ["up", "is", "pick", "to", "it"]}, {"sent": "pick another one .", "words": ["pick", "another"]}, {"sent": "um yeah I guess the zebra was one thing he did pick out .", "words": ["the", "zebra", "pick", "was", "he", "did", "out"]}, {"sent": "well thats why you hafta pick your box up .", "words": ["up", "box", "your", "pick", "you", "why"]}, {"sent": "lets pick them all up .", "words": ["pick", "all", "up", "them"]}, {"sent": "you gotta pick this up .", "words": ["pick", "this", "up", "you"]}, {"sent": "well if its still there when we go out and nobody else has picked it up well pick it up wont we ?", "words": ["up", "if", "go", "we", "pick", "and", "there", "it", "out", "when"]}, {"sent": "pick them up Bro .", "words": ["pick", "up", "them"]}, {"sent": "you did pick up snow did you eat snow ?", "words": ["up", "pick", "snow", "you", "eat", "did"]}, {"sent": "you hafta pick it up .", "words": ["pick", "it", "up", "you"]}, {"sent": "go ahead pick it up .", "words": ["pick", "it", "up", "go"]}, {"sent": "why theres my button he cried and he tried to pick it up .", "words": ["up", "my", "pick", "to", "and", "it", "he", "button", "why"]}, {"sent": "where that old pick up move ?", "words": ["where", "up", "old", "pick", "that"]}, {"sent": "are you gonna pick it up ?", "words": ["up", "pick", "you", "it", "are"]}, {"sent": "as I said while youre down there will you please pick up the pencils .", "words": ["up", "the", "pick", "you", "there", "will", "down"]}, {"sent": "well you pick it up then .", "words": ["up", "pick", "you", "it", "then"]}, {"sent": "and then you walk along with your tray and you pick up a teaspoon and you pay the lady at the till and you sit down and eat it .", "words": ["up", "the", "your", "pick", "sit", "with", "you", "and", "eat", "at", "it", "tray", "then", "down", "walk", "a"]}, {"sent": "did you pick it out just especially for dada ?", "words": ["for", "pick", "you", "it", "did", "out"]}, {"sent": "pick a crayon that matches your hair .", "words": ["your", "pick", "crayon", "hair", "that", "a"]}, {"sent": "pick blue .", "words": ["pick", "blue"]}, {"sent": "pick out your story .", "words": ["pick", "story", "your", "out"]}, {"sent": "pick it up .", "words": ["pick", "it", "up"]}, {"sent": "pick it up and put it on the table please .", "words": ["up", "the", "put", "table", "pick", "and", "it", "on"]}, {"sent": "here pick those pieces up .", "words": ["here", "up", "those", "pick"]}, {"sent": "okay let me go find xxx whoops Honey gotta pick that box up .", "words": ["up", "box", "go", "pick", "find", "me", "that"]}, {"sent": "well pick them up off the floor .", "words": ["the", "up", "off", "pick", "them"]}, {"sent": "were gonna pick some people up .", "words": ["pick", "some", "up", "were"]}, {"sent": "pick up your balls .", "words": ["pick", "up", "your"]}, {"sent": "if you hadta pick a book which book did she prefer to look at with you ?", "words": ["if", "pick", "to", "with", "you", "at", "did", "look", "book", "which", "she", "a"]}, {"sent": "lets pick the pieces all up and then put them on .", "words": ["the", "up", "all", "put", "pick", "them", "and", "on", "then"]}, {"sent": "and pick one up from the pile .", "words": ["pick", "the", "up", "and"]}, {"sent": "pick the car you want .", "words": ["pick", "the", "car", "you"]}, {"sent": "pick em up please .", "words": ["pick", "up"]}, {"sent": "pick up stuff easily right now .", "words": ["pick", "up"]}, {"sent": "do they pick up lot of kids to school ?", "words": ["up", "do", "pick", "of", "to", "school", "they"]}, {"sent": "room if you wanna and you can talk and we can still pick it up .", "words": ["can", "up", "if", "we", "pick", "and", "you", "room", "it", "wanna", "talk"]}, {"sent": "and put it next to pilchards and when people try and pick the two up .", "words": ["the", "up", "put", "try", "pick", "to", "and", "it", "when"]}, {"sent": "okay now Im gonna pick you up .", "words": ["pick", "up", "you"]}, {"sent": "but I still want you to pick the bits up .", "words": ["the", "up", "pick", "to", "you", "but"]}, {"sent": "pick out pick out the blue one .", "words": ["pick", "the", "out", "blue"]}, {"sent": "pick up that other stuff first .", "words": ["up", "pick", "first", "other", "that"]}, {"sent": "pick it up please .", "words": ["pick", "it", "up"]}, {"sent": "dont pick him up .", "words": ["pick", "dont", "him", "up"]}, {"sent": "and ya push a button and she puts her hands up for you to pick her up .", "words": ["up", "for", "push", "pick", "to", "her", "and", "you", "button", "she", "a"]}, {"sent": "pick them up .", "words": ["pick", "up", "them"]}, {"sent": "I was gonna pick her up Friday up to see Robin and Richard and our new little Danny ya know ?", "words": ["up", "pick", "was", "to", "and", "our", "little", "see", "new", "her"]}, {"sent": "well ask Daddy to pick some up on his way home from work .", "words": ["up", "work", "pick", "to", "his", "on", "some", "home"]}, {"sent": "Ill pick up the blue .", "words": ["pick", "the", "up", "blue"]}, {"sent": "well pick up the one you can see and then have a look for the other one .", "words": ["the", "can", "up", "for", "pick", "you", "and", "have", "other", "look", "see", "then", "a"]}, {"sent": "as long as you pick it up I dont mind .", "words": ["up", "pick", "long", "you", "dont", "it"]}, {"sent": "and if the telephone rings you mustnt pick it up .", "words": ["the", "up", "if", "pick", "you", "and", "it", "telephone"]}, {"sent": "what do ya pick ?", "words": ["pick", "what", "do"]}, {"sent": "you gonna pick up the crayons ?", "words": ["pick", "the", "up", "you"]}, {"sent": "pick it up .", "words": ["pick", "it", "up"]}, {"sent": "pick up the pencil please .", "words": ["pick", "the", "up", "pencil"]}, {"sent": "pick them up .", "words": ["pick", "up", "them"]}, {"sent": "alright well Ill finish reading the book when you pick your Playdoh up .", "words": ["the", "up", "your", "pick", "finish", "you", "book", "when"]}, {"sent": "pick up your sock ?", "words": ["pick", "up", "your", "sock"]}, {"sent": "you hafta pick the puzzle up first .", "words": ["the", "up", "pick", "first", "you", "puzzle"]}, {"sent": "you can lose all of them but you cant pick em all up .", "words": ["can", "all", "up", "of", "pick", "them", "you", "but"]}, {"sent": "pick them up everyone .", "words": ["pick", "up", "them"]}, {"sent": "you pick it up .", "words": ["pick", "it", "up", "you"]}, {"sent": "pick up the money .", "words": ["pick", "the", "up", "money"]}, {"sent": "and the mailman will pick it up .", "words": ["the", "up", "pick", "and", "it", "will"]}, {"sent": "here Ill bring it to you and you can pick what you want .", "words": ["can", "what", "pick", "to", "bring", "you", "here", "and", "it"]}, {"sent": "did Erin come to the airport to pick us up ?", "words": ["the", "up", "pick", "to", "us", "did"]}, {"sent": "pick up your apple .", "words": ["pick", "apple", "up", "your"]}, {"sent": "xxx please pick that up .", "words": ["pick", "up", "that"]}, {"sent": "should we go pick some out of your tea set ?", "words": ["your", "go", "pick", "we", "of", "out", "some"]}, {"sent": "and then when I came to pick you up he was getting into a car with his Mummy .", "words": ["up", "into", "pick", "to", "was", "car", "with", "and", "you", "he", "his", "then", "when", "a"]}, {"sent": "ya gonna pick one out for me ?", "words": ["pick", "me", "for", "out"]}, {"sent": "high foot low foot .", "words": ["foot", "high"]}, {"sent": "I thought you were going to see her foot .", "words": ["to", "you", "foot", "see", "her", "were"]}, {"sent": "what colors TinkeyWinkeys foot ?", "words": ["what", "foot"]}, {"sent": "a foot ?", "words": ["foot", "a"]}, {"sent": "the foot book ?", "words": ["the", "book", "foot"]}, {"sent": "I need that foot .", "words": ["foot", "that", "need"]}, {"sent": "Im glad you found that funny that you just swept it at my foot .", "words": ["my", "you", "foot", "it", "at", "that"]}, {"sent": "dont take her foot off .", "words": ["off", "dont", "foot", "take", "her"]}, {"sent": "foot whose foot is this ?", "words": ["this", "foot", "is"]}, {"sent": "look shes got one red foot and one pink foot .", "words": ["red", "foot", "and", "look"]}, {"sent": "I thought it was Ronalds foot foot birthday .", "words": ["was", "foot", "it"]}, {"sent": "I bugged your foot .", "words": ["foot", "your"]}, {"sent": "wrong foot .", "words": ["foot"]}, {"sent": "two foot .", "words": ["foot"]}, {"sent": "put it on her foot .", "words": ["put", "foot", "it", "on", "her"]}, {"sent": "just get back over there foot .", "words": ["back", "get", "there", "foot", "over"]}, {"sent": "shall I move my foot ?", "words": ["my", "foot"]}, {"sent": "are you done with that foot ?", "words": ["with", "you", "foot", "are", "that"]}, {"sent": "yyy the foot does .", "words": ["the", "does", "foot"]}, {"sent": "theres only one thing that goes in there Aran and that is your foot isnt it ?", "words": ["your", "is", "and", "there", "in", "foot", "it", "that"]}, {"sent": "what about the other foot ?", "words": ["the", "what", "other", "foot", "about"]}, {"sent": "shes gonna hurt your foot you better move .", "words": ["better", "your", "hurt", "you", "foot"]}, {"sent": "will you sit further back xxx I dont wanna go through your foot .", "words": ["your", "back", "go", "you", "dont", "foot", "wanna", "will", "sit"]}, {"sent": "thats your foot .", "words": ["foot", "your"]}, {"sent": "when that jar fell down a piece of glass hit me in the foot .", "words": ["the", "when", "jar", "of", "hit", "in", "foot", "glass", "me", "down", "that", "a"]}, {"sent": "and on the foot we have and this is quite a difficult one .", "words": ["the", "we", "is", "and", "have", "this", "foot", "on", "a"]}, {"sent": "do you see your foot ?", "words": ["do", "your", "you", "foot", "see"]}, {"sent": "left foot right foot feet feet feet .", "words": ["foot"]}, {"sent": "by Dimitras foot theres a telephone .", "words": ["telephone", "foot", "a", "by"]}, {"sent": "this foot off the ground .", "words": ["off", "this", "foot", "the"]}, {"sent": "a cats probably easier to deal with than a boy with a poorly foot .", "words": ["to", "foot", "with", "a"]}, {"sent": "wet foot dry foot front feet back feet .", "words": ["wet", "foot", "dry", "back"]}, {"sent": "where are you foot ?", "words": ["where", "foot", "are", "you"]}, {"sent": "whats that hes got on his foot ?", "words": ["on", "his", "foot", "that"]}, {"sent": "uh can you put it on her foot ?", "words": ["can", "put", "you", "foot", "it", "on", "her"]}, {"sent": "Im back to get your foot .", "words": ["your", "back", "to", "get", "foot"]}, {"sent": "theres an f word Justin foot .", "words": ["an", "foot"]}, {"sent": "xxx Courtneys foot .", "words": ["foot"]}, {"sent": "did you bump your foot ?", "words": ["bump", "your", "you", "foot", "did"]}, {"sent": "well hafta untie it to get it on your foot wont we ?", "words": ["your", "we", "to", "get", "foot", "it", "on"]}, {"sent": "stop playing with your foot ?", "words": ["your", "foot", "stop", "with"]}, {"sent": "will you take your foot out of my tin please ?", "words": ["my", "your", "of", "you", "foot", "out", "will", "take"]}, {"sent": "John its down here by my foot .", "words": ["my", "by", "here", "foot", "down"]}, {"sent": "oh I think you had better be careful putting your foot in it .", "words": ["better", "think", "your", "be", "you", "foot", "in", "it", "careful"]}, {"sent": "look at your foot .", "words": ["foot", "at", "your", "look"]}, {"sent": "and you say new words like foot ?", "words": ["say", "like", "you", "and", "foot", "new"]}, {"sent": "and wheres his other foot ?", "words": ["his", "foot", "other", "and"]}, {"sent": "that was Mummys foot .", "words": ["was", "foot", "that"]}, {"sent": "and if if it spills all over your foot then it will hurt .", "words": ["all", "your", "if", "hurt", "and", "foot", "it", "will", "then", "over"]}, {"sent": "can you put your foot out the door .", "words": ["can", "the", "put", "your", "you", "foot", "out", "door"]}, {"sent": "okay Ill get it if you hop on one show me if you can hop on one foot .", "words": ["can", "if", "show", "you", "get", "foot", "it", "on", "me"]}, {"sent": "where did your foot go ?", "words": ["where", "your", "go", "foot", "did"]}, {"sent": "this is your right foot .", "words": ["this", "foot", "your", "is"]}, {"sent": "a foot .", "words": ["foot", "a"]}, {"sent": "put it on her foot .", "words": ["put", "foot", "it", "on", "her"]}, {"sent": "wheres your foot ?", "words": ["foot", "your"]}, {"sent": "now put her foot in .", "words": ["foot", "put", "her", "in"]}, {"sent": "its only because Ive just moved my foot but I can still .", "words": ["can", "my", "because", "foot", "but"]}, {"sent": "do you want your big pink foot ?", "words": ["do", "your", "you", "foot", "big"]}, {"sent": "left foot .", "words": ["foot"]}, {"sent": "because ow I just hit my foot on the rockingchair too here .", "words": ["the", "my", "here", "foot", "too", "on", "hit", "because"]}, {"sent": "did Jeannine hurt your foot ?", "words": ["hurt", "foot", "your", "did"]}, {"sent": "you have a dirty foot .", "words": ["you", "have", "foot", "dirty", "a"]}, {"sent": "left foot .", "words": ["foot"]}, {"sent": "mind your foot .", "words": ["foot", "your"]}, {"sent": "now just put her foot in .", "words": ["foot", "put", "her", "in"]}, {"sent": "one foot is out .", "words": ["foot", "out", "is"]}, {"sent": "put one foot up .", "words": ["up", "put", "foot"]}, {"sent": "on that foot .", "words": ["on", "foot", "that"]}, {"sent": "left foot left foot right foot right .", "words": ["foot"]}, {"sent": "he was able to find a friends foot which was shorter .", "words": ["to", "was", "foot", "he", "find", "which", "a"]}, {"sent": "whats that foot ?", "words": ["foot", "that"]}, {"sent": "see now push your foot in .", "words": ["your", "push", "foot", "in", "see"]}, {"sent": "next time well start with the hand sothat they maybe if they know the hand theyll know the foot .", "words": ["the", "if", "hand", "with", "they", "foot"]}, {"sent": "if you dont hes going to tickle your foot .", "words": ["your", "if", "to", "tickle", "you", "dont", "foot"]}, {"sent": "yeah and thats his foot .", "words": ["his", "foot", "and"]}, {"sent": "bottom of his foot ?", "words": ["of", "his", "foot", "bottom"]}, {"sent": "a foot Im getting a little tired of things that make you fall .", "words": ["of", "fall", "tired", "little", "you", "foot", "make", "that", "a"]}, {"sent": "you cant try the other foot because the socks still on huh .", "words": ["the", "other", "you", "foot", "on", "try", "because"]}, {"sent": "your left foot .", "words": ["foot", "your"]}, {"sent": "dont put your foot on the video .", "words": ["the", "put", "your", "dont", "foot", "on"]}, {"sent": "theres your other foot .", "words": ["foot", "your", "other"]}, {"sent": "is your foot getting caught in your pants ?", "words": ["your", "is", "in", "foot", "pants"]}, {"sent": "wheres teddys foot ?", "words": ["foot"]}, {"sent": "dry foot .", "words": ["dry", "foot"]}, {"sent": "wheres the bees foot ?", "words": ["the", "foot"]}, {"sent": "here gimme your foot .", "words": ["here", "foot", "your"]}, {"sent": "with a foot on it doesnt it ?", "words": ["with", "foot", "it", "on", "a"]}, {"sent": "who did do you know someone else who got a thorn in their foot ?", "words": ["do", "their", "you", "in", "foot", "did", "who", "a"]}, {"sent": "you cant walk around with nothing on your foot .", "words": ["your", "with", "you", "foot", "around", "on", "walk"]}, {"sent": "left foot .", "words": ["foot"]}, {"sent": "low foot high foot .", "words": ["foot", "high"]}, {"sent": "right foot .", "words": ["foot"]}, {"sent": "what goes on your foot besides socks ?", "words": ["on", "what", "foot", "your"]}, {"sent": "right foot .", "words": ["foot"]}, {"sent": "youre an elephant .", "words": ["an", "elephant"]}, {"sent": "where did you see a big elephant ?", "words": ["where", "you", "elephant", "did", "see", "big", "a"]}, {"sent": "nobody stayed to see the elephant .", "words": ["see", "to", "elephant", "the"]}, {"sent": "its the one with the elephant on .", "words": ["the", "on", "with", "elephant"]}, {"sent": "this was all on the elephant ?", "words": ["the", "all", "was", "elephant", "this", "on"]}, {"sent": "could an elephant ride a bicycle ?", "words": ["bicycle", "elephant", "an", "could", "ride", "a"]}, {"sent": "I dont think it was an elephant .", "words": ["think", "was", "elephant", "dont", "an", "it"]}, {"sent": "is that an elephant on anywhere on your boards ?", "words": ["your", "is", "elephant", "an", "on", "that"]}, {"sent": "elephant .", "words": ["elephant"]}, {"sent": "the elephant .", "words": ["the", "elephant"]}, {"sent": "wheres the elephant ?", "words": ["the", "elephant"]}, {"sent": "I think the elephant would survive a fall by a crab .", "words": ["the", "think", "fall", "elephant", "by", "would", "a"]}, {"sent": "look at the giraffe and the elephant .", "words": ["the", "look", "giraffe", "and", "elephant", "at"]}, {"sent": "crazy elephant .", "words": ["elephant"]}, {"sent": "youre going to hurt the elephant .", "words": ["hurt", "to", "elephant", "the"]}, {"sent": "but then the elephants turn around and then Mowgli crashes into that baby elephant doesnt he ?", "words": ["the", "into", "and", "elephant", "he", "around", "then", "that", "but"]}, {"sent": "cant you see the elephant ?", "words": ["the", "see", "elephant", "you"]}, {"sent": "the elephant has got toys and boxes you think .", "words": ["the", "think", "you", "and", "elephant"]}, {"sent": "elephant elephant elephant .", "words": ["elephant"]}, {"sent": "hes found an elephant .", "words": ["an", "elephant"]}, {"sent": "an elephant eating eggplants ?", "words": ["an", "elephant"]}, {"sent": "shall I draw a elephant ?", "words": ["elephant", "draw", "a"]}, {"sent": "elephant ?", "words": ["elephant"]}, {"sent": "you know who goes with the giraffe maybe the elephant ?", "words": ["the", "giraffe", "with", "you", "elephant", "who"]}, {"sent": "heres the elephant .", "words": ["the", "elephant"]}, {"sent": "did you fall off the elephant ?", "words": ["off", "the", "fall", "you", "elephant", "did"]}, {"sent": "is that an elephant there ?", "words": ["is", "elephant", "there", "an", "that"]}, {"sent": "ah and the baby elephant .", "words": ["the", "elephant", "and"]}, {"sent": "elephant right there yeah I see it what else ?", "words": ["what", "elephant", "there", "it", "see"]}, {"sent": "wrinkly elephant cow kisses .", "words": ["cow", "elephant"]}, {"sent": "Ellie the elephant .", "words": ["the", "elephant"]}, {"sent": "its got an elephant on the back hasnt it ?", "words": ["the", "back", "elephant", "an", "it", "on"]}, {"sent": "where the elephant goes right here look .", "words": ["where", "the", "elephant", "here", "look"]}, {"sent": "leave elephant alone .", "words": ["elephant"]}, {"sent": "there is an elephant taking a bath .", "words": ["is", "elephant", "there", "an", "a"]}, {"sent": "thats the baby elephant isnt it ?", "words": ["the", "it", "elephant"]}, {"sent": "elephant .", "words": ["elephant"]}, {"sent": "an elephant yeah .", "words": ["an", "elephant"]}, {"sent": "is the elephant nice and clean now ?", "words": ["the", "clean", "is", "nice", "elephant", "and"]}, {"sent": "you can you dont hafta keep the elephant and the swan if you dont like .", "words": ["can", "the", "if", "like", "elephant", "you", "dont", "and"]}, {"sent": "not a very good elephant is it ?", "words": ["is", "elephant", "it", "good", "not", "a"]}, {"sent": "he was taking you to the elephant slide today wasnt he ?", "words": ["the", "slide", "to", "was", "you", "elephant", "he"]}, {"sent": "and then they hafta get the elephant out ?", "words": ["the", "and", "get", "they", "elephant", "out", "then"]}, {"sent": "uhhuh thats an elephant .", "words": ["an", "elephant"]}, {"sent": "when elephant and worker meet they hope the elephant minds its feet .", "words": ["the", "and", "elephant", "they", "when"]}, {"sent": "and she had stroke big elephant ears .", "words": ["elephant", "big", "she", "and"]}, {"sent": "ticktock whats the time asked little elephant .", "words": ["the", "elephant", "little"]}, {"sent": "an elephant ?", "words": ["an", "elephant"]}, {"sent": "what is that elephant doing ?", "words": ["what", "elephant", "that", "is"]}, {"sent": "an elephant .", "words": ["an", "elephant"]}, {"sent": "elephant .", "words": ["elephant"]}, {"sent": "what does the elephant eat ?", "words": ["the", "what", "does", "elephant", "eat"]}, {"sent": "pink elephant .", "words": ["elephant"]}, {"sent": "in her cars were all kinds of toy elephant .", "words": ["all", "of", "elephant", "in", "toy", "her", "were"]}, {"sent": "elephant has fallen down .", "words": ["down", "elephant"]}, {"sent": "dont you like the elephant ?", "words": ["the", "like", "elephant", "you", "dont"]}, {"sent": "by the elephant .", "words": ["the", "elephant", "by"]}, {"sent": "elephant has a trunk .", "words": ["a", "elephant"]}, {"sent": "thats a big elephant isnt it .", "words": ["it", "elephant", "big", "a"]}, {"sent": "do you remember in your book your book on prehistoric animals whats the name of the animal that looks like an elephant but hes really hairy ?", "words": ["the", "do", "your", "of", "like", "you", "elephant", "in", "on", "animal", "an", "book", "that", "but"]}, {"sent": "elephant .", "words": ["elephant"]}, {"sent": "elephant .", "words": ["elephant"]}, {"sent": "but sometime after midnight the elephant goes snore .", "words": ["the", "but", "elephant"]}, {"sent": "is he an elephant ?", "words": ["an", "elephant", "he", "is"]}, {"sent": "baby elephant .", "words": ["elephant"]}, {"sent": "oh hes gonna ask someone if he can use that apple car thats nice thats pretty good manners for an elephant .", "words": ["can", "for", "if", "car", "nice", "elephant", "an", "he", "pretty", "good", "apple", "that"]}, {"sent": "what kind of food does the elephant eat ?", "words": ["the", "what", "does", "of", "elephant", "eat", "food"]}, {"sent": "hes not trying to eat the elephant .", "words": ["the", "to", "elephant", "eat", "not"]}, {"sent": "elephant trumpeted so loud that baby tiger .", "words": ["so", "elephant", "tiger", "that", "loud"]}, {"sent": "oh baby elephant is an animal youre right .", "words": ["an", "animal", "is", "elephant"]}, {"sent": "zebra kisses elephant .", "words": ["zebra", "elephant"]}, {"sent": "will you get your elephant self off me .", "words": ["off", "your", "elephant", "get", "you", "will", "me"]}, {"sent": "thats Eddy the elephant .", "words": ["the", "elephant"]}, {"sent": "the elephant says oh .", "words": ["the", "elephant"]}, {"sent": "an elephant ?", "words": ["an", "elephant"]}, {"sent": "you know that Dumbos an elephant .", "words": ["an", "elephant", "that", "you"]}, {"sent": "elephant ?", "words": ["elephant"]}, {"sent": "an elephant ?", "words": ["an", "elephant"]}, {"sent": "I thought that when the elephant was laying down he heard a strange noise .", "words": ["the", "when", "was", "elephant", "he", "down", "that", "a"]}, {"sent": "and the elephant today I like elephant .", "words": ["the", "elephant", "like", "and"]}, {"sent": "thats elephant again because its the same stories isnt it ?", "words": ["the", "elephant", "it", "same", "because"]}, {"sent": "that a elephant ?", "words": ["elephant", "that", "a"]}, {"sent": "do you wanna have the elephant go through the ring ?", "words": ["the", "do", "go", "you", "have", "elephant", "wanna"]}, {"sent": "so its got to be the cow or the elephant then .", "words": ["the", "to", "so", "be", "elephant", "then", "cow"]}, {"sent": "have you found elephant yet ?", "words": ["have", "elephant", "you"]}, {"sent": "I wanna take a ride on you elephant .", "words": ["you", "elephant", "wanna", "on", "take", "ride", "a"]}, {"sent": "Max says good morning when he spills water on his red rubber elephant at bedtime .", "words": ["red", "elephant", "his", "at", "he", "good", "water", "on", "when"]}, {"sent": "wheres your elephant ?", "words": ["your", "elephant"]}, {"sent": "an elephant ?", "words": ["an", "elephant"]}, {"sent": "lookit I found another elephant .", "words": ["another", "elephant"]}, {"sent": "elephant yes .", "words": ["elephant"]}, {"sent": "think um that elephant must be hungry .", "words": ["think", "be", "hungry", "elephant", "that"]}, {"sent": "the elephant wants some .", "words": ["the", "some", "elephant"]}, {"sent": "heres your friend the elephant .", "words": ["the", "your", "elephant"]}, {"sent": "you wanna put the elephant down ?", "words": ["the", "put", "elephant", "you", "wanna", "down"]}, {"sent": "long and short tails uh chirpy chipmunks and elephant trunks .", "words": ["elephant", "long", "and"]}, {"sent": "and Ill get the elephant up .", "words": ["the", "up", "elephant", "get", "and"]}, {"sent": "wheres the elephant ?", "words": ["the", "elephant"]}, {"sent": "an elephant in a bathtub .", "words": ["elephant", "in", "an", "bathtub", "a"]}, {"sent": "and how about the little elephant ?", "words": ["the", "how", "little", "and", "elephant", "about"]}, {"sent": "lets put the telephone back shall we ?", "words": ["the", "put", "back", "we", "telephone"]}, {"sent": "thats one of the most confusing telephone conversations Ive had .", "words": ["of", "the", "telephone"]}, {"sent": "who was it on the telephone ?", "words": ["the", "was", "it", "who", "on", "telephone"]}, {"sent": "your blue mobile telephone that UncleDavid bought you ?", "words": ["your", "you", "blue", "that", "telephone"]}, {"sent": "look a telephone .", "words": ["telephone", "a", "look"]}, {"sent": "I hear telephone .", "words": ["telephone", "hear"]}, {"sent": "you didnt talk to Nonna on the telephone ?", "words": ["the", "to", "you", "talk", "on", "telephone"]}, {"sent": "I dont think they can use the telephone .", "words": ["can", "the", "think", "dont", "they", "telephone"]}, {"sent": "the wires go in there that make the telephone work .", "words": ["the", "work", "go", "there", "in", "make", "that", "telephone"]}, {"sent": "can you find the telephone ?", "words": ["can", "the", "you", "find", "telephone"]}, {"sent": "is that your telephone ?", "words": ["your", "that", "telephone", "is"]}, {"sent": "is that disco dancing with telephone ?", "words": ["telephone", "that", "with", "is"]}, {"sent": "you talk to Charlie on the telephone or youll be talking to him next week on holiday .", "words": ["the", "to", "be", "you", "him", "talk", "on", "telephone"]}, {"sent": "thats a telephone .", "words": ["telephone", "a"]}, {"sent": "well lets find her a telephone to use .", "words": ["to", "find", "her", "telephone", "a"]}, {"sent": "are you talking on the telephone ?", "words": ["the", "you", "are", "on", "telephone"]}, {"sent": "telephone ?", "words": ["telephone"]}, {"sent": "heres Henrys telephone .", "words": ["telephone"]}, {"sent": "its a little telephone .", "words": ["telephone", "little", "a"]}, {"sent": "Mamas on the telephone darling .", "words": ["on", "the", "telephone"]}, {"sent": "let us can you find a telephone ?", "words": ["can", "you", "us", "find", "telephone", "a"]}, {"sent": "you talk on the telephone .", "words": ["the", "you", "talk", "on", "telephone"]}, {"sent": "in the great green room there was a telephone .", "words": ["the", "was", "there", "in", "room", "green", "telephone", "a"]}, {"sent": "oh thats somebody on the telephone .", "words": ["on", "the", "telephone"]}, {"sent": "just lets see answer the telephone .", "words": ["the", "see", "telephone"]}, {"sent": "have you found the telephone now ?", "words": ["have", "the", "telephone", "you"]}, {"sent": "hes too busy with the telephone .", "words": ["the", "with", "telephone", "too"]}, {"sent": "who are you gonna call on the telephone ?", "words": ["the", "you", "are", "who", "on", "telephone"]}, {"sent": "now you know where the telephone is for later .", "words": ["where", "the", "for", "is", "you", "telephone"]}, {"sent": "on the telephone right okay .", "words": ["on", "the", "telephone"]}, {"sent": "get the telephone .", "words": ["get", "the", "telephone"]}, {"sent": "Amy answer your telephone .", "words": ["your", "telephone"]}, {"sent": "telephone booth .", "words": ["telephone"]}, {"sent": "the first thing we hafta do is make them call each other on the telephone Hilda says .", "words": ["the", "do", "each", "is", "we", "first", "them", "other", "on", "make", "telephone"]}, {"sent": "where did your telephone go ?", "words": ["where", "your", "go", "did", "telephone"]}, {"sent": "put the telephone down now .", "words": ["the", "put", "down", "telephone"]}, {"sent": "its a telephone booth .", "words": ["telephone", "a"]}, {"sent": "oh are you talking to Linda and Brian on the telephone now are you ?", "words": ["the", "to", "and", "you", "are", "on", "telephone"]}, {"sent": "and theres the telephone .", "words": ["the", "telephone", "and"]}, {"sent": "why dont cha come and call me up on the telephone .", "words": ["up", "the", "and", "dont", "on", "me", "telephone", "why"]}, {"sent": "he helped this little boy get his kite outof the telephone lines do you know what a kite is ?", "words": ["the", "do", "what", "is", "little", "get", "this", "you", "he", "his", "telephone", "a"]}, {"sent": "could you give me the telephone Mama ?", "words": ["the", "you", "me", "give", "telephone", "could"]}, {"sent": "telephone .", "words": ["telephone"]}, {"sent": "that was a telephone call for Daddy .", "words": ["for", "was", "that", "telephone", "a"]}, {"sent": "the telephone .", "words": ["the", "telephone"]}, {"sent": "why dont you bring your telephone down here Peter ?", "words": ["your", "bring", "you", "here", "dont", "telephone", "down", "why"]}, {"sent": "theres the telephone .", "words": ["the", "telephone"]}, {"sent": "talking on the telephone .", "words": ["on", "the", "telephone"]}, {"sent": "well Ill hang up the telephone .", "words": ["up", "the", "telephone"]}, {"sent": "you wanna go put the telephone back now ?", "words": ["the", "put", "back", "go", "you", "wanna", "telephone"]}, {"sent": "a telephone .", "words": ["telephone", "a"]}, {"sent": "play telephone .", "words": ["play", "telephone"]}, {"sent": "and theres a telephone number there .", "words": ["there", "telephone", "a", "and"]}, {"sent": "want the telephone ?", "words": ["the", "telephone"]}, {"sent": "telephone .", "words": ["telephone"]}, {"sent": "he stands up oh let me get the telephone .", "words": ["up", "the", "get", "he", "me", "telephone"]}, {"sent": "and telephone number .", "words": ["telephone", "and"]}, {"sent": "can you dial the telephone ?", "words": ["can", "the", "telephone", "you"]}, {"sent": "do you wanna talk on the telephone ?", "words": ["the", "do", "you", "talk", "wanna", "on", "telephone"]}, {"sent": "who did you talk to on the telephone ?", "words": ["the", "to", "you", "did", "talk", "who", "on", "telephone"]}, {"sent": "thats theyve got their telephone on the wall .", "words": ["on", "the", "their", "telephone"]}, {"sent": "right here that is a telephone .", "words": ["is", "here", "that", "telephone", "a"]}, {"sent": "heres the toy telephone .", "words": ["the", "toy", "telephone"]}, {"sent": "youre taking your telephone with you .", "words": ["with", "your", "telephone", "you"]}, {"sent": "oh do you remember the red telephone ?", "words": ["the", "do", "red", "you", "telephone"]}, {"sent": "the telephone ?", "words": ["the", "telephone"]}, {"sent": "I am going to talk to Linda on the telephone .", "words": ["the", "to", "am", "talk", "on", "telephone"]}, {"sent": "I think you were looking at them while I was on the telephone werent you ?", "words": ["the", "think", "was", "them", "you", "at", "on", "were", "telephone"]}, {"sent": "what ever happened to her telephone do you know ?", "words": ["what", "do", "to", "you", "her", "telephone"]}, {"sent": "is that what you have on the telephone honey a commersation ?", "words": ["the", "what", "is", "you", "have", "on", "that", "telephone", "a"]}, {"sent": "telephone booth .", "words": ["telephone"]}, {"sent": "leave the telephone there .", "words": ["the", "there", "telephone"]}, {"sent": "so we hadta go and answer the telephone and go and get Jamie from school .", "words": ["the", "go", "we", "so", "school", "and", "get", "telephone"]}, {"sent": "the telephone .", "words": ["the", "telephone"]}, {"sent": "telephone .", "words": ["telephone"]}, {"sent": "like its a telephone .", "words": ["like", "telephone", "a"]}, {"sent": "are you saying hello on the telephone ?", "words": ["the", "you", "are", "on", "telephone"]}, {"sent": "oh there goes the telephone again .", "words": ["the", "there", "telephone"]}, {"sent": "then shes going to tell him that shes by the telephone box .", "words": ["the", "box", "to", "by", "him", "then", "that", "telephone"]}, {"sent": "what do you mean telephone man ?", "words": ["what", "do", "telephone", "you"]}, {"sent": "I I said what a dog doing barking on the telephone ?", "words": ["the", "what", "on", "dog", "telephone", "a"]}, {"sent": "youre brushing the telephone ?", "words": ["the", "telephone"]}, {"sent": "shes on the telephone .", "words": ["on", "the", "telephone"]}, {"sent": "yeah the telephone huh ?", "words": ["the", "telephone"]}, {"sent": "wheres the telephone ?", "words": ["the", "telephone"]}, {"sent": "telephone .", "words": ["telephone"]}, {"sent": "its a telephone .", "words": ["telephone", "a"]}, {"sent": "see the telephone ?", "words": ["the", "see", "telephone"]}, {"sent": "its Mummys telephone now .", "words": ["telephone"]}, {"sent": "whose telephone is that ?", "words": ["that", "telephone", "is"]}, {"sent": "whats your telephone number then Marie ?", "words": ["then", "your", "telephone"]}, {"sent": "was it when Mummy was on the telephone that you disappeared behind the settee ?", "words": ["the", "was", "behind", "you", "that", "it", "on", "when", "telephone"]}, {"sent": "where did the telephone go ?", "words": ["where", "the", "go", "did", "telephone"]}, {"sent": "thats the telephone .", "words": ["the", "telephone"]}, {"sent": "you want my telephone ?", "words": ["my", "telephone", "you"]}, {"sent": "oh thats the telephone .", "words": ["the", "telephone"]}, {"sent": "oh theres the telephone .", "words": ["the", "telephone"]}, {"sent": "can you get the telephone ?", "words": ["can", "the", "you", "get", "telephone"]}, {"sent": "wheres the telephone in your house ?", "words": ["the", "your", "in", "house", "telephone"]}, {"sent": "swing swing swing swing whoa .", "words": ["swing"]}, {"sent": "swing you up .", "words": ["swing", "up", "you"]}, {"sent": "swing .", "words": ["swing"]}, {"sent": "does she like to swing ?", "words": ["does", "swing", "to", "like", "she"]}, {"sent": "look shes gonna swing .", "words": ["swing", "look"]}, {"sent": "theres no swing .", "words": ["swing"]}, {"sent": "hes playing on the swing .", "words": ["on", "the", "swing"]}, {"sent": "swing swing .", "words": ["swing"]}, {"sent": "lemme find that swing page .", "words": ["swing", "that", "find"]}, {"sent": "would you like to swing on a star .", "words": ["swing", "to", "star", "like", "you", "would", "on", "a"]}, {"sent": "swing you up .", "words": ["swing", "up", "you"]}, {"sent": "thats a swing .", "words": ["swing", "a"]}, {"sent": "you needta be more careful how you swing that head around .", "words": ["how", "around", "swing", "be", "head", "you", "more", "careful", "that"]}, {"sent": "oop you hold the swing for me okay and Ill put him in .", "words": ["the", "put", "for", "hold", "swing", "you", "and", "him", "in", "me"]}, {"sent": "uhoh the swing fell off of there didnt it ?", "words": ["the", "off", "swing", "of", "there", "it"]}, {"sent": "Ella fell off the swing and broke her arm ?", "words": ["off", "the", "swing", "and", "arm", "her"]}, {"sent": "yeah well just because you swing and hit something with a stick right ?", "words": ["stick", "swing", "with", "and", "you", "hit", "because", "a"]}, {"sent": "thats a swing .", "words": ["swing", "a"]}, {"sent": "and this little boy and a girl sitting on a swing .", "words": ["swing", "little", "and", "this", "on", "a"]}, {"sent": "its a swing .", "words": ["swing", "a"]}, {"sent": "swing swing swing .", "words": ["swing"]}, {"sent": "baby Mickey likes to swing .", "words": ["swing", "to"]}, {"sent": "thats your little tiny puppy with your black xxx oh whos on the swing ?", "words": ["tiny", "the", "your", "swing", "with", "little", "black", "puppy", "on"]}, {"sent": "well is anyone else gonna go on the swing ?", "words": ["the", "go", "swing", "is", "on"]}, {"sent": "easy when you swing it around .", "words": ["swing", "you", "it", "around", "when"]}, {"sent": "Mommy is a swing .", "words": ["swing", "a", "is"]}, {"sent": "its like a swing do you want a swing ?", "words": ["do", "swing", "like", "you", "a"]}, {"sent": "swing ?", "words": ["swing"]}, {"sent": "youknow swing low swing high .", "words": ["swing", "high"]}, {"sent": "oh it looks like the sun is gonna be right on your face if we put you in the swing .", "words": ["the", "put", "your", "face", "if", "is", "we", "be", "sun", "swing", "like", "you", "in", "it", "on"]}, {"sent": "how many chains do we need for the swing ?", "words": ["the", "do", "how", "for", "need", "we", "swing"]}, {"sent": "well go whee on the tyre swing .", "words": ["on", "the", "swing", "go"]}, {"sent": "theyre going from a swing ?", "words": ["swing", "a"]}, {"sent": "we washed the slide and the swing .", "words": ["the", "slide", "we", "swing", "and"]}, {"sent": "on the swing in the park .", "words": ["the", "swing", "in", "park", "on"]}, {"sent": "you wanna go in your swing here in the house ?", "words": ["the", "your", "go", "swing", "you", "here", "in", "house", "wanna"]}, {"sent": "your next door neighbor got a swing ?", "words": ["swing", "your", "a", "door"]}, {"sent": "you want to swing high ?", "words": ["swing", "to", "you", "want to", "high"]}, {"sent": "is this your swing ?", "words": ["swing", "this", "your", "is"]}, {"sent": "ya know we can go outside on the swing and sing songs .", "words": ["can", "the", "go", "we", "outside", "swing", "and", "sing", "on"]}, {"sent": "thats the swing song .", "words": ["the", "swing"]}, {"sent": "is he going to swing ?", "words": ["swing", "to", "he", "is"]}, {"sent": "oh your slide and swing .", "words": ["swing", "slide", "your", "and"]}, {"sent": "Laras in the swing .", "words": ["the", "swing", "in"]}, {"sent": "swing going the bridge .", "words": ["swing", "the"]}, {"sent": "looks like you were on a swing .", "words": ["swing", "like", "you", "on", "were", "a"]}, {"sent": "I remember that swing .", "words": ["swing", "that"]}, {"sent": "look whats on your swing ?", "words": ["on", "swing", "your", "look"]}, {"sent": "shes underneath the swing isnt she ?", "words": ["the", "swing", "she"]}, {"sent": "say swing .", "words": ["swing", "say"]}, {"sent": "well go swing on Haleys swing .", "words": ["swing", "on", "go"]}, {"sent": "here is the swing .", "words": ["here", "the", "swing", "is"]}, {"sent": "do you want to go back to the house and swing ?", "words": ["the", "do", "back", "go", "swing", "to", "you", "and", "house", "want to"]}, {"sent": "swing .", "words": ["swing"]}, {"sent": "because hes playing football and pushing her on the swing in his red tshirt .", "words": ["the", "red", "swing", "and", "his", "in", "on", "her", "because"]}, {"sent": "oh look there he is hes got a swing .", "words": ["is", "swing", "there", "he", "look", "a"]}, {"sent": "youve got a swing now .", "words": ["swing", "a"]}, {"sent": "there is your swing .", "words": ["swing", "there", "your", "is"]}, {"sent": "a swing .", "words": ["swing", "a"]}, {"sent": "take your hands and swing .", "words": ["your", "swing", "take", "and"]}, {"sent": "its a swing huh ?", "words": ["swing", "a"]}, {"sent": "swing on the swings ?", "words": ["swing", "the", "on"]}, {"sent": "swing swing .", "words": ["swing"]}, {"sent": "wanna come swing with me a few minutes my friend ?", "words": ["my", "swing", "with", "wanna", "me", "a"]}, {"sent": "you go out and sit in your swing or something huh ?", "words": ["your", "go", "swing", "and", "you", "in", "out", "sit"]}, {"sent": "and you swing your arms backwards and forwards dont you ?", "words": ["your", "swing", "you", "and", "dont"]}, {"sent": "the little boy on the swing .", "words": ["the", "swing", "on", "little"]}, {"sent": "did we go in the swing ?", "words": ["the", "go", "we", "swing", "in", "did"]}, {"sent": "I need somebody to swing with me if Im going to swing on that one .", "words": ["need", "if", "swing", "to", "with", "on", "me", "that"]}, {"sent": "you cant swing them .", "words": ["swing", "them", "you"]}, {"sent": "the little boys having a go on the swing .", "words": ["the", "go", "swing", "little", "on", "a"]}, {"sent": "who do you want me to put on the swing ?", "words": ["the", "do", "put", "swing", "to", "you", "who", "on", "me"]}, {"sent": "swing ?", "words": ["swing"]}, {"sent": "youre swinging the swing .", "words": ["the", "swing"]}, {"sent": "swing swing ?", "words": ["swing"]}, {"sent": "gonna put her in the swing .", "words": ["the", "put", "swing", "in", "her"]}, {"sent": "swing .", "words": ["swing"]}, {"sent": "youre swinging the swing .", "words": ["the", "swing"]}, {"sent": "yeah he likes to swing his crane around and then go down down down down down down down .", "words": ["go", "swing", "to", "and", "he", "around", "his", "then", "down"]}, {"sent": "ready to swing up again ?", "words": ["swing", "to", "up"]}, {"sent": "wanna sit in your swing ?", "words": ["your", "swing", "in", "wanna", "sit"]}, {"sent": "you ready to swing up ?", "words": ["swing", "to", "up", "you"]}, {"sent": "we can play basketball we can play baseball we can swing we can go down the slide play in the sandbox can play in the fort .", "words": ["can", "the", "sandbox", "slide", "go", "swing", "we", "play", "in", "down"]}, {"sent": "swing .", "words": ["swing"]}, {"sent": "it looks like he needs someone else to swing with .", "words": ["swing", "to", "with", "like", "it", "he"]}, {"sent": "dont swing the bricks round do you ?", "words": ["the", "do", "swing", "you", "dont"]}, {"sent": "you have a swing like that dont you ?", "words": ["swing", "like", "you", "have", "dont", "that", "a"]}, {"sent": "theres no needta swing .", "words": ["swing"]}, {"sent": "swing .", "words": ["swing"]}, {"sent": "no dont swing it .", "words": ["swing", "dont", "it"]}, {"sent": "do you think theres room for two people on that swing ?", "words": ["do", "think", "for", "swing", "you", "room", "on", "that"]}, {"sent": "push swing .", "words": ["swing", "push"]}, {"sent": "do you want to stop and get out of the swing ?", "words": ["the", "do", "of", "to", "swing", "stop", "and", "you", "get", "want to", "out"]}, {"sent": "let us find the swing page .", "words": ["the", "us", "swing", "find"]}, {"sent": "I know you like the swing best of all huh ?", "words": ["the", "all", "swing", "of", "like", "you"]}, {"sent": "put Amy in this swing .", "words": ["swing", "in", "put", "this"]}, {"sent": "a swing .", "words": ["swing", "a"]}, {"sent": "they cant swing too high .", "words": ["swing", "they", "high", "too"]}, {"sent": "was bunny was bunny awake when you woke up ?", "words": ["up", "bunny", "was", "awake", "you", "when"]}, {"sent": "hes awake .", "words": ["awake"]}, {"sent": "everybodys awake now .", "words": ["awake"]}, {"sent": "is Eleanor awake good girl .", "words": ["awake", "good", "is"]}, {"sent": "not like you awake .", "words": ["awake", "not", "like", "you"]}, {"sent": "well they didnt know you were awake .", "words": ["they", "awake", "were", "you"]}, {"sent": "and you know they all have to be nursed and Andrew sometimes is cranky and hes and hes and Andrews mostly awake and playing around with the other babies .", "words": ["the", "all", "is", "to", "be", "awake", "with", "you", "have", "they", "and", "other", "around"]}, {"sent": "did you know that owls sleep in the daytime and theyre awake all night .", "words": ["the", "all", "sleep", "awake", "you", "and", "in", "did", "that"]}, {"sent": "and keep mummy and daddy awake ?", "words": ["awake", "and"]}, {"sent": "are you awake Nicole ?", "words": ["are", "awake", "you"]}, {"sent": "theyre still awake there .", "words": ["there", "awake"]}, {"sent": "the mans awake yes .", "words": ["the", "awake"]}, {"sent": "wide awake .", "words": ["awake"]}, {"sent": "were you awake ?", "words": ["awake", "were", "you"]}, {"sent": "shes not awake yet is she not ?", "words": ["she", "awake", "not", "is"]}, {"sent": "so hes awake ?", "words": ["so", "awake"]}, {"sent": "that mans awake look .", "words": ["awake", "that", "look"]}, {"sent": "youre keeping your sister awake .", "words": ["awake", "your"]}, {"sent": "is she awake now ?", "words": ["awake", "she", "is"]}, {"sent": "just because it makes you feel awake .", "words": ["it", "awake", "because", "you"]}, {"sent": "Ill awake now .", "words": ["awake"]}, {"sent": "Lwww is awake now isnt she ?", "words": ["awake", "she", "is"]}, {"sent": "youve got to stop awake a little while longer .", "words": ["to", "awake", "stop", "little", "a"]}, {"sent": "is she sleeping or is she awake .", "words": ["awake", "she", "is"]}, {"sent": "I am awake .", "words": ["awake", "am"]}, {"sent": "truth is I think Jennys awake .", "words": ["think", "awake", "is"]}, {"sent": "oh god bless are we keeping you awake ?", "words": ["are", "awake", "you", "we"]}, {"sent": "that mommy is awake .", "words": ["awake", "that", "is"]}, {"sent": "have you been awake for a long time ?", "words": ["for", "awake", "long", "you", "have", "a"]}, {"sent": "thanks for staying awake so you can hear my prayers .", "words": ["can", "my", "for", "so", "awake", "you", "hear"]}, {"sent": "and he was awake awake at two oclock .", "words": ["was", "awake", "and", "he", "at"]}, {"sent": "hes awake .", "words": ["awake"]}, {"sent": "fat cat Ive been awake all night .", "words": ["all", "cat", "awake"]}, {"sent": "awake .", "words": ["awake"]}, {"sent": "are you awake now ?", "words": ["are", "awake", "you"]}, {"sent": "oh Little Bird you were awake all night .", "words": ["all", "awake", "were", "you"]}, {"sent": "now hes awake huh ?", "words": ["awake"]}, {"sent": "is bunny awake ?", "words": ["bunny", "awake", "is"]}, {"sent": "am I keeping you awake ?", "words": ["awake", "am", "you"]}, {"sent": "oh hes awake ?", "words": ["awake"]}, {"sent": "before you were awake .", "words": ["awake", "were", "you"]}, {"sent": "youre awake .", "words": ["awake"]}, {"sent": "very much awake .", "words": ["awake", "much"]}, {"sent": "because if he doesnt get his wind up he stays awake a while .", "words": ["up", "if", "a", "awake", "get", "he", "his", "because", "wind"]}, {"sent": "comeon you awake ?", "words": ["awake", "you"]}, {"sent": "oh look Eleanor Lalas awake .", "words": ["awake", "look"]}, {"sent": "well now Anas awake .", "words": ["awake"]}, {"sent": "hes a hes awake .", "words": ["awake", "a"]}, {"sent": "rabbits bunnys awake .", "words": ["awake"]}, {"sent": "Fat Cat Ive been awake all night .", "words": ["all", "awake"]}, {"sent": "Jwwws awake .", "words": ["awake"]}, {"sent": "bunnys awake .", "words": ["awake"]}, {"sent": "then we go to see if Mummy and Daddy are awake .", "words": ["if", "go", "we", "to", "awake", "and", "are", "see", "then"]}, {"sent": "now Amys awake .", "words": ["awake"]}, {"sent": "tiggy looks awake to me .", "words": ["to", "me", "awake"]}, {"sent": "sort of awake .", "words": ["of", "awake"]}, {"sent": "is she awake now ?", "words": ["awake", "she", "is"]}, {"sent": "shes only awake because you keep making a noise .", "words": ["awake", "a", "because", "you"]}, {"sent": "one cat is asleep and one is awake .", "words": ["is", "awake", "and", "cat", "asleep"]}, {"sent": "is she awake as well ?", "words": ["awake", "she", "is"]}, {"sent": "are you awake Ethan ?", "words": ["are", "awake", "you"]}, {"sent": "theyre awake .", "words": ["awake"]}, {"sent": "was he awake ?", "words": ["was", "awake", "he"]}, {"sent": "awake .", "words": ["awake"]}, {"sent": "is she awake ?", "words": ["awake", "she", "is"]}, {"sent": "asleep awake .", "words": ["asleep", "awake"]}, {"sent": "Im not surprised shes awake .", "words": ["awake", "not"]}, {"sent": "no hes awake now .", "words": ["awake"]}, {"sent": "are you awake ?", "words": ["are", "awake", "you"]}, {"sent": "dont stay awake ?", "words": ["dont", "stay", "awake"]}, {"sent": "is the doggies asleep or awake ?", "words": ["the", "asleep", "awake", "is"]}, {"sent": "are they awake on this side ?", "words": ["awake", "they", "this", "are", "on"]}, {"sent": "Jwwws awake .", "words": ["awake"]}, {"sent": "I think Amys awake .", "words": ["think", "awake"]}, {"sent": "are you not quite awake yet ?", "words": ["are", "awake", "not", "you"]}, {"sent": "my babys awake .", "words": ["my", "awake"]}, {"sent": "did you did you feel wide awake you werent tired ?", "words": ["tired", "awake", "did", "you"]}, {"sent": "awake .", "words": ["awake"]}, {"sent": "she will be awake if you shout like that .", "words": ["if", "be", "awake", "like", "you", "will", "that", "she"]}, {"sent": "awake .", "words": ["awake"]}, {"sent": "the babys awake .", "words": ["the", "awake"]}, {"sent": "yes bunnys awake now .", "words": ["awake"]}, {"sent": "Cinderellas awake .", "words": ["awake"]}, {"sent": "xxx stay awake .", "words": ["stay", "awake"]}, {"sent": "hes always awake and working hard .", "words": ["awake", "hard", "and"]}, {"sent": "because you were awake you were awake when we set off .", "words": ["off", "when", "we", "awake", "you", "were", "because"]}, {"sent": "you oh okay are you more awake now ?", "words": ["more", "are", "awake", "you"]}, {"sent": "well Im glad theyre all awake now .", "words": ["all", "awake"]}, {"sent": "sisters awake ?", "words": ["awake"]}, {"sent": "Bunnys awake .", "words": ["awake"]}, {"sent": "awake .", "words": ["awake"]}, {"sent": "yeah hes awake but theyre all alseep except shes awake .", "words": ["all", "awake", "but"]}, {"sent": "even the teachers not awake she laughed its too early .", "words": ["the", "awake", "she", "too", "not"]}, {"sent": "shes awake .", "words": ["awake"]}, {"sent": "hes awake okay .", "words": ["awake"]}, {"sent": "everybodys awake now arent they ?", "words": ["they", "awake"]}, {"sent": "when she sits with her eyes open shes wide awake .", "words": ["when", "open", "awake", "with", "her", "she"]}, {"sent": "youre sposta lick em ?", "words": ["lick"]}, {"sent": "give it a good lick .", "words": ["it", "good", "give", "lick", "a"]}, {"sent": "so piggy said animal um Ill let you have one of my balloons if youll let me have a lick of your icecream .", "words": ["my", "your", "if", "of", "so", "you", "have", "animal", "me", "lick", "a"]}, {"sent": "youre not singing till I lick your hand ?", "words": ["your", "lick", "not", "hand"]}, {"sent": "itll teach you how to lick .", "words": ["to", "how", "lick", "you"]}, {"sent": "youre gonna lick it ?", "words": ["it", "lick"]}, {"sent": "is he gonna lick the feet of this little boy ?", "words": ["the", "is", "of", "little", "this", "he", "lick"]}, {"sent": "the the lick of the billygoat was kind of wet I think .", "words": ["the", "wet", "think", "of", "was", "lick"]}, {"sent": "trying to lick it .", "words": ["to", "it", "lick"]}, {"sent": "dont lick it .", "words": ["dont", "it", "lick"]}, {"sent": "you wanna lick the bowl ?", "words": ["the", "bowl", "you", "wanna", "lick"]}, {"sent": "lick it off just like you do tapioca .", "words": ["off", "do", "like", "you", "it", "lick"]}, {"sent": "you dont lick it if youre pretending .", "words": ["if", "you", "dont", "it", "lick"]}, {"sent": "lick it ?", "words": ["it", "lick"]}, {"sent": "should I lick it ?", "words": ["it", "lick"]}, {"sent": "you lick it .", "words": ["it", "lick", "you"]}, {"sent": "Im not really gonna lick it .", "words": ["it", "lick", "not"]}, {"sent": "now we got to lick each one and put it on a square .", "words": ["put", "each", "we", "to", "and", "it", "on", "lick", "a"]}, {"sent": "was kind of lick in the boys face huh ?", "words": ["the", "face", "of", "was", "in", "lick"]}, {"sent": "you can lick it out at the end .", "words": ["can", "the", "you", "it", "out", "at", "lick"]}, {"sent": "did he let her have a lick of his icecream ?", "words": ["of", "have", "he", "did", "his", "lick", "her", "a"]}, {"sent": "okay Im going to lick it .", "words": ["to", "it", "lick"]}, {"sent": "and then you lick .", "words": ["then", "lick", "you", "and"]}, {"sent": "is he going to lick you ?", "words": ["is", "to", "you", "he", "lick"]}, {"sent": "oh did he lick you ?", "words": ["lick", "he", "did", "you"]}, {"sent": "youd like to lick the bowl ?", "words": ["the", "bowl", "to", "like", "lick"]}, {"sent": "ca can you lick it at the back ?", "words": ["can", "the", "back", "you", "it", "at", "lick"]}, {"sent": "did you lick them and stick them on .", "words": ["stick", "them", "and", "you", "did", "on", "lick"]}, {"sent": "would you like to lick that green one ?", "words": ["to", "like", "you", "that", "green", "would", "lick"]}, {"sent": "its good to lick it .", "words": ["to", "it", "lick", "good"]}, {"sent": "lick you with the tongue .", "words": ["the", "with", "you", "tongue", "lick"]}, {"sent": "give it a good lick .", "words": ["it", "good", "give", "lick", "a"]}, {"sent": "lick the end .", "words": ["the", "lick"]}, {"sent": "no you dont needta lick it .", "words": ["dont", "it", "lick", "you"]}, {"sent": "would you like to lick everything in here before were done ?", "words": ["to", "like", "you", "here", "in", "would", "lick", "were"]}, {"sent": "you can lick this instead .", "words": ["can", "this", "lick", "you"]}, {"sent": "you wanna lick it do you ?", "words": ["do", "you", "it", "wanna", "lick"]}, {"sent": "ah lick me too Purdie .", "words": ["me", "lick", "too"]}, {"sent": "Thomas you would not lick your fingers .", "words": ["your", "you", "would", "lick", "not"]}, {"sent": "oh can I have a lick ?", "words": ["can", "have", "lick", "a"]}, {"sent": "lick ?", "words": ["lick"]}, {"sent": "you wont lick it off .", "words": ["off", "it", "lick", "you"]}, {"sent": "is he gonna lick you ?", "words": ["lick", "he", "you", "is"]}, {"sent": "you lick your finger .", "words": ["your", "finger", "lick", "you"]}, {"sent": "would you like to lick a bit of jam off that ?", "words": ["off", "of", "to", "like", "you", "that", "would", "lick", "a"]}, {"sent": "good maybe SantaClaus can bring you a big wet tongue for Christmas that can lick you all the .", "words": ["can", "wet", "all", "for", "the", "bring", "you", "that", "good", "tongue", "lick", "big", "a"]}, {"sent": "yeah lick it xxx .", "words": ["it", "lick"]}, {"sent": "I dont wanna lick your finger .", "words": ["your", "dont", "finger", "wanna", "lick"]}, {"sent": "there is nothing on that plate to lick .", "words": ["plate", "is", "to", "there", "on", "lick", "that"]}, {"sent": "did he lick you ?", "words": ["lick", "he", "did", "you"]}, {"sent": "or lick a lolly scrub your teeth .", "words": ["your", "lick", "a"]}, {"sent": "lick your icecream .", "words": ["your", "lick"]}, {"sent": "give your mommy a little lick .", "words": ["your", "little", "give", "lick", "a"]}, {"sent": "and then you can lick the will you be able to lick the envelope for me ?", "words": ["can", "the", "for", "be", "to", "you", "and", "will", "me", "then", "lick"]}, {"sent": "pretend you dont lick it .", "words": ["you", "dont", "it", "pretend", "lick"]}, {"sent": "you dont lick the ah book .", "words": ["the", "you", "dont", "book", "lick"]}, {"sent": "first I got to lick .", "words": ["to", "first", "lick"]}, {"sent": "youll hafta lick them Lara .", "words": ["lick", "them"]}, {"sent": "Im not really sure its a good idea for you to lick it is it if youre not very well ?", "words": ["for", "if", "is", "to", "you", "it", "good", "lick", "not", "a"]}, {"sent": "oh can I have a lick ?", "words": ["can", "have", "lick", "a"]}, {"sent": "can I have a lick ?", "words": ["can", "have", "lick", "a"]}, {"sent": "oh the cats gonna lick the mommy oh .", "words": ["the", "lick"]}, {"sent": "yeah she really did lick the microphone .", "words": ["the", "did", "lick", "she"]}, {"sent": "were going to both lick it .", "words": ["to", "it", "were", "lick"]}, {"sent": "dont lick your knee .", "words": ["your", "dont", "knee", "lick"]}, {"sent": "precut stickers lick stick and color .", "words": ["stick", "lick", "and"]}, {"sent": "lick your hands go like this .", "words": ["your", "go", "like", "this", "lick"]}, {"sent": "just pretend to lick it please .", "words": ["to", "pretend", "it", "lick"]}, {"sent": "are you going to lick it up ?", "words": ["up", "to", "you", "it", "are", "lick"]}, {"sent": "why does she lick you do you know ?", "words": ["does", "do", "you", "lick", "she", "why"]}, {"sent": "if youre going to lick it dont then put it on a biscuit .", "words": ["put", "if", "to", "dont", "it", "on", "then", "lick", "a"]}, {"sent": "lick the underside Jeremy or lean over one .", "words": ["the", "over", "lick"]}, {"sent": "well lets get the bowl scrapped first to make the buns then you can lick lick out afterwards .", "words": ["the", "bowl", "can", "to", "first", "you", "get", "out", "lick", "then", "make"]}, {"sent": "now would you like to lick that and stick it on the envelope for Grandma ?", "words": ["the", "stick", "for", "to", "like", "you", "and", "that", "it", "would", "on", "lick"]}, {"sent": "you lick this .", "words": ["this", "lick", "you"]}, {"sent": "you going to lick it off ?", "words": ["off", "to", "you", "it", "lick"]}, {"sent": "Eleanor youre not sposta lick them all .", "words": ["all", "lick", "not", "them"]}, {"sent": "hes gonna go thpt thpt thpt lick his feet ?", "words": ["his", "lick", "go"]}, {"sent": "just lick it a little .", "words": ["it", "lick", "little", "a"]}, {"sent": "you can lick it .", "words": ["can", "it", "lick", "you"]}, {"sent": "you dont want him to lick your face Becky .", "words": ["your", "face", "to", "you", "dont", "him", "lick"]}, {"sent": "a lick on Purdies face .", "words": ["on", "lick", "face", "a"]}, {"sent": "lick it .", "words": ["it", "lick"]}, {"sent": "lick .", "words": ["lick"]}, {"sent": "you can lick the spoon .", "words": ["can", "the", "you", "spoon", "lick"]}, {"sent": "these are some stamps that Grandma has given you that you can lick and put on Christmas cards .", "words": ["can", "these", "put", "you", "and", "are", "lick", "on", "some", "that"]}, {"sent": "lick it down now now stick it down .", "words": ["down", "stick", "it", "lick"]}, {"sent": "do you wanna lick it ?", "words": ["do", "you", "it", "wanna", "lick"]}, {"sent": "you dont hafta lick the spoon .", "words": ["the", "you", "dont", "spoon", "lick"]}, {"sent": "I just hafta lick it with my tongue .", "words": ["my", "with", "it", "tongue", "lick"]}, {"sent": "does he lick you ?", "words": ["does", "he", "lick", "you"]}, {"sent": "if you had that you could lick all those wee bits off couldnt you ?", "words": ["off", "all", "those", "if", "you", "that", "could", "lick"]}, {"sent": "you lick them and put them right in the middle okay ?", "words": ["the", "put", "them", "and", "you", "in", "lick"]}, {"sent": "dont lick it .", "words": ["dont", "it", "lick"]}, {"sent": "no were not gonna lick the apple juice Ally .", "words": ["the", "apple", "juice", "lick", "not", "were"]}, {"sent": "are you going to lick those ?", "words": ["those", "to", "you", "are", "lick"]}, {"sent": "you hafta lick it and put it inside a square .", "words": ["put", "inside", "and", "you", "it", "lick", "a"]}, {"sent": "dont lick it .", "words": ["dont", "it", "lick"]}, {"sent": "he unbuttoned Corduroys shoulder straps and put his overalls in the dryer .", "words": ["the", "put", "and", "shoulder", "in", "he", "his", "dryer"]}, {"sent": "and look at the sheep mark on the shoulder .", "words": ["the", "and", "shoulder", "at", "sheep", "look", "on"]}, {"sent": "or head shoulder knees and toes ?", "words": ["head", "and", "shoulder"]}, {"sent": "shoulder \u5462 ?", "words": ["shoulder"]}, {"sent": "Ill meet you in a week in the public garden he quacked over his shoulder .", "words": ["the", "you", "shoulder", "garden", "in", "he", "his", "over", "a"]}, {"sent": "oh you look like Santa now that you have your sack over your shoulder .", "words": ["your", "like", "you", "have", "shoulder", "look", "over", "that"]}, {"sent": "lay your head on Mommys shoulder .", "words": ["on", "head", "your", "shoulder"]}, {"sent": "I put yours on your shirt too but its on your shoulder .", "words": ["put", "your", "shirt", "shoulder", "too", "on", "but"]}, {"sent": "on her shoulder ?", "words": ["on", "her", "shoulder"]}, {"sent": "your left shoulder .", "words": ["your", "shoulder"]}, {"sent": "on my shoulder .", "words": ["on", "my", "shoulder"]}, {"sent": "xxx xxx my shoulder Honey .", "words": ["my", "shoulder"]}, {"sent": "Annes shoulder or Mummys shoulder ?", "words": ["shoulder"]}, {"sent": "doesnt go on your shoulder ?", "words": ["on", "shoulder", "your", "go"]}, {"sent": "ride on my shoulder .", "words": ["on", "my", "ride", "shoulder"]}, {"sent": "theres a big dinosaur on my shoulder .", "words": ["my", "shoulder", "on", "big", "a"]}, {"sent": "so the itsy bitsy spider walked across Dillons shoulder .", "words": ["the", "so", "shoulder"]}, {"sent": "theres another baby with all kinds of little bluebirds sitting on his shoulder .", "words": ["all", "of", "with", "little", "another", "his", "shoulder", "on"]}, {"sent": "youre hitting me on the shoulder .", "words": ["on", "me", "shoulder", "the"]}, {"sent": "those are the cloths that we useta put on our shoulder when you were a baby .", "words": ["the", "put", "those", "when", "we", "our", "shoulder", "you", "are", "on", "that", "were", "a"]}, {"sent": "going on your shoulder now .", "words": ["on", "your", "shoulder"]}, {"sent": "its on his shoulder .", "words": ["on", "his", "shoulder"]}, {"sent": "shoulder \u662f \u80a9\u8180 .", "words": ["shoulder"]}, {"sent": "this man has got a blue bag over his shoulder .", "words": ["blue", "shoulder", "this", "his", "over", "a"]}, {"sent": "whats that you put on your shoulder ?", "words": ["put", "your", "you", "shoulder", "on", "that"]}, {"sent": "your shoulder ?", "words": ["your", "shoulder"]}, {"sent": "and this man has got a bag over his shoulder .", "words": ["and", "shoulder", "this", "his", "over", "a"]}, {"sent": "your shoulder .", "words": ["your", "shoulder"]}, {"sent": "you wanna ride Daddys shoulder and .", "words": ["and", "shoulder", "you", "wanna", "ride"]}, {"sent": "this this is your left shoulder no put this hand down this is your left shoulder .", "words": ["put", "your", "is", "hand", "shoulder", "this", "down"]}, {"sent": "here sweetie let me help you bring that up over your shoulder .", "words": ["up", "help", "your", "bring", "you", "here", "shoulder", "over", "me", "that"]}, {"sent": "thas your left shoulder .", "words": ["your", "shoulder"]}, {"sent": "on Frasers shoulder .", "words": ["on", "shoulder"]}, {"sent": "so let me put a diaper on my shoulder so I can take Ana so all my clothes wont be spit up on hi sweetheart .", "words": ["can", "all", "put", "my", "up", "diaper", "so", "be", "shoulder", "on", "me", "take", "a"]}, {"sent": "Matthew will you tap Matthew on the shoulder please .", "words": ["the", "shoulder", "you", "on", "will"]}, {"sent": "shoulder \u5462 ?", "words": ["shoulder"]}, {"sent": "this man has got a blue bag over his shoulder .", "words": ["blue", "shoulder", "this", "his", "over", "a"]}, {"sent": "that toy I handed it to her and she was like threw it over her shoulder I was like oh .", "words": ["to", "was", "her", "like", "and", "shoulder", "toy", "it", "over", "that", "she"]}, {"sent": "so Charles father took him to town and bought him some beautiful football things a shirt with scarlet with shoulder pads and the helmet had silver wings .", "words": ["the", "to", "so", "shirt", "with", "and", "shoulder", "him", "some", "a"]}, {"sent": "no I dont want you to put it on my shoulder .", "words": ["put", "my", "to", "you", "shoulder", "dont", "it", "on"]}, {"sent": "my bluebird on my shoulder what shape does this look like ?", "words": ["what", "my", "does", "like", "shoulder", "this", "look", "on"]}, {"sent": "you go on my shoulder first ?", "words": ["my", "go", "first", "shoulder", "you", "on"]}, {"sent": "slip it off the shoulder .", "words": ["off", "the", "it", "shoulder"]}, {"sent": "shoulder .", "words": ["shoulder"]}, {"sent": "dolly over your shoulder .", "words": ["your", "over", "shoulder"]}, {"sent": "shoulder \u662f \u4ec0\u4e48 ?", "words": ["shoulder"]}, {"sent": "this man has got a blue bag over his shoulder .", "words": ["blue", "shoulder", "this", "his", "over", "a"]}, {"sent": "and this man has got a blue bag over his shoulder .", "words": ["blue", "and", "this", "shoulder", "his", "over", "a"]}, {"sent": "shoulder blades spine .", "words": ["shoulder"]}, {"sent": "and theres the lions shoulder .", "words": ["the", "shoulder", "and"]}, {"sent": "you well Im not doing a lot with you sitting on my shoulder .", "words": ["my", "a lot", "with", "you", "shoulder", "on", "not", "a"]}, {"sent": "over her shoulder .", "words": ["her", "over", "shoulder"]}, {"sent": "oh that hurt Mummys shoulder that hurt Mummys shoulder .", "words": ["hurt", "that", "shoulder"]}, {"sent": "come and grab Daddys shoulder ?", "words": ["shoulder", "and"]}, {"sent": "thats my shoulder thats my cheek .", "words": ["cheek", "my", "shoulder"]}, {"sent": "this man has got a blue bag over his shoulder .", "words": ["blue", "shoulder", "this", "his", "over", "a"]}, {"sent": "you almost touch your mothers shoulder ?", "words": ["shoulder", "your", "touch", "you"]}, {"sent": "the man whos got a blue bag over his shoulder right .", "words": ["the", "blue", "shoulder", "his", "over", "a"]}, {"sent": "this man has got a blue bag over his shoulder .", "words": ["blue", "shoulder", "this", "his", "over", "a"]}, {"sent": "I will hold it and what you wanna do is stretch it way out until it gets unto her shoulder like that .", "words": ["what", "do", "hold", "is", "like", "and", "you", "shoulder", "that", "it", "wanna", "out", "will", "her"]}, {"sent": "this man has got a blue bag over his shoulder .", "words": ["blue", "shoulder", "this", "his", "over", "a"]}, {"sent": "whats those big planks Thomas on your shoulder ?", "words": ["those", "your", "shoulder", "on", "big"]}, {"sent": "I could touch my mother shoulder without tippyingtoe .", "words": ["my", "could", "touch", "shoulder"]}, {"sent": "if I look on your shoulder it says dustman Ray ?", "words": ["your", "if", "shoulder", "it", "look", "on"]}, {"sent": "shoulder \u662f \u80a9\u8180 .", "words": ["shoulder"]}, {"sent": "yeah and hes got a big bag on his shoulder .", "words": ["and", "his", "shoulder", "on", "big", "a"]}, {"sent": "right near your shoulder .", "words": ["your", "shoulder"]}, {"sent": "its not for banging on Mummys shoulder .", "words": ["on", "for", "not", "shoulder"]}, {"sent": "he gonna shoot over the shoulder ?", "words": ["over", "the", "he", "shoulder"]}, {"sent": "heres your shoulder bones .", "words": ["your", "shoulder"]}, {"sent": "its not really a shoulder bag John .", "words": ["shoulder", "not", "a"]}, {"sent": "you want Polly on your shoulder ?", "words": ["on", "shoulder", "your", "you"]}, {"sent": "now put it on this shoulder up here .", "words": ["up", "put", "shoulder", "here", "this", "it", "on"]}, {"sent": "oh you act so brave end when he gets okay your shoulder you get afraid .", "words": ["your", "so", "you", "shoulder", "get", "he", "when"]}, {"sent": "this man has got a blue bag over his shoulder .", "words": ["blue", "shoulder", "this", "his", "over", "a"]}, {"sent": "this man has got a blue bag over his shoulder .", "words": ["blue", "shoulder", "this", "his", "over", "a"]}, {"sent": "Mummy hurt her shoulder .", "words": ["hurt", "her", "shoulder"]}, {"sent": "this man has got a blue bag over his shoulder .", "words": ["blue", "shoulder", "this", "his", "over", "a"]}, {"sent": "the guard tossed Norman up on his shoulder .", "words": ["the", "up", "shoulder", "on", "his"]}, {"sent": "and in the oval picture it shows a teddybear with a toothbrush and some toothpaste and a towel slung over his shoulder .", "words": ["the", "toothbrush", "picture", "with", "teddybear", "and", "shoulder", "in", "it", "towel", "his", "some", "over", "a"]}, {"sent": "your left shoulder very good Benita .", "words": ["your", "good", "shoulder"]}, {"sent": "well you could do really all you hafta do is slip this strap off your shoulder .", "words": ["off", "all", "do", "your", "is", "you", "shoulder", "this", "could"]}, {"sent": "this man has got a bag over his shoulder .", "words": ["shoulder", "this", "his", "over", "a"]}, {"sent": "wheres your shoulder ?", "words": ["your", "shoulder"]}, {"sent": "put the baby on your shoulder baby like to burp .", "words": ["the", "put", "your", "to", "like", "shoulder", "on"]}, {"sent": "thats your left shoulder .", "words": ["your", "shoulder"]}, {"sent": "shes sitting down looking over her shoulder .", "words": ["her", "down", "over", "shoulder"]}, {"sent": "this mans got a blue bag over his shoulder .", "words": ["blue", "shoulder", "this", "his", "over", "a"]}, {"sent": "xxx shoulder .", "words": ["shoulder"]}, {"sent": "well put your bag properly on your shoulder .", "words": ["on", "put", "your", "shoulder"]}, {"sent": "then Dopey started pacing back and forth and pointing over his shoulder .", "words": ["back", "and", "shoulder", "his", "then", "over"]}, {"sent": "do your shoulder .", "words": ["do", "your", "shoulder"]}, {"sent": "Annes shoulder .", "words": ["shoulder"]}, {"sent": "your left shoulder .", "words": ["your", "shoulder"]}, {"sent": "and Ive got a Tigger on my shoulder .", "words": ["my", "shoulder", "and", "on", "a"]}, {"sent": "this shoulder .", "words": ["this", "shoulder"]}, {"sent": "PostmanPats van is going on your shoulder now .", "words": ["on", "shoulder", "your", "is"]}, {"sent": "and on each one theyve piped icing .", "words": ["on", "each", "and"]}, {"sent": "well why dont you each take one pillow ?", "words": ["each", "you", "dont", "take", "why", "pillow"]}, {"sent": "so each plant pot has a sticker .", "words": ["so", "plant", "each", "a"]}, {"sent": "oh theyre pushing each other away .", "words": ["away", "each", "other"]}, {"sent": "oh each one gets it and each one eats it ?", "words": ["it", "each", "and"]}, {"sent": "they are made up of many tiny compartments with an individual animal in each .", "words": ["up", "tiny", "each", "of", "with", "they", "an", "are", "animal", "in"]}, {"sent": "can you see each of the kings is bringing a little gift for baby Jesus ?", "words": ["can", "the", "for", "each", "is", "of", "you", "little", "see", "a"]}, {"sent": "just that these toys are different see the prior to todays session each time we came it was the same toys .", "words": ["the", "these", "each", "we", "to", "was", "are", "it", "same", "see", "that"]}, {"sent": "can they jump over each other ?", "words": ["can", "each", "other", "they", "jump", "over"]}, {"sent": "you know with the hosepipes youre not sposta hit each other .", "words": ["the", "each", "with", "you", "other", "hit", "not"]}, {"sent": "okay they each get to pick their favorites .", "words": ["their", "each", "pick", "to", "get", "they"]}, {"sent": "it has a reading for each day .", "words": ["it", "each", "for", "a"]}, {"sent": "five people and one from each .", "words": ["each", "and"]}, {"sent": "each day we open the boxes .", "words": ["open", "the", "each", "we"]}, {"sent": "and each the same color as the Teletubbies .", "words": ["the", "same", "each", "and"]}, {"sent": "one for each hand yay oh .", "words": ["for", "each", "hand"]}, {"sent": "theres one each .", "words": ["each"]}, {"sent": "he handled each masterpiece with as much care and respect as if he had painted it himself .", "words": ["if", "each", "with", "and", "it", "he", "much"]}, {"sent": "can you imagine sitting there and knitting and then pulling each string out ?", "words": ["can", "each", "and", "you", "there", "out", "then"]}, {"sent": "it says find the tane tiny bear in each picture .", "words": ["the", "tiny", "each", "picture", "bear", "in", "it", "find"]}, {"sent": "each time .", "words": ["each"]}, {"sent": "and each son had a wife .", "words": ["a", "each", "and"]}, {"sent": "well you hafta look at each piece .", "words": ["look", "at", "each", "you"]}, {"sent": "its just its more interesting and more broken up but you know if you go by ferry I mean if its say fifteen minutes for the ferry each ferry thats half an hour of a wait plus each one is about twenty five minutes thats an hour and a half plus the drive time so .", "words": ["the", "go", "of", "more", "an", "about", "each", "by", "up", "for", "if", "is", "so", "broken", "and", "you", "drive", "a", "wait", "say", "but"]}, {"sent": "theyre each going in the box .", "words": ["the", "in", "box", "each"]}, {"sent": "and its different colored checks and crosses stripes and things and each family has a different tartan .", "words": ["a", "each", "and"]}, {"sent": "but Albert Ames was first each day because his name began with .", "words": ["each", "was", "first", "with", "but", "his", "because"]}, {"sent": "we hafta wrap each of these very very carefully in tissue paper dont we ?", "words": ["these", "each", "we", "of", "tissue", "paper", "dont", "in"]}, {"sent": "one for each side .", "words": ["for", "each"]}, {"sent": "so should we put those there next to each other ?", "words": ["put", "those", "each", "we", "so", "to", "other", "there"]}, {"sent": "lets put two eggs on top of each .", "words": ["on", "of", "put", "each"]}, {"sent": "each person one by one and hes counting as he does it .", "words": ["does", "each", "and", "by", "it", "he"]}, {"sent": "oh weve each got these horns right on top of our head .", "words": ["these", "each", "of", "head", "our", "on"]}, {"sent": "I have lotsof fun each day .", "words": ["have", "each"]}, {"sent": "babies probably even talk to each other because they know what googa is and that kind of stuff means .", "words": ["what", "each", "is", "of", "to", "other", "and", "they", "talk", "that", "because"]}, {"sent": "Ill slice it in half and they can each have a half .", "words": ["can", "each", "and", "have", "they", "in", "it", "a"]}, {"sent": "one on each side .", "words": ["on", "each"]}, {"sent": "you hadta wear something each time we got in a boat .", "words": ["each", "we", "you", "in", "boat", "a"]}, {"sent": "with each questionnaire I felt like he wasnt .", "words": ["he", "with", "like", "each"]}, {"sent": "one for each kid and and they dont know about it .", "words": ["for", "each", "and", "dont", "they", "it", "about"]}, {"sent": "and they each had a .", "words": ["they", "a", "each", "and"]}, {"sent": "since each of its toes were hoofed Machrauchenia wouldve been able to run quickly to escape predators .", "words": ["each", "of", "to", "run", "were"]}, {"sent": "two each ?", "words": ["each"]}, {"sent": "and we each had two cookies .", "words": ["we", "each", "and"]}, {"sent": "does she play with each toy the same way as she would at .", "words": ["the", "does", "each", "play", "with", "toy", "at", "same", "would", "she"]}, {"sent": "there was about four bottles of wine each to maybe four of us .", "words": ["each", "of", "was", "to", "there", "us", "about"]}, {"sent": "but they work fine with one in each hole .", "words": ["work", "each", "fine", "with", "they", "in", "but"]}, {"sent": "you had an armband on each arm didnt you ?", "words": ["each", "you", "an", "arm", "on"]}, {"sent": "you said you wanted a chocolate one and I gave you one of each .", "words": ["each", "of", "chocolate", "and", "you", "a"]}, {"sent": "all things in different places each time you play it .", "words": ["all", "each", "play", "you", "in", "it"]}, {"sent": "baby angels join my play but most of all throughout each day .", "words": ["all", "my", "each", "of", "play", "but"]}, {"sent": "I told you that they have different colors all next to each other .", "words": ["all", "each", "to", "you", "have", "they", "other", "that"]}, {"sent": "but I mean if youre saying we only got one each at Gap all he has is one blue navy .", "words": ["all", "if", "each", "we", "is", "blue", "he", "at", "but"]}, {"sent": "PostmanPat and his friends have each lost their hat .", "words": ["their", "each", "and", "have", "hat", "his"]}, {"sent": "all the birdies are talking to each other arent they ?", "words": ["the", "all", "each", "to", "other", "they", "are"]}, {"sent": "can you see theyre all standing on each others shoulders .", "words": ["can", "all", "each", "you", "on", "see"]}, {"sent": "how many are those each ?", "words": ["how", "are", "those", "each"]}, {"sent": "did you give cup cakes to each child ?", "words": ["cup", "each", "to", "you", "did", "give"]}, {"sent": "but we thought you oughta have one each .", "words": ["each", "we", "you", "have", "but"]}, {"sent": "I think theyre talking to each other cardinal must be Carmen must be talking to the other cardinal that are outside .", "words": ["the", "think", "each", "to", "be", "outside", "other", "are", "that"]}, {"sent": "each on one side of the duck .", "words": ["the", "each", "of", "duck", "on"]}, {"sent": "mother tiger washes each of the three cubs with her rough tongue .", "words": ["the", "each", "of", "with", "tiger", "tongue", "her"]}, {"sent": "what xxx car do they sit next to each other .", "words": ["what", "do", "each", "car", "to", "other", "they", "sit"]}, {"sent": "Cinderella has no money to spend on clothes so her animal friends help her to patch up a simple dress an apron each morning and shine her worn out shoes .", "words": ["money", "up", "help", "each", "dress", "to", "so", "and", "an", "animal", "out", "on", "her", "a"]}, {"sent": "and put a couple of the peanutbutter in this basket so everybody can get one of each .", "words": ["the", "can", "put", "each", "of", "so", "and", "get", "this", "in", "basket", "a"]}, {"sent": "so you get very very few sweets from me because Im trying to look after your teeth and I buy little jelly snakes that cost a penny each .", "words": ["jelly", "your", "each", "so", "to", "you", "get", "and", "little", "penny", "look", "me", "that", "buy", "because", "a"]}, {"sent": "theres a ladybird hiding in you know each each page .", "words": ["in", "each", "you", "a"]}, {"sent": "we only wants one each .", "words": ["each", "we"]}, {"sent": "and each week they look at something dont they ?", "words": ["each", "and", "they", "dont", "at", "look"]}, {"sent": "cause its fun to make the snowballs and then they throw them at each other and they go phwack .", "words": ["the", "throw", "each", "go", "to", "them", "and", "other", "they", "at", "then", "make"]}, {"sent": "now therere two one in each box .", "words": ["in", "box", "each"]}, {"sent": "then they can each have their own space .", "words": ["can", "their", "each", "have", "they", "then"]}, {"sent": "because they each build a house .", "words": ["each", "they", "build", "house", "because", "a"]}, {"sent": "we shouldnt have trouble finding each other .", "words": ["have", "other", "each", "we"]}, {"sent": "because they each wanted the same thing .", "words": ["the", "each", "they", "same", "because"]}, {"sent": "well each have half a pack .", "words": ["have", "each", "a"]}, {"sent": "now each take one from each pile .", "words": ["take", "each"]}, {"sent": "and theyve each got something on them .", "words": ["on", "them", "each", "and"]}, {"sent": "as each tooth falls out .", "words": ["tooth", "out", "each"]}, {"sent": "or one of each .", "words": ["of", "each"]}, {"sent": "we can have a page each cant we ?", "words": ["can", "each", "we", "have", "a"]}, {"sent": "oh one for each hand now .", "words": ["for", "each", "hand"]}, {"sent": "yeah well I was scanned early on in each xxx definitely pregnant youknow ?", "words": ["on", "was", "in", "each"]}, {"sent": "we have the big one and we each finished it and they didnt have a bigger one I brought this before this reminds me of sociology we were talking about .", "words": ["the", "each", "we", "of", "and", "have", "they", "this", "it", "about", "me", "big", "were", "a"]}, {"sent": "each one gets a little money ?", "words": ["money", "little", "each", "a"]}, {"sent": "theyre all telling each other to be quiet because .", "words": ["all", "each", "to", "be", "other", "quiet", "because"]}, {"sent": "with each five dollar purchase you get a free glass of lemonade .", "words": ["each", "of", "with", "you", "get", "glass", "a"]}, {"sent": "so on each of these tickets Ive chosen six numbers .", "words": ["these", "each", "of", "so", "on"]}, {"sent": "Isabelle had the same idea and it was not long before they found each other .", "words": ["the", "each", "was", "long", "and", "other", "they", "it", "same", "not"]}, {"sent": "I hope you I hope we can see each other sometime .", "words": ["can", "each", "we", "you", "other", "see"]}, {"sent": "one each ?", "words": ["each"]}, {"sent": "did you bump into each other .", "words": ["bump", "into", "each", "you", "other", "did"]}, {"sent": "are you gonna get that pizza out the oven ?", "words": ["the", "pizza", "you", "get", "are", "out", "that", "oven"]}, {"sent": "put it in the oven .", "words": ["the", "put", "in", "it", "oven"]}, {"sent": "should you have an oven ?", "words": ["have", "an", "oven", "you"]}, {"sent": "and put it in the oven for .", "words": ["the", "put", "for", "and", "in", "it", "oven"]}, {"sent": "how about if I use the oven you do the mixing .", "words": ["the", "do", "how", "if", "you", "about", "oven"]}, {"sent": "putting xxx in the oven and Grettel puts the witch in the oven .", "words": ["the", "in", "oven", "and"]}, {"sent": "put it in the oven .", "words": ["the", "put", "in", "it", "oven"]}, {"sent": "put it in the oven .", "words": ["the", "put", "in", "it", "oven"]}, {"sent": "going in duh aht oven ?", "words": ["in", "oven"]}, {"sent": "lets get the oven lit and they can go in and well be back later to put some icing on top .", "words": ["the", "can", "put", "back", "go", "be", "to", "and", "get", "they", "in", "on", "some", "oven"]}, {"sent": "ha ha you put cakes in the oven .", "words": ["the", "put", "you", "in", "oven"]}, {"sent": "okay ready to put it in the oven ?", "words": ["the", "put", "to", "in", "it", "oven"]}, {"sent": "well she Grettal shut the oven door .", "words": ["the", "she", "oven", "door"]}, {"sent": "an oven mitt .", "words": ["an", "oven"]}, {"sent": "switch the oven off .", "words": ["the", "oven", "off"]}, {"sent": "a chair on the oven ?", "words": ["the", "chair", "on", "oven", "a"]}, {"sent": "oven .", "words": ["oven"]}, {"sent": "yeah you may not know that since we dont have an oven .", "words": ["we", "oven", "you", "have", "dont", "an", "that", "not"]}, {"sent": "then the little old man opened the oven .", "words": ["the", "old", "little", "then", "oven"]}, {"sent": "Mummy will take them out of the oven and put them on a cooling rack and then when theyre completely cold we can decorate them with that nice icing .", "words": ["the", "put", "of", "nice", "when", "cold", "oven", "can", "with", "on", "then", "that", "take", "and", "will", "a", "we", "them", "out"]}, {"sent": "and whats whats that your oven ?", "words": ["your", "that", "oven", "and"]}, {"sent": "oven and freezer proof .", "words": ["oven", "and"]}, {"sent": "that pigs name is Humper Dink put the uncooked loaves of bread in the hot oven to bake .", "words": ["the", "put", "bread", "is", "of", "to", "in", "hot", "that", "oven"]}, {"sent": "and put it in the oven for .", "words": ["the", "put", "for", "and", "in", "it", "oven"]}, {"sent": "meet you in the oven ?", "words": ["the", "in", "oven", "you"]}, {"sent": "and then we can put it on the oven .", "words": ["can", "the", "put", "we", "and", "it", "on", "then", "oven"]}, {"sent": "weve got an oven mitt hanging up there .", "words": ["up", "there", "an", "oven"]}, {"sent": "throw it in the oven for Theo and me .", "words": ["the", "for", "throw", "and", "in", "it", "me", "oven"]}, {"sent": "would you like to have some of them as oven gloves ?", "words": ["of", "to", "them", "like", "you", "have", "gloves", "would", "some", "oven"]}, {"sent": "its not quite as hot as the oven is it ?", "words": ["the", "is", "oven", "hot", "it", "not"]}, {"sent": "look theres an oven .", "words": ["an", "oven", "look"]}, {"sent": "you gonna cook with your oven set ?", "words": ["your", "with", "you", "cook", "oven"]}, {"sent": "put it in the oven yeah .", "words": ["the", "put", "in", "it", "oven"]}, {"sent": "she puts things in pots and puts them in the oven .", "words": ["the", "them", "oven", "and", "in", "she"]}, {"sent": "and I think its the timer on the oven dont you ?", "words": ["the", "think", "and", "you", "dont", "on", "oven"]}, {"sent": "if we have chips we put them in the oven dont we ?", "words": ["the", "put", "if", "we", "them", "have", "dont", "in", "oven"]}, {"sent": "its in the oven .", "words": ["the", "in", "oven"]}, {"sent": "we can have a little break and a drink because these have got to bake and well hafta turn the tape recorder off while the ovens on because its a fan oven .", "words": ["can", "these", "the", "off", "drink", "we", "to", "tape", "little", "have", "and", "on", "break", "oven", "because", "a"]}, {"sent": "lets pop them in the oven its time to bake .", "words": ["the", "pop", "to", "them", "in", "oven"]}, {"sent": "even buy a couple of bags of those things of oven chips or something .", "words": ["those", "of", "oven", "buy", "a"]}, {"sent": "and we cook it in the oven ?", "words": ["the", "we", "and", "in", "cook", "it", "oven"]}, {"sent": "should we get the bread ready to go in the oven ?", "words": ["the", "bread", "we", "go", "to", "get", "in", "oven"]}, {"sent": "you gonna put the ice cream in the oven ?", "words": ["the", "ice cream", "put", "you", "in", "ice", "oven"]}, {"sent": "you need one for the oven and one for the stove the vegetables .", "words": ["the", "for", "need", "you", "and", "stove", "oven"]}, {"sent": "whyve they all got to go in the oven ?", "words": ["the", "all", "go", "to", "they", "in", "oven"]}, {"sent": "a xxx in the oven .", "words": ["the", "in", "oven", "a"]}, {"sent": "and we hafta put them in the oven .", "words": ["the", "put", "we", "them", "and", "in", "oven"]}, {"sent": "and mark it with a b and throw it in the oven for Henry and me .", "words": ["the", "for", "throw", "with", "and", "in", "it", "me", "oven", "a"]}, {"sent": "or even your oven .", "words": ["your", "oven"]}, {"sent": "theres your pan for the oven .", "words": ["the", "for", "your", "oven"]}, {"sent": "are you going to put the oven on ?", "words": ["the", "put", "to", "you", "are", "on", "oven"]}, {"sent": "its in the oven .", "words": ["the", "in", "oven"]}, {"sent": "no open the oven and no cook it well just eat the cake without cooking it thats okay right ?", "words": ["the", "cake", "open", "and", "eat", "cook", "it", "oven"]}, {"sent": "theres banana on the oven .", "words": ["on", "banana", "oven", "the"]}, {"sent": "the oven is hot ?", "words": ["the", "hot", "oven", "is"]}, {"sent": "put it in the oven ?", "words": ["the", "put", "in", "it", "oven"]}, {"sent": "thats the oven .", "words": ["the", "oven"]}, {"sent": "thats the oven .", "words": ["the", "oven"]}, {"sent": "ice cream in the oven ?", "words": ["the", "ice cream", "in", "ice", "oven"]}, {"sent": "chips that you just spread on a baking sheet and bake in the oven .", "words": ["the", "and", "you", "in", "on", "that", "oven", "a"]}, {"sent": "this is this is the oven .", "words": ["the", "this", "oven", "is"]}, {"sent": "well weve got two more to do and then we can pop them in the oven .", "words": ["can", "do", "the", "pop", "we", "to", "them", "and", "more", "in", "then", "oven"]}, {"sent": "dont put them all in the oven .", "words": ["the", "all", "put", "them", "dont", "in", "oven"]}, {"sent": "no er thats a oven .", "words": ["oven", "a"]}, {"sent": "then we turn oven off .", "words": ["off", "then", "oven", "we"]}, {"sent": "what did we make in the oven ?", "words": ["the", "what", "we", "in", "did", "make", "oven"]}, {"sent": "ooh get him off your oven your stove .", "words": ["off", "your", "get", "him", "stove", "oven"]}, {"sent": "an oven .", "words": ["an", "oven"]}, {"sent": "would you like to see it before it goes in the oven ?", "words": ["the", "to", "like", "you", "in", "it", "would", "see", "oven"]}, {"sent": "Ill put the oven on .", "words": ["the", "put", "on", "oven"]}, {"sent": "you get it in the oven .", "words": ["the", "you", "get", "in", "it", "oven"]}, {"sent": "you put ice cream cones in the oven ?", "words": ["the", "ice cream", "put", "you", "in", "ice", "oven"]}, {"sent": "lets take the potatos out of the oven .", "words": ["the", "of", "out", "take", "oven"]}, {"sent": "the oven and this is what the oven looks like ?", "words": ["the", "what", "is", "like", "and", "this", "oven"]}, {"sent": "but okay you can have some applesauce before dinner but first you wanna put your dinner in the oven .", "words": ["can", "the", "put", "your", "first", "you", "have", "in", "wanna", "some", "applesauce", "oven", "but"]}, {"sent": "your oven ?", "words": ["your", "oven"]}, {"sent": "just put it in the oven and come on back .", "words": ["the", "put", "back", "and", "in", "it", "on", "oven"]}, {"sent": "you wanna put the chair by the oven ?", "words": ["the", "put", "chair", "you", "by", "wanna", "oven"]}, {"sent": "are you gonna open the oven door then ?", "words": ["the", "open", "you", "are", "then", "oven", "door"]}, {"sent": "that is the oven .", "words": ["the", "that", "oven", "is"]}, {"sent": "oh the mixing bowl that comes with your baking oven .", "words": ["the", "bowl", "your", "with", "that", "oven"]}, {"sent": "oven is that the oven ?", "words": ["the", "that", "oven", "is"]}, {"sent": "isnt it and is that pie going in the oven ?", "words": ["the", "is", "and", "in", "it", "that", "oven"]}, {"sent": "shake them throw them in the oven .", "words": ["the", "throw", "them", "shake", "in", "oven"]}, {"sent": "it just came out of the oven right ?", "words": ["the", "of", "it", "out", "oven"]}, {"sent": "Thomas would you like to have a look at the flapjack now it has come out of the oven ?", "words": ["the", "look", "of", "to", "like", "you", "have", "at", "it", "out", "would", "oven", "a"]}, {"sent": "the piece I had in the oven I changed my mind xxx .", "words": ["the", "in", "oven", "my"]}, {"sent": "straight in the oven look .", "words": ["the", "in", "oven", "look"]}, {"sent": "put it in the oven for .", "words": ["the", "put", "for", "in", "it", "oven"]}, {"sent": "push the witch in the oven .", "words": ["the", "in", "push", "oven"]}, {"sent": "and put in the oven for .", "words": ["the", "put", "for", "and", "in", "oven"]}, {"sent": "Mom opened the oven she sniffed and she looked .", "words": ["the", "oven", "and", "she"]}, {"sent": "let me just take something outof the oven .", "words": ["the", "me", "take", "oven"]}, {"sent": "you slice it up put it in the oven and then it will come out as a crab ?", "words": ["up", "the", "put", "you", "and", "in", "it", "out", "will", "then", "oven", "a"]}, {"sent": "no we hafta put them in the oven first .", "words": ["the", "put", "we", "first", "them", "in", "oven"]}, {"sent": "are they going in Mummys oven or your oven ?", "words": ["your", "they", "in", "are", "oven"]}, {"sent": "um we got the chocolate chip cookies in the oven .", "words": ["the", "we", "chocolate", "in", "oven"]}, {"sent": "oops theres the oven timer .", "words": ["the", "oven"]}, {"sent": "put it in the oven ?", "words": ["the", "put", "in", "it", "oven"]}, {"sent": "it got broken .", "words": ["it", "broken"]}, {"sent": "xxx the lights broken .", "words": ["the", "broken"]}, {"sent": "the broken hearted dwarfs watched over Snow White day and night one day the prince appeared as he had been searching for the beautiful princess .", "words": ["the", "for", "and", "broken", "he", "over"]}, {"sent": "you know my cars broken Eleanor ?", "words": ["my", "broken", "you"]}, {"sent": "no its broken right now .", "words": ["broken"]}, {"sent": "broken handle .", "words": ["broken"]}, {"sent": "are they broken ?", "words": ["they", "are", "broken"]}, {"sent": "another broken thing .", "words": ["another", "broken"]}, {"sent": "I think its broken permanently .", "words": ["think", "broken"]}, {"sent": "must be broken because they fell from the truck .", "words": ["the", "be", "truck", "broken", "they", "because"]}, {"sent": "have your legs broken ?", "words": ["have", "your", "broken"]}, {"sent": "why is the car broken Warren ?", "words": ["the", "is", "car", "broken", "why"]}, {"sent": "are they broken ?", "words": ["they", "are", "broken"]}, {"sent": "I think thats broken off your sunglasses hasnt it ?", "words": ["off", "think", "your", "broken", "it"]}, {"sent": "its broken as well .", "words": ["broken"]}, {"sent": "uhoh I think I have broken down actually .", "words": ["have", "think", "down", "broken"]}, {"sent": "the hoovers broken .", "words": ["the", "broken"]}, {"sent": "oh no the gate got broken ?", "words": ["the", "broken"]}, {"sent": "he had a broken leg ?", "words": ["he", "leg", "broken", "a"]}, {"sent": "broken my nail .", "words": ["my", "nail", "broken"]}, {"sent": "it doesnt sound very broken to me .", "words": ["to", "me", "it", "broken"]}, {"sent": "I think hes probably broken Becky .", "words": ["think", "broken"]}, {"sent": "that was broken already honey .", "words": ["was", "that", "broken"]}, {"sent": "you want a broken chair ?", "words": ["chair", "broken", "a", "you"]}, {"sent": "has he broken his leg ?", "words": ["his", "he", "leg", "broken"]}, {"sent": "its broken ?", "words": ["broken"]}, {"sent": "oh that is a broken cup .", "words": ["cup", "is", "broken", "that", "a"]}, {"sent": "yeah its broken .", "words": ["broken"]}, {"sent": "ah yellow ones are broken .", "words": ["yellow", "are", "broken"]}, {"sent": "if they were real eggs theyd be broken .", "words": ["if", "be", "broken", "they", "were"]}, {"sent": "I think it was something that got broken when we moved house .", "words": ["think", "when", "we", "was", "broken", "it", "house", "that"]}, {"sent": "you havent broken it .", "words": ["it", "broken", "you"]}, {"sent": "Ces where should we take it if it is broken down ?", "words": ["where", "if", "we", "is", "broken", "it", "down", "take"]}, {"sent": "the tow ropes broken again hasnt it ?", "words": ["the", "it", "broken"]}, {"sent": "and if this parts broken .", "words": ["this", "if", "broken", "and"]}, {"sent": "has it broken down ?", "words": ["down", "it", "broken"]}, {"sent": "poor Mummy will get into trouble if Ive broken it .", "words": ["if", "into", "poor", "get", "broken", "it", "will"]}, {"sent": "why are the wheels broken ?", "words": ["the", "are", "why", "broken"]}, {"sent": "have you broken it again ?", "words": ["have", "it", "broken", "you"]}, {"sent": "well we know that BluesClues one is broken .", "words": ["that", "broken", "is", "we"]}, {"sent": "because this one is broken now .", "words": ["this", "broken", "because", "is"]}, {"sent": "this is really broken .", "words": ["this", "broken", "is"]}, {"sent": "what we say is its something that broke or that has broken been broken .", "words": ["what", "is", "we", "say", "broken", "that"]}, {"sent": "alotof them are broken .", "words": ["are", "them", "broken"]}, {"sent": "yes its broken .", "words": ["broken"]}, {"sent": "no you havent broken it .", "words": ["it", "broken", "you"]}, {"sent": "theyre not broken .", "words": ["not", "broken"]}, {"sent": "can I have a fresh loaf please thats broken .", "words": ["can", "have", "broken", "a"]}, {"sent": "oh its got a bit broken this .", "words": ["this", "broken", "a"]}, {"sent": "everythings broken .", "words": ["broken"]}, {"sent": "its broken .", "words": ["broken"]}, {"sent": "because its broken a little bit .", "words": ["a", "little", "because", "broken"]}, {"sent": "youve broken them all .", "words": ["all", "them", "broken"]}, {"sent": "youve broken it .", "words": ["it", "broken"]}, {"sent": "yah got a broken arm ?", "words": ["arm", "broken", "a"]}, {"sent": "hes broken it .", "words": ["it", "broken"]}, {"sent": "broken ?", "words": ["broken"]}, {"sent": "well you cant go around in you cant drive around a crane thats broken can you ?", "words": ["can", "go", "you", "drive", "broken", "in", "around", "a"]}, {"sent": "its not broken .", "words": ["not", "broken"]}, {"sent": "its broken isnt it ?", "words": ["it", "broken"]}, {"sent": "and baby bear found out his chair was broken .", "words": ["was", "chair", "bear", "and", "broken", "out", "his"]}, {"sent": "the steering wheels broken .", "words": ["the", "broken"]}, {"sent": "are they all broken now ?", "words": ["all", "they", "are", "broken"]}, {"sent": "its broken .", "words": ["broken"]}, {"sent": "I think that sides broken .", "words": ["think", "that", "broken"]}, {"sent": "shall we have a broken down car instead ?", "words": ["we", "car", "broken", "have", "down", "a"]}, {"sent": "oh dear its broken .", "words": ["broken"]}, {"sent": "any more broken cars ?", "words": ["more", "any", "broken"]}, {"sent": "no this ones not broken .", "words": ["this", "not", "broken"]}, {"sent": "I really have one piece broken in half .", "words": ["have", "in", "broken"]}, {"sent": "youve broken it ?", "words": ["it", "broken"]}, {"sent": "no that that televisions broken .", "words": ["that", "broken"]}, {"sent": "oh is that one broken ?", "words": ["that", "broken", "is"]}, {"sent": "no its not broken .", "words": ["not", "broken"]}, {"sent": "or is he going to go and help the broken down car ?", "words": ["the", "help", "go", "is", "to", "car", "and", "broken", "he", "down"]}, {"sent": "you cant get the bat out because the zip is broken .", "words": ["the", "is", "you", "get", "broken", "out", "bat", "because"]}, {"sent": "its not broken Dominic .", "words": ["not", "broken"]}, {"sent": "it hasnt really broken has it ?", "words": ["it", "broken"]}, {"sent": "okay watch see this is broken .", "words": ["watch", "is", "broken", "this", "see"]}, {"sent": "that looks like a Land rover that has broken down .", "words": ["like", "broken", "down", "that", "a"]}, {"sent": "is that button broken ?", "words": ["that", "broken", "button", "is"]}, {"sent": "what is broken again ?", "words": ["what", "broken", "is"]}, {"sent": "this is broken Thomas this um Mecado man .", "words": ["this", "broken", "is"]}, {"sent": "dont want his hand to get broken do we ?", "words": ["do", "hand", "we", "to", "broken", "get", "dont", "his"]}, {"sent": "and the next thing we have broken you know broken xxx .", "words": ["the", "we", "broken", "have", "and", "you"]}, {"sent": "now hes broken his crayon .", "words": ["his", "crayon", "broken"]}, {"sent": "is it broken ?", "words": ["it", "broken", "is"]}, {"sent": "oh the red ones broken isnt it ?", "words": ["the", "it", "red", "broken"]}, {"sent": "is it broken ?", "words": ["it", "broken", "is"]}, {"sent": "so was anything broken when you went .", "words": ["was", "so", "you", "broken", "when"]}, {"sent": "broken heart .", "words": ["broken"]}, {"sent": "but thats broken isnt it ?", "words": ["it", "but", "broken"]}, {"sent": "its broken .", "words": ["broken"]}, {"sent": "how can you see when youve broken it .", "words": ["can", "how", "you", "broken", "it", "see", "when"]}, {"sent": "I wont be able to bring this one because it will be broken .", "words": ["to", "be", "bring", "broken", "this", "it", "will", "because"]}, {"sent": "is it the engine thats broken down ?", "words": ["the", "is", "broken", "it", "down"]}, {"sent": "the Cherios were broken .", "words": ["the", "were", "broken"]}, {"sent": "somethings broken ?", "words": ["broken"]}, {"sent": "it has broken .", "words": ["it", "broken"]}, {"sent": "thats cute .", "words": ["cute"]}, {"sent": "werent you cute then ?", "words": ["cute", "then", "you"]}, {"sent": "thats a nice box I like those boxes those are cute .", "words": ["box", "those", "cute", "nice", "like", "are", "a"]}, {"sent": "yeah I think hes really cute .", "words": ["cute", "think"]}, {"sent": "cute huh ?", "words": ["cute"]}, {"sent": "cute stuff .", "words": ["cute"]}, {"sent": "mm how cute .", "words": ["cute", "how"]}, {"sent": "youre cute stuff arent you ?", "words": ["cute", "you"]}, {"sent": "wow so cute oh so cute .", "words": ["cute", "so"]}, {"sent": "was he cute ?", "words": ["cute", "was", "he"]}, {"sent": "I thought that was cute his hers and ours .", "words": ["cute", "was", "hers", "and", "his", "that"]}, {"sent": "tell her how cute she looks today .", "words": ["cute", "how", "her", "she"]}, {"sent": "yeah more in theory than in actuality a lot of time you dont want to watch Roly Poly Oly but hes cute .", "words": ["watch", "of", "to", "cute", "a lot", "you", "dont", "in", "more", "want to", "but", "a"]}, {"sent": "its cute .", "words": ["cute"]}, {"sent": "oh thats cute .", "words": ["cute"]}, {"sent": "this one is so cute .", "words": ["cute", "this", "so", "is"]}, {"sent": "is it cute ?", "words": ["cute", "it", "is"]}, {"sent": "thats the baby Nemo look how cute .", "words": ["the", "cute", "how", "look"]}, {"sent": "aw thats so cute wheres his ears ?", "words": ["cute", "so", "his"]}, {"sent": "its cute ?", "words": ["cute"]}, {"sent": "isnt he a cute squirrel ?", "words": ["cute", "he", "squirrel", "a"]}, {"sent": "my cutie cutie cute ?", "words": ["cute", "my"]}, {"sent": "its pretty cute .", "words": ["cute", "pretty"]}, {"sent": "is he cute ?", "words": ["cute", "he", "is"]}, {"sent": "youre a cute little girl .", "words": ["cute", "little", "a"]}, {"sent": "xxx cute little baby .", "words": ["cute", "little"]}, {"sent": "these are cute pictures of you Seth .", "words": ["these", "cute", "of", "you", "are"]}, {"sent": "that looks cute .", "words": ["cute", "that"]}, {"sent": "cute little duck .", "words": ["cute", "duck", "little"]}, {"sent": "hes a cute little guy .", "words": ["cute", "little", "a"]}, {"sent": "dinosaurs cute dinosaurs not cute dinosaurs early dinosaurs later look at that .", "words": ["look", "cute", "at", "that", "not"]}, {"sent": "cute babies .", "words": ["cute"]}, {"sent": "isnt that cute ?", "words": ["cute", "that"]}, {"sent": "isnt that cute ?", "words": ["cute", "that"]}, {"sent": "theyre very cute .", "words": ["cute"]}, {"sent": "look at these little purple slippers theyre so cute .", "words": ["these", "look", "cute", "so", "little", "at"]}, {"sent": "that was cute .", "words": ["cute", "was", "that"]}, {"sent": "are you cute ?", "words": ["cute", "are", "you"]}, {"sent": "aw shes so cute .", "words": ["cute", "so"]}, {"sent": "its cute .", "words": ["cute"]}, {"sent": "oh this a very cute tea set .", "words": ["cute", "this", "a"]}, {"sent": "aw its so cute .", "words": ["cute", "so"]}, {"sent": "isnt that a cute name for a little girl ?", "words": ["for", "cute", "little", "that", "a"]}, {"sent": "this would be cute .", "words": ["cute", "this", "be", "would"]}, {"sent": "aw that LizzieLou is so cute .", "words": ["cute", "so", "that", "is"]}, {"sent": "oh he is cute he is cute .", "words": ["cute", "he", "is"]}, {"sent": "its very cute .", "words": ["cute"]}, {"sent": "isnt that cute ?", "words": ["cute", "that"]}, {"sent": "youre very cute .", "words": ["cute"]}, {"sent": "its cute to see them kinda like trying to figure out what goes where .", "words": ["where", "what", "cute", "to", "them", "like", "out", "see"]}, {"sent": "arent you cute ?", "words": ["cute", "you"]}, {"sent": "look at that cute little girl .", "words": ["look", "cute", "little", "at", "that"]}, {"sent": "I know these are words that like you dont necessarily want her saying in public but its just so cute we cant help ourselves .", "words": ["these", "help", "we", "cute", "so", "her", "like", "you", "dont", "in", "are", "that", "but"]}, {"sent": "I do too I think thats really cute .", "words": ["cute", "do", "think", "too"]}, {"sent": "so cute .", "words": ["cute", "so"]}, {"sent": "right cute .", "words": ["cute"]}, {"sent": "is that a cute picture of Nina ?", "words": ["is", "cute", "of", "picture", "that", "a"]}, {"sent": "theyre cute .", "words": ["cute"]}, {"sent": "its no good putting your head on one side looking cute .", "words": ["your", "cute", "head", "good", "on"]}, {"sent": "look how cute .", "words": ["cute", "how", "look"]}, {"sent": "that is so cute .", "words": ["cute", "so", "that", "is"]}, {"sent": "ah arent they cute ?", "words": ["cute", "they"]}, {"sent": "a cute little rabbit .", "words": ["cute", "little", "a"]}, {"sent": "look how cute it came out .", "words": ["how", "cute", "it", "out", "look"]}, {"sent": "oh that is cute .", "words": ["cute", "that", "is"]}, {"sent": "so he pokes his head in that room and hello cute little small hall .", "words": ["cute", "so", "head", "and", "little", "in", "room", "he", "his", "that"]}, {"sent": "hes so cute and all alone can we keep him ?", "words": ["can", "all", "we", "cute", "so", "and", "him"]}, {"sent": "thats very cute .", "words": ["cute"]}, {"sent": "oh cute .", "words": ["cute"]}, {"sent": "thats a cute little car isnt it ?", "words": ["cute", "car", "little", "it", "a"]}, {"sent": "oh thats cute little pjs .", "words": ["cute", "little"]}, {"sent": "its a very cute one .", "words": ["cute", "a"]}, {"sent": "isnt that cute ?", "words": ["cute", "that"]}, {"sent": "arent you cute ?", "words": ["cute", "you"]}, {"sent": "I saw some cute little Navajo babas today .", "words": ["cute", "some", "little"]}, {"sent": "mm kissing Daddy through the veil thats very cute .", "words": ["the", "cute"]}, {"sent": "those are cute .", "words": ["cute", "are", "those"]}, {"sent": "because you are so cute in this box .", "words": ["box", "cute", "so", "you", "this", "in", "are", "because"]}, {"sent": "its cute .", "words": ["cute"]}, {"sent": "I think theyre cute too .", "words": ["cute", "think", "too"]}, {"sent": "how cute .", "words": ["cute", "how"]}, {"sent": "Purdie has got a cute little nose hasnt she ?", "words": ["cute", "little", "she", "a", "nose"]}, {"sent": "thats a pretty cute little sthing .", "words": ["cute", "pretty", "little", "a"]}, {"sent": "youre so cute .", "words": ["cute", "so"]}, {"sent": "is he cute ?", "words": ["cute", "he", "is"]}, {"sent": "these are cute .", "words": ["cute", "these", "are"]}, {"sent": "cute .", "words": ["cute"]}, {"sent": "because I was saying its cute .", "words": ["cute", "was", "because"]}, {"sent": "what isnt that cute ?", "words": ["cute", "what", "that"]}, {"sent": "oh arent they cute ?", "words": ["cute", "they"]}, {"sent": "thats a cute baby .", "words": ["cute", "a"]}, {"sent": "I was thinking maybe wed buy her some cute clothes .", "words": ["cute", "was", "some", "her", "buy"]}, {"sent": "aw thats cute .", "words": ["cute"]}, {"sent": "wow I love this Olaf its so cute .", "words": ["cute", "this", "so", "love"]}, {"sent": "thats cute .", "words": ["cute"]}, {"sent": "youre cute stuff .", "words": ["cute"]}, {"sent": "how cute .", "words": ["cute", "how"]}, {"sent": "youre so cute .", "words": ["cute", "so"]}, {"sent": "you go a cute little belly .", "words": ["go", "cute", "little", "you", "a"]}, {"sent": "its a cute book .", "words": ["cute", "book", "a"]}, {"sent": "Whiskers with tuna in jelly .", "words": ["jelly", "in", "with", "tuna"]}, {"sent": "and jelly ?", "words": ["jelly", "and"]}, {"sent": "jelly on the plate jelly on the plate .", "words": ["jelly", "the", "on", "plate"]}, {"sent": "and we go and buy jelly crocodiles .", "words": ["jelly", "go", "we", "and", "buy"]}, {"sent": "a jelly teddy out of yellow jelly .", "words": ["jelly", "of", "out", "yellow", "a"]}, {"sent": "its the only snake with a jelly ring and its the only snake called Thomas .", "words": ["the", "jelly", "with", "and", "a"]}, {"sent": "jelly .", "words": ["jelly"]}, {"sent": "jelly spider .", "words": ["jelly"]}, {"sent": "if its not that its jelly .", "words": ["jelly", "not", "that", "if"]}, {"sent": "you like jelly dont you ?", "words": ["jelly", "dont", "like", "you"]}, {"sent": "mint jelly yuck .", "words": ["jelly"]}, {"sent": "oh youre not having a jelly sweet for breakfast .", "words": ["jelly", "for", "not", "a"]}, {"sent": "jelly was pink ?", "words": ["jelly", "was"]}, {"sent": "jelly we saw a big thing of jellyfish didnt we ?", "words": ["jelly", "we", "of", "big", "a"]}, {"sent": "peanutbutter and jelly .", "words": ["jelly", "and"]}, {"sent": "you like jelly and ice cream dont you ?", "words": ["jelly", "ice cream", "like", "and", "you", "dont", "ice"]}, {"sent": "do you wanna make some jelly ?", "words": ["jelly", "do", "you", "wanna", "some", "make"]}, {"sent": "no dont touch that because if you knock the jelly over youll get burnt .", "words": ["the", "jelly", "if", "touch", "you", "get", "dont", "knock", "over", "that", "because"]}, {"sent": "and you had jelly babies one day .", "words": ["jelly", "you", "and"]}, {"sent": "ya gonna put the jelly where it goes ?", "words": ["the", "jelly", "put", "where", "it"]}, {"sent": "no more jelly babies for little boys who say that .", "words": ["jelly", "for", "say", "little", "more", "who", "that"]}, {"sent": "we I think you were saying jelly on the plate then .", "words": ["jelly", "the", "think", "plate", "we", "you", "on", "then", "were"]}, {"sent": "jelly mouse .", "words": ["jelly", "mouse"]}, {"sent": "you dont have jelly cakes darling .", "words": ["have", "dont", "jelly", "you"]}, {"sent": "you had jelly ?", "words": ["jelly", "you"]}, {"sent": "jelly on the plate .", "words": ["jelly", "the", "on", "plate"]}, {"sent": "jelly that is right .", "words": ["jelly", "that", "is"]}, {"sent": "and you pour your jelly into it and then when .", "words": ["jelly", "your", "into", "pour", "you", "and", "it", "then", "when"]}, {"sent": "theres no jelly in it .", "words": ["jelly", "in", "it"]}, {"sent": "jelly belly .", "words": ["jelly"]}, {"sent": "I dont think weve got any jelly babies .", "words": ["jelly", "dont", "think", "any"]}, {"sent": "your treat today were the jelly babies youve just eaten and this .", "words": ["the", "jelly", "your", "and", "this", "were"]}, {"sent": "theyre like jumping jelly beans arent they ?", "words": ["jelly", "they", "beans", "like"]}, {"sent": "xxx jelly .", "words": ["jelly"]}, {"sent": "can I have a jelly tot ?", "words": ["can", "jelly", "have", "a"]}, {"sent": "I think youre more into jelly type sweets now arent you ?", "words": ["jelly", "think", "into", "you", "more"]}, {"sent": "jelly that dont look like jelly .", "words": ["jelly", "like", "dont", "look", "that"]}, {"sent": "can you see any jelly anywhere ?", "words": ["can", "jelly", "any", "you", "see"]}, {"sent": "youve just had a little jelly snake .", "words": ["jelly", "little", "a"]}, {"sent": "do you know why weve got so many pussy cat jelly moulds ?", "words": ["jelly", "do", "so", "you", "cat", "why"]}, {"sent": "one jelly sweet .", "words": ["jelly"]}, {"sent": "he had jelly as well didnt he ?", "words": ["jelly", "he"]}, {"sent": "did you enjoy your jelly ?", "words": ["jelly", "your", "did", "you"]}, {"sent": "she had a bag of jelly babies red and green and yellow .", "words": ["jelly", "red", "of", "and", "green", "yellow", "she", "a"]}, {"sent": "jelly jelly jelly jellys .", "words": ["jelly"]}, {"sent": "they had a big bowl of green jelly and a big bowl of mud .", "words": ["bowl", "jelly", "of", "and", "they", "green", "big", "a"]}, {"sent": "spitting out jelly balls .", "words": ["jelly", "out"]}, {"sent": "jelly oh I loved jelly .", "words": ["jelly"]}, {"sent": "well I think theyre still reading the book about dont stick your finger in the jelly Nellie .", "words": ["the", "jelly", "think", "stick", "your", "dont", "finger", "in", "about", "book"]}, {"sent": "jelly on .", "words": ["jelly", "on"]}, {"sent": "jelly on the plate .", "words": ["jelly", "the", "on", "plate"]}, {"sent": "shall I have jelly ?", "words": ["have", "jelly"]}, {"sent": "would you like some ice cream with your jelly at tea time ?", "words": ["jelly", "ice cream", "your", "with", "like", "you", "at", "would", "ice", "some"]}, {"sent": "we dont have any jelly .", "words": ["jelly", "we", "any", "have", "dont"]}, {"sent": "but I thought that jelly ring was going to make you happier .", "words": ["jelly", "was", "to", "you", "make", "that", "but"]}, {"sent": "its like jelly .", "words": ["jelly", "like"]}, {"sent": "but its not the same sort of jelly as jelly sweets .", "words": ["the", "jelly", "of", "same", "not", "but"]}, {"sent": "would you like a jelly baby now ?", "words": ["jelly", "like", "you", "would", "a"]}, {"sent": "the peanut butter and jelly song ?", "words": ["the", "jelly", "and", "peanut butter", "butter"]}, {"sent": "thats a little fairy cake with a jelly tot on .", "words": ["jelly", "cake", "with", "little", "on", "a"]}, {"sent": "you had jelly at school ?", "words": ["jelly", "school", "at", "you"]}, {"sent": "j for jelly .", "words": ["jelly", "for"]}, {"sent": "right these are jelly moulds .", "words": ["jelly", "these", "are"]}, {"sent": "do you like jelly and custard Thomas ?", "words": ["jelly", "do", "like", "and", "you"]}, {"sent": "grape jelly .", "words": ["jelly"]}, {"sent": "you can be chief jelly maker .", "words": ["can", "be", "jelly", "you"]}, {"sent": "okay lets find the jelly .", "words": ["the", "jelly", "find"]}, {"sent": "another sachet of jelly .", "words": ["another", "of", "jelly"]}, {"sent": "wheres the jelly ?", "words": ["the", "jelly"]}, {"sent": "frogs lay their eggs in clumps of clear jelly which float to the surface of the pond .", "words": ["jelly", "the", "their", "of", "to", "in", "which"]}, {"sent": "we havent any jelly babies Thomas .", "words": ["jelly", "any", "we"]}, {"sent": "I think it must be um off the um jelly babies off the jelly babies dont you ?", "words": ["off", "the", "think", "jelly", "be", "you", "dont", "it"]}, {"sent": "jelly .", "words": ["jelly"]}, {"sent": "peanutbutter and jelly .", "words": ["jelly", "and"]}, {"sent": "dont put your finger in the jelly Nelly .", "words": ["the", "jelly", "put", "your", "dont", "finger", "in"]}, {"sent": "oh jelly .", "words": ["jelly"]}, {"sent": "jelly ?", "words": ["jelly"]}, {"sent": "oh look theres one jelly sweet left .", "words": ["jelly", "look"]}, {"sent": "I like um jelly with mayonnaise .", "words": ["jelly", "with", "like"]}, {"sent": "cranberry jelly and honey baked hams .", "words": ["jelly", "and"]}, {"sent": "thats a jelly apple .", "words": ["jelly", "apple", "a"]}, {"sent": "peanut butter and jelly .", "words": ["jelly", "peanut butter", "butter", "and"]}, {"sent": "jelly spoon .", "words": ["jelly", "spoon"]}, {"sent": "wanna go get the jelly .", "words": ["the", "jelly", "go", "get", "wanna"]}, {"sent": "jelly on the plate .", "words": ["jelly", "the", "on", "plate"]}, {"sent": "daddy wherere you going with gonna do with the jelly ?", "words": ["the", "do", "jelly", "with", "you"]}, {"sent": "because theyre like little jelly sweets .", "words": ["jelly", "like", "because", "little"]}, {"sent": "does it look like a jelly ?", "words": ["jelly", "does", "like", "it", "look", "a"]}, {"sent": "some jelly ?", "words": ["jelly", "some"]}, {"sent": "jelly beans .", "words": ["jelly", "beans"]}, {"sent": "xxx jelly today .", "words": ["jelly"]}, {"sent": "I thought we could put a jelly baby on each one .", "words": ["jelly", "put", "each", "we", "on", "could", "a"]}, {"sent": "grandma makes you jelly doesnt she ?", "words": ["jelly", "she", "you"]}, {"sent": "peanutbutter and jelly .", "words": ["jelly", "and"]}, {"sent": "now this particular day you came home with three absolutely huge jelly snakes that had cost twenty five pence each because when you went in the sweet shop with Daddy what did you tell Daddy ?", "words": ["jelly", "the", "what", "when", "each", "with", "you", "this", "in", "did", "that", "because", "home"]}, {"sent": "has Jwww got your toy ?", "words": ["toy", "your"]}, {"sent": "toy train how many blocks in this pattern ?", "words": ["how", "train", "this", "in", "toy"]}, {"sent": "she knows what the toy some of the toys but she doesnt .", "words": ["the", "what", "of", "toy", "some", "she", "but"]}, {"sent": "its a toy .", "words": ["toy", "a"]}, {"sent": "theyre all in here in your toy box .", "words": ["all", "your", "box", "here", "in", "toy"]}, {"sent": "want me to spell toy ?", "words": ["to", "me", "toy"]}, {"sent": "whos his favorite toy ?", "words": ["his", "toy"]}, {"sent": "thats the little pram toy that we useta have .", "words": ["the", "we", "little", "have", "toy", "that"]}, {"sent": "get off my get off my toy .", "words": ["get", "toy", "my", "off"]}, {"sent": "its not a real one its a toy one is it ?", "words": ["is", "toy", "it", "not", "a"]}, {"sent": "a toy ?", "words": ["toy", "a"]}, {"sent": "what did you do with that toy ?", "words": ["what", "do", "with", "you", "toy", "did", "that"]}, {"sent": "its a toy one .", "words": ["toy", "a"]}, {"sent": "what pink toy you mean ?", "words": ["what", "toy", "you"]}, {"sent": "and this back in your toy cupboard .", "words": ["your", "back", "and", "this", "in", "toy"]}, {"sent": "is he still in the toy box ?", "words": ["the", "box", "is", "toy", "in", "he"]}, {"sent": "which is basically a toy right now .", "words": ["toy", "which", "a", "is"]}, {"sent": "or not even toy because the others theyre werent toys theyre just objects .", "words": ["the", "toy", "not", "because"]}, {"sent": "yeah um and uh lets see and bang the toy and um can we fit the toy in this is a lot of fun and uhoh whered the toy go you know we put it in bins are you talking about just here or at home ?", "words": ["the", "put", "go", "of", "about", "can", "fit", "it", "are", "home", "is", "a lot", "and", "you", "here", "toy", "see", "a", "we", "this", "in", "at"]}, {"sent": "now that you can walk this toy will be very amusing .", "words": ["can", "be", "you", "this", "that", "toy", "will", "walk"]}, {"sent": "its just a pretend toy .", "words": ["toy", "pretend", "a"]}, {"sent": "yeah its a little toy windup toy .", "words": ["toy", "little", "a"]}, {"sent": "you want that toy ?", "words": ["toy", "that", "you"]}, {"sent": "not a toy .", "words": ["toy", "not", "a"]}, {"sent": "its not for its not a toy though right because you gotta be careful it cuts you it hurts .", "words": ["for", "be", "you", "toy", "it", "careful", "not", "because", "a"]}, {"sent": "show me how big toy is .", "words": ["how", "is", "show", "toy", "me", "big"]}, {"sent": "in your toy box ?", "words": ["toy", "in", "box", "your"]}, {"sent": "then theyve got bubbles a balloon chocolate bar piece of cake and instead of putting a little toy in which we dont know what to put we could put in one of those windmills .", "words": ["what", "put", "of", "balloon", "could", "which", "bubbles", "those", "then", "cake", "to", "and", "little", "dont", "toy", "a", "we", "chocolate", "in"]}, {"sent": "and ten toy houses .", "words": ["toy", "and"]}, {"sent": "now what is the little girl going to take to camp for a toy ?", "words": ["the", "what", "for", "is", "to", "little", "toy", "take", "a"]}, {"sent": "here toy .", "words": ["here", "toy"]}, {"sent": "lets try to get a different toy .", "words": ["to", "get", "toy", "try", "try to", "a"]}, {"sent": "somebody hit it with a toy maybe .", "words": ["with", "toy", "it", "hit", "a"]}, {"sent": "its just a toy it cant crawl .", "words": ["toy", "it", "a"]}, {"sent": "what toy do you want ?", "words": ["what", "toy", "do", "you"]}, {"sent": "you have a toy box .", "words": ["box", "you", "have", "toy", "a"]}, {"sent": "that seems like the most successful toy shes ever had too practically .", "words": ["the", "like", "toy", "too", "that"]}, {"sent": "you going to get another toy ?", "words": ["to", "you", "get", "another", "toy"]}, {"sent": "I told her the other night xxx when I found the toy .", "words": ["the", "when", "other", "toy", "her"]}, {"sent": "what toy what does Dora have ?", "words": ["have", "what", "toy", "does"]}, {"sent": "you wanna get a toy ?", "words": ["you", "get", "toy", "wanna", "a"]}, {"sent": "another soft toy .", "words": ["another", "toy", "soft"]}, {"sent": "and basically anything thats not designed to be a toy she prefers .", "words": ["to", "be", "and", "toy", "she", "not", "a"]}, {"sent": "I think he likes the texture of that bumblebee stuffed toy .", "words": ["the", "think", "of", "toy", "he", "that"]}, {"sent": "on your toy box and it has ripped my blouse .", "words": ["my", "box", "your", "and", "toy", "it", "on"]}, {"sent": "well yeah he Imean he probably recognizes peas because we both eat peas and have toy peas at home .", "words": ["we", "peas", "and", "eat", "have", "toy", "he", "at", "because", "home"]}, {"sent": "toy gerbil ?", "words": ["toy"]}, {"sent": "look at this toy .", "words": ["this", "toy", "at", "look"]}, {"sent": "we saved it in the toy box .", "words": ["the", "box", "we", "toy", "in", "it"]}, {"sent": "because my daddy spended a hundred dollars on this mess and its just thrown all up in the toy box .", "words": ["up", "all", "my", "the", "box", "and", "this", "in", "toy", "on", "because", "a"]}, {"sent": "youve got a telephone in your toy box .", "words": ["your", "box", "toy", "in", "telephone", "a"]}, {"sent": "can you say toy ?", "words": ["can", "toy", "say", "you"]}, {"sent": "thats a very very good toy isnt it ?", "words": ["toy", "it", "good", "a"]}, {"sent": "and that rabbits a toy .", "words": ["toy", "that", "a", "and"]}, {"sent": "which toy ?", "words": ["toy", "which"]}, {"sent": "what kind of toy a rooster ?", "words": ["what", "of", "rooster", "toy", "a"]}, {"sent": "that spells toy right there .", "words": ["there", "toy", "that"]}, {"sent": "toy apple .", "words": ["apple", "toy"]}, {"sent": "look at that toy .", "words": ["toy", "at", "that", "look"]}, {"sent": "toy bricks .", "words": ["toy"]}, {"sent": "no you know what please dont play with that thats not a toy .", "words": ["what", "play", "with", "you", "dont", "toy", "that", "not", "a"]}, {"sent": "a toy one do you mean .", "words": ["do", "toy", "you", "a"]}, {"sent": "she can have one toy .", "words": ["can", "toy", "have", "she"]}, {"sent": "you know he wants a new toy .", "words": ["you", "toy", "he", "new", "a"]}, {"sent": "theyre in the toy box .", "words": ["the", "toy", "in", "box"]}, {"sent": "no no thats not a toy .", "words": ["toy", "not", "a"]}, {"sent": "wheres the toy ?", "words": ["the", "toy"]}, {"sent": "*GR2: your toy cry and because you broke the F .", "words": ["the", "your", "cry", "and", "you", "toy", "because"]}, {"sent": "because when you look at your toy things as well the diggers and dumper trucks are all yellow and red and orange arent they ?", "words": ["the", "all", "red", "look", "your", "you", "and", "they", "toy", "are", "orange", "at", "yellow", "when", "because"]}, {"sent": "anything that I should know that um about his toy preferences um other than what youve told me ?", "words": ["what", "other", "toy", "about", "his", "me", "that"]}, {"sent": "xxx xxx xxx xxx this toy .", "words": ["this", "toy"]}, {"sent": "dont you have a toy like that hm ?", "words": ["like", "you", "have", "dont", "toy", "that", "a"]}, {"sent": "tell me whos been to a toy shop today ?", "words": ["to", "me", "toy", "a"]}, {"sent": "well hafta sort out that toy cupboard before Christmas make room for new presents .", "words": ["for", "toy", "that", "room", "out", "new", "make"]}, {"sent": "oh number fours the toy shop .", "words": ["the", "toy"]}, {"sent": "his tinker toy .", "words": ["his", "toy"]}, {"sent": "getting an Actionman toy .", "words": ["toy", "an"]}, {"sent": "I took the mouse toy away from you a couple weeks ago .", "words": ["the", "you", "toy", "away", "mouse", "a"]}, {"sent": "toy ?", "words": ["toy"]}, {"sent": "its a toy company .", "words": ["toy", "a"]}, {"sent": "is there any other type of toy that you think that she liked alot ?", "words": ["think", "is", "of", "any", "other", "you", "there", "toy", "that", "she"]}, {"sent": "and it spoilt the toy .", "words": ["the", "toy", "it", "and"]}, {"sent": "try to find a new toy .", "words": ["to", "toy", "find", "new", "try", "try to", "a"]}, {"sent": "this is AuntCareys toy huh ?", "words": ["this", "toy", "is"]}, {"sent": "its like the toy itself is not so interesting its just the fact that theres water there you know ?", "words": ["the", "is", "so", "like", "you", "there", "toy", "water", "that", "not"]}, {"sent": "a toy and then what happened next ?", "words": ["what", "and", "toy", "then", "a"]}, {"sent": "and a toy .", "words": ["toy", "a", "and"]}, {"sent": "toy box .", "words": ["toy", "box"]}, {"sent": "you have a toy lawnmower dont you ?", "words": ["you", "have", "dont", "toy", "a"]}, {"sent": "I have a bathrobe a toy slippers and a rug .", "words": ["have", "toy", "and", "a"]}, {"sent": "you show me which toy you think its from .", "words": ["think", "show", "you", "toy", "me", "which"]}, {"sent": "yes its a toy .", "words": ["toy", "a"]}, {"sent": "one evening when Eleanor was going to bed she looked in her toy cupboard .", "words": ["to", "was", "her", "bed", "in", "toy", "when", "she"]}, {"sent": "she didnt really I mean for the most part she didnt interact with most of the toy animals or the uh the play I dunno what you call that but .", "words": ["the", "what", "for", "of", "play", "with", "you", "toy", "that", "she", "but"]}, {"sent": "its an old toy .", "words": ["old", "toy", "an"]}, {"sent": "let me see what toy Ive got then .", "words": ["what", "me", "toy", "see", "then"]}, {"sent": "Eleanor on the paper please .", "words": ["on", "the", "paper"]}, {"sent": "on the paper ?", "words": ["on", "the", "paper"]}, {"sent": "well sometimes yes sometimes they get a spot in the paper when theyre making the paper .", "words": ["the", "paper", "get", "they", "in", "when", "a"]}, {"sent": "come and get us some paper towels .", "words": ["paper", "and", "get", "us", "some"]}, {"sent": "because I was just thinking we could make our own wrapping paper another day if we had plenty of paper couldnt we ?", "words": ["if", "we", "of", "was", "paper", "our", "another", "could", "make", "because"]}, {"sent": "you cut paper only .", "words": ["paper", "cut", "you"]}, {"sent": "oh heres your paper daddy .", "words": ["paper", "your"]}, {"sent": "well put the paper down here .", "words": ["the", "put", "paper", "here", "down"]}, {"sent": "and you dropped all those pieces of paper on the floor didnt you ?", "words": ["the", "all", "those", "of", "paper", "you", "and", "on"]}, {"sent": "only on the paper .", "words": ["on", "the", "paper"]}, {"sent": "heres a lot of xxx paper .", "words": ["of", "paper", "a lot", "a"]}, {"sent": "heres the paper .", "words": ["the", "paper"]}, {"sent": "do you wanna do you want me to get you some paper Darling ?", "words": ["do", "to", "paper", "you", "get", "wanna", "me", "some"]}, {"sent": "do you have any paper ?", "words": ["do", "paper", "any", "you", "have"]}, {"sent": "this is your paper .", "words": ["this", "paper", "your", "is"]}, {"sent": "draw on the paper please .", "words": ["on", "the", "paper", "draw"]}, {"sent": "draw on the paper .", "words": ["on", "the", "paper", "draw"]}, {"sent": "here is the paper .", "words": ["here", "the", "paper", "is"]}, {"sent": "this is my paper here .", "words": ["my", "is", "paper", "here", "this"]}, {"sent": "stand on the paper a bit more .", "words": ["the", "stand", "paper", "more", "on", "a"]}, {"sent": "no dont bite the paper Thomas .", "words": ["bite", "dont", "paper", "the"]}, {"sent": "the dog doesnt want any paper does he ?", "words": ["the", "does", "paper", "any", "he", "dog"]}, {"sent": "thats a big order for one little piece of paper .", "words": ["for", "of", "paper", "little", "big", "a"]}, {"sent": "okay how about we turn the paper over ?", "words": ["the", "how", "we", "paper", "about", "over"]}, {"sent": "and why dont you pick up all those raggedy pieces of paper and put them in the garbage .", "words": ["up", "all", "put", "the", "those", "pick", "of", "paper", "them", "you", "and", "dont", "in", "garbage", "why"]}, {"sent": "paper paper paper paper .", "words": ["paper"]}, {"sent": "you you see all of this paper and you dont wanna do nothing without all of this paper right ?", "words": ["all", "do", "of", "paper", "you", "and", "this", "dont", "wanna", "see"]}, {"sent": "paper paper paper .", "words": ["paper"]}, {"sent": "on the paper on the paper .", "words": ["on", "the", "paper"]}, {"sent": "let him give you some paper okay ?", "words": ["paper", "you", "him", "some", "give"]}, {"sent": "are you eating the paper now ?", "words": ["the", "paper", "are", "you"]}, {"sent": "whats your paper on ?", "words": ["on", "paper", "your"]}, {"sent": "get the paper and Ill get the frog .", "words": ["the", "paper", "frog", "get", "and"]}, {"sent": "it was a long list and he had almost reached the end of the paper when he suddenly realized he left out one of the nicest things of all .", "words": ["the", "all", "of", "was", "paper", "long", "and", "it", "he", "out", "when", "a"]}, {"sent": "you want another piece of paper ?", "words": ["another", "of", "paper", "you"]}, {"sent": "take the paper you can put it on your dress .", "words": ["the", "can", "put", "your", "dress", "paper", "you", "it", "on", "take"]}, {"sent": "on this paper .", "words": ["on", "this", "paper"]}, {"sent": "now dont take the paper off them .", "words": ["the", "off", "paper", "them", "dont", "take"]}, {"sent": "xxx the paper say .", "words": ["the", "paper", "say"]}, {"sent": "Id also like to play with your paper .", "words": ["your", "to", "paper", "play", "with", "like"]}, {"sent": "want your paper ?", "words": ["paper", "your"]}, {"sent": "and then she showed all sorts of other different paper things didnt she ?", "words": ["all", "of", "paper", "and", "other", "then", "she"]}, {"sent": "weve got some tissue paper .", "words": ["some", "paper", "tissue"]}, {"sent": "so you need another sheet of paper right ?", "words": ["need", "of", "so", "paper", "you", "another"]}, {"sent": "no put the paper on the book .", "words": ["the", "put", "paper", "on", "book"]}, {"sent": "well what we could do we could do a collage sometime from pieces of paper like this .", "words": ["what", "do", "we", "of", "paper", "like", "this", "could", "a"]}, {"sent": "wheres your construction paper ?", "words": ["paper", "your"]}, {"sent": "so we dont waste so much paper .", "words": ["we", "so", "paper", "dont", "much"]}, {"sent": "oh you found the other rolls of toilet paper huh ?", "words": ["the", "of", "paper", "other", "you"]}, {"sent": "one more paper .", "words": ["more", "paper"]}, {"sent": "I havent got any different color paper Will Harry .", "words": ["paper", "any"]}, {"sent": "I need another piece of paper .", "words": ["another", "of", "paper", "need"]}, {"sent": "this paper ?", "words": ["this", "paper"]}, {"sent": "hes a paper hanger .", "words": ["paper", "a"]}, {"sent": "little paper cases .", "words": ["paper", "little"]}, {"sent": "paper .", "words": ["paper"]}, {"sent": "Im gonna get the construction paper .", "words": ["get", "the", "paper"]}, {"sent": "do you need some paper for this ?", "words": ["do", "for", "need", "paper", "you", "this", "some"]}, {"sent": "with wrapping paper ?", "words": ["paper", "with"]}, {"sent": "can you have some paper ?", "words": ["can", "paper", "you", "have", "some"]}, {"sent": "weve got lotsof white paper here havent we ?", "words": ["here", "paper", "white", "we"]}, {"sent": "when we buy pears from the fruit and veggie man they come in a big brown paper bag dont they ?", "words": ["the", "we", "brown", "paper", "and", "they", "in", "big", "dont", "when", "buy", "a"]}, {"sent": "you dropped your piece of paper ?", "words": ["of", "paper", "your", "you"]}, {"sent": "what can you use for paper ?", "words": ["can", "what", "for", "paper", "you"]}, {"sent": "I want a piece of paper .", "words": ["of", "paper", "a"]}, {"sent": "youre drawing on the paper ?", "words": ["on", "the", "paper"]}, {"sent": "I need another piece of paper .", "words": ["another", "of", "paper", "need"]}, {"sent": "do you want this piece of paper inside ?", "words": ["do", "of", "paper", "inside", "you", "this"]}, {"sent": "did you put the paper towel in the wastebasket ?", "words": ["the", "put", "paper", "you", "towel", "in", "did"]}, {"sent": "draw on the paper Margaret .", "words": ["on", "the", "paper", "draw"]}, {"sent": "okay well get a paper bag and you can put all the food into a paper bag ?", "words": ["can", "all", "put", "the", "into", "paper", "you", "get", "and", "food", "a"]}, {"sent": "lets take this piece of paper out so we can see this picture .", "words": ["can", "we", "of", "so", "paper", "picture", "this", "out", "see", "take"]}, {"sent": "the paper ?", "words": ["the", "paper"]}, {"sent": "why dont you ask Gail if shell give you a piece of paper and you can try .", "words": ["can", "if", "of", "paper", "you", "and", "dont", "give", "try", "why", "a"]}, {"sent": "its to hold the paper on .", "words": ["the", "hold", "to", "paper", "on"]}, {"sent": "paper what else ?", "words": ["what", "paper"]}, {"sent": "oh is this my bit of paper ?", "words": ["my", "is", "of", "paper", "this"]}, {"sent": "on a piece of paper .", "words": ["on", "of", "paper", "a"]}, {"sent": "can you make one on your paper ?", "words": ["can", "your", "paper", "you", "on", "make"]}, {"sent": "oh there are letters in the paper .", "words": ["the", "paper", "there", "in", "are"]}, {"sent": "this is fingerpaint paper its a little bit slippery .", "words": ["is", "paper", "little", "this", "a"]}, {"sent": "xxx you gonna paper your hall ?", "words": ["paper", "your", "you"]}, {"sent": "youre playing with the piece of paper arent you ?", "words": ["the", "of", "paper", "with", "you"]}, {"sent": "and a piece of paper .", "words": ["of", "paper", "a", "and"]}, {"sent": "did you get all the milk Eve with the paper towel ?", "words": ["the", "all", "paper", "with", "you", "get", "towel", "did", "milk"]}, {"sent": "you took the paper off .", "words": ["the", "paper", "off", "you"]}, {"sent": "hey honey let us get you another piece of paper .", "words": ["of", "paper", "you", "get", "us", "another"]}, {"sent": "that can go in the bag too you can put the pads the paper in the bag .", "words": ["can", "the", "put", "go", "paper", "you", "in", "too", "that"]}, {"sent": "I got some more paper for that .", "words": ["for", "paper", "more", "some", "that"]}, {"sent": "you can try one color on your paper towel .", "words": ["can", "your", "paper", "you", "towel", "on", "try"]}, {"sent": "Ill get another paper ?", "words": ["get", "paper", "another"]}, {"sent": "and were having that paper arent we ?", "words": ["we", "paper", "and", "that", "were"]}, {"sent": "do you want some pen and paper ?", "words": ["do", "paper", "and", "you", "pen", "some"]}, {"sent": "what were what did you write on the paper ?", "words": ["the", "what", "paper", "write", "you", "did", "on", "were"]}, {"sent": "on the paper please .", "words": ["on", "the", "paper"]}, {"sent": "the wrapping paper was standing in the vase wasnt it ?", "words": ["the", "was", "paper", "in", "it"]}, {"sent": "and I think the the paper cards here .", "words": ["the", "think", "paper", "and", "here"]}, {"sent": "heres your tractor .", "words": ["tractor", "your"]}, {"sent": "tractor .", "words": ["tractor"]}, {"sent": "is daddy driving the tractor now ?", "words": ["the", "tractor", "is"]}, {"sent": "get outof the way please tractor .", "words": ["get", "the", "tractor"]}, {"sent": "thats a tractor .", "words": ["tractor", "a"]}, {"sent": "a tractor tidy up ?", "words": ["up", "tractor", "a"]}, {"sent": "heres another green tractor coming through .", "words": ["another", "tractor", "green"]}, {"sent": "go and see if its in the kitchen near your tractor .", "words": ["the", "your", "if", "go", "and", "in", "tractor", "kitchen", "see"]}, {"sent": "wheres that tractor off to ?", "words": ["off", "tractor", "to", "that"]}, {"sent": "heres another OldMacDonald from your tractor set .", "words": ["another", "tractor", "your"]}, {"sent": "well tractor tyres are very very big arent they ?", "words": ["they", "big", "are", "tractor"]}, {"sent": "make that our tractor .", "words": ["tractor", "that", "make", "our"]}, {"sent": "tractor shovel it is called there .", "words": ["is", "there", "it", "tractor", "shovel"]}, {"sent": "what happened to the tractor ?", "words": ["to", "what", "tractor", "the"]}, {"sent": "somebody could take the tractor .", "words": ["the", "tractor", "could", "take"]}, {"sent": "a tractor cab .", "words": ["tractor", "a"]}, {"sent": "and you and Daddy went on a tractor didnt you ?", "words": ["you", "and", "tractor", "on", "a"]}, {"sent": "xxx the tractor .", "words": ["the", "tractor"]}, {"sent": "what if the tractor gets stuck would the horse hafta help ?", "words": ["the", "what", "help", "if", "stuck", "tractor", "would", "horse"]}, {"sent": "theres a person sitting on a tractor thats mowing the grass .", "words": ["the", "tractor", "on", "grass", "a"]}, {"sent": "big big tractor and a little tractor .", "words": ["little", "and", "tractor", "big", "a"]}, {"sent": "tractor .", "words": ["tractor"]}, {"sent": "why dont you stand the lego tractor and trailer in the back ?", "words": ["the", "back", "stand", "you", "and", "dont", "in", "tractor", "why"]}, {"sent": "is it a tractor or a truck ?", "words": ["is", "truck", "it", "tractor", "a"]}, {"sent": "shes hiding behind a tractor trailer .", "words": ["tractor", "behind", "a"]}, {"sent": "where do you think hes going in his tractor ?", "words": ["where", "do", "think", "you", "in", "tractor", "his"]}, {"sent": "you gonna take the tractor ?", "words": ["the", "tractor", "take", "you"]}, {"sent": "tractor yay .", "words": ["tractor"]}, {"sent": "youre putting it under the tractor ?", "words": ["the", "under", "it", "tractor"]}, {"sent": "lets get this tractor puzzle out .", "words": ["get", "this", "puzzle", "tractor", "out"]}, {"sent": "oh look at this tractor .", "words": ["this", "at", "tractor", "look"]}, {"sent": "heres another tractor .", "words": ["another", "tractor"]}, {"sent": "a tractor .", "words": ["tractor", "a"]}, {"sent": "lets eat the candy tractor .", "words": ["eat", "the", "candy", "tractor"]}, {"sent": "the tractor .", "words": ["the", "tractor"]}, {"sent": "lets see if we can turn this into a tractor .", "words": ["can", "if", "into", "we", "this", "tractor", "see", "a"]}, {"sent": "the tractor .", "words": ["the", "tractor"]}, {"sent": "what happened to the tractor ?", "words": ["to", "what", "tractor", "the"]}, {"sent": "and what were you doing in the tractor ?", "words": ["the", "what", "you", "and", "in", "tractor", "were"]}, {"sent": "who should drive the tractor ?", "words": ["drive", "the", "tractor", "who"]}, {"sent": "here are the wheels of the tractor .", "words": ["the", "of", "here", "are", "tractor"]}, {"sent": "a tractor didnt we ?", "words": ["tractor", "we", "a"]}, {"sent": "tractor .", "words": ["tractor"]}, {"sent": "theres a tractor in the way .", "words": ["the", "in", "tractor", "a"]}, {"sent": "right hes driving the tractor .", "words": ["the", "tractor"]}, {"sent": "the tractor ?", "words": ["the", "tractor"]}, {"sent": "you playing with the train now as well as the tractor and trailer ?", "words": ["the", "train", "with", "you", "and", "tractor"]}, {"sent": "tractor .", "words": ["tractor"]}, {"sent": "tractor whats that ?", "words": ["that", "tractor"]}, {"sent": "if you put the tractor on your book wont you tear it up ?", "words": ["the", "up", "put", "your", "if", "tear", "you", "it", "tractor", "on", "book"]}, {"sent": "Warren draw a tractor then .", "words": ["then", "draw", "tractor", "a"]}, {"sent": "a tractor .", "words": ["tractor", "a"]}, {"sent": "I might be able to drive a tractor .", "words": ["to", "be", "drive", "tractor", "a"]}, {"sent": "lets build something up for the tractor ?", "words": ["up", "the", "for", "build", "tractor"]}, {"sent": "you cant put a whole field in a tractor .", "words": ["put", "you", "in", "tractor", "a"]}, {"sent": "and Percy the tractor hadta go out and pull this car back up the bank .", "words": ["the", "up", "back", "go", "car", "and", "pull", "this", "tractor", "out"]}, {"sent": "or how about using the big tractor ?", "words": ["the", "how", "tractor", "about", "big"]}, {"sent": "tower and a tractor .", "words": ["tractor", "a", "and"]}, {"sent": "and they brought a little blue tractor .", "words": ["blue", "little", "they", "and", "tractor", "a"]}, {"sent": "we saw the tractor moving the snow on PostmanPat didnt we ?", "words": ["the", "we", "snow", "tractor", "on"]}, {"sent": "so its a tractor again now isnt it ?", "words": ["so", "it", "tractor", "a"]}, {"sent": "Spots Daddys driving the tractor isnt he ?", "words": ["the", "he", "tractor"]}, {"sent": "its a tractor isnt it ?", "words": ["it", "tractor", "a"]}, {"sent": "where does another tractor go ?", "words": ["where", "does", "go", "another", "tractor"]}, {"sent": "what about the tractor ?", "words": ["the", "what", "tractor", "about"]}, {"sent": "well maybe the babys tired of driving the tractor .", "words": ["the", "of", "tractor", "tired"]}, {"sent": "put her in your tractor and take her to the hospital quick because her heads bleeding .", "words": ["the", "put", "your", "to", "and", "in", "tractor", "take", "her", "because"]}, {"sent": "you have a tractor too that you ride sometimes .", "words": ["ride", "you", "have", "tractor", "too", "that", "a"]}, {"sent": "tractor where are you ?", "words": ["where", "are", "tractor", "you"]}, {"sent": "we got him a tractor and a tape recorder and tapes and HappyStreet .", "words": ["we", "tape", "and", "him", "tractor", "a"]}, {"sent": "Maisy drives the tractor .", "words": ["the", "tractor"]}, {"sent": "a tractor .", "words": ["tractor", "a"]}, {"sent": "is that your tractor ?", "words": ["your", "tractor", "that", "is"]}, {"sent": "heres the tractor .", "words": ["the", "tractor"]}, {"sent": "is that like when you rode the tractor at Grampas ?", "words": ["the", "is", "like", "you", "that", "tractor", "at", "when"]}, {"sent": "is driving the tractor .", "words": ["the", "tractor", "is"]}, {"sent": "I know lets take the cow for a ride in the tractor .", "words": ["the", "for", "ride", "in", "tractor", "take", "cow", "a"]}, {"sent": "no thats the tractor .", "words": ["the", "tractor"]}, {"sent": "will that go on the tractor ?", "words": ["the", "go", "on", "tractor", "will", "that"]}, {"sent": "wheres the tractor ?", "words": ["the", "tractor"]}, {"sent": "tractor did .", "words": ["tractor", "did"]}, {"sent": "go and pull the tractor outof the outof the ditch .", "words": ["the", "go", "pull", "and", "tractor"]}, {"sent": "that man is carrying an umbrella while hes riding on the tractor ?", "words": ["the", "is", "an", "tractor", "on", "that"]}, {"sent": "hes driving the tractor right .", "words": ["the", "tractor"]}, {"sent": "wheres the tractor going ?", "words": ["the", "tractor"]}, {"sent": "you want Momma do it on the tractor ?", "words": ["the", "do", "you", "it", "tractor", "on"]}, {"sent": "tractor .", "words": ["tractor"]}, {"sent": "the man in the tractor ?", "words": ["the", "in", "tractor"]}, {"sent": "and whats that next to the tractor ?", "words": ["the", "to", "and", "tractor", "that"]}, {"sent": "we have a tractor here .", "words": ["we", "have", "here", "tractor", "a"]}, {"sent": "last night you insisted that that crane hadta be a tractor .", "words": ["be", "last", "you", "tractor", "that", "a"]}, {"sent": "theyre gonna watch out for the tractor ?", "words": ["the", "watch", "for", "tractor", "out"]}, {"sent": "tractor .", "words": ["tractor"]}, {"sent": "driving tractor okay .", "words": ["tractor"]}, {"sent": "is this tractor dirty ?", "words": ["this", "tractor", "dirty", "is"]}, {"sent": "whats the tractor man called ?", "words": ["the", "tractor"]}, {"sent": "theres no one driving that tractor though .", "words": ["tractor", "that"]}, {"sent": "the tractor and the trailer are going to hafta be the ambulance to bring all the bodys back .", "words": ["the", "all", "back", "to", "be", "bring", "and", "are", "tractor"]}, {"sent": "same tractor .", "words": ["tractor", "same"]}, {"sent": "what do you do with a hammer ?", "words": ["what", "do", "hammer", "with", "you", "a"]}, {"sent": "do you wanna do the puzzle or do your hammer ?", "words": ["the", "do", "hammer", "your", "you", "puzzle", "wanna"]}, {"sent": "the hammer is .", "words": ["the", "hammer", "is"]}, {"sent": "take that hammer and hammer it in .", "words": ["hammer", "and", "in", "it", "take", "that"]}, {"sent": "theres the hammer .", "words": ["the", "hammer"]}, {"sent": "what colors the hammer ?", "words": ["the", "what", "hammer"]}, {"sent": "hammer .", "words": ["hammer"]}, {"sent": "thats a hammer .", "words": ["hammer", "a"]}, {"sent": "with a hammer ?", "words": ["hammer", "with", "a"]}, {"sent": "ya wanna hammer ?", "words": ["hammer", "wanna"]}, {"sent": "he has got a toy hammer and hes banging the bicycle .", "words": ["the", "hammer", "bicycle", "and", "toy", "he", "a"]}, {"sent": "it can be a hammer .", "words": ["can", "hammer", "be", "it", "a"]}, {"sent": "ready hammer hammer hammer good hammer you hold the hammer there there you go .", "words": ["the", "hammer", "hold", "go", "you", "there", "good"]}, {"sent": "you were using a hammer today too werent you ?", "words": ["hammer", "you", "too", "were", "a"]}, {"sent": "hammer .", "words": ["hammer"]}, {"sent": "is there sposta be a hammer with it ?", "words": ["hammer", "is", "be", "with", "there", "it", "a"]}, {"sent": "wanna put the hammer back in the box ?", "words": ["the", "hammer", "put", "box", "back", "in", "wanna"]}, {"sent": "Im gonna hammer xxx .", "words": ["hammer"]}, {"sent": "you were hitting something with a hammer .", "words": ["hammer", "with", "you", "were", "a"]}, {"sent": "heres you hammer .", "words": ["hammer", "you"]}, {"sent": "you need a hammer .", "words": ["hammer", "need", "a", "you"]}, {"sent": "you see Olivia with her hammer ?", "words": ["hammer", "with", "you", "see", "her"]}, {"sent": "what its a hammer .", "words": ["what", "hammer", "a"]}, {"sent": "you think its a hammer do you ?", "words": ["hammer", "think", "do", "you", "a"]}, {"sent": "a hammer ?", "words": ["hammer", "a"]}, {"sent": "uh oh something needs sawing down that I can hammer .", "words": ["can", "hammer", "down", "that"]}, {"sent": "theres a hammer .", "words": ["hammer", "a"]}, {"sent": "see the hammer ?", "words": ["the", "see", "hammer"]}, {"sent": "the other hammer .", "words": ["the", "hammer", "other"]}, {"sent": "going to go round and hammer everybody are you ?", "words": ["hammer", "go", "to", "and", "you", "are"]}, {"sent": "a hammer .", "words": ["hammer", "a"]}, {"sent": "you wanna hammer it ?", "words": ["hammer", "it", "wanna", "you"]}, {"sent": "my hammer .", "words": ["hammer", "my"]}, {"sent": "show mommy how you use the hammer .", "words": ["the", "hammer", "how", "show", "you"]}, {"sent": "whenever the telly goes wrong you just hit it with your hammer .", "words": ["the", "hammer", "your", "with", "you", "it", "hit"]}, {"sent": "you got a hammer ?", "words": ["hammer", "a", "you"]}, {"sent": "whos banging a hammer ?", "words": ["hammer", "a"]}, {"sent": "show us with the hammer .", "words": ["the", "hammer", "show", "with", "us"]}, {"sent": "I wanna hammer .", "words": ["hammer", "wanna"]}, {"sent": "wheres your hammer honey ?", "words": ["hammer", "your"]}, {"sent": "oh it looks like a hammer ?", "words": ["hammer", "it", "like", "a"]}, {"sent": "are you gonna be able to hammer with this ?", "words": ["hammer", "to", "be", "with", "you", "this", "are"]}, {"sent": "do you have the hammer that goes on this ?", "words": ["the", "do", "hammer", "you", "have", "this", "on", "that"]}, {"sent": "thats not a hammer .", "words": ["hammer", "not", "a"]}, {"sent": "no were just going to hammer huh ?", "words": ["to", "hammer", "were"]}, {"sent": "are you going to hammer this ?", "words": ["hammer", "to", "you", "this", "are"]}, {"sent": "now just very gently with your hammer if you wanna tap it down you can .", "words": ["can", "hammer", "your", "if", "with", "you", "it", "wanna", "down"]}, {"sent": "hammer .", "words": ["hammer"]}, {"sent": "where is the hammer one I dont know ?", "words": ["where", "the", "hammer", "is", "dont"]}, {"sent": "a hammer ?", "words": ["hammer", "a"]}, {"sent": "a hammer .", "words": ["hammer", "a"]}, {"sent": "would you like to color the hammer in ?", "words": ["the", "hammer", "to", "like", "you", "in", "would"]}, {"sent": "thats a hammer yeah .", "words": ["hammer", "a"]}, {"sent": "thats not a thing to hammer .", "words": ["to", "hammer", "not", "a"]}, {"sent": "hammer .", "words": ["hammer"]}, {"sent": "it doesnt have a hammer and without a hammer its just not a very good toy .", "words": ["hammer", "and", "have", "toy", "it", "good", "not", "a"]}, {"sent": "show Kate your hammer .", "words": ["show", "hammer", "your"]}, {"sent": "put the hammer in the box .", "words": ["the", "hammer", "put", "box", "in"]}, {"sent": "say bang bang bang bang bang with your hammer .", "words": ["your", "hammer", "say", "with"]}, {"sent": "why do you need a hammer ?", "words": ["do", "hammer", "need", "you", "why", "a"]}, {"sent": "let him have a hammer please .", "words": ["have", "hammer", "him", "a"]}, {"sent": "I think youre sposta use the hammer .", "words": ["the", "hammer", "think"]}, {"sent": "a hammer .", "words": ["hammer", "a"]}, {"sent": "do you need a hammer too ?", "words": ["do", "hammer", "need", "you", "too", "a"]}, {"sent": "with a hammer not a screwdriver .", "words": ["hammer", "with", "not", "a"]}, {"sent": "make that hammer work .", "words": ["hammer", "that", "make", "work"]}, {"sent": "hammer .", "words": ["hammer"]}, {"sent": "hammer hammer hammer .", "words": ["hammer"]}, {"sent": "we dont need a hammer Aran .", "words": ["hammer", "need", "we", "dont", "a"]}, {"sent": "hammer xxx on the wall .", "words": ["on", "hammer", "the"]}, {"sent": "look at what my hammer .", "words": ["what", "my", "hammer", "look", "at"]}, {"sent": "that hammer I needta smash the door is in Thomass mouth .", "words": ["the", "hammer", "is", "mouth", "in", "that", "door"]}, {"sent": "hammer nails in do you ?", "words": ["hammer", "in", "do", "you"]}, {"sent": "what are you what are you doing with the hammer ?", "words": ["the", "what", "hammer", "with", "you", "are"]}, {"sent": "hammer .", "words": ["hammer"]}, {"sent": "do we need a hammer ?", "words": ["do", "hammer", "need", "we", "a"]}, {"sent": "just hammer that one down as well a bit .", "words": ["hammer", "down", "that", "a"]}, {"sent": "what do you do with the hammer ?", "words": ["the", "what", "do", "hammer", "with", "you"]}, {"sent": "well what well what do you do with a hammer tell us or show us ?", "words": ["what", "do", "hammer", "show", "with", "you", "us", "a"]}, {"sent": "lets hammer something else .", "words": ["hammer"]}, {"sent": "well hammer it later .", "words": ["hammer", "it"]}, {"sent": "wheres the hammer ?", "words": ["the", "hammer"]}, {"sent": "so you can hammer and saw and stuff .", "words": ["can", "hammer", "so", "and", "you"]}, {"sent": "the hammer and stuff .", "words": ["the", "hammer", "and"]}, {"sent": "do you have the hammer ?", "words": ["the", "do", "hammer", "you", "have"]}, {"sent": "whats Miffy gonna do with the wood and hammer ?", "words": ["the", "do", "hammer", "with", "and"]}, {"sent": "Thomass usual response is I know a hammer .", "words": ["hammer", "a", "is"]}, {"sent": "okay dont hit it with the hammer .", "words": ["the", "hammer", "with", "dont", "it", "hit"]}, {"sent": "thats a hammer like daddys .", "words": ["hammer", "like", "a"]}, {"sent": "hammer .", "words": ["hammer"]}, {"sent": "Bob the builder hammer .", "words": ["the", "hammer"]}, {"sent": "theres his hammer .", "words": ["his", "hammer"]}, {"sent": "my hammer .", "words": ["hammer", "my"]}, {"sent": "is that a hammer ?", "words": ["hammer", "that", "a", "is"]}, {"sent": "hammer the block .", "words": ["the", "hammer", "block"]}, {"sent": "what do you do with the hammer ?", "words": ["the", "what", "do", "hammer", "with", "you"]}, {"sent": "hammer .", "words": ["hammer"]}, {"sent": "a hammer yes .", "words": ["hammer", "a"]}, {"sent": "Peter works with one hammer one hammer one hammer .", "words": ["hammer", "with"]}, {"sent": "spaghetti .", "words": ["spaghetti"]}, {"sent": "you eat spaghetti at home dont ya ?", "words": ["spaghetti", "you", "eat", "dont", "at", "home"]}, {"sent": "were back to the spaghetti huh ?", "words": ["the", "back", "spaghetti", "to", "were"]}, {"sent": "there is no spaghetti on it .", "words": ["spaghetti", "is", "there", "it", "on"]}, {"sent": "spaghetti .", "words": ["spaghetti"]}, {"sent": "spaghetti bolognese in Austria .", "words": ["in", "spaghetti"]}, {"sent": "spaghetti isnt it ?", "words": ["it", "spaghetti"]}, {"sent": "okay I will need some money before I can give you the spaghetti .", "words": ["can", "money", "the", "need", "spaghetti", "you", "will", "some", "give"]}, {"sent": "spaghetti bolognese .", "words": ["spaghetti"]}, {"sent": "she liked the spaghetti before too I remember that .", "words": ["the", "spaghetti", "too", "that", "she"]}, {"sent": "toast and spaghetti and ham you want ?", "words": ["you", "spaghetti", "toast", "and"]}, {"sent": "ooh I think we have more spaghetti .", "words": ["think", "spaghetti", "we", "have", "more"]}, {"sent": "see thats not where spaghetti goes .", "words": ["where", "see", "not", "spaghetti"]}, {"sent": "and they eat spaghetti ?", "words": ["eat", "they", "spaghetti", "and"]}, {"sent": "spaghetti ?", "words": ["spaghetti"]}, {"sent": "and I thought well maybe its food around the house but she was really into the spaghetti .", "words": ["the", "into", "spaghetti", "was", "and", "food", "house", "around", "she", "but"]}, {"sent": "whos eating spaghetti ?", "words": ["spaghetti"]}, {"sent": "mm spaghetti .", "words": ["spaghetti"]}, {"sent": "youve got spaghetti all over your face Fraser .", "words": ["all", "your", "face", "spaghetti", "over"]}, {"sent": "thats spaghetti .", "words": ["spaghetti"]}, {"sent": "more spaghetti ?", "words": ["more", "spaghetti"]}, {"sent": "spaghetti ?", "words": ["spaghetti"]}, {"sent": "thats spaghetti with meat sauce .", "words": ["sauce", "with", "meat", "spaghetti"]}, {"sent": "different spaghetti .", "words": ["spaghetti"]}, {"sent": "hot potato hot potato hot potato hot potato hot potato hot potato potato potato potato potato cold spaghetti cold spaghetti cold spaghetti cold spaghetti .", "words": ["hot", "spaghetti", "cold", "potato"]}, {"sent": "you need spaghetti ?", "words": ["spaghetti", "need", "you"]}, {"sent": "and put it in the tofu and the spaghetti spaghetti foe feti .", "words": ["the", "put", "spaghetti", "and", "in", "it"]}, {"sent": "can you make her some spaghetti to drink ?", "words": ["can", "drink", "spaghetti", "to", "you", "make", "some", "her"]}, {"sent": "spaghetti by putting it in the pasta .", "words": ["the", "spaghetti", "by", "in", "it"]}, {"sent": "youre having spaghetti ?", "words": ["spaghetti"]}, {"sent": "does that taste like spaghetti ?", "words": ["does", "spaghetti", "taste", "like", "that"]}, {"sent": "spaghetti ?", "words": ["spaghetti"]}, {"sent": "you have the spaghetti ?", "words": ["have", "the", "spaghetti", "you"]}, {"sent": "spaghetti ?", "words": ["spaghetti"]}, {"sent": "and eat your spaghetti ?", "words": ["eat", "spaghetti", "your", "and"]}, {"sent": "look at the spaghetti Lee .", "words": ["the", "spaghetti", "at", "look"]}, {"sent": "the spaghetti ?", "words": ["the", "spaghetti"]}, {"sent": "hes eating spaghetti .", "words": ["spaghetti"]}, {"sent": "she could be looking at that spaghetti and thinking it looks similar .", "words": ["spaghetti", "be", "and", "it", "at", "could", "that", "she"]}, {"sent": "thats spaghetti .", "words": ["spaghetti"]}, {"sent": "spaghetti .", "words": ["spaghetti"]}, {"sent": "spaghetti .", "words": ["spaghetti"]}, {"sent": "spaghetti .", "words": ["spaghetti"]}, {"sent": "youve got spaghetti Fraser .", "words": ["spaghetti"]}, {"sent": "would you like spaghetti ?", "words": ["spaghetti", "like", "you", "would"]}, {"sent": "I do I don we didnt really go over the spaghetti Im not sure if he would recognize this as spaghetti .", "words": ["the", "do", "if", "go", "we", "spaghetti", "this", "he", "would", "over", "not"]}, {"sent": "and spaghetti you had spaghetti too .", "words": ["too", "spaghetti", "you", "and"]}, {"sent": "you wanna give her spaghetti ?", "words": ["spaghetti", "you", "wanna", "give", "her"]}, {"sent": "its little alphabet spaghetti .", "words": ["spaghetti", "little"]}, {"sent": "I dont think anybody wants any spaghetti .", "words": ["dont", "think", "any", "spaghetti"]}, {"sent": "and these little spaghetti type ones .", "words": ["these", "spaghetti", "little", "and"]}, {"sent": "yum you had spaghetti last night for dinner .", "words": ["spaghetti", "for", "last", "you"]}, {"sent": "spaghetti and meatballs .", "words": ["and", "spaghetti"]}, {"sent": "thats Frasers spaghetti .", "words": ["spaghetti"]}, {"sent": "what your spaghetti ?", "words": ["what", "your", "spaghetti"]}, {"sent": "bread peas spaghetti .", "words": ["bread", "peas", "spaghetti"]}, {"sent": "xxx spaghetti .", "words": ["spaghetti"]}, {"sent": "youll hafta just eat spaghetti .", "words": ["eat", "spaghetti"]}, {"sent": "only spaghetti today .", "words": ["spaghetti"]}, {"sent": "hes never had spaghetti so .", "words": ["so", "spaghetti"]}, {"sent": "mashed potato mashed potato mashed banana mashed banana spaghetti spaghetti spaghetti spaghetti .", "words": ["banana", "spaghetti", "potato"]}, {"sent": "want some spaghetti and meat sauce ?", "words": ["sauce", "spaghetti", "meat", "and", "some"]}, {"sent": "looks like the spaghetti we had yesterday .", "words": ["the", "we", "like", "spaghetti"]}, {"sent": "some of your spaghetti ?", "words": ["of", "some", "your", "spaghetti"]}, {"sent": "spaghetti .", "words": ["spaghetti"]}, {"sent": "soup with your spaghetti ?", "words": ["soup", "spaghetti", "with", "your"]}, {"sent": "shes having spaghetti bolognese without the rice and with some pasta this time .", "words": ["the", "spaghetti", "with", "and", "this", "some"]}, {"sent": "is that nice spaghetti and toast Fraser ?", "words": ["toast", "spaghetti", "is", "nice", "and", "that"]}, {"sent": "so I didnt know if maybe thats why she gravitated to the spaghetti because she was getting noodles we were giving her little bits of noodles last night .", "words": ["the", "if", "spaghetti", "we", "to", "so", "was", "her", "because", "of", "little", "last", "noodles", "were", "she", "why"]}, {"sent": "spaghetti ?", "words": ["spaghetti"]}, {"sent": "more spaghetti .", "words": ["more", "spaghetti"]}, {"sent": "is that spaghetti ?", "words": ["spaghetti", "that", "is"]}, {"sent": "that spaghetti sure is fascinating isnt it ?", "words": ["it", "that", "is", "spaghetti"]}, {"sent": "spaghetti ?", "words": ["spaghetti"]}, {"sent": "mm you have meat sauce in your spaghetti .", "words": ["your", "sauce", "spaghetti", "meat", "you", "have", "in"]}, {"sent": "you like spaghetti dont you ?", "words": ["dont", "spaghetti", "like", "you"]}, {"sent": "you had spaghetti didnt you ?", "words": ["spaghetti", "you"]}, {"sent": "were having spaghetti .", "words": ["were", "spaghetti"]}, {"sent": "but you didnt eat your spaghetti .", "words": ["your", "spaghetti", "you", "eat", "but"]}, {"sent": "you know Mommy bought some baby food spaghetti for you to have for lunch .", "words": ["for", "spaghetti", "to", "you", "have", "food", "some"]}, {"sent": "spaghetti .", "words": ["spaghetti"]}, {"sent": "I put plates down too because I feed him like green beans and stuff on plates and and he didnt want his spaghetti on the plate .", "words": ["the", "put", "plate", "spaghetti", "like", "green beans", "and", "his", "him", "beans", "he", "green", "feed", "on", "too", "down", "because"]}, {"sent": "oh you already answered that about the she liked the spaghetti .", "words": ["the", "spaghetti", "you", "about", "that", "she"]}, {"sent": "spaghetti ?", "words": ["spaghetti"]}, {"sent": "love that spaghetti .", "words": ["that", "love", "spaghetti"]}, {"sent": "oh some spaghetti .", "words": ["some", "spaghetti"]}, {"sent": "you like that spaghetti .", "words": ["spaghetti", "that", "like", "you"]}, {"sent": "you see the spaghetti ?", "words": ["the", "see", "spaghetti", "you"]}, {"sent": "spaghetti ?", "words": ["spaghetti"]}, {"sent": "we can twirl spaghetti on a fork .", "words": ["can", "spaghetti", "we", "fork", "on", "a"]}, {"sent": "whos going to eat spaghetti ?", "words": ["eat", "to", "spaghetti"]}, {"sent": "well touch the spaghetti and feel how hot it is .", "words": ["the", "how", "spaghetti", "is", "touch", "and", "hot", "it"]}, {"sent": "you want me to turn your spaghetti around in there .", "words": ["your", "spaghetti", "to", "you", "there", "in", "around", "me"]}, {"sent": "and wipe off the spaghetti sauce .", "words": ["off", "the", "spaghetti", "sauce", "and", "wipe"]}, {"sent": "it was a big long spaghetti wasnt it Ruth ?", "words": ["spaghetti", "was", "long", "it", "big", "a"]}, {"sent": "you love the spaghetti .", "words": ["the", "spaghetti", "love", "you"]}, {"sent": "I see theyre going to eat rice and spaghetti too ?", "words": ["spaghetti", "to", "and", "eat", "too", "see"]}, {"sent": "noodles .", "words": ["noodles"]}, {"sent": "noodles ?", "words": ["noodles"]}, {"sent": "its got noodles and its got some sort of sauce .", "words": ["sauce", "of", "and", "noodles", "some"]}, {"sent": "you like noodles very much .", "words": ["noodles", "much", "like", "you"]}, {"sent": "do you want some more noodles ?", "words": ["do", "you", "more", "noodles", "some"]}, {"sent": "chicken noodles and broccoli .", "words": ["noodles", "chicken", "and"]}, {"sent": "you ate some noodles ?", "words": ["noodles", "some", "you"]}, {"sent": "yes Im giving you some noodles .", "words": ["noodles", "some", "you"]}, {"sent": "well the noodles are kindof little .", "words": ["the", "noodles", "are", "little"]}, {"sent": "noodles did you eat those ?", "words": ["those", "you", "eat", "did", "noodles"]}, {"sent": "it doesnt have any noodles .", "words": ["have", "noodles", "it", "any"]}, {"sent": "noodles or ?", "words": ["noodles"]}, {"sent": "do you want them with some juice or just with uh just noodles ?", "words": ["do", "them", "with", "you", "noodles", "some", "juice"]}, {"sent": "and there is spaghetti and noodles on rice .", "words": ["spaghetti", "is", "and", "there", "on", "noodles"]}, {"sent": "and I called her up for supper and she wanted noodles for supper .", "words": ["up", "for", "and", "noodles", "her", "she"]}, {"sent": "I like noodles .", "words": ["noodles", "like"]}, {"sent": "first youre going to have your noodles .", "words": ["your", "to", "first", "have", "noodles"]}, {"sent": "Ive got some noodles cooking and you love noodles .", "words": ["you", "and", "love", "noodles", "some"]}, {"sent": "does anybody want more noodles ?", "words": ["noodles", "does", "more"]}, {"sent": "noodles .", "words": ["noodles"]}, {"sent": "because you have a few problems with noodles dont you ?", "words": ["with", "you", "have", "dont", "noodles", "because", "a"]}, {"sent": "yeah youve got some noodles on your clothes .", "words": ["noodles", "some", "on", "your"]}, {"sent": "you made so many noodles .", "words": ["noodles", "so", "you"]}, {"sent": "because that looks a bit like noodles .", "words": ["like", "noodles", "that", "because", "a"]}, {"sent": "who wants ketchup on their noodles ?", "words": ["on", "noodles", "their", "who"]}, {"sent": "yeah how do I cook noodles ?", "words": ["noodles", "do", "how", "cook"]}, {"sent": "go and talk to Fraser for a minute until the noodles cook .", "words": ["the", "for", "go", "to", "and", "cook", "talk", "noodles", "a"]}, {"sent": "the noodles ?", "words": ["the", "noodles"]}, {"sent": "any noodles on the new one huh ?", "words": ["the", "any", "on", "noodles", "new"]}, {"sent": "I didnt have any of your noodles did I ?", "words": ["your", "of", "any", "have", "did", "noodles"]}, {"sent": "I love noodles .", "words": ["noodles", "love"]}, {"sent": "we had noodles and chicken .", "words": ["noodles", "chicken", "and", "we"]}, {"sent": "noodles .", "words": ["noodles"]}, {"sent": "doodles or noodles ?", "words": ["noodles"]}, {"sent": "hes never seen actual spaghetti and noodles but .", "words": ["noodles", "but", "and", "spaghetti"]}, {"sent": "six noodles .", "words": ["noodles"]}, {"sent": "chicken and noodles .", "words": ["noodles", "chicken", "and"]}, {"sent": "oh we had um stir fried noodles in JohnLewis didnt we ?", "words": ["noodles", "in", "we"]}, {"sent": "chicken noodles water vinegar broccoli and carrots .", "words": ["and", "carrots", "water", "noodles", "chicken"]}, {"sent": "we can have some alphabet noodles for lunch .", "words": ["can", "for", "we", "have", "noodles", "some"]}, {"sent": "you put noodles .", "words": ["noodles", "put", "you"]}, {"sent": "you dont mind the chicken but you dont like the noodles .", "words": ["the", "like", "you", "dont", "noodles", "chicken", "but"]}, {"sent": "how about bite of chicken and noodles .", "words": ["how", "of", "noodles", "and", "about", "bite", "chicken"]}, {"sent": "shall daddy go and get some noodles and we can munch them all up ?", "words": ["can", "all", "up", "go", "we", "them", "and", "get", "noodles", "some"]}, {"sent": "you like noodles .", "words": ["noodles", "like", "you"]}, {"sent": "its noodles .", "words": ["noodles"]}, {"sent": "I might make noodles for you .", "words": ["noodles", "for", "make", "you"]}, {"sent": "noodles and what else ?", "words": ["noodles", "what", "and"]}, {"sent": "mommys gonna have a bite of noodles you want try some noodles ?", "words": ["noodles", "of", "you", "have", "bite", "some", "try", "a"]}, {"sent": "well Eve you cant eat darling until the noodles cook .", "words": ["the", "you", "eat", "cook", "noodles"]}, {"sent": "its noodles on your head .", "words": ["noodles", "on", "your", "head"]}, {"sent": "here I can scoop some chicken noodles off your back .", "words": ["can", "off", "your", "back", "chicken", "here", "noodles", "some"]}, {"sent": "when I cook the noodles what do I cook the noodles on ?", "words": ["the", "what", "do", "cook", "on", "noodles", "when"]}, {"sent": "noodles for the baby .", "words": ["noodles", "the", "for"]}, {"sent": "alphabet noodles yeah .", "words": ["noodles"]}, {"sent": "noodles of any kind veggies fruit .", "words": ["noodles", "of", "any"]}, {"sent": "thats suppose to be noodles .", "words": ["noodles", "to", "be"]}, {"sent": "can we have some chicken and noodles and mashed potatoes ?", "words": ["can", "chicken", "we", "and", "have", "noodles", "some"]}, {"sent": "more noodles you like those noodles .", "words": ["those", "like", "you", "more", "noodles"]}, {"sent": "want some noodles ?", "words": ["noodles", "some"]}, {"sent": "were both eating noodles arent we ?", "words": ["noodles", "were", "we"]}, {"sent": "Ive finished all my noodles so Ill go and get you some milk in a cup .", "words": ["cup", "all", "my", "milk", "go", "so", "and", "get", "you", "in", "noodles", "some", "a"]}, {"sent": "want some more noodles ?", "words": ["noodles", "some", "more"]}, {"sent": "and he could compare the different sounds slurping noodles .", "words": ["the", "and", "he", "noodles", "could"]}, {"sent": "therere your noodles .", "words": ["noodles", "your"]}, {"sent": "noodles how many noodles ?", "words": ["noodles", "how"]}, {"sent": "or noodles ?", "words": ["noodles"]}, {"sent": "we have noodles .", "words": ["have", "noodles", "we"]}, {"sent": "noodles ?", "words": ["noodles"]}, {"sent": "to the noodles .", "words": ["the", "to", "noodles"]}, {"sent": "oh some noodles .", "words": ["noodles", "some"]}, {"sent": "lemme check on my noodles Boneen .", "words": ["on", "my", "noodles"]}, {"sent": "and then we have some other little noodles in here .", "words": ["we", "and", "have", "other", "little", "in", "here", "noodles", "some", "then"]}, {"sent": "do you not want these noodles ?", "words": ["do", "these", "you", "noodles", "not"]}, {"sent": "chicken and noodles cheese and gum .", "words": ["and", "gum", "noodles", "chicken", "cheese"]}, {"sent": "we have noodles .", "words": ["have", "noodles", "we"]}, {"sent": "but for the most part its noodles and uh nothing is youknow looking like the noodles he eats anyway .", "words": ["the", "for", "is", "like", "and", "he", "noodles", "but"]}, {"sent": "we eat noodles dont we ?", "words": ["eat", "dont", "noodles", "we"]}, {"sent": "you love noodles .", "words": ["noodles", "love", "you"]}, {"sent": "no you eat your noodles and your celery .", "words": ["your", "and", "eat", "you", "noodles"]}, {"sent": "do you want some more noodles ?", "words": ["do", "you", "more", "noodles", "some"]}, {"sent": "not again youre always eating noodles .", "words": ["noodles", "not"]}, {"sent": "you hafta wait while the noodles get cold .", "words": ["the", "wait", "you", "get", "noodles", "cold"]}, {"sent": "heres your noodles darling .", "words": ["noodles", "your"]}, {"sent": "for noodles .", "words": ["noodles", "for"]}, {"sent": "you put gravy on the noodles .", "words": ["the", "put", "noodles", "you", "on"]}, {"sent": "thats noodles uhhuh .", "words": ["noodles"]}, {"sent": "heres some noodles .", "words": ["noodles", "some"]}, {"sent": "ah she does like noodles .", "words": ["noodles", "does", "she", "like"]}, {"sent": "noodles noodles .", "words": ["noodles"]}, {"sent": "xxx noodles gone .", "words": ["noodles"]}, {"sent": "noodles .", "words": ["noodles"]}, {"sent": "would you like Mama to make your noodles for you ?", "words": ["for", "your", "to", "like", "you", "would", "noodles", "make"]}, {"sent": "you ate alotof noodles and rice and beans today .", "words": ["noodles", "beans", "and", "you"]}, {"sent": "you dont have any noodles ?", "words": ["any", "you", "have", "dont", "noodles"]}, {"sent": "are you finishing the lion one ?", "words": ["the", "are", "lion", "you"]}, {"sent": "if Elmo were a lion he could roar loud enough for everyone to hear .", "words": ["for", "if", "to", "hear", "he", "could", "lion", "loud", "were", "a"]}, {"sent": "thats not a horse thats a lion .", "words": ["horse", "lion", "not", "a"]}, {"sent": "has the lion got a big tail ?", "words": ["the", "big", "lion", "a"]}, {"sent": "yes a lion growls .", "words": ["lion", "a"]}, {"sent": "T the lion went away .", "words": ["the", "away", "lion"]}, {"sent": "whats that lion doing in town ?", "words": ["lion", "in", "that"]}, {"sent": "Im being attacked my a lion .", "words": ["my", "lion", "a"]}, {"sent": "got one friendly lion .", "words": ["lion"]}, {"sent": "what does the lion say ?", "words": ["the", "what", "does", "say", "lion"]}, {"sent": "a lion .", "words": ["lion", "a"]}, {"sent": "now this lion stays in the .", "words": ["the", "this", "in", "lion"]}, {"sent": "lion .", "words": ["lion"]}, {"sent": "is there a lion coming ?", "words": ["there", "lion", "a", "is"]}, {"sent": "lion ?", "words": ["lion"]}, {"sent": "thats your toy lion .", "words": ["lion", "toy", "your"]}, {"sent": "a sheep and a lion .", "words": ["lion", "sheep", "and", "a"]}, {"sent": "we wanna see the lion .", "words": ["the", "we", "wanna", "see", "lion"]}, {"sent": "maybe the the dinosaursll eat the lion .", "words": ["the", "lion", "eat"]}, {"sent": "well hafta make it a very secure lion house wont we ?", "words": ["we", "it", "house", "make", "lion", "a"]}, {"sent": "www is this a lion www is this a lion ?", "words": ["this", "lion", "a", "is"]}, {"sent": "little lion better go with his mummy then .", "words": ["better", "go", "with", "little", "his", "then", "lion"]}, {"sent": "yes the lion is listening to tigers heart huh ?", "words": ["the", "to", "lion", "is"]}, {"sent": "weve got a nice lion .", "words": ["nice", "lion", "a"]}, {"sent": "oh thats a lion yeah .", "words": ["lion", "a"]}, {"sent": "snake and a lion .", "words": ["lion", "a", "and"]}, {"sent": "lion ?", "words": ["lion"]}, {"sent": "lion .", "words": ["lion"]}, {"sent": "you like the lion you mean James Lion ?", "words": ["the", "lion", "like", "you"]}, {"sent": "and what does the lion eat ?", "words": ["the", "what", "does", "and", "eat", "lion"]}, {"sent": "I said come on over here lion .", "words": ["on", "lion", "here", "over"]}, {"sent": "who sent the lion to Nina ?", "words": ["the", "to", "lion", "who"]}, {"sent": "where little lion gone .", "words": ["where", "lion", "little"]}, {"sent": "not far from the ceremony in a cave at the back side of Pride rock a scraggly lion with a dark mane grumbled lifes not fair I shall never be king .", "words": ["the", "back", "of", "be", "dark", "with", "in", "at", "rock", "lion", "not", "a"]}, {"sent": "hes a nice soft lion isnt he ?", "words": ["soft", "nice", "he", "lion", "a"]}, {"sent": "and a lion would go grr .", "words": ["go", "and", "would", "lion", "a"]}, {"sent": "look on here look under the lion .", "words": ["the", "here", "under", "look", "on", "lion"]}, {"sent": "and we were looking for the crocodile and then the lion ate the crocodile .", "words": ["the", "for", "we", "lion", "and", "then", "were"]}, {"sent": "thats the lion .", "words": ["the", "lion"]}, {"sent": "its an old lion .", "words": ["old", "an", "lion"]}, {"sent": "a lion cub crying ?", "words": ["lion", "a"]}, {"sent": "look at this funny lion .", "words": ["this", "at", "lion", "look"]}, {"sent": "isnt that a lion ?", "words": ["lion", "that", "a"]}, {"sent": "is there a lion coming ?", "words": ["there", "lion", "a", "is"]}, {"sent": "lion .", "words": ["lion"]}, {"sent": "or just a lion ?", "words": ["lion", "a"]}, {"sent": "Derek you wanna be the lion tamer ?", "words": ["the", "be", "you", "wanna", "lion"]}, {"sent": "youre gonna bang your head on the lion tiger whatever .", "words": ["the", "your", "head", "tiger", "on", "lion"]}, {"sent": "well youre drinking from a lion king beaker arent you ?", "words": ["lion", "you", "a"]}, {"sent": "theres a lion in your garage .", "words": ["your", "garage", "in", "lion", "a"]}, {"sent": "where did you hear a lion ?", "words": ["where", "hear", "you", "did", "lion", "a"]}, {"sent": "yes shall we send the lion for a trip on this train ?", "words": ["the", "for", "we", "train", "this", "on", "lion", "a"]}, {"sent": "here comes the lion .", "words": ["here", "the", "lion"]}, {"sent": "not a lion .", "words": ["lion", "not", "a"]}, {"sent": "really a lion goes roar .", "words": ["lion", "a"]}, {"sent": "thats a lion .", "words": ["lion", "a"]}, {"sent": "lion .", "words": ["lion"]}, {"sent": "its a lion .", "words": ["lion", "a"]}, {"sent": "but hes smaller than your lion .", "words": ["lion", "your", "but"]}, {"sent": "a lion .", "words": ["lion", "a"]}, {"sent": "there is a lion .", "words": ["there", "lion", "a", "is"]}, {"sent": "thats a funny lion .", "words": ["lion", "a"]}, {"sent": "now where is baby lion ?", "words": ["where", "lion", "is"]}, {"sent": "whats a lion say ?", "words": ["lion", "say", "a"]}, {"sent": "does the lion say baahbabaah ?", "words": ["the", "does", "lion", "say"]}, {"sent": "lion yeah .", "words": ["lion"]}, {"sent": "lion .", "words": ["lion"]}, {"sent": "and you saw a lion ?", "words": ["a", "lion", "you", "and"]}, {"sent": "what do you think the lion would like to eat ?", "words": ["the", "what", "do", "think", "to", "like", "you", "eat", "would", "lion"]}, {"sent": "make the lion jump through the hoop .", "words": ["the", "lion", "jump", "make"]}, {"sent": "sure I can make a lion .", "words": ["can", "lion", "make", "a"]}, {"sent": "lion .", "words": ["lion"]}, {"sent": "lion says Im going to make the lion talk okay .", "words": ["the", "to", "lion", "talk", "make"]}, {"sent": "putting the lion back on top of the lion right ?", "words": ["the", "back", "of", "on", "lion"]}, {"sent": "theres a lion coming to get me .", "words": ["to", "get", "me", "lion", "a"]}, {"sent": "when did you see a lion eat a KitKat ?", "words": ["lion", "you", "eat", "did", "see", "when", "a"]}, {"sent": "its a kind of lion .", "words": ["of", "lion", "a"]}, {"sent": "this chicken talked to the lion and said you know what you look like a big yellow banana .", "words": ["the", "what", "to", "yellow", "like", "and", "you", "this", "big", "banana", "look", "chicken", "lion", "a"]}, {"sent": "a lion .", "words": ["lion", "a"]}, {"sent": "right lets go downstairs and show Rachel that lion picture that you drew .", "words": ["go", "show", "picture", "and", "you", "that", "lion"]}, {"sent": "you saw a lion ?", "words": ["lion", "a", "you"]}, {"sent": "no its a lion .", "words": ["lion", "a"]}, {"sent": "right well it can be a lion then cant it ?", "words": ["can", "be", "it", "then", "lion", "a"]}, {"sent": "a lion .", "words": ["lion", "a"]}, {"sent": "is it a lion ?", "words": ["it", "lion", "a", "is"]}, {"sent": "the lion said .", "words": ["the", "lion"]}, {"sent": "a lion ?", "words": ["lion", "a"]}, {"sent": "I hear a raar lion .", "words": ["lion", "a", "hear"]}, {"sent": "look at this funny lion .", "words": ["this", "at", "lion", "look"]}, {"sent": "you like lion ?", "words": ["lion", "like", "you"]}, {"sent": "it had the lion .", "words": ["the", "it", "lion"]}, {"sent": "Mummy doesnt want the lion .", "words": ["the", "lion"]}, {"sent": "a lion ?", "words": ["lion", "a"]}, {"sent": "the lion was on the little cup cake yesterday wasnt it ?", "words": ["the", "cup", "cake", "was", "little", "it", "on", "lion"]}, {"sent": "is that a lion ?", "words": ["lion", "that", "a", "is"]}, {"sent": "thats a lion a daddy lion .", "words": ["lion", "a"]}, {"sent": "what does the lion say ?", "words": ["the", "what", "does", "say", "lion"]}, {"sent": "whats a lion say ?", "words": ["lion", "say", "a"]}, {"sent": "licking a lion club cub .", "words": ["lion", "a"]}, {"sent": "you put the lion cutter with the cutters that youve already used .", "words": ["the", "put", "with", "you", "that", "lion"]}, {"sent": "so I believe youre going to xxx for a lawn mower .", "words": ["for", "to", "so", "lawn mower", "a"]}, {"sent": "that man across the street is doing the lawn mower .", "words": ["the", "is", "lawn mower", "street", "that"]}, {"sent": "now youve got a lawn mower outside havent you ?", "words": ["outside", "lawn mower", "you", "a"]}, {"sent": "and he put his lawn mower away didnt he ?", "words": ["put", "lawn mower", "and", "away", "he", "his"]}, {"sent": "our last lawn mower before the Flymo had a xxx on it .", "words": ["the", "lawn mower", "last", "our", "it", "on", "a"]}, {"sent": "it wasnt christmas yet it couldnt be a christmas present couldnt be a lawn mower daddy still had a lawn mower couldnt be a tricycle mike still had his red tricycle .", "words": ["red", "present", "be", "lawn mower", "it", "tricycle", "his", "a"]}, {"sent": "no thats the um lawn mower .", "words": ["the", "lawn mower"]}, {"sent": "so he has a lawn mower .", "words": ["so", "he", "lawn mower", "a"]}, {"sent": "a lawn mower .", "words": ["lawn mower", "a"]}, {"sent": "a lawn mower .", "words": ["lawn mower", "a"]}, {"sent": "thats a lawn mower .", "words": ["lawn mower", "a"]}, {"sent": "oh do you hear the lawn mower ?", "words": ["the", "do", "lawn mower", "hear", "you"]}, {"sent": "lawn mower .", "words": ["lawn mower"]}, {"sent": "Daddy has a lawn mower .", "words": ["lawn mower", "a"]}, {"sent": "lawn mower .", "words": ["lawn mower"]}, {"sent": "I see the lawn mower .", "words": ["the", "see", "lawn mower"]}, {"sent": "is he doing the lawn mower ?", "words": ["the", "lawn mower", "he", "is"]}, {"sent": "and you can use your toy lawn mower on the garden cant you ?", "words": ["can", "the", "your", "lawn mower", "you", "and", "toy", "garden", "on"]}, {"sent": "no lawn mower ?", "words": ["lawn mower"]}, {"sent": "things like your lawn mower and your wheelbarrow ?", "words": ["lawn mower", "your", "like", "and"]}, {"sent": "weve got all the tools and the lawn mower .", "words": ["the", "all", "lawn mower", "and"]}, {"sent": "youre just going up and down with the lawn mower .", "words": ["up", "the", "lawn mower", "with", "and", "down"]}, {"sent": "you have a lawn mower outside havent you ?", "words": ["outside", "lawn mower", "you", "have", "a"]}, {"sent": "uh not the wheelbarrow the lawn mower .", "words": ["the", "lawn mower", "not"]}, {"sent": "theres the lawn mower .", "words": ["the", "lawn mower"]}, {"sent": "theres a lawn mower outside of Sues house .", "words": ["of", "outside", "lawn mower", "house", "a"]}, {"sent": "and theyve come back again today with the lawn mower .", "words": ["the", "back", "lawn mower", "with", "and"]}, {"sent": "the gardeners have arrived now with the lawn mower and theyre going to cut Sues grass .", "words": ["the", "cut", "to", "lawn mower", "with", "and", "have", "grass"]}, {"sent": "can you hear the sound of the lawn mower ?", "words": ["can", "the", "of", "lawn mower", "hear", "you"]}, {"sent": "its not a lawn mower now .", "words": ["lawn mower", "not", "a"]}, {"sent": "he has used the lawn mower its a like an edger isnt it ?", "words": ["the", "lawn mower", "like", "an", "it", "he", "a"]}, {"sent": "that fits on the back of the lawn mower .", "words": ["the", "back", "of", "lawn mower", "on", "that"]}, {"sent": "lawn mowing the gr grass shes mowing the grass with the lawn mower isnt she ?", "words": ["the", "lawn mower", "with", "she", "grass"]}, {"sent": "now he can use his new lawn mower .", "words": ["can", "lawn mower", "he", "his", "new"]}, {"sent": "why did he buy the lawn mower ?", "words": ["the", "lawn mower", "he", "did", "buy", "why"]}, {"sent": "are you gonna blow Daddy a kiss ?", "words": ["you", "are", "kiss", "blow", "a"]}, {"sent": "blow it .", "words": ["it", "blow"]}, {"sent": "yes it means blow too does not it .", "words": ["blow", "does", "it", "too", "not"]}, {"sent": "you blow and let me go get Paul and Diandra .", "words": ["go", "and", "you", "get", "me", "blow"]}, {"sent": "blow it again .", "words": ["it", "blow"]}, {"sent": "what blow away ?", "words": ["what", "away", "blow"]}, {"sent": "can you blow into it ?", "words": ["can", "into", "you", "it", "blow"]}, {"sent": "and theyre gonna blow up the balloon see the fires going straight into the balloon .", "words": ["up", "the", "into", "and", "balloon", "see", "blow"]}, {"sent": "I will blow it .", "words": ["will", "it", "blow"]}, {"sent": "you can make a small breeze by yourself when you make air move you are making wind blow blow a piece of tissue across the table .", "words": ["can", "the", "yourself", "when", "table", "tissue", "of", "wind", "by", "you", "are", "make", "blow", "a"]}, {"sent": "and shell blow it out .", "words": ["it", "blow", "out", "and"]}, {"sent": "go blow on it .", "words": ["on", "it", "blow", "go"]}, {"sent": "blow blow your nose blow .", "words": ["your", "blow", "nose"]}, {"sent": "you want Mummy to blow her nose ?", "words": ["to", "you", "her", "blow", "nose"]}, {"sent": "yeah you blow into the kleenex .", "words": ["the", "into", "kleenex", "you", "blow"]}, {"sent": "lets blow a really big bubble .", "words": ["big", "blow", "a"]}, {"sent": "you blow a xxx .", "words": ["blow", "a", "you"]}, {"sent": "he hasta blow the sausage before he eats it .", "words": ["the", "it", "he", "blow"]}, {"sent": "too hot again oo we better blow .", "words": ["better", "we", "hot", "too", "blow"]}, {"sent": "do you wanna me to blow some more bubbles ?", "words": ["do", "bubbles", "to", "you", "more", "wanna", "me", "blow", "some"]}, {"sent": "blow .", "words": ["blow"]}, {"sent": "you can blow aswellas Daddy .", "words": ["can", "blow", "you"]}, {"sent": "and blow and blow .", "words": ["blow", "and"]}, {"sent": "big blow .", "words": ["big", "blow"]}, {"sent": "what comes out if you blow ?", "words": ["what", "if", "you", "out", "blow"]}, {"sent": "did you blow them ?", "words": ["blow", "them", "did", "you"]}, {"sent": "you blow down your nose .", "words": ["your", "you", "down", "blow", "nose"]}, {"sent": "you want ta blow ?", "words": ["blow", "you"]}, {"sent": "blow on it .", "words": ["on", "it", "blow"]}, {"sent": "blow ?", "words": ["blow"]}, {"sent": "well if you blow the horn its usually a warning to other people .", "words": ["the", "if", "to", "you", "other", "blow", "a"]}, {"sent": "its nice to blow bubbles outside isnt it ?", "words": ["to", "outside", "nice", "it", "blow", "bubbles"]}, {"sent": "you come back and blow on the carrots .", "words": ["the", "back", "and", "you", "carrots", "on", "blow"]}, {"sent": "can you blow a bubble ?", "words": ["can", "blow", "a", "you"]}, {"sent": "thats what happens when when things blow up .", "words": ["up", "what", "when", "blow"]}, {"sent": "do you wanna blow your nose ?", "words": ["do", "your", "you", "wanna", "blow", "nose"]}, {"sent": "you blow it .", "words": ["it", "blow", "you"]}, {"sent": "say no Adam I cant blow yet .", "words": ["say", "blow"]}, {"sent": "youre going to blow ?", "words": ["to", "blow"]}, {"sent": "you wanna blow some more ?", "words": ["you", "more", "wanna", "some", "blow"]}, {"sent": "give it a blow so it whistles .", "words": ["so", "it", "give", "blow", "a"]}, {"sent": "blow that nose .", "words": ["that", "blow", "nose"]}, {"sent": "big blow .", "words": ["big", "blow"]}, {"sent": "blow it that way .", "words": ["it", "that", "blow"]}, {"sent": "no she doesnt wanna blow that .", "words": ["blow", "that", "she", "wanna"]}, {"sent": "would you like to blow some bubbles ?", "words": ["to", "like", "you", "would", "some", "blow", "bubbles"]}, {"sent": "cant hear you properly unless I can blow your nose .", "words": ["can", "your", "you", "hear", "blow", "nose"]}, {"sent": "blow .", "words": ["blow"]}, {"sent": "blow blow blow .", "words": ["blow"]}, {"sent": "blow it out .", "words": ["it", "blow", "out"]}, {"sent": "big blow .", "words": ["big", "blow"]}, {"sent": "you can blow it yourself little sweetie .", "words": ["can", "yourself", "you", "little", "it", "blow"]}, {"sent": "oh that was a wet blow .", "words": ["wet", "was", "that", "blow", "a"]}, {"sent": "are you blowing as if to blow on hot chips to cool them down ?", "words": ["if", "to", "them", "you", "hot", "are", "on", "down", "blow"]}, {"sent": "blow your nose ?", "words": ["your", "blow", "nose"]}, {"sent": "just blow gently .", "words": ["blow"]}, {"sent": "blow .", "words": ["blow"]}, {"sent": "Ill blow on it .", "words": ["on", "it", "blow"]}, {"sent": "there it is you wana blow it a kiss ?", "words": ["is", "you", "there", "it", "kiss", "blow", "a"]}, {"sent": "blow it over .", "words": ["it", "over", "blow"]}, {"sent": "Betty likes to blow bubbles .", "words": ["to", "blow", "bubbles"]}, {"sent": "here blow your nose .", "words": ["here", "your", "blow", "nose"]}, {"sent": "blow .", "words": ["blow"]}, {"sent": "do you want to blow it ?", "words": ["do", "to", "you", "it", "want to", "blow"]}, {"sent": "*SI1: blow .", "words": ["blow"]}, {"sent": "can you blow ?", "words": ["can", "blow", "you"]}, {"sent": "and gently blow .", "words": ["blow", "and"]}, {"sent": "you wanna blow the whistles instead ?", "words": ["the", "blow", "wanna", "you"]}, {"sent": "blow blow blow .", "words": ["blow"]}, {"sent": "you dont hafta blow the grapes Thomas .", "words": ["the", "you", "dont", "grapes", "blow"]}, {"sent": "can you blow it ?", "words": ["can", "it", "blow", "you"]}, {"sent": "blow them out blow them out .", "words": ["them", "blow", "out"]}, {"sent": "did you wanna blow the candle ?", "words": ["the", "you", "did", "wanna", "blow"]}, {"sent": "blow .", "words": ["blow"]}, {"sent": "oh I think you blow them out first because if you were if they were burning when you jumped over them then they might burn you .", "words": ["think", "if", "when", "first", "them", "you", "they", "out", "over", "then", "were", "blow", "because"]}, {"sent": "Ill blow one .", "words": ["blow"]}, {"sent": "blow the .", "words": ["the", "blow"]}, {"sent": "blow hard .", "words": ["blow", "hard"]}, {"sent": "Dinah blow your horn .", "words": ["your", "blow"]}, {"sent": "did you wanna blow the candles ?", "words": ["the", "you", "did", "wanna", "blow"]}, {"sent": "blow your nose .", "words": ["your", "blow", "nose"]}, {"sent": "hey hey blow the man down .", "words": ["the", "down", "blow"]}, {"sent": "Dinah wont you blow .", "words": ["blow", "you"]}, {"sent": "youre just gonna blow bubbles huh ?", "words": ["blow", "bubbles"]}, {"sent": "now youre going to watch and then you can blow yourself .", "words": ["can", "watch", "yourself", "to", "you", "and", "then", "blow"]}, {"sent": "blow out the candles .", "words": ["the", "blow", "out"]}, {"sent": "oh blow with the spoon .", "words": ["the", "with", "blow", "spoon"]}, {"sent": "lets blow on them .", "words": ["on", "them", "blow"]}, {"sent": "is that what you do with hot stuff is you blow .", "words": ["what", "do", "is", "with", "you", "hot", "that", "blow"]}, {"sent": "did you play with the bubbles did she blow some bubbles for you ?", "words": ["the", "blow", "for", "play", "with", "you", "did", "some", "she", "bubbles"]}, {"sent": "blow on it .", "words": ["on", "it", "blow"]}, {"sent": "oh my gosh its going to blow away ?", "words": ["to", "my", "away", "blow"]}, {"sent": "see hands with handkerchiefs blow blow blow .", "words": ["see", "with", "blow"]}, {"sent": "blow it Olivia itll be okay .", "words": ["be", "it", "blow"]}, {"sent": "Ill blow it up xxx .", "words": ["up", "it", "blow"]}, {"sent": "*SI1: could blow blow bubbles .", "words": ["could", "blow", "bubbles"]}, {"sent": "well finish reading this and then well blow again .", "words": ["finish", "and", "this", "then", "blow"]}, {"sent": "blow first and then wipe .", "words": ["first", "and", "then", "wipe", "blow"]}, {"sent": "blow that food .", "words": ["food", "that", "blow"]}, {"sent": "no its soft .", "words": ["soft"]}, {"sent": "like the soft food toys .", "words": ["the", "food", "soft", "like"]}, {"sent": "were they really soft ?", "words": ["they", "soft", "were"]}, {"sent": "lightly and silently it floats above the soft planet .", "words": ["the", "soft", "and", "it", "above"]}, {"sent": "but because your skins so soft .", "words": ["soft", "your", "so", "but", "because"]}, {"sent": "soft .", "words": ["soft"]}, {"sent": "very soft .", "words": ["soft"]}, {"sent": "xxx with the soft shapes in them .", "words": ["the", "soft", "with", "them", "in"]}, {"sent": "soft soft .", "words": ["soft"]}, {"sent": "soft ?", "words": ["soft"]}, {"sent": "xxx soft .", "words": ["soft"]}, {"sent": "MisterBrown can whisper whisper very soft very high .", "words": ["can", "soft", "high"]}, {"sent": "soft isnt it ?", "words": ["it", "soft"]}, {"sent": "the soft stuff ?", "words": ["the", "soft"]}, {"sent": "here so soft .", "words": ["here", "so", "soft"]}, {"sent": "hes so soft .", "words": ["so", "soft"]}, {"sent": "nice and soft .", "words": ["soft", "nice", "and"]}, {"sent": "its beautiful soft PlayDoh that .", "words": ["soft", "that"]}, {"sent": "soft .", "words": ["soft"]}, {"sent": "oh yes and he must be soft like Bert .", "words": ["soft", "be", "like", "and", "he"]}, {"sent": "you can play with soft stuff anytime .", "words": ["can", "soft", "play", "with", "you"]}, {"sent": "it looks nice and soft the white one as well .", "words": ["the", "soft", "nice", "and", "it", "white"]}, {"sent": "thats a kitty with soft fur .", "words": ["soft", "kitty", "with", "a"]}, {"sent": "oh that feels nice and soft .", "words": ["soft", "nice", "that", "and"]}, {"sent": "you feel soft fur ?", "words": ["soft", "you"]}, {"sent": "and I think thats part of the reason he likes to play with the pla the hard toys compared to the soft toys .", "words": ["the", "think", "soft", "of", "to", "play", "with", "and", "he", "hard"]}, {"sent": "that silly soft chicken .", "words": ["chicken", "soft", "that"]}, {"sent": "look at how soft that oh that is nice .", "words": ["how", "soft", "look", "is", "nice", "at", "that"]}, {"sent": "oh your skin is so soft .", "words": ["so", "soft", "your", "is"]}, {"sent": "its soft .", "words": ["soft"]}, {"sent": "that ones quite a soft one .", "words": ["soft", "that", "a"]}, {"sent": "its big its soft its Andrex .", "words": ["soft", "big"]}, {"sent": "soft isnt it ?", "words": ["it", "soft"]}, {"sent": "because thats because its soft right ?", "words": ["soft", "because"]}, {"sent": "magic soft .", "words": ["soft"]}, {"sent": "soft .", "words": ["soft"]}, {"sent": "its nice and soft .", "words": ["soft", "nice", "and"]}, {"sent": "it is nice and soft .", "words": ["soft", "is", "nice", "and", "it"]}, {"sent": "suddenly Miffy heard a soft .", "words": ["soft", "a"]}, {"sent": "yeah the books um and we have like a little soft toys but none of them are the same .", "words": ["the", "soft", "we", "of", "none", "them", "like", "and", "have", "little", "are", "same", "but", "a"]}, {"sent": "you and Fraser you and Jwww have got your soft blankets havent you ?", "words": ["soft", "your", "and", "have", "you"]}, {"sent": "oh mines soft .", "words": ["soft"]}, {"sent": "and a real soft ball .", "words": ["ball", "soft", "a", "and"]}, {"sent": "a little soft toy .", "words": ["toy", "soft", "little", "a"]}, {"sent": "you know its funny we have um two soft my first baby dolls .", "words": ["my", "soft", "we", "first", "you", "have"]}, {"sent": "theyre very soft .", "words": ["soft"]}, {"sent": "why youre right its not very soft cause I didnt put this one in my bra .", "words": ["put", "soft", "my", "this", "in", "not", "why"]}, {"sent": "your skin feels nice and soft .", "words": ["soft", "nice", "your", "and"]}, {"sent": "its nice and soft do you feel it ?", "words": ["do", "soft", "nice", "you", "and", "it"]}, {"sent": "oh very soft .", "words": ["soft"]}, {"sent": "this is nice nice and soft .", "words": ["soft", "is", "nice", "and", "this"]}, {"sent": "and a soft tail .", "words": ["soft", "a", "and"]}, {"sent": "its soft .", "words": ["soft"]}, {"sent": "feel how soft it is .", "words": ["it", "how", "soft", "is"]}, {"sent": "youve got a lot of soft toys in the bedroom and Im tired of having to move them everytime I make the bed .", "words": ["the", "soft", "bedroom", "of", "to", "a lot", "them", "bed", "and", "tired", "in", "make", "a"]}, {"sent": "she likes her little soft blocks that we I make the tower .", "words": ["the", "soft", "we", "her", "little", "make", "that", "she"]}, {"sent": "soft .", "words": ["soft"]}, {"sent": "think thats really feathery and soft .", "words": ["think", "soft", "and"]}, {"sent": "you are soft .", "words": ["soft", "are", "you"]}, {"sent": "soft .", "words": ["soft"]}, {"sent": "okay but blow soft .", "words": ["soft", "blow", "but"]}, {"sent": "nice and soft these arent they ?", "words": ["these", "soft", "nice", "and", "they"]}, {"sent": "aw yeah its a soft doggie .", "words": ["soft", "a"]}, {"sent": "soft .", "words": ["soft"]}, {"sent": "this is soft .", "words": ["this", "soft", "is"]}, {"sent": "oh soft .", "words": ["soft"]}, {"sent": "so it gets soft for you .", "words": ["for", "soft", "so", "you", "it"]}, {"sent": "and maybe the soft texture of it .", "words": ["the", "soft", "of", "and", "it"]}, {"sent": "this playdough is too soft .", "words": ["this", "too", "soft", "is"]}, {"sent": "its not too bad with the soft things but not the hard .", "words": ["the", "soft", "with", "bad", "hard", "too", "not", "but"]}, {"sent": "so soft .", "words": ["so", "soft"]}, {"sent": "we have like the soft blocks .", "words": ["the", "soft", "we", "like", "have"]}, {"sent": "well if you if you throw the seeds on soft plowed earth then they grow better than if you sow them on hard caked earth .", "words": ["the", "better", "soft", "throw", "if", "them", "you", "they", "hard", "on", "then"]}, {"sent": "oh is it nice and soft ?", "words": ["soft", "is", "nice", "and", "it"]}, {"sent": "no and when he is its kinda soft .", "words": ["soft", "is", "and", "he", "when"]}, {"sent": "if you talk too soft I cant hear you then either .", "words": ["soft", "if", "you", "hear", "talk", "too", "then"]}, {"sent": "this looks like a soft pink .", "words": ["this", "soft", "like", "a"]}, {"sent": "soft .", "words": ["soft"]}, {"sent": "he is soft .", "words": ["soft", "he", "is"]}, {"sent": "like the soft soft whisper of a butterfly .", "words": ["the", "butterfly", "soft", "of", "like", "a"]}, {"sent": "look how soft .", "words": ["how", "soft", "look"]}, {"sent": "all soft and snuggly .", "words": ["all", "soft", "and"]}, {"sent": "soft fruit .", "words": ["soft"]}, {"sent": "in a soft fluffy snowbank answered Stella .", "words": ["in", "soft", "a"]}, {"sent": "that nice and soft ?", "words": ["soft", "nice", "that", "and"]}, {"sent": "soft stuff alert .", "words": ["soft"]}, {"sent": "next she sat down in the middle sized chair but it was too soft when Goldilocks sat down in the wee little chair it was neither too hard nor too soft but just right but she was so heavy the chair broke all in pieces down she came plump on the floor and that made her very cross .", "words": ["the", "hard", "when", "soft", "chair", "it", "on", "down", "that", "she", "all", "was", "so", "little", "and", "too", "heavy", "in", "her", "but"]}, {"sent": "does it feel nice and soft ?", "words": ["does", "soft", "nice", "and", "it"]}, {"sent": "soft wooly coat .", "words": ["soft", "coat"]}, {"sent": "soft .", "words": ["soft"]}, {"sent": "oh isnt that soft ?", "words": ["soft", "that"]}, {"sent": "soft like a puppy dog .", "words": ["soft", "like", "puppy", "dog", "a"]}, {"sent": "was she interested in any of the soft toys at all ?", "words": ["the", "all", "soft", "of", "was", "any", "in", "at", "she"]}, {"sent": "is that soft ?", "words": ["soft", "that", "is"]}, {"sent": "Mister Brown can whisper whisper very soft very high .", "words": ["can", "soft", "high"]}, {"sent": "but it has soft skin .", "words": ["soft", "it", "but"]}, {"sent": "this soft .", "words": ["this", "soft"]}, {"sent": "soft kitty .", "words": ["kitty", "soft"]}, {"sent": "did you say thats a cow ?", "words": ["say", "you", "did", "cow", "a"]}, {"sent": "cow .", "words": ["cow"]}, {"sent": "Lubriderm for the cow ?", "words": ["the", "for", "cow"]}, {"sent": "dont think there ever was a lying down cow was there ?", "words": ["think", "was", "dont", "there", "down", "cow", "a"]}, {"sent": "a big brown cow came out to play .", "words": ["brown", "to", "play", "out", "big", "cow", "a"]}, {"sent": "cow .", "words": ["cow"]}, {"sent": "feel the cow .", "words": ["the", "cow"]}, {"sent": "heres the cow .", "words": ["the", "cow"]}, {"sent": "a calf and a cow .", "words": ["cow", "and", "a"]}, {"sent": "thats a cow .", "words": ["cow", "a"]}, {"sent": "lets see if we can do it can do downward cow here .", "words": ["can", "do", "if", "we", "here", "it", "see", "cow"]}, {"sent": "we got the momma cow .", "words": ["the", "cow", "we"]}, {"sent": "you little cow or you little donkey or whatever you are .", "words": ["little", "you", "are", "donkey", "cow"]}, {"sent": "a cow .", "words": ["cow", "a"]}, {"sent": "what does the cow say ?", "words": ["the", "what", "does", "say", "cow"]}, {"sent": "they hadta sell the cow didnt they ?", "words": ["the", "they", "cow"]}, {"sent": "yeah where do you see a cow ?", "words": ["where", "do", "you", "see", "cow", "a"]}, {"sent": "you wanna hear the cow ?", "words": ["the", "hear", "you", "wanna", "cow"]}, {"sent": "thats for pushing the cow outof the way .", "words": ["the", "for", "cow"]}, {"sent": "I gave mine to the cow didnt I ?", "words": ["to", "mine", "cow", "the"]}, {"sent": "a cow ?", "words": ["cow", "a"]}, {"sent": "its a cow .", "words": ["cow", "a"]}, {"sent": "goodnight cow jumping over the moon .", "words": ["the", "over", "moon", "cow"]}, {"sent": "another cow .", "words": ["another", "cow"]}, {"sent": "is that a cow ?", "words": ["that", "cow", "a", "is"]}, {"sent": "Thomas the cow is wandering away .", "words": ["the", "away", "cow", "is"]}, {"sent": "cow ?", "words": ["cow"]}, {"sent": "I weve seen cow is there a cow in here weve seen pigs .", "words": ["is", "here", "there", "in", "cow", "a"]}, {"sent": "what cow ?", "words": ["what", "cow"]}, {"sent": "a cow jumping over the moon thats right .", "words": ["the", "moon", "over", "cow", "a"]}, {"sent": "not a cow .", "words": ["not", "cow", "a"]}, {"sent": "now do I have a cow ?", "words": ["have", "do", "cow", "a"]}, {"sent": "is this a cow ?", "words": ["this", "cow", "a", "is"]}, {"sent": "a cow wha does a cow say ?", "words": ["does", "say", "cow", "a"]}, {"sent": "the cow catcher looks a bit like the snowplow that Thomas had didnt it in the story about the village getting cut off .", "words": ["the", "off", "cut", "like", "in", "it", "about", "story", "that", "cow", "a"]}, {"sent": "that looks like a little baby cow .", "words": ["like", "little", "that", "cow", "a"]}, {"sent": "its a cow .", "words": ["cow", "a"]}, {"sent": "the cow then won first prize as the best cow in the country .", "words": ["the", "first", "in", "then", "cow"]}, {"sent": "theres a cow there that needs to go somewhere .", "words": ["go", "to", "there", "that", "cow", "a"]}, {"sent": "the cow ?", "words": ["the", "cow"]}, {"sent": "you might find a cow over here .", "words": ["you", "here", "find", "over", "cow", "a"]}, {"sent": "the cow says moo .", "words": ["the", "cow"]}, {"sent": "pretend you milk the cow .", "words": ["the", "you", "milk", "pretend", "cow"]}, {"sent": "five beans for our cow cried his mother .", "words": ["for", "our", "beans", "his", "cow"]}, {"sent": "the cow .", "words": ["the", "cow"]}, {"sent": "why do you like the cow ?", "words": ["the", "do", "like", "cow", "you", "why"]}, {"sent": "hes really a crippled cow .", "words": ["cow", "a"]}, {"sent": "are you looking for the cow ?", "words": ["the", "for", "you", "are", "cow"]}, {"sent": "sell the cow ?", "words": ["the", "cow"]}, {"sent": "what does the cow .", "words": ["the", "what", "does", "cow"]}, {"sent": "can you find that cow can you find that cow ?", "words": ["can", "you", "find", "that", "cow"]}, {"sent": "theres a cow .", "words": ["cow", "a"]}, {"sent": "a little baby cow .", "words": ["cow", "little", "a"]}, {"sent": "what does what sound does a cow make ?", "words": ["what", "does", "make", "cow", "a"]}, {"sent": "shall I get the shall I get the plastic cow out ?", "words": ["get", "the", "out", "cow"]}, {"sent": "see the cow ?", "words": ["the", "see", "cow"]}, {"sent": "theres the cow .", "words": ["the", "cow"]}, {"sent": "its like a cow .", "words": ["like", "cow", "a"]}, {"sent": "lets look for the cow .", "words": ["the", "for", "cow", "look"]}, {"sent": "the cow is eating flowers .", "words": ["the", "cow", "is"]}, {"sent": "cow .", "words": ["cow"]}, {"sent": "can you put the cow on ?", "words": ["can", "the", "put", "you", "on", "cow"]}, {"sent": "yeah good cow .", "words": ["cow", "good"]}, {"sent": "is he a long horned cow ?", "words": ["is", "long", "he", "cow", "a"]}, {"sent": "theres a cow .", "words": ["cow", "a"]}, {"sent": "is that a cow ?", "words": ["that", "cow", "a", "is"]}, {"sent": "a cow .", "words": ["cow", "a"]}, {"sent": "the cow is chasing ?", "words": ["the", "cow", "is"]}, {"sent": "baby cow called a calf .", "words": ["cow", "a"]}, {"sent": "how does the cow go ?", "words": ["the", "does", "how", "go", "cow"]}, {"sent": "what does the cow do ?", "words": ["the", "what", "do", "does", "cow"]}, {"sent": "a cow .", "words": ["cow", "a"]}, {"sent": "its a big cow .", "words": ["big", "cow", "a"]}, {"sent": "he can sound like a cow .", "words": ["can", "like", "he", "cow", "a"]}, {"sent": "its a cow .", "words": ["cow", "a"]}, {"sent": "they were so poor that she said to him one day well hafta sell our cow its the only way .", "words": ["the", "so", "to", "poor", "our", "they", "that", "him", "were", "she", "cow"]}, {"sent": "he can sound like a cow he can go moo moo moo .", "words": ["can", "go", "like", "he", "cow", "a"]}, {"sent": "cow .", "words": ["cow"]}, {"sent": "cow .", "words": ["cow"]}, {"sent": "whats a cow say ?", "words": ["say", "cow", "a"]}, {"sent": "and what do we call a Daddy cow ?", "words": ["what", "do", "we", "and", "cow", "a"]}, {"sent": "its another picture of the cow .", "words": ["the", "of", "picture", "another", "cow"]}, {"sent": "its a baby cow .", "words": ["cow", "a"]}, {"sent": "I can see a cow .", "words": ["can", "see", "cow", "a"]}, {"sent": "there now hes on the mommy cow .", "words": ["on", "there", "cow", "the"]}, {"sent": "go on then little cow which ones do you wanna turn over ?", "words": ["do", "over", "go", "little", "you", "wanna", "on", "then", "which", "cow"]}, {"sent": "moo a cow .", "words": ["cow", "a"]}, {"sent": "is the little cow coming to the birthday party ?", "words": ["the", "is", "to", "little", "party", "cow"]}, {"sent": "I know a cow goes moo .", "words": ["cow", "a"]}, {"sent": "you havent finished the cow one yet .", "words": ["the", "cow", "you"]}, {"sent": "cow ?", "words": ["cow"]}, {"sent": "whys he sitting on top of the cow ?", "words": ["the", "of", "he", "on", "cow"]}, {"sent": "heres an old cow .", "words": ["old", "an", "cow"]}, {"sent": "the cow can go in cant it ?", "words": ["the", "can", "go", "in", "it", "cow"]}, {"sent": "say cow .", "words": ["say", "cow"]}, {"sent": "its a cow .", "words": ["cow", "a"]}, {"sent": "theres the cow .", "words": ["the", "cow"]}, {"sent": "oh theres not going to be very much for the cow .", "words": ["the", "for", "to", "be", "much", "not", "cow"]}, {"sent": "let me see your tongue .", "words": ["see", "me", "your", "tongue"]}, {"sent": "youre using your tongue to lick your lips arent you ?", "words": ["your", "to", "you", "tongue", "lick"]}, {"sent": "now now you have it on your tongue .", "words": ["your", "you", "have", "it", "on", "tongue"]}, {"sent": "get your tongue in your mouth .", "words": ["your", "mouth", "get", "in", "tongue"]}, {"sent": "Ill do his tongue then .", "words": ["his", "do", "then", "tongue"]}, {"sent": "oh good well its a good thing they fit on your tongue .", "words": ["your", "fit", "they", "good", "on", "tongue", "a"]}, {"sent": "theyre seeing whose tongue is longer .", "words": ["tongue", "is"]}, {"sent": "Im cut that tongue out .", "words": ["tongue", "that", "cut", "out"]}, {"sent": "xxx take your tongue outof the way .", "words": ["your", "the", "take", "tongue"]}, {"sent": "didnt you bite your tongue this morning ?", "words": ["your", "you", "this", "bite", "tongue"]}, {"sent": "I didnt know he had a tongue .", "words": ["tongue", "he", "a"]}, {"sent": "under your tongue .", "words": ["tongue", "under", "your"]}, {"sent": "let me see your tongue .", "words": ["see", "me", "your", "tongue"]}, {"sent": "youre going to bite your tongue off .", "words": ["off", "your", "to", "bite", "tongue"]}, {"sent": "your tongue .", "words": ["tongue", "your"]}, {"sent": "not reading it until you put your tongue away and talk properly .", "words": ["put", "your", "you", "and", "it", "away", "talk", "tongue", "not"]}, {"sent": "you would take one mouthful of it and then you would take your tongue and spray it all over the place .", "words": ["the", "all", "over", "your", "of", "you", "and", "it", "would", "tongue", "then", "take"]}, {"sent": "stop sticking your tongue xxx .", "words": ["your", "stop", "tongue"]}, {"sent": "not your tongue your mouth you gonna stick out your tongue ?", "words": ["stick", "your", "mouth", "you", "out", "tongue", "not"]}, {"sent": "hes sticking out his tongue .", "words": ["his", "tongue", "out"]}, {"sent": "youre putting your tongue out .", "words": ["tongue", "your", "out"]}, {"sent": "the cat got your tongue ?", "words": ["the", "cat", "your", "tongue"]}, {"sent": "see his tongue ?", "words": ["his", "see", "tongue"]}, {"sent": "that kittys got a rough tongue .", "words": ["tongue", "that", "a"]}, {"sent": "Naomi you really upset that lady the other day in the boat honey when you stuck your tongue out at her .", "words": ["the", "your", "when", "boat", "her", "other", "you", "in", "stuck", "out", "at", "tongue", "that"]}, {"sent": "red tongue .", "words": ["red", "tongue"]}, {"sent": "this could be an anteaters tongue too .", "words": ["be", "this", "an", "too", "tongue", "could"]}, {"sent": "whos gonna do his tongue ?", "words": ["his", "do", "tongue"]}, {"sent": "wheres your tongue ?", "words": ["tongue", "your"]}, {"sent": "a long red tongue oh .", "words": ["red", "tongue", "long", "a"]}, {"sent": "your tongue is very blue from all of those blueberries .", "words": ["all", "your", "those", "is", "of", "blue", "tongue"]}, {"sent": "it would would it stick to his tongue ?", "words": ["stick", "to", "it", "would", "his", "tongue"]}, {"sent": "I think thats his tongue dont you .", "words": ["think", "you", "dont", "his", "tongue"]}, {"sent": "play with my tongue xxx .", "words": ["tongue", "my", "play", "with"]}, {"sent": "I think theyre just sticking out their tongue .", "words": ["tongue", "think", "their", "out"]}, {"sent": "it looks like this guys tongue is coming out and his tongue is alive is alive .", "words": ["is", "like", "and", "his", "this", "it", "out", "tongue"]}, {"sent": "is it his tongue ?", "words": ["his", "it", "tongue", "is"]}, {"sent": "wheres the cookie monsters tongue ?", "words": ["the", "cookie", "tongue"]}, {"sent": "Im gonna get your tongue .", "words": ["get", "your", "tongue"]}, {"sent": "everybody has a tongue .", "words": ["tongue", "a"]}, {"sent": "does this dog have a tongue ?", "words": ["does", "have", "this", "tongue", "dog", "a"]}, {"sent": "put that tongue in .", "words": ["tongue", "in", "put", "that"]}, {"sent": "hes opening his tongue hes showing his tongue hes sticking his tongue right out actually .", "words": ["his", "tongue", "out"]}, {"sent": "thats the frogs tongue .", "words": ["the", "tongue"]}, {"sent": "here will you wipe your tongue off please ?", "words": ["off", "your", "you", "here", "will", "wipe", "tongue"]}, {"sent": "is it making your lips go cold and your tongue go cold ?", "words": ["your", "go", "is", "and", "it", "tongue", "cold"]}, {"sent": "should we give the dog a tongue ?", "words": ["the", "we", "tongue", "give", "dog", "a"]}, {"sent": "whats wrong with your tongue ?", "words": ["your", "with", "tongue"]}, {"sent": "it twists your tongue .", "words": ["tongue", "it", "your"]}, {"sent": "thats the whales tongue .", "words": ["the", "tongue"]}, {"sent": "I dont like this tongue .", "words": ["tongue", "dont", "like", "this"]}, {"sent": "wheres your tongue ?", "words": ["tongue", "your"]}, {"sent": "look his tongue is up and his tongue is down .", "words": ["up", "is", "and", "look", "his", "down", "tongue"]}, {"sent": "you better keep that tongue in your mouth .", "words": ["better", "your", "mouth", "you", "in", "tongue", "that"]}, {"sent": "wheres your tongue ?", "words": ["tongue", "your"]}, {"sent": "it is a twist tongue .", "words": ["tongue", "it", "a", "is"]}, {"sent": "he had a tongue sandwich didnt he Daddy did ?", "words": ["he", "did", "tongue", "sandwich", "a"]}, {"sent": "my tongue isnt rubber its only a tongue .", "words": ["tongue", "my", "a"]}, {"sent": "um and tongue .", "words": ["tongue", "and"]}, {"sent": "I think thats a tongue if you wanna stick your tongue out you going to put it on the top there ?", "words": ["the", "think", "stick", "your", "if", "put", "to", "you", "there", "on", "it", "wanna", "out", "tongue", "a"]}, {"sent": "xxx at her tongue .", "words": ["tongue", "her", "at"]}, {"sent": "stick out your tongue .", "words": ["tongue", "stick", "your", "out"]}, {"sent": "big wet tongue lick your face .", "words": ["wet", "your", "face", "tongue", "lick", "big"]}, {"sent": "ah thats his tongue .", "words": ["his", "tongue"]}, {"sent": "you just put a bunch of that teddy bears hair on my tongue .", "words": ["put", "my", "of", "you", "on", "hair", "that", "tongue", "a"]}, {"sent": "I got your tongue .", "words": ["tongue", "your"]}, {"sent": "wheres his tongue ?", "words": ["his", "tongue"]}, {"sent": "this doggys sticking his tongue out isnt he ?", "words": ["this", "he", "out", "his", "tongue"]}, {"sent": "you taste with your tongue .", "words": ["your", "with", "taste", "you", "tongue"]}, {"sent": "she must have bit her tongue or her mouth or something .", "words": ["mouth", "have", "tongue", "her", "she"]}, {"sent": "tongue ?", "words": ["tongue"]}, {"sent": "move your tongue .", "words": ["tongue", "your"]}, {"sent": "are you doing silly things with your tongue ?", "words": ["your", "with", "you", "are", "tongue"]}, {"sent": "wheres your tongue ?", "words": ["tongue", "your"]}, {"sent": "put the tongue in .", "words": ["the", "in", "put", "tongue"]}, {"sent": "washing your tongue ?", "words": ["tongue", "your"]}, {"sent": "I want xxx my tongue came out ?", "words": ["tongue", "my", "out"]}, {"sent": "who has a big tongue like that ?", "words": ["like", "that", "who", "tongue", "big", "a"]}, {"sent": "hold your tongue Thomas .", "words": ["tongue", "your", "hold"]}, {"sent": "no tongue no tongue .", "words": ["tongue"]}, {"sent": "where is my tongue ?", "words": ["where", "my", "tongue", "is"]}, {"sent": "theres a big tongue .", "words": ["tongue", "big", "a"]}, {"sent": "she had her tongue gone .", "words": ["tongue", "her", "she"]}, {"sent": "how about the tongue ?", "words": ["the", "how", "tongue", "about"]}, {"sent": "xxx has a red tongue .", "words": ["red", "tongue", "a"]}, {"sent": "oh he is sticking his tongue out .", "words": ["is", "he", "out", "his", "tongue"]}, {"sent": "oh you have one tongue .", "words": ["have", "tongue", "you"]}, {"sent": "move your tongue .", "words": ["tongue", "your"]}, {"sent": "kittys tongue is rough .", "words": ["tongue", "is"]}, {"sent": "put your tongue in your mouth .", "words": ["put", "your", "mouth", "in", "tongue"]}, {"sent": "under my tongue ?", "words": ["tongue", "under", "my"]}, {"sent": "dont pull his tongue off .", "words": ["off", "pull", "dont", "his", "tongue"]}, {"sent": "shes licking her paw with her tongue .", "words": ["tongue", "her", "with"]}, {"sent": "tongue in .", "words": ["tongue", "in"]}, {"sent": "is Mommys tongue blue ?", "words": ["tongue", "blue", "is"]}, {"sent": "ah we can tell its a snake because of the tongue cant we ?", "words": ["can", "the", "we", "of", "tongue", "because", "a"]}, {"sent": "it stings your tongue ?", "words": ["tongue", "it", "your"]}, {"sent": "you hafta lick it with your tongue like this .", "words": ["your", "with", "like", "you", "this", "it", "tongue", "lick"]}, {"sent": "I bet Im gonna burn my tongue .", "words": ["tongue", "my"]}, {"sent": "I see your little tongue .", "words": ["tongue", "see", "your", "little"]}, {"sent": "or do you ride with your parents ?", "words": ["do", "your", "with", "you", "ride"]}, {"sent": "xxx wanna ride in the truck ?", "words": ["the", "truck", "in", "wanna", "ride"]}, {"sent": "lets go for a ride .", "words": ["for", "ride", "a", "go"]}, {"sent": "but I can ride on top .", "words": ["can", "on", "ride", "but"]}, {"sent": "they can sit in there and have a ride .", "words": ["can", "and", "have", "they", "in", "there", "sit", "ride", "a"]}, {"sent": "here well go see mommy here Ill give you a little ride .", "words": ["go", "you", "here", "little", "see", "give", "ride", "a"]}, {"sent": "take him for a ride David .", "words": ["for", "ride", "him", "take", "a"]}, {"sent": "does that mean its going to give you a nice ride or is it going to kick you ?", "words": ["does", "is", "to", "nice", "ride", "you", "it", "kick", "give", "that", "a"]}, {"sent": "ya wanna ride it ?", "words": ["it", "ride", "wanna"]}, {"sent": "Roman do you think the baby can ride that horsie what do you think ?", "words": ["the", "do", "think", "can", "what", "you", "that", "ride"]}, {"sent": "Id love to ride a horse .", "words": ["to", "love", "horse", "ride", "a"]}, {"sent": "on and whos Dan having a ride on ?", "words": ["on", "ride", "a", "and"]}, {"sent": "did you ride round on some bikes ?", "words": ["you", "did", "on", "some", "ride"]}, {"sent": "he wants to go for a ride .", "words": ["for", "go", "to", "he", "ride", "a"]}, {"sent": "is anyone going to ride those horses ?", "words": ["to", "ride", "those", "is"]}, {"sent": "youre gonna ride your tyke bike at the grocery store ?", "words": ["the", "your", "store", "at", "ride"]}, {"sent": "he could ride .", "words": ["could", "he", "ride"]}, {"sent": "hes taking a ride on another horse right ?", "words": ["another", "horse", "on", "ride", "a"]}, {"sent": "oh no you ride it and show him how to ride it .", "words": ["how", "show", "to", "and", "you", "him", "it", "ride"]}, {"sent": "did we ride up on the lift ?", "words": ["up", "the", "we", "did", "on", "ride"]}, {"sent": "a ride .", "words": ["ride", "a"]}, {"sent": "a ride in the vroomvroom .", "words": ["the", "in", "ride", "a"]}, {"sent": "some children walk to school and some ride the bus .", "words": ["the", "to", "school", "ride", "and", "some", "walk", "bus"]}, {"sent": "remember when you took the ride on the fire engine ?", "words": ["the", "ride", "you", "on", "when"]}, {"sent": "hes having a ride is he ?", "words": ["he", "ride", "is", "a"]}, {"sent": "you cant ride on the toys the car wont go .", "words": ["the", "go", "car", "you", "on", "ride"]}, {"sent": "wanna take the doggie for a ride ?", "words": ["the", "for", "ride", "wanna", "take", "a"]}, {"sent": "ride a xxx to BanburyCross .", "words": ["to", "ride", "a"]}, {"sent": "she wants the money to ride in it .", "words": ["the", "money", "to", "in", "it", "ride", "she"]}, {"sent": "dyou wanna take them to play school for a ride ?", "words": ["for", "to", "school", "play", "them", "ride", "wanna", "take", "a"]}, {"sent": "oh that was your best ride ?", "words": ["your", "was", "that", "ride"]}, {"sent": "please can I sit on for a ride ?", "words": ["can", "for", "on", "sit", "ride", "a"]}, {"sent": "there you can take her for a ride on the train .", "words": ["can", "the", "for", "train", "ride", "you", "there", "on", "take", "her", "a"]}, {"sent": "do you like to ride your bike ?", "words": ["do", "your", "to", "like", "you", "ride"]}, {"sent": "lets get on the horsie and go for a ride .", "words": ["the", "for", "go", "and", "get", "on", "ride", "a"]}, {"sent": "did you go for a ride ?", "words": ["for", "go", "you", "did", "ride", "a"]}, {"sent": "did you ride in it ?", "words": ["you", "in", "it", "did", "ride"]}, {"sent": "cats dont usually like to ride too well so hes trying to run outof the carriage when shes not looking .", "words": ["the", "when", "to", "so", "like", "dont", "run", "too", "ride", "not"]}, {"sent": "and the monkey asks can we have a ride ?", "words": ["the", "can", "we", "monkey", "and", "have", "ride", "a"]}, {"sent": "ah is baby having a ride on Woodys horse ?", "words": ["is", "horse", "on", "ride", "a"]}, {"sent": "wow did Naima ride a pony ?", "words": ["pony", "ride", "did", "a"]}, {"sent": "can I come for a ride with you ?", "words": ["can", "for", "with", "you", "ride", "a"]}, {"sent": "Id love to ride a horse wouldnt you ?", "words": ["to", "you", "love", "horse", "ride", "a"]}, {"sent": "no you didnt ride near your school .", "words": ["your", "school", "ride", "you"]}, {"sent": "I forget what that ride was called .", "words": ["was", "what", "that", "ride"]}, {"sent": "youre going to take them for a ride ?", "words": ["for", "to", "them", "ride", "take", "a"]}, {"sent": "horsie ride .", "words": ["ride"]}, {"sent": "bouncy ride ?", "words": ["ride"]}, {"sent": "to give the people a ride .", "words": ["the", "to", "give", "ride", "a"]}, {"sent": "you wanna ride on the bike itd be actually kind of hard for you to ride on it right now because you have that on your waist .", "words": ["the", "for", "your", "of", "be", "to", "you", "have", "that", "it", "wanna", "hard", "on", "ride", "because"]}, {"sent": "shall they all ride around in a circle ?", "words": ["all", "they", "in", "around", "ride", "a"]}, {"sent": "you you ride past me and Ill try and get them in the back .", "words": ["the", "back", "them", "ride", "and", "you", "get", "in", "me", "try"]}, {"sent": "you had a ride in helicopter at the airport .", "words": ["the", "helicopter", "you", "in", "at", "ride", "a"]}, {"sent": "thats a ride .", "words": ["ride", "a"]}, {"sent": "a ride on a tractor .", "words": ["on", "tractor", "ride", "a"]}, {"sent": "and thats the end of the ride .", "words": ["the", "of", "ride", "and"]}, {"sent": "do you ride every morning ?", "words": ["do", "ride", "every", "you"]}, {"sent": "because that thats where the carpets given you a bit of a rough ride .", "words": ["where", "the", "of", "ride", "you", "that", "because", "a"]}, {"sent": "he wanted to ride all by himself .", "words": ["all", "to", "by", "he", "ride"]}, {"sent": "yeah do you wanna go for a ride in the car with mommy and daddy ?", "words": ["the", "do", "for", "go", "car", "with", "you", "and", "in", "wanna", "ride", "a"]}, {"sent": "taking a ride taking a ride .", "words": ["ride", "a"]}, {"sent": "shes the daddys giving the baby a piggieback ride actually the babys ridding on the daddys shoulders .", "words": ["the", "on", "ride", "a"]}, {"sent": "are we going to give your pussy cat a ride ?", "words": ["your", "we", "to", "cat", "are", "give", "ride", "a"]}, {"sent": "would you like a ride ?", "words": ["like", "you", "would", "ride", "a"]}, {"sent": "maybe this elephant can give this man a ride can he ?", "words": ["can", "elephant", "this", "he", "give", "ride", "a"]}, {"sent": "do I get to have a nice ride in your van ?", "words": ["do", "your", "to", "nice", "get", "have", "in", "ride", "a"]}, {"sent": "is he having a ride on the back ?", "words": ["the", "back", "is", "he", "on", "ride", "a"]}, {"sent": "maybe the baby would like to ride this sick horse .", "words": ["the", "to", "like", "this", "sick", "would", "horse", "ride"]}, {"sent": "and I did ride a horse too .", "words": ["and", "did", "too", "horse", "ride", "a"]}, {"sent": "and go for a little ride ?", "words": ["for", "go", "little", "and", "ride", "a"]}, {"sent": "where is the ball gonna take a ride to ?", "words": ["where", "the", "is", "to", "ride", "ball", "take", "a"]}, {"sent": "does he wanna have a ride down the chute ?", "words": ["the", "does", "have", "he", "wanna", "down", "ride", "a"]}, {"sent": "maybe the puppy can ride the horse .", "words": ["the", "can", "puppy", "horse", "ride"]}, {"sent": "what about giving the little man a ride ?", "words": ["the", "what", "little", "about", "ride", "a"]}, {"sent": "horses have decorations on them too because theyre going to ride in the circus .", "words": ["the", "to", "them", "have", "in", "too", "on", "ride", "because"]}, {"sent": "give him a ride .", "words": ["give", "ride", "him", "a"]}, {"sent": "dont you think they might want to go for a ride ?", "words": ["think", "for", "go", "to", "you", "dont", "they", "want to", "ride", "a"]}, {"sent": "you put the chocolate in the back of the train and ride around with it .", "words": ["the", "put", "back", "train", "of", "chocolate", "with", "you", "and", "in", "it", "around", "ride"]}, {"sent": "he hasta ride back there with all the monkey food ?", "words": ["the", "all", "back", "monkey", "with", "there", "food", "he", "ride"]}, {"sent": "look it here theyre all taking a ride on the merrygoround .", "words": ["the", "all", "here", "it", "look", "on", "ride", "a"]}, {"sent": "this is the way the ladies ride and this is the way the gentlemen ride .", "words": ["the", "is", "and", "this", "ride"]}, {"sent": "are you going to give him a ride ?", "words": ["to", "you", "him", "are", "give", "ride", "a"]}, {"sent": "choo choo the barrettes going for a train ride .", "words": ["the", "for", "train", "ride", "a"]}, {"sent": "when the ladys ride this is the way the ladys ride clippety clop clippety clop .", "words": ["the", "is", "ride", "this", "when"]}, {"sent": "are you gonna ride in the car ?", "words": ["the", "car", "you", "in", "are", "ride"]}, {"sent": "he go for a ride .", "words": ["for", "go", "he", "ride", "a"]}, {"sent": "that big train ride made them all hungry .", "words": ["all", "train", "them", "hungry", "ride", "that", "big"]}, {"sent": "do you wanna ride in the trolley ride in the trolley round the shop ?", "words": ["the", "do", "you", "in", "wanna", "ride"]}, {"sent": "you going for a ride ?", "words": ["for", "ride", "a", "you"]}, {"sent": "my mother is still looking at books so I ride the train Jane is riding on the turtle here comes my mother she has two books I have three and a record too .", "words": ["the", "my", "is", "train", "so", "and", "here", "have", "at", "on", "turtle", "too", "ride", "she", "a"]}, {"sent": "is the horse going on a ride ?", "words": ["the", "is", "on", "horse", "ride", "a"]}, {"sent": "I think you should ride it .", "words": ["think", "it", "ride", "you"]}, {"sent": "what did you ride on ?", "words": ["what", "you", "did", "on", "ride"]}, {"sent": "did you ride your bicycle today ?", "words": ["your", "bicycle", "you", "did", "ride"]}, {"sent": "would teddy like a ride in the car dont you think ?", "words": ["the", "think", "car", "like", "you", "dont", "in", "would", "ride", "a"]}, {"sent": "ride in it .", "words": ["in", "it", "ride"]}, {"sent": "is the l is the lady going for a ride in the taxi ?", "words": ["the", "for", "is", "in", "ride", "a"]}, {"sent": "I Im tellin you I swear weve been out to ride it the other day he wanted me to I thought I could push him he got on and he he rode the bike by himself .", "words": ["the", "push", "to", "you", "other", "and", "him", "it", "he", "by", "out", "on", "me", "could", "ride"]}, {"sent": "M take them for a ride .", "words": ["for", "them", "take", "ride", "a"]}, {"sent": "Sarahs noisy today isnt she ?", "words": ["noisy", "she"]}, {"sent": "tractors are noisy though uhn arent they ?", "words": ["they", "are", "noisy"]}, {"sent": "are you a noisy boy ?", "words": ["are", "noisy", "a", "you"]}, {"sent": "those are very noisy animals .", "words": ["are", "those", "noisy"]}, {"sent": "what a noisy boy you are .", "words": ["what", "noisy", "you", "are", "a"]}, {"sent": "it is very noisy .", "words": ["it", "noisy", "is"]}, {"sent": "that noisy timer .", "words": ["that", "noisy"]}, {"sent": "because its noisy .", "words": ["noisy", "because"]}, {"sent": "they are noisy .", "words": ["they", "are", "noisy"]}, {"sent": "Isabel likes Milo and I think Daddy said he she likes Milo because Milo is the noisy one .", "words": ["the", "think", "is", "noisy", "and", "he", "she", "because"]}, {"sent": "thats noisy .", "words": ["noisy"]}, {"sent": "youre noisy .", "words": ["noisy"]}, {"sent": "boy it is noisy outside .", "words": ["outside", "it", "noisy", "is"]}, {"sent": "you noisy thing .", "words": ["noisy", "you"]}, {"sent": "you could be noisy as well but when he told us to be quiet we hadta be quiet didnt we ?", "words": ["we", "be", "to", "noisy", "you", "us", "he", "quiet", "could", "when", "but"]}, {"sent": "and two very big noisy tractors came past .", "words": ["big", "noisy", "and"]}, {"sent": "your sisters noisy today isnt she ?", "words": ["your", "noisy", "she"]}, {"sent": "thats a noisy airplane .", "words": ["airplane", "noisy", "a"]}, {"sent": "very noisy .", "words": ["noisy"]}, {"sent": "yesterday you didnt want ta go see daddy you wanted to stay with Nana because you didnt want ta go on the noisy train .", "words": ["the", "stay", "go", "train", "to", "with", "noisy", "you", "on", "see", "because"]}, {"sent": "noisy cash register .", "words": ["noisy"]}, {"sent": "hes awful noisy isnt he today .", "words": ["he", "noisy"]}, {"sent": "its a bit noisy isnt it ?", "words": ["it", "noisy", "a"]}, {"sent": "are they very very noisy ones ?", "words": ["they", "are", "noisy"]}, {"sent": "those are probably too noisy for a small area with a microphone .", "words": ["for", "those", "with", "noisy", "are", "too", "a"]}, {"sent": "too noisy .", "words": ["noisy", "too"]}, {"sent": "if I threw a banana onto the kennel youre so noisy I bet you couldnt get it without waking the dog .", "words": ["the", "if", "so", "noisy", "you", "get", "banana", "it", "dog", "a"]}, {"sent": "oh thats noisy .", "words": ["noisy"]}, {"sent": "youre a noisy rowdy boy arent you ?", "words": ["noisy", "you", "a"]}, {"sent": "thats a noisy thunder .", "words": ["noisy", "a"]}, {"sent": "see its noisy huh ?", "words": ["see", "noisy"]}, {"sent": "hey noisy ?", "words": ["noisy"]}, {"sent": "its too noisy .", "words": ["noisy", "too"]}, {"sent": "noisy what ?", "words": ["what", "noisy"]}, {"sent": "its a bit noisy Harry .", "words": ["noisy", "a"]}, {"sent": "it is a noisy train isnt it ?", "words": ["train", "is", "noisy", "it", "a"]}, {"sent": "this noisy book .", "words": ["book", "this", "noisy"]}, {"sent": "bunnys noisy book .", "words": ["book", "noisy"]}, {"sent": "but he said as he sat in that hot noisy tent .", "words": ["noisy", "in", "hot", "he", "that", "but"]}, {"sent": "is she not she not noisy ?", "words": ["noisy", "not", "she", "is"]}, {"sent": "I dont know why theyre being so noisy .", "words": ["dont", "so", "noisy", "why"]}, {"sent": "a very noisy digger was it ?", "words": ["was", "it", "noisy", "a"]}, {"sent": "he likes noisy .", "words": ["he", "noisy"]}, {"sent": "hes a bit of a noisy thing isnt he ?", "words": ["of", "he", "noisy", "a"]}, {"sent": "oh my thats noisy .", "words": ["my", "noisy"]}, {"sent": "I think um Daisycats a bit noisy though isnt he ?", "words": ["think", "he", "noisy", "a"]}, {"sent": "youre noisy .", "words": ["noisy"]}, {"sent": "so we dont want noisy babies and children on the plane do we ?", "words": ["the", "do", "we", "so", "noisy", "and", "dont", "on"]}, {"sent": "more like one person whos very noisy .", "words": ["more", "like", "noisy"]}, {"sent": "because its a bit noisy .", "words": ["noisy", "because", "a"]}, {"sent": "do you wanna read Cliffords noisy day ?", "words": ["do", "noisy", "you", "read", "wanna"]}, {"sent": "whats noisy ?", "words": ["noisy"]}, {"sent": "noisy .", "words": ["noisy"]}, {"sent": "noisy quick noisy quick noisy quick .", "words": ["noisy"]}, {"sent": "he went back to sleep hes so noisy .", "words": ["back", "sleep", "to", "so", "noisy", "he"]}, {"sent": "theyre very noisy .", "words": ["noisy"]}, {"sent": "oh youre noisy tonight arent you ?", "words": ["noisy", "you"]}, {"sent": "she likes noisy toys .", "words": ["she", "noisy"]}, {"sent": "too noisy .", "words": ["noisy", "too"]}, {"sent": "oh truckie Im sorry you said noisy truckie .", "words": ["noisy", "you"]}, {"sent": "Dakota noisy ?", "words": ["noisy"]}, {"sent": "is it noisy on the subway ?", "words": ["the", "is", "noisy", "it", "on"]}, {"sent": "very noisy .", "words": ["noisy"]}, {"sent": "were you very noisy ?", "words": ["were", "noisy", "you"]}, {"sent": "thats very noisy Fraser .", "words": ["noisy"]}, {"sent": "xxx playing with noisy toys today .", "words": ["with", "noisy"]}, {"sent": "is it noisy in the hairdressers ?", "words": ["the", "is", "noisy", "in", "it"]}, {"sent": "its too noisy that .", "words": ["that", "noisy", "too"]}, {"sent": "youre very noisy arent you ?", "words": ["noisy", "you"]}, {"sent": "were not going on a noisy train .", "words": ["train", "noisy", "on", "were", "not", "a"]}, {"sent": "its noisy isnt it ?", "words": ["it", "noisy"]}, {"sent": "the apartments were awful noisy so they were .", "words": ["the", "so", "noisy", "they", "were"]}, {"sent": "its too noisy I think .", "words": ["think", "noisy", "too"]}, {"sent": "too noisy said the man .", "words": ["the", "noisy", "too"]}, {"sent": "its too noisy .", "words": ["noisy", "too"]}, {"sent": "its too noisy .", "words": ["noisy", "too"]}, {"sent": "it is noisy isnt it ?", "words": ["it", "noisy", "is"]}, {"sent": "I just hope its not going to be too noisy .", "words": ["to", "be", "noisy", "too", "not"]}, {"sent": "the books called noisy .", "words": ["the", "noisy"]}, {"sent": "Im gonna shut that door because the fan is so noisy upstairs .", "words": ["the", "is", "so", "noisy", "that", "because", "door"]}, {"sent": "lets turn it off because its too noisy .", "words": ["off", "noisy", "it", "too", "because"]}, {"sent": "isnt she noisy .", "words": ["she", "noisy"]}, {"sent": "noisy yucky arent they ?", "words": ["they", "yucky", "noisy"]}, {"sent": "is it those noisy boys ?", "words": ["it", "those", "noisy", "is"]}, {"sent": "are you being noisy ?", "words": ["are", "noisy", "you"]}, {"sent": "its so noisy when you do that .", "words": ["do", "so", "noisy", "you", "that", "when"]}, {"sent": "who said it was too noisy ?", "words": ["was", "noisy", "it", "too", "who"]}, {"sent": "is it noisy ?", "words": ["it", "noisy", "is"]}, {"sent": "Bens noisy .", "words": ["noisy"]}, {"sent": "Im always a little bit worried that this is too noisy mixing the sugar and the margarine .", "words": ["the", "is", "noisy", "little", "and", "this", "too", "that", "a"]}, {"sent": "hello noisy Thomas .", "words": ["noisy"]}, {"sent": "oh I think youre a noisy boy now .", "words": ["think", "noisy", "a"]}, {"sent": "ones noisy and ones quick .", "words": ["noisy", "and"]}, {"sent": "a noisy motor ?", "words": ["noisy", "a"]}, {"sent": "I think well just turn the oven down a little bit because its a noisy oven .", "words": ["the", "think", "noisy", "little", "down", "oven", "because", "a"]}, {"sent": "oh thats very noisy .", "words": ["noisy"]}, {"sent": "its a bit noisy .", "words": ["noisy", "a"]}, {"sent": "thats noisy .", "words": ["noisy"]}, {"sent": "wheres your rooster ?", "words": ["rooster", "your"]}, {"sent": "I think thats a rooster .", "words": ["rooster", "think", "a"]}, {"sent": "theres a rooster .", "words": ["rooster", "a"]}, {"sent": "ah okay rooster rooster .", "words": ["rooster"]}, {"sent": "rooster .", "words": ["rooster"]}, {"sent": "rooster rooster .", "words": ["rooster"]}, {"sent": "doo like a rooster .", "words": ["rooster", "like", "a"]}, {"sent": "oh thats a rooster .", "words": ["rooster", "a"]}, {"sent": "thats a rooster .", "words": ["rooster", "a"]}, {"sent": "a rooster .", "words": ["rooster", "a"]}, {"sent": "maybe hes trying to be a rooster to wake everyone up .", "words": ["up", "to", "be", "rooster", "wake", "a"]}, {"sent": "there was a bantam rooster and a little bantam hen with a big clutch of eggs .", "words": ["of", "was", "with", "and", "little", "there", "rooster", "hen", "big", "a"]}, {"sent": "the rooster .", "words": ["the", "rooster"]}, {"sent": "some chickens and a rooster .", "words": ["rooster", "some", "a", "and"]}, {"sent": "Elvis the rooster almost goes to heaven .", "words": ["the", "rooster", "to"]}, {"sent": "thats a rooster .", "words": ["rooster", "a"]}, {"sent": "oh look heres a rooster .", "words": ["rooster", "a", "look"]}, {"sent": "maybe its the rooster .", "words": ["the", "rooster"]}, {"sent": "the zebra um she came over with the frog and the bee and the chicken rooster whatever but she didnt really play much with it .", "words": ["the", "zebra", "bee", "with", "play", "much", "frog", "and", "rooster", "it", "chicken", "over", "she", "but"]}, {"sent": "even a rooster .", "words": ["rooster", "a"]}, {"sent": "when they grow up called rooster .", "words": ["up", "they", "when", "rooster"]}, {"sent": "oh the rooster doesnt wanna talk ?", "words": ["the", "rooster", "talk", "wanna"]}, {"sent": "he can sound like a rooster .", "words": ["can", "like", "rooster", "he", "a"]}, {"sent": "do rooster now .", "words": ["do", "rooster"]}, {"sent": "rooster .", "words": ["rooster"]}, {"sent": "its a rooster .", "words": ["rooster", "a"]}, {"sent": "it is a rooster .", "words": ["rooster", "it", "a", "is"]}, {"sent": "and what does a rooster say ?", "words": ["what", "does", "say", "and", "rooster", "a"]}, {"sent": "was a rooster say ?", "words": ["rooster", "was", "say", "a"]}, {"sent": "um the rooster .", "words": ["the", "rooster"]}, {"sent": "thats a rooster isnt ?", "words": ["rooster", "a"]}, {"sent": "thats a rooster .", "words": ["rooster", "a"]}, {"sent": "whos that is it a rooster ?", "words": ["is", "rooster", "it", "that", "a"]}, {"sent": "we got the little rooster here .", "words": ["the", "we", "little", "here", "rooster"]}, {"sent": "I think he wishes he was a rooster .", "words": ["think", "was", "rooster", "he", "a"]}, {"sent": "what does a rooster say ?", "words": ["what", "does", "say", "rooster", "a"]}, {"sent": "and rooster ?", "words": ["rooster", "and"]}, {"sent": "can you put the rooster in the wagon ?", "words": ["can", "the", "put", "you", "rooster", "in"]}, {"sent": "you gotta put the rooster down to pick up the keys .", "words": ["the", "up", "put", "keys", "pick", "to", "you", "rooster", "down"]}, {"sent": "rooster .", "words": ["rooster"]}, {"sent": "what does that rooster see first thing in the morning ?", "words": ["the", "what", "does", "first", "rooster", "in", "see", "that"]}, {"sent": "thats a rooster .", "words": ["rooster", "a"]}, {"sent": "heres the rooster .", "words": ["the", "rooster"]}, {"sent": "or is it a its a chicken or a rooster or something .", "words": ["is", "rooster", "it", "chicken", "a"]}, {"sent": "hm its the rooster .", "words": ["the", "rooster"]}, {"sent": "oh do you remember this your the rooster ?", "words": ["the", "do", "your", "you", "this", "rooster"]}, {"sent": "and the rooster .", "words": ["the", "rooster", "and"]}, {"sent": "rooster .", "words": ["rooster"]}, {"sent": "its a rooster .", "words": ["rooster", "a"]}, {"sent": "what does a rooster say ?", "words": ["what", "does", "say", "rooster", "a"]}, {"sent": "oh the rooster ?", "words": ["the", "rooster"]}, {"sent": "youre right on his farm he had a rooster with a .", "words": ["with", "his", "rooster", "he", "on", "a"]}, {"sent": "the rooster ?", "words": ["the", "rooster"]}, {"sent": "Danny your turn to find the rooster .", "words": ["the", "your", "to", "rooster", "find"]}, {"sent": "or is it a rooster ?", "words": ["rooster", "it", "a", "is"]}, {"sent": "a rooster .", "words": ["rooster", "a"]}, {"sent": "rooster says .", "words": ["rooster"]}, {"sent": "thats a rooster .", "words": ["rooster", "a"]}, {"sent": "can the rooster go on the farm ?", "words": ["can", "the", "go", "rooster", "on"]}, {"sent": "thats a rooster .", "words": ["rooster", "a"]}, {"sent": "cockadoodledoo crowed the rooster .", "words": ["the", "rooster"]}, {"sent": "what does a rooster say ?", "words": ["what", "does", "say", "rooster", "a"]}, {"sent": "the rooster .", "words": ["the", "rooster"]}, {"sent": "thats a rooster .", "words": ["rooster", "a"]}, {"sent": "I saw a red rooster and the rooster saw me .", "words": ["red", "the", "and", "rooster", "me", "a"]}, {"sent": "cockadoodledoo says the rooster .", "words": ["the", "rooster"]}, {"sent": "this is the this is the part of the railing its a stairs you can sit on are we gonna leave my rooster here ?", "words": ["the", "can", "my", "is", "of", "we", "you", "here", "this", "rooster", "are", "on", "stairs", "sit", "a"]}, {"sent": "yes thats a rooster .", "words": ["rooster", "a"]}, {"sent": "are you sure youre not gonna be too hot with the chicken blanket rooster ?", "words": ["the", "be", "with", "you", "rooster", "hot", "are", "too", "blanket", "chicken", "not"]}, {"sent": "rooster rooster duck .", "words": ["duck", "rooster"]}, {"sent": "when the rooster says cock a doodle doo the people get up right ?", "words": ["the", "up", "get", "rooster", "when", "a"]}, {"sent": "the rooster says .", "words": ["the", "rooster"]}, {"sent": "wheres that rooster ?", "words": ["rooster", "that"]}, {"sent": "in the barn there was a rooster and a pigeon too .", "words": ["the", "was", "and", "there", "in", "rooster", "too", "a"]}, {"sent": "he can sound like a rooster cockadoodledoo .", "words": ["can", "like", "rooster", "he", "a"]}, {"sent": "Mwwws not a rooster .", "words": ["rooster", "not", "a"]}, {"sent": "are you a rooster ?", "words": ["rooster", "are", "a", "you"]}, {"sent": "those are chickens and thats a rooster .", "words": ["those", "and", "rooster", "are", "a"]}, {"sent": "heres your rooster .", "words": ["rooster", "your"]}, {"sent": "yes like a rooster .", "words": ["rooster", "like", "a"]}, {"sent": "he can sound like a rooster .", "words": ["can", "like", "rooster", "he", "a"]}, {"sent": "how does the rooster go ?", "words": ["the", "does", "how", "go", "rooster"]}, {"sent": "tell Daddy about the rooster .", "words": ["the", "rooster", "about"]}, {"sent": "a rooster says bonk ?", "words": ["rooster", "a"]}, {"sent": "what happened to the rooster ?", "words": ["to", "what", "rooster", "the"]}, {"sent": "Elvis the rooster almost goes to heaven .", "words": ["the", "rooster", "to"]}, {"sent": "no its not the rooster .", "words": ["the", "rooster", "not"]}, {"sent": "a rooster .", "words": ["rooster", "a"]}, {"sent": "rooster .", "words": ["rooster"]}, {"sent": "theres a rooster and heres a hen .", "words": ["rooster", "hen", "and", "a"]}, {"sent": "a rooster says cockadoodledoo .", "words": ["rooster", "a"]}, {"sent": "rooster .", "words": ["rooster"]}, {"sent": "what does a rooster say ?", "words": ["what", "does", "say", "rooster", "a"]}, {"sent": "I said the wrong word I said roaster instead of rooster .", "words": ["the", "of", "rooster"]}, {"sent": "rooster thats right .", "words": ["rooster"]}, {"sent": "where is your rooster tail ?", "words": ["where", "rooster", "your", "is"]}, {"sent": "tickle tickle tickle .", "words": ["tickle"]}, {"sent": "because I hafta tickle the baby .", "words": ["the", "tickle", "because"]}, {"sent": "tickle tickle tickle .", "words": ["tickle"]}, {"sent": "tickle tickle tickle tickle .", "words": ["tickle"]}, {"sent": "it does tickle .", "words": ["does", "it", "tickle"]}, {"sent": "tickle me too .", "words": ["me", "tickle", "too"]}, {"sent": "tickle tickle tickle tickle .", "words": ["tickle"]}, {"sent": "tickle tickle .", "words": ["tickle"]}, {"sent": "tickle tickle tickle .", "words": ["tickle"]}, {"sent": "Ill tickle you .", "words": ["tickle", "you"]}, {"sent": "tickle tickle tickle .", "words": ["tickle"]}, {"sent": "does that tickle ?", "words": ["does", "that", "tickle"]}, {"sent": "does it tickle ?", "words": ["does", "it", "tickle"]}, {"sent": "shall I tickle you ?", "words": ["tickle", "you"]}, {"sent": "if you leave your vest out peoplell come and tickle you willnt they ?", "words": ["your", "if", "tickle", "and", "you", "they", "out"]}, {"sent": "dont dont dont tickle his feet .", "words": ["his", "dont", "tickle"]}, {"sent": "shall we tickle moony ?", "words": ["tickle", "we"]}, {"sent": "oh tickle .", "words": ["tickle"]}, {"sent": "tickle tickle tickle Mama .", "words": ["tickle"]}, {"sent": "tickle tickle on the knee .", "words": ["on", "the", "knee", "tickle"]}, {"sent": "ooh tickle tickle .", "words": ["tickle"]}, {"sent": "how about I tickle you instead .", "words": ["how", "tickle", "about", "you"]}, {"sent": "tickle tickle tickle .", "words": ["tickle"]}, {"sent": "you wanna tickle Amys hand ?", "words": ["hand", "tickle", "wanna", "you"]}, {"sent": "tickle tickle tickle tickle tickle tickle tickle tickle tickle tickle tickle tickle tickle tickle tickle .", "words": ["tickle"]}, {"sent": "lets tickle her .", "words": ["her", "tickle"]}, {"sent": "I gonna tickle you .", "words": ["tickle", "you"]}, {"sent": "tickle your toes .", "words": ["your", "tickle"]}, {"sent": "tickle tickle tickle tickle .", "words": ["tickle"]}, {"sent": "they dont tickle do they ?", "words": ["dont", "they", "do", "tickle"]}, {"sent": "tickle tickle tickle .", "words": ["tickle"]}, {"sent": "is he gonna tickle you ?", "words": ["he", "tickle", "you", "is"]}, {"sent": "tickle tickle tickle tickle the wheels going around and around .", "words": ["the", "tickle", "and", "around"]}, {"sent": "tickle tickle tickle tickle .", "words": ["tickle"]}, {"sent": "tickle tickle tickle .", "words": ["tickle"]}, {"sent": "tickle tickle tickle tickle tickle tickle tickle .", "words": ["tickle"]}, {"sent": "tickle me .", "words": ["me", "tickle"]}, {"sent": "tickle baby .", "words": ["tickle"]}, {"sent": "tickle tickle tickle tickle tickle tickle tickle .", "words": ["tickle"]}, {"sent": "tickle tickle .", "words": ["tickle"]}, {"sent": "tickle .", "words": ["tickle"]}, {"sent": "now the babys going to tickle you .", "words": ["the", "to", "tickle", "you"]}, {"sent": "tickle ?", "words": ["tickle"]}, {"sent": "do you tickle behind your knee ?", "words": ["do", "knee", "your", "behind", "tickle", "you"]}, {"sent": "tickle tickle tickle .", "words": ["tickle"]}, {"sent": "Carl tickle teddy .", "words": ["tickle"]}, {"sent": "oh dont tickle me .", "words": ["dont", "me", "tickle"]}, {"sent": "tickle tickle tickle tickle .", "words": ["tickle"]}, {"sent": "tickle your leg ?", "words": ["your", "tickle", "leg"]}, {"sent": "tickle tickle .", "words": ["tickle"]}, {"sent": "tickle tickle on the cheek .", "words": ["on", "the", "cheek", "tickle"]}, {"sent": "tickle tickle tickle tickle tickle tickle tickle tickle .", "words": ["tickle"]}, {"sent": "Im going to tickle you .", "words": ["to", "tickle", "you"]}, {"sent": "tickle tickle tickle .", "words": ["tickle"]}, {"sent": "making Mummys nose tickle isnt it ?", "words": ["it", "tickle", "nose"]}, {"sent": "tickle tickle tickle .", "words": ["tickle"]}, {"sent": "tickle tickle tickle .", "words": ["tickle"]}, {"sent": "Ill tickle tickle tickle you .", "words": ["tickle", "you"]}, {"sent": "a tickle tickle tickle .", "words": ["tickle", "a"]}, {"sent": "tickle tickle tickle .", "words": ["tickle"]}, {"sent": "tickle tickle tickle tickle tickle tickle tickle tickle .", "words": ["tickle"]}, {"sent": "tickle your feet .", "words": ["your", "tickle"]}, {"sent": "tick tickle tickle .", "words": ["tickle"]}, {"sent": "tickle .", "words": ["tickle"]}, {"sent": "tickle tickle tickle tickle .", "words": ["tickle"]}, {"sent": "tickle tickle tickle tickle tickle .", "words": ["tickle"]}, {"sent": "wheres big bear and tickle me too before you go to sleep .", "words": ["go", "sleep", "to", "tickle", "and", "bear", "you", "too", "me", "big"]}, {"sent": "tickle tickle on the elbow .", "words": ["on", "the", "tickle"]}, {"sent": "tickle tickle tickle tickle .", "words": ["tickle"]}, {"sent": "tickle tickle tickle .", "words": ["tickle"]}, {"sent": "I can tickle you .", "words": ["can", "tickle", "you"]}, {"sent": "I gonna tickle your chichi buttons .", "words": ["your", "tickle"]}, {"sent": "tickle tickle do you wanna tickle Jwww ?", "words": ["do", "tickle", "wanna", "you"]}, {"sent": "gonna tickle your feet .", "words": ["your", "tickle"]}, {"sent": "I tickle you .", "words": ["tickle", "you"]}, {"sent": "tickle .", "words": ["tickle"]}, {"sent": "tickle tickle tickle .", "words": ["tickle"]}, {"sent": "tickle tickle tickle .", "words": ["tickle"]}, {"sent": "yes tickle tickle tickle .", "words": ["tickle"]}, {"sent": "tickle tickle tickle tickle tickle tickle tickle tickle tickle .", "words": ["tickle"]}, {"sent": "tickle tickle tickle .", "words": ["tickle"]}, {"sent": "tickle tickle tickle .", "words": ["tickle"]}, {"sent": "xxx a tickle .", "words": ["tickle", "a"]}, {"sent": "going to tickle you .", "words": ["to", "tickle", "you"]}, {"sent": "tickle tickle tickle .", "words": ["tickle"]}, {"sent": "tickle tickle tickle .", "words": ["tickle"]}, {"sent": "oh tickle tickle .", "words": ["tickle"]}, {"sent": "does it tickle ?", "words": ["does", "it", "tickle"]}, {"sent": "tickle tickle ?", "words": ["tickle"]}, {"sent": "tickle tickle tickle .", "words": ["tickle"]}, {"sent": "tickle .", "words": ["tickle"]}, {"sent": "here is a little chick on a lovely sunny day tickle her tummy what will she say ?", "words": ["what", "is", "will", "say", "tickle", "little", "here", "tummy", "on", "her", "she", "a"]}, {"sent": "oh why are you going to give me a little tickle on my tummy button ?", "words": ["my", "to", "tickle", "you", "little", "are", "tummy", "on", "me", "give", "button", "why", "a"]}, {"sent": "tickle tickle tickle .", "words": ["tickle"]}, {"sent": "tickle tickle .", "words": ["tickle"]}, {"sent": "Im gonna tickle you Im gonna tickle you .", "words": ["tickle", "you"]}, {"sent": "theres Eleanors nice slipper .", "words": ["slipper", "nice"]}, {"sent": "no it cant go in the slipper .", "words": ["the", "go", "in", "it", "slipper"]}, {"sent": "wheres my slipper ?", "words": ["slipper", "my"]}, {"sent": "slipper oh these slippers I dont know .", "words": ["dont", "slipper", "these"]}, {"sent": "with one one red slipper inside Daddys roller blade boot .", "words": ["red", "slipper", "inside", "with"]}, {"sent": "using the hair brush and I said let me find your other slipper and I realized youre using your other slipper to brush my hair .", "words": ["the", "my", "your", "brush", "to", "and", "other", "find", "me", "hair", "slipper"]}, {"sent": "Mummy just sat on your slipper Fraser .", "words": ["on", "slipper", "your"]}, {"sent": "Baaees other slipper .", "words": ["slipper", "other"]}, {"sent": "when the grand duke put the slipper on Cinderellas foot it fit perfectly .", "words": ["the", "put", "fit", "foot", "it", "on", "slipper", "when"]}, {"sent": "put on your other slipper honey .", "words": ["put", "your", "other", "on", "slipper"]}, {"sent": "here can I have your foot so I can put your other slipper on ?", "words": ["can", "put", "your", "so", "other", "here", "have", "foot", "on", "slipper"]}, {"sent": "because his slipper has fallen off his foot and its sitting on the carpet .", "words": ["off", "the", "and", "foot", "on", "his", "slipper", "because"]}, {"sent": "Cinderellas slipper .", "words": ["slipper"]}, {"sent": "I take my slipper off .", "words": ["off", "slipper", "my", "take"]}, {"sent": "well lets find the other dancing slipper then should we ?", "words": ["the", "we", "other", "find", "slipper", "then"]}, {"sent": "youve just tidied up so whats the point of pulling my slipper off my foot and throwing it down ?", "words": ["up", "the", "my", "off", "of", "so", "and", "foot", "it", "slipper", "down"]}, {"sent": "see the big one next to the slipper ?", "words": ["the", "to", "see", "big", "slipper"]}, {"sent": "a slipper .", "words": ["slipper", "a"]}, {"sent": "who who finds the slipper ?", "words": ["the", "slipper", "who"]}, {"sent": "slipper .", "words": ["slipper"]}, {"sent": "use your slipper .", "words": ["slipper", "your"]}, {"sent": "Baaees slipper .", "words": ["slipper"]}, {"sent": "theres your other slipper .", "words": ["slipper", "your", "other"]}, {"sent": "find your slipper for Mummy .", "words": ["slipper", "for", "your", "find"]}, {"sent": "yes slipper .", "words": ["slipper"]}, {"sent": "never did find your slipper did we ?", "words": ["your", "we", "did", "find", "slipper"]}, {"sent": "what slipper ?", "words": ["what", "slipper"]}, {"sent": "thats a pretty slipper .", "words": ["slipper", "pretty", "a"]}, {"sent": "by your slipper .", "words": ["slipper", "your", "by"]}, {"sent": "how about slipper ?", "words": ["slipper", "how", "about"]}, {"sent": "the next day the prince announced he wanted to marry the girl who had lost her glass slipper .", "words": ["the", "to", "he", "glass", "who", "slipper", "her"]}, {"sent": "come here and have your slipper on please .", "words": ["your", "and", "here", "have", "on", "slipper"]}, {"sent": "she took off her slipper and put her foot on top of there so she could tie it .", "words": ["off", "put", "of", "so", "and", "there", "foot", "it", "on", "slipper", "could", "her", "she"]}, {"sent": "I think we forgot the slipper .", "words": ["the", "slipper", "think", "we"]}, {"sent": "and the grand duke is absolutely delighted when Cinderella produces the matching glass slipper and puts it on her dainty foot .", "words": ["the", "is", "her", "and", "foot", "it", "glass", "on", "slipper", "when"]}, {"sent": "one slipper and heres the other slipper .", "words": ["the", "slipper", "other", "and"]}, {"sent": "you go and get your slipper on .", "words": ["your", "go", "and", "get", "you", "on", "slipper"]}, {"sent": "you just run over my slipper ?", "words": ["my", "you", "run", "slipper", "over"]}, {"sent": "put that slipper on my foot ?", "words": ["put", "my", "foot", "on", "slipper", "that"]}, {"sent": "and then put your slipper back on .", "words": ["put", "your", "back", "and", "on", "slipper", "then"]}, {"sent": "he found her slipper .", "words": ["slipper", "her", "he"]}, {"sent": "bring Mummy the other slipper please and well put it on .", "words": ["the", "put", "bring", "and", "other", "it", "on", "slipper"]}, {"sent": "Thomas do you know what there is on the floor just behind your slipper ?", "words": ["the", "do", "what", "your", "is", "behind", "you", "there", "on", "slipper"]}, {"sent": "want your other slipper ?", "words": ["slipper", "your", "other"]}, {"sent": "when the grand duke put the slipper on Cinderellas foot .", "words": ["the", "put", "foot", "on", "slipper", "when"]}, {"sent": "little slipper .", "words": ["slipper", "little"]}, {"sent": "oh I think youve just put your slipper on it .", "words": ["think", "put", "your", "it", "on", "slipper"]}, {"sent": "your slipper going to come off ?", "words": ["to", "slipper", "your", "off"]}, {"sent": "weve another slipper to put on havent we ?", "words": ["put", "we", "to", "another", "on", "slipper"]}, {"sent": "fuzzy slipper yeah and thats a rubbery ball .", "words": ["ball", "slipper", "a", "and"]}, {"sent": "he has one slipper on and one slipper off .", "words": ["off", "and", "he", "on", "slipper"]}, {"sent": "where does the slipper go ?", "words": ["where", "does", "the", "go", "slipper"]}, {"sent": "you leave your slipper off now while youre working .", "words": ["off", "slipper", "your", "you"]}, {"sent": "another slipper .", "words": ["another", "slipper"]}, {"sent": "that is a slipper .", "words": ["slipper", "that", "a", "is"]}, {"sent": "I made myself a slipper .", "words": ["slipper", "myself", "a"]}, {"sent": "my slipper or my foot .", "words": ["slipper", "my", "foot"]}, {"sent": "or perhaps its more inside you slipper thinking about it because theyre blue arent they ?", "words": ["inside", "you", "blue", "they", "more", "it", "about", "slipper", "because"]}, {"sent": "wheres your other slipper ?", "words": ["slipper", "your", "other"]}, {"sent": "yeah slipper and duckie is taking a nap .", "words": ["slipper", "a", "is", "and"]}, {"sent": "ah maybe now she can wear her slipper .", "words": ["can", "slipper", "her", "she"]}, {"sent": "wheres your other slipper ?", "words": ["slipper", "your", "other"]}, {"sent": "no its a slipper .", "words": ["slipper", "a"]}, {"sent": "play with mommas slipper .", "words": ["slipper", "play", "with"]}, {"sent": "you put your slipper back on or you go back in .", "words": ["put", "your", "back", "go", "you", "in", "on", "slipper"]}, {"sent": "slipper .", "words": ["slipper"]}, {"sent": "one slipper on and one slipper off .", "words": ["on", "slipper", "off", "and"]}, {"sent": "you can hold my slipper ?", "words": ["can", "my", "hold", "you", "slipper"]}, {"sent": "and now youve just spoilt it by throwing your slipper .", "words": ["your", "by", "and", "it", "slipper"]}, {"sent": "a fuzzy slipper .", "words": ["slipper", "a"]}, {"sent": "I dont have on slipper .", "words": ["have", "dont", "on", "slipper"]}, {"sent": "where is my other slipper ?", "words": ["where", "my", "is", "other", "slipper"]}, {"sent": "wheres your slipper first ?", "words": ["slipper", "first", "your"]}, {"sent": "well I definitely can see one red slipper behind the rocking chair .", "words": ["can", "red", "the", "rocking chair", "behind", "chair", "see", "slipper"]}, {"sent": "lets put this other slipper on .", "words": ["put", "other", "this", "on", "slipper"]}, {"sent": "you still dont have your slipper on babe .", "words": ["your", "you", "have", "dont", "on", "slipper"]}, {"sent": "did you know that that doggie was a really a slipper that you could wear on your foot ?", "words": ["your", "was", "you", "foot", "did", "on", "slipper", "could", "that", "a"]}, {"sent": "did you leave the other slipper outside Nomi ?", "words": ["the", "outside", "other", "you", "did", "slipper"]}, {"sent": "does she want the slipper on ?", "words": ["the", "does", "on", "slipper", "she"]}, {"sent": "I made a slipper .", "words": ["slipper", "a"]}, {"sent": "slipper .", "words": ["slipper"]}, {"sent": "and Ive got a slipper .", "words": ["slipper", "a", "and"]}, {"sent": "lets put your slipper back on .", "words": ["put", "your", "back", "on", "slipper"]}, {"sent": "you banged the slipper down and knocked my leg .", "words": ["the", "my", "and", "you", "leg", "slipper", "down"]}, {"sent": "have you pinched my slipper again ?", "words": ["have", "slipper", "my", "you"]}, {"sent": "oh slipper ?", "words": ["slipper"]}, {"sent": "wheres the other slipper John ?", "words": ["the", "slipper", "other"]}, {"sent": "no slipper can you see a slipper ?", "words": ["can", "see", "you", "slipper", "a"]}, {"sent": "so she is asking Cinderella to try on the glass slipper .", "words": ["the", "is", "to", "so", "glass", "on", "slipper", "try", "she"]}, {"sent": "ah just because I took a slipper off doesnt mean you can .", "words": ["off", "can", "you", "slipper", "because", "a"]}, {"sent": "if theres any on the table Im coming out there right now with my slipper off .", "words": ["the", "off", "my", "if", "table", "any", "with", "there", "out", "on", "slipper"]}, {"sent": "she did she lost her slipper .", "words": ["did", "her", "she", "slipper"]}, {"sent": "xxx slipper .", "words": ["slipper"]}, {"sent": "put the other slipper on .", "words": ["the", "put", "other", "on", "slipper"]}, {"sent": "one slipper .", "words": ["slipper"]}, {"sent": "just trying to find my slipper .", "words": ["to", "my", "slipper", "find"]}, {"sent": "a broom .", "words": ["broom", "a"]}, {"sent": "okay Ill help you with my broom .", "words": ["help", "my", "with", "you", "broom"]}, {"sent": "where are you broom ?", "words": ["where", "broom", "are", "you"]}, {"sent": "is he flying on the broom ?", "words": ["the", "is", "broom", "he", "on"]}, {"sent": "get broom .", "words": ["get", "broom"]}, {"sent": "where did you get your broom ?", "words": ["where", "your", "you", "get", "broom", "did"]}, {"sent": "someone is riding on the broom with him .", "words": ["the", "is", "with", "broom", "him", "on"]}, {"sent": "thats a broom .", "words": ["broom", "a"]}, {"sent": "where are you broom ?", "words": ["where", "broom", "are", "you"]}, {"sent": "broom broom .", "words": ["broom"]}, {"sent": "youre getting your broom .", "words": ["broom", "your"]}, {"sent": "do you want a long broom a medium broom a short broom ?", "words": ["do", "long", "you", "broom", "a"]}, {"sent": "and oh do you wanna take my broom for me ?", "words": ["do", "my", "for", "you", "and", "broom", "wanna", "me", "take"]}, {"sent": "she wished that she had a broom .", "words": ["broom", "that", "she", "a"]}, {"sent": "have you got the big broom now ?", "words": ["the", "you", "have", "broom", "big"]}, {"sent": "where should I put the broom ?", "words": ["where", "the", "put", "broom"]}, {"sent": "thats the broom .", "words": ["the", "broom"]}, {"sent": "mops and a broom .", "words": ["broom", "a", "and"]}, {"sent": "you gonna climb up here with your big mop and your broom ?", "words": ["up", "your", "with", "you", "here", "and", "broom", "mop", "climb", "big"]}, {"sent": "why dont you put your broom back ?", "words": ["put", "your", "back", "you", "dont", "broom", "why"]}, {"sent": "lemme get you your broom .", "words": ["get", "broom", "your", "you"]}, {"sent": "no right there whats he doing with that broom ?", "words": ["with", "there", "broom", "he", "that"]}, {"sent": "well what happens do you remember when we saw the movie what happened when hes flying on his broom and he was trying to catch the golden snitch what happened to the broom ?", "words": ["the", "what", "do", "we", "was", "to", "catch", "you", "his", "broom", "and", "he", "on", "when"]}, {"sent": "look weve got a little broom here .", "words": ["little", "here", "broom", "look", "a"]}, {"sent": "excuse me Ursula part of the broom ?", "words": ["of", "me", "broom", "the"]}, {"sent": "broom .", "words": ["broom"]}, {"sent": "broom .", "words": ["broom"]}, {"sent": "where do you put the broom ?", "words": ["where", "do", "put", "the", "you", "broom"]}, {"sent": "broom .", "words": ["broom"]}, {"sent": "broom .", "words": ["broom"]}, {"sent": "what are you going to do with the broom ?", "words": ["the", "what", "do", "to", "with", "you", "broom", "are"]}, {"sent": "and in the broom cupboard you have a broom and a vacuum cleaner .", "words": ["the", "you", "and", "broom", "in", "have", "vacuum", "a"]}, {"sent": "that man has a broom and he has a little truck .", "words": ["truck", "and", "little", "broom", "he", "that", "a"]}, {"sent": "lets get the broom .", "words": ["get", "the", "broom"]}, {"sent": "a broom .", "words": ["broom", "a"]}, {"sent": "lets put the broom back .", "words": ["the", "broom", "put", "back"]}, {"sent": "look at the broom .", "words": ["the", "broom", "at", "look"]}, {"sent": "youve seen a broom before .", "words": ["broom", "a"]}, {"sent": "heres your broom .", "words": ["broom", "your"]}, {"sent": "he hit you with a big broom ?", "words": ["with", "you", "broom", "he", "hit", "big", "a"]}, {"sent": "hey the broom sweep .", "words": ["the", "broom", "sweep"]}, {"sent": "now can I do something with the broom for a second ?", "words": ["can", "do", "the", "for", "with", "broom", "a"]}, {"sent": "leave that broom alone .", "words": ["broom", "that"]}, {"sent": "well that was good now we can sweep in here wheres your broom ?", "words": ["can", "your", "we", "was", "sweep", "here", "broom", "in", "good", "that"]}, {"sent": "thats a broom for sweeping .", "words": ["broom", "for", "a"]}, {"sent": "it looks like a broom handle and some old sacks and straw .", "words": ["old", "like", "and", "broom", "it", "some", "a"]}, {"sent": "thats part of the broom .", "words": ["of", "the", "broom"]}, {"sent": "Cookie Monster hasta clean it up so you put him by the broom ?", "words": ["clean", "up", "put", "the", "so", "you", "by", "broom", "him", "it"]}, {"sent": "wheres the broom .", "words": ["the", "broom"]}, {"sent": "so you could thats like just like the big purple broom that they bought at the store remember ?", "words": ["the", "so", "store", "like", "you", "broom", "that", "they", "at", "could", "big"]}, {"sent": "what do you do with a broom ?", "words": ["what", "do", "with", "you", "broom", "a"]}, {"sent": "a broom .", "words": ["broom", "a"]}, {"sent": "broom .", "words": ["broom"]}, {"sent": "then why did you buy a broom and a cauldron ?", "words": ["you", "and", "broom", "did", "then", "buy", "why", "a"]}, {"sent": "a broom ?", "words": ["broom", "a"]}, {"sent": "oh it looks like a broom does it ?", "words": ["does", "like", "broom", "it", "a"]}, {"sent": "you heard another broom ?", "words": ["another", "broom", "you"]}, {"sent": "heres a broom .", "words": ["broom", "a"]}, {"sent": "broom ?", "words": ["broom"]}, {"sent": "yeah with a broom .", "words": ["broom", "with", "a"]}, {"sent": "I will hit that old spot with this broom .", "words": ["old", "with", "this", "broom", "will", "hit", "that"]}, {"sent": "see the broom .", "words": ["the", "see", "broom"]}, {"sent": "put the broom right there .", "words": ["the", "broom", "put", "there"]}, {"sent": "can you say broom ?", "words": ["can", "broom", "say", "you"]}, {"sent": "two broom .", "words": ["broom"]}, {"sent": "you havent had your broom for .", "words": ["broom", "for", "your", "you"]}, {"sent": "dont cut the broom off .", "words": ["the", "off", "cut", "dont", "broom"]}, {"sent": "theres a squirrel with the broom .", "words": ["the", "with", "broom", "squirrel", "a"]}, {"sent": "he has a broom yeah .", "words": ["broom", "he", "a"]}, {"sent": "your broom .", "words": ["broom", "your"]}, {"sent": "broom .", "words": ["broom"]}, {"sent": "take the broom .", "words": ["the", "broom", "take"]}, {"sent": "the broom .", "words": ["the", "broom"]}, {"sent": "broom .", "words": ["broom"]}, {"sent": "thats a squirrel xxx with a broom .", "words": ["broom", "with", "squirrel", "a"]}, {"sent": "wheres that broom ?", "words": ["broom", "that"]}, {"sent": "*SI1: broom ?", "words": ["broom"]}, {"sent": "broom .", "words": ["broom"]}, {"sent": "broom broom broom .", "words": ["broom"]}, {"sent": "broom .", "words": ["broom"]}, {"sent": "broom .", "words": ["broom"]}, {"sent": "broom .", "words": ["broom"]}, {"sent": "a broom .", "words": ["broom", "a"]}, {"sent": "broom Max say broom .", "words": ["broom", "say"]}, {"sent": "shes got a broom in her hand .", "words": ["hand", "broom", "in", "her", "a"]}, {"sent": "using her broom as an imaginary partner Cinderella danced and twirled gracefully through the room she led everyone across the floor sweeping and cleaning as they went .", "words": ["the", "and", "broom", "an", "room", "they", "her", "she"]}, {"sent": "is that part of the broom ?", "words": ["the", "is", "of", "broom", "that"]}, {"sent": "broom .", "words": ["broom"]}, {"sent": "dyou sweep with the broom ?", "words": ["the", "broom", "sweep", "with"]}, {"sent": "broom .", "words": ["broom"]}, {"sent": "my broom .", "words": ["broom", "my"]}, {"sent": "thats a broom .", "words": ["broom", "a"]}, {"sent": "I dont know you can put the broom in the kitchen maybe .", "words": ["can", "the", "put", "you", "dont", "broom", "in", "kitchen"]}, {"sent": "Donna hit you and AnnMarie with the broom ?", "words": ["the", "with", "and", "you", "broom", "hit"]}, {"sent": "broom .", "words": ["broom"]}, {"sent": "shes got a broom in her hand .", "words": ["hand", "broom", "in", "her", "a"]}, {"sent": "is the clown gonna even reach there wheres yer broom ?", "words": ["the", "there", "broom", "is"]}, {"sent": "lets go up this way Patrick we cant make it across the street there .", "words": ["up", "the", "we", "go", "this", "there", "it", "street", "make"]}, {"sent": "oh youre stopped for a street light ?", "words": ["light", "for", "street", "a"]}, {"sent": "he went up the street .", "words": ["up", "the", "street", "he"]}, {"sent": "in my street theres an old lady who lives on my street and she does everyones rubbish .", "words": ["old", "does", "my", "and", "in", "an", "who", "on", "street", "she"]}, {"sent": "its right across the street .", "words": ["the", "street"]}, {"sent": "that sign means that theres a school nearby and children are crossing the street .", "words": ["the", "school", "and", "are", "street", "that", "a"]}, {"sent": "quick quick because hes walking down the street .", "words": ["the", "down", "because", "street"]}, {"sent": "right there are three street cleaners .", "words": ["there", "are", "street"]}, {"sent": "oh this Kay down the street .", "words": ["the", "this", "down", "street"]}, {"sent": "down the street go the cars and lorrys .", "words": ["the", "go", "and", "street", "down"]}, {"sent": "shall we put a car on the street ?", "words": ["the", "put", "we", "car", "on", "street", "a"]}, {"sent": "you live on York street .", "words": ["on", "street", "you"]}, {"sent": "what does Oscar do on sesame street ?", "words": ["what", "do", "does", "on", "street"]}, {"sent": "do you do it in the street ?", "words": ["the", "do", "you", "in", "it", "street"]}, {"sent": "shes not that far away down the street she might actually hear you .", "words": ["the", "hear", "you", "she", "away", "street", "down", "that", "not"]}, {"sent": "now giraffe cant get out walk into the street and get hurt .", "words": ["the", "giraffe", "into", "hurt", "and", "get", "out", "street", "walk"]}, {"sent": "mhm right on one hundred thirty first street on Madison .", "words": ["on", "first", "street"]}, {"sent": "hes on the street .", "words": ["on", "the", "street"]}, {"sent": "a different Nancy not the one across the street .", "words": ["the", "street", "not", "a"]}, {"sent": "who could I get in the street xxx .", "words": ["the", "get", "in", "who", "could", "street"]}, {"sent": "fifty five east ninetieth ninetieth street .", "words": ["street"]}, {"sent": "put the car in the street .", "words": ["the", "put", "car", "in", "street"]}, {"sent": "well you see that bit hasnt got street lamps .", "words": ["see", "street", "that", "you"]}, {"sent": "they came down the street and in the gate .", "words": ["the", "and", "they", "in", "street", "down"]}, {"sent": "AuntieMabel said you must never touch street lamps in the street didnt she ?", "words": ["the", "touch", "you", "in", "street", "she"]}, {"sent": "at oop you know where oop is on Thayer street ?", "words": ["where", "is", "you", "at", "on", "street"]}, {"sent": "maybe when we go to frog and toad well go to the bakery across the street .", "words": ["the", "go", "we", "to", "frog", "and", "street", "when"]}, {"sent": "you know the other day when I was walking down the street I saw a lady who had long hair like yours and she had it braided .", "words": ["the", "was", "hair", "like", "long", "other", "you", "and", "it", "street", "who", "down", "when", "she", "a"]}, {"sent": "into the street ?", "words": ["the", "street", "into"]}, {"sent": "yes on the high street .", "words": ["on", "the", "street", "high"]}, {"sent": "Betty and Georgie and Kathy up the street .", "words": ["up", "the", "street", "and"]}, {"sent": "from Sesame street .", "words": ["street"]}, {"sent": "lets cut across the street right quick .", "words": ["the", "street", "cut"]}, {"sent": "you see dogs out in the street .", "words": ["the", "you", "in", "out", "see", "street"]}, {"sent": "oh I forgot to go to the house across the street .", "words": ["the", "go", "to", "house", "street"]}, {"sent": "xxx if I get off at water street .", "words": ["water", "off", "if", "get", "at", "street"]}, {"sent": "youve watched pigeon street on the television have you ?", "words": ["the", "you", "have", "on", "street"]}, {"sent": "theyre gonna rush down the street now arent they ?", "words": ["the", "they", "down", "street"]}, {"sent": "thats a hundred and twenty fifth street .", "words": ["street", "and", "a"]}, {"sent": "thats a street light .", "words": ["light", "street", "a"]}, {"sent": "I have a street ?", "words": ["have", "street", "a"]}, {"sent": "I live a hundred and thirty third street .", "words": ["street", "and", "a"]}, {"sent": "Pippin wanted to sniff the street light did he ?", "words": ["the", "light", "to", "he", "did", "street"]}, {"sent": "Gabbys street is called xxx Street .", "words": ["street", "is"]}, {"sent": "and youve got some street lamps here .", "words": ["here", "some", "street", "and"]}, {"sent": "you know Shem knows what street he lives on .", "words": ["what", "you", "he", "on", "street"]}, {"sent": "one five five street .", "words": ["street"]}, {"sent": "did you ever see sesame street ?", "words": ["see", "street", "did", "you"]}, {"sent": "what else happens on sesame street ?", "words": ["on", "what", "street"]}, {"sent": "oh Graeme street .", "words": ["street"]}, {"sent": "dont you think Robin hasta look both ways when he crosses the street ?", "words": ["the", "think", "you", "dont", "he", "look", "street", "when"]}, {"sent": "hey then Sesame street will be on later .", "words": ["be", "on", "will", "then", "street"]}, {"sent": "um and they have like little restaurants and stores and and Frog and Toad is right across the street from the big C V S .", "words": ["the", "is", "like", "little", "have", "they", "and", "street", "big"]}, {"sent": "fifty five east ninetieth street .", "words": ["street"]}, {"sent": "can you put the car in the street ?", "words": ["can", "the", "put", "car", "you", "in", "street"]}, {"sent": "you just youre just across the street .", "words": ["the", "street", "you"]}, {"sent": "are you saying its your street lamp because when you were watching AuntyMabel and Pippin they watching a man who was testing the street lamps .", "words": ["the", "were", "your", "was", "you", "and", "they", "are", "lamp", "who", "street", "when", "because", "a"]}, {"sent": "well we cant see now but he must be checking the lamps further down the street .", "words": ["the", "we", "be", "he", "street", "see", "down", "but"]}, {"sent": "street .", "words": ["street"]}, {"sent": "down na street .", "words": ["down", "street"]}, {"sent": "tell your guy to circle around and wait and xxx the street .", "words": ["the", "wait", "your", "to", "and", "around", "street"]}, {"sent": "I was across the street an I turn in front a your house .", "words": ["the", "your", "was", "an", "in", "house", "street", "a"]}, {"sent": "what do you hafta do when you cross the street ?", "words": ["the", "what", "do", "you", "street", "when"]}, {"sent": "wouldnt it be funny if they named a street after Bungle ?", "words": ["if", "be", "they", "it", "street", "a"]}, {"sent": "the little kids are crossing the street .", "words": ["the", "are", "street", "little"]}, {"sent": "a flowering pair like they use for a street tree ?", "words": ["tree", "for", "like", "they", "street", "a"]}, {"sent": "in the dark dark town theres a dark dark street .", "words": ["the", "in", "dark", "street", "a"]}, {"sent": "right on the sidewalk dont go in the street with them because theres cars in the street .", "words": ["the", "sidewalk", "go", "with", "them", "dont", "in", "on", "street", "because"]}, {"sent": "this is a street in a busy town .", "words": ["is", "this", "in", "street", "a"]}, {"sent": "oh these rollers come to put that bumpy tar stuff on the street ?", "words": ["the", "these", "put", "to", "on", "street", "that"]}, {"sent": "when they were showing AuntieMabel and the street lights episode .", "words": ["the", "and", "they", "street", "when", "were"]}, {"sent": "and Tom went howling down the street .", "words": ["the", "down", "street", "and"]}, {"sent": "oh does it stop for a street light ?", "words": ["does", "for", "light", "stop", "it", "street", "a"]}, {"sent": "did he play with you on the street ?", "words": ["the", "play", "with", "you", "he", "did", "on", "street"]}, {"sent": "hes taking it all the way up the street .", "words": ["the", "all", "up", "it", "street"]}, {"sent": "yeah the sun is now on this side of the street .", "words": ["the", "is", "of", "sun", "this", "on", "street"]}, {"sent": "and so she goes running down the street .", "words": ["the", "so", "and", "street", "down", "she"]}, {"sent": "I dont know I was just walking on the street .", "words": ["the", "was", "dont", "on", "street"]}, {"sent": "yeah yeah we get to walk in the street .", "words": ["the", "we", "to", "get", "in", "street", "walk"]}, {"sent": "it came from that and some little boy was playing with it on the street .", "words": ["the", "was", "with", "little", "and", "it", "on", "some", "street", "that"]}, {"sent": "what animals got in the middle of the street here ?", "words": ["the", "what", "of", "here", "in", "street"]}, {"sent": "it starts at seventy seventh street and goes south on Central park west then turns onto Broadway and ends at Macys on thirty fourth street .", "words": ["and", "it", "park", "at", "on", "then", "street"]}, {"sent": "the one when we go home it goes to west street .", "words": ["the", "go", "we", "to", "it", "street", "when", "home"]}, {"sent": "cars go on the street .", "words": ["on", "the", "street", "go"]}, {"sent": "you saw a sign with seagulls on Waterman street .", "words": ["with", "you", "on", "street", "a"]}, {"sent": "this is the story of a toy maker Matthew and the dolls he makes for Mary his daughter who is not strong enough to play in the street with the other children .", "words": ["the", "of", "not", "with", "who", "for", "is", "to", "play", "and", "toy", "a", "other", "this", "in", "he", "his", "story", "street"]}, {"sent": "paints lines on the street right .", "words": ["on", "the", "street"]}, {"sent": "downstairs in the street ?", "words": ["the", "in", "street"]}, {"sent": "why shouldnt you play in the street ?", "words": ["the", "play", "you", "in", "street", "why"]}, {"sent": "eth ethical culturess located central park west and sixty third street .", "words": ["park", "street", "and"]}, {"sent": "Matthew lives on a street called xxx .", "words": ["on", "street", "a"]}, {"sent": "we gonna down a hundred and thirty fifth street to Saint Nicholas to a Lenox avenue and then we gonna take the train and to ninety sixth street we get off and then well xxx go over to take the a Broadway train get off at a hundred and sixteenth street and then well walk over to Riverside Church .", "words": ["the", "off", "we", "train", "to", "go", "and", "get", "walk", "at", "street", "over", "then", "down", "take", "a"]}, {"sent": "sesame street ?", "words": ["street"]}, {"sent": "and then I saw her on the street right in front of the school .", "words": ["the", "of", "school", "and", "in", "on", "then", "street", "her"]}, {"sent": "uh no Im not connected with sesame street .", "words": ["street", "with", "not"]}, {"sent": "see this is the other side of the street .", "words": ["the", "is", "of", "other", "this", "see", "street"]}, {"sent": "downtown Boston street ?", "words": ["street"]}, {"sent": "stand up a minute .", "words": ["up", "stand", "a"]}, {"sent": "youre welcome to stand here .", "words": ["here", "to", "stand"]}, {"sent": "you see thats not really flat so theyre not gonna stand up .", "words": ["up", "stand", "so", "you", "see", "not"]}, {"sent": "Mom can I stand up on slide like this .", "words": ["can", "up", "slide", "stand", "like", "this", "on"]}, {"sent": "stand up .", "words": ["up", "stand"]}, {"sent": "yeah you stand there and Ill stand over here okay ?", "words": ["stand", "and", "you", "there", "here", "over"]}, {"sent": "what does n stand for ?", "words": ["what", "does", "stand", "for"]}, {"sent": "xxx I cant stand to watch you do that .", "words": ["do", "watch", "to", "stand", "you", "that"]}, {"sent": "you stand there .", "words": ["stand", "there", "you"]}, {"sent": "Cathy somebody in the xxx a girl that knows I hafta stand and explain to her a hundred times .", "words": ["the", "stand", "to", "her", "and", "in", "that", "a"]}, {"sent": "get a man to stand up there xxx .", "words": ["up", "to", "stand", "get", "there", "a"]}, {"sent": "will it stand on the lid ?", "words": ["the", "stand", "on", "it", "will"]}, {"sent": "Mummy Mummy Mummy stand ?", "words": ["stand"]}, {"sent": "stand .", "words": ["stand"]}, {"sent": "it wont stand up in the bath .", "words": ["up", "the", "stand", "in", "it"]}, {"sent": "stand up .", "words": ["up", "stand"]}, {"sent": "its a bit difficult to stand these on the carpet .", "words": ["the", "these", "to", "stand", "on", "a"]}, {"sent": "stand up .", "words": ["up", "stand"]}, {"sent": "well make him stand .", "words": ["stand", "him", "make"]}, {"sent": "make him stand up .", "words": ["up", "stand", "him", "make"]}, {"sent": "stand up and bounce ?", "words": ["up", "stand", "and"]}, {"sent": "let him stand up ?", "words": ["up", "stand", "him"]}, {"sent": "stand them up .", "words": ["up", "stand", "them"]}, {"sent": "she she used to stand in the middle of that green room and stare at the bookcase .", "words": ["the", "of", "to", "stand", "and", "in", "room", "green", "at", "that", "she"]}, {"sent": "what does xxx stand for ?", "words": ["what", "does", "stand", "for"]}, {"sent": "stand up there .", "words": ["up", "stand", "there"]}, {"sent": "stand up Barbara until I see you .", "words": ["up", "stand", "see", "you"]}, {"sent": "hm hm can you stand ?", "words": ["can", "stand", "you"]}, {"sent": "because if you stand on it itll tip up .", "words": ["up", "if", "stand", "you", "it", "on", "because"]}, {"sent": "you wanna go stand over there ?", "words": ["go", "stand", "you", "there", "wanna", "over"]}, {"sent": "okay you stand next to me for a second and let me see something .", "words": ["for", "see", "to", "stand", "you", "and", "me", "a"]}, {"sent": "all the shelves behind where he stand stands .", "words": ["the", "all", "where", "stand", "behind", "he"]}, {"sent": "Ill stand you up .", "words": ["up", "stand", "you"]}, {"sent": "do we go and stand on the puzzle ?", "words": ["the", "do", "go", "we", "stand", "and", "puzzle", "on"]}, {"sent": "my elephant can stand up and be even taller .", "words": ["can", "up", "my", "stand", "be", "elephant", "and"]}, {"sent": "can you make the man stand up John in the train ?", "words": ["can", "the", "up", "train", "stand", "you", "in", "make"]}, {"sent": "stand up .", "words": ["up", "stand"]}, {"sent": "okay stand up now .", "words": ["up", "stand"]}, {"sent": "stand up and just check youre not sat on anything .", "words": ["up", "stand", "and", "on", "not"]}, {"sent": "stand up .", "words": ["up", "stand"]}, {"sent": "shall I stand up the goose ?", "words": ["up", "stand", "goose", "the"]}, {"sent": "can he stand up ?", "words": ["can", "stand", "up", "he"]}, {"sent": "can you stand up now ?", "words": ["can", "stand", "up", "you"]}, {"sent": "shall I stand on you ?", "words": ["on", "stand", "you"]}, {"sent": "a horsie cant stand up .", "words": ["up", "stand", "a"]}, {"sent": "stand here Mamas watching you .", "words": ["here", "stand", "you"]}, {"sent": "you stand the animals up .", "words": ["the", "stand", "up", "you"]}, {"sent": "stand up .", "words": ["up", "stand"]}, {"sent": "stand on the table .", "words": ["on", "stand", "table", "the"]}, {"sent": "oh stand up .", "words": ["up", "stand"]}, {"sent": "which bit does Pingu stand in ?", "words": ["does", "in", "which", "stand"]}, {"sent": "youd better stand up then .", "words": ["up", "better", "then", "stand"]}, {"sent": "stand up .", "words": ["up", "stand"]}, {"sent": "others cant stand getting wet .", "words": ["stand", "wet"]}, {"sent": "you couldnt stand up again could you ?", "words": ["up", "stand", "could", "you"]}, {"sent": "stand it up .", "words": ["up", "stand", "it"]}, {"sent": "oh Pete dont you think that cow wants the mommy cow should stand up .", "words": ["the", "up", "think", "stand", "you", "dont", "that", "cow"]}, {"sent": "wanna stand there and look at this for a second ?", "words": ["for", "stand", "and", "there", "this", "at", "wanna", "look", "a"]}, {"sent": "stand still .", "words": ["stand"]}, {"sent": "Anna doesnt wanna stand on there .", "words": ["on", "stand", "wanna", "there"]}, {"sent": "you just stand right there .", "words": ["stand", "there", "you"]}, {"sent": "now stand up .", "words": ["up", "stand"]}, {"sent": "you wanna sit on the chair or you wanna stand up ?", "words": ["the", "up", "stand", "chair", "you", "wanna", "on", "sit"]}, {"sent": "well itll stand in it loose .", "words": ["stand", "in", "it"]}, {"sent": "xxx stand in the tree will he ?", "words": ["the", "tree", "stand", "in", "he", "will"]}, {"sent": "no you couldnt swim but you could stand up in that pool couldnt you .", "words": ["up", "stand", "swim", "pool", "you", "in", "could", "that", "but"]}, {"sent": "xxx stand up .", "words": ["up", "stand"]}, {"sent": "yeah you can see it when you stand up in your bed in the morning .", "words": ["can", "up", "the", "your", "stand", "bed", "you", "in", "it", "see", "when"]}, {"sent": "and theres a stand here .", "words": ["here", "stand", "a", "and"]}, {"sent": "would you stand over there ?", "words": ["stand", "you", "there", "would", "over"]}, {"sent": "dont stand on the mud .", "words": ["on", "dont", "stand", "the"]}, {"sent": "c what does c stand for ?", "words": ["what", "does", "stand", "for"]}, {"sent": "stand off the wire incase you trip .", "words": ["off", "stand", "you", "the"]}, {"sent": "and they can stand on the mantlepiece .", "words": ["can", "the", "stand", "and", "they", "on"]}, {"sent": "I couldnt stand it .", "words": ["stand", "it"]}, {"sent": "stand up .", "words": ["up", "stand"]}, {"sent": "they dont stand up very well do they ?", "words": ["up", "do", "stand", "dont", "they"]}, {"sent": "you stand up .", "words": ["up", "stand", "you"]}, {"sent": "this guy wants to go up on a pedestal like a victory stand .", "words": ["up", "go", "to", "stand", "like", "this", "on", "a"]}, {"sent": "we stand we stand it over here by the stereo .", "words": ["the", "we", "stand", "by", "here", "it", "over"]}, {"sent": "because sometimes when you go into tea shops and restaurants and cafes they have a stand with all todays newspapers on .", "words": ["all", "into", "go", "stand", "with", "you", "and", "they", "have", "on", "when", "because", "a"]}, {"sent": "you hafta stand up high and you can feel them .", "words": ["up", "can", "stand", "them", "and", "you", "high"]}, {"sent": "well that wont stand up very well because its soft .", "words": ["up", "soft", "stand", "that", "because"]}, {"sent": "dont let him stand on the flower .", "words": ["the", "stand", "flower", "dont", "him", "on"]}, {"sent": "can you stand up ?", "words": ["can", "stand", "up", "you"]}, {"sent": "dont stand up on it please .", "words": ["up", "stand", "dont", "it", "on"]}, {"sent": "I made the colt stand up .", "words": ["the", "stand", "up"]}, {"sent": "no you stand in front of it .", "words": ["of", "stand", "you", "in", "it"]}, {"sent": "can you stand up Mark ?", "words": ["can", "stand", "up", "you"]}, {"sent": "it was to get the cobra to stand up .", "words": ["the", "up", "to", "was", "stand", "get", "it"]}, {"sent": "stand up please .", "words": ["up", "stand"]}, {"sent": "or just stand ?", "words": ["stand"]}, {"sent": "do you wanna stand it in the back of there ?", "words": ["the", "do", "back", "of", "stand", "you", "there", "in", "it", "wanna"]}, {"sent": "dont stand on it .", "words": ["on", "dont", "it", "stand"]}, {"sent": "dont stand on Mummys legs .", "words": ["on", "dont", "stand"]}, {"sent": "I wanna stand up .", "words": ["up", "stand", "wanna"]}, {"sent": "I could xxx put a stand right here .", "words": ["put", "stand", "here", "could", "a"]}, {"sent": "his hose is at the back .", "words": ["the", "back", "is", "hose", "at", "his"]}, {"sent": "thats the firemans hose .", "words": ["the", "hose"]}, {"sent": "dont hose down our dinner .", "words": ["dont", "hose", "down", "our"]}, {"sent": "okay heres the hose and heres the little thing on the end of it that the water comes outof right ?", "words": ["the", "of", "little", "and", "hose", "it", "water", "on", "that"]}, {"sent": "FiremanSam Im terribly sorry but you left your hose pipe there and its just lying in the road .", "words": ["the", "your", "you", "and", "hose", "there", "in", "but"]}, {"sent": "is he using his hose ?", "words": ["his", "hose", "he", "is"]}, {"sent": "see some yellow wound up hose .", "words": ["up", "yellow", "hose", "see", "some"]}, {"sent": "a piece of hose .", "words": ["of", "hose", "a"]}, {"sent": "your firemans hose ?", "words": ["hose", "your"]}, {"sent": "we put lotsof things like pillows and blankets in and then we put the vacuum cleaner hose on the bag and then we go .", "words": ["the", "put", "we", "go", "like", "and", "vacuum", "hose", "in", "on", "then"]}, {"sent": "I cant hear what youre saying why are you hopping around why have you wrapped that hose pipe around your foot ?", "words": ["what", "your", "you", "hear", "have", "hose", "are", "foot", "around", "that", "why"]}, {"sent": "Ill get my hose out .", "words": ["get", "hose", "my", "out"]}, {"sent": "hey Pete wheres the hose ?", "words": ["the", "hose"]}, {"sent": "have you lost your hose ?", "words": ["have", "hose", "your", "you"]}, {"sent": "put the hose into the holes of the gas pump ?", "words": ["the", "put", "into", "of", "hose"]}, {"sent": "Liz attached a hose from the engine to the nearest water hydrant .", "words": ["the", "to", "hose", "water", "a"]}, {"sent": "but a hose is full of water and it puts fires out .", "words": ["full", "is", "of", "and", "hose", "it", "out", "water", "but", "a"]}, {"sent": "theyre all holding a hose but that fireman look that firemans hose is the one thats connected to the water .", "words": ["the", "all", "water", "is", "to", "hose", "look", "that", "but", "a"]}, {"sent": "is that what the hose pipe is ?", "words": ["the", "what", "is", "hose", "that"]}, {"sent": "I can hear somebody with a hose pipe .", "words": ["can", "with", "hear", "hose", "a"]}, {"sent": "that is how we hose huh ?", "words": ["how", "we", "is", "hose", "that"]}, {"sent": "I bet the hose goes in the firemans hand dont you ?", "words": ["the", "hand", "you", "hose", "in", "dont"]}, {"sent": "we could use that for a hose couldnt we ?", "words": ["for", "we", "hose", "could", "that", "a"]}, {"sent": "your um hose ?", "words": ["hose", "your"]}, {"sent": "they put a hose on the fire hydrant .", "words": ["the", "put", "they", "hose", "on", "a"]}, {"sent": "or the hose .", "words": ["the", "hose"]}, {"sent": "oh okay you gonna pretend its a water hose to put the fire out .", "words": ["the", "put", "to", "you", "hose", "out", "water", "pretend", "a"]}, {"sent": "youll leave the hose in your tummy ?", "words": ["the", "your", "hose", "in", "tummy"]}, {"sent": "we could pretend this is a hose oop .", "words": ["is", "we", "this", "hose", "pretend", "a", "could"]}, {"sent": "heres your hose .", "words": ["hose", "your"]}, {"sent": "theres some hose there .", "words": ["some", "hose", "there"]}, {"sent": "if theres a fire they attach the hose to it and then they put out the fire .", "words": ["the", "put", "if", "to", "and", "they", "hose", "it", "out", "then", "a"]}, {"sent": "like Daddys gardening hose .", "words": ["hose", "like"]}, {"sent": "theres the hose .", "words": ["the", "hose"]}, {"sent": "maybe well sprinkle water on this part of the lawn first this is the hose .", "words": ["the", "is", "of", "first", "this", "hose", "water", "on"]}, {"sent": "and ah you you attach a hose to the bottom .", "words": ["the", "bottom", "to", "you", "and", "hose", "a"]}, {"sent": "Ill go hook up the hose .", "words": ["up", "the", "hose", "go"]}, {"sent": "do you want the hose ?", "words": ["the", "do", "hose", "you"]}, {"sent": "youre giving me a hose pipe ?", "words": ["me", "hose", "a"]}, {"sent": "what what er what do you use the hose for ?", "words": ["the", "what", "do", "for", "you", "hose"]}, {"sent": "okay theres your you mean a hose and a sprinkler ?", "words": ["your", "and", "you", "hose", "sprinkler", "a"]}, {"sent": "the other end of the hose is connected to the pump .", "words": ["the", "is", "of", "to", "other", "hose"]}, {"sent": "oh and now youve got the extension hose from the Dyson vacuum cleaner .", "words": ["the", "hose", "vacuum", "and"]}, {"sent": "thats er the nozzle on the hose .", "words": ["the", "hose", "on"]}, {"sent": "something very strange Ive just been to the cash machine and Ive actually found this hose sticking out of the hole in the wall now why is that ?", "words": ["the", "is", "of", "to", "and", "hose", "this", "in", "out", "that", "why"]}, {"sent": "Henry holds a hose .", "words": ["hose", "a"]}, {"sent": "now can you see the rest of the hose ?", "words": ["can", "the", "of", "you", "hose", "see"]}, {"sent": "looks like hes strangling himself with his own hose .", "words": ["his", "hose", "with", "like"]}, {"sent": "because youve poured water all over me from the from the fire engine hose .", "words": ["the", "all", "hose", "water", "me", "over", "because"]}, {"sent": "he grabbed a hose lets put the wet stuff on the red stuff said Dan .", "words": ["the", "wet", "put", "red", "hose", "he", "on", "a"]}, {"sent": "theyre putting out the fire with the water and the hose arent they ?", "words": ["the", "with", "and", "hose", "they", "out", "water"]}, {"sent": "go from his head right down to his toes than give him an arm so he can hold his hose .", "words": ["can", "hold", "go", "to", "head", "so", "hose", "him", "an", "he", "arm", "his", "give", "down"]}, {"sent": "Ive got a very special hose now .", "words": ["hose", "a"]}, {"sent": "and thats his hose .", "words": ["his", "hose", "and"]}, {"sent": "and theres a piece here thats got the rest of the hose on .", "words": ["the", "of", "and", "here", "hose", "on", "a"]}, {"sent": "perhaps somebody has got a tumble dryer or something that they hafta out a hose out of the window .", "words": ["the", "of", "they", "hose", "out", "window", "that", "a", "dryer"]}, {"sent": "you mean the hose ?", "words": ["the", "hose", "you"]}, {"sent": "or shaped like a block or a garden hose .", "words": ["block", "like", "hose", "garden", "a"]}, {"sent": "you had a hose .", "words": ["hose", "a", "you"]}, {"sent": "you were asked several times not to play with the hose .", "words": ["the", "to", "play", "with", "you", "hose", "were", "not"]}, {"sent": "now you have a fuel pump good I mean a fuel whatever it is hose .", "words": ["is", "you", "have", "hose", "it", "good", "a"]}, {"sent": "and then youd be going to a fire and youd get there and think oh I havent brought my hose Ive brought my handcuffs .", "words": ["think", "my", "to", "be", "and", "get", "there", "hose", "then", "a"]}, {"sent": "one time we went shopping and the turned the water hose on all the people .", "words": ["the", "all", "we", "and", "hose", "water", "on"]}, {"sent": "and what does the hose do ?", "words": ["the", "what", "does", "do", "and", "hose"]}, {"sent": "hose .", "words": ["hose"]}, {"sent": "thats the wheel the big hose is wound onto .", "words": ["the", "hose", "big", "is"]}, {"sent": "or you gonna use it as a water hose .", "words": ["you", "hose", "it", "water", "a"]}, {"sent": "can you find which fireman is holding the hose that will put out the fire ?", "words": ["can", "the", "put", "is", "you", "hose", "that", "out", "find", "will", "which"]}, {"sent": "whats this thing in here is that the hose ?", "words": ["the", "is", "here", "this", "in", "hose", "that"]}, {"sent": "you can put your hose in that hole and squirt the water and it puts out a fire .", "words": ["can", "the", "put", "your", "you", "and", "hose", "in", "it", "out", "water", "that", "a"]}, {"sent": "theres a hose on the fire engine .", "words": ["on", "hose", "the", "a"]}, {"sent": "is the hose pipe the anchor ?", "words": ["the", "hose", "is"]}, {"sent": "the bit that slides down and covers the hose ?", "words": ["the", "and", "hose", "down", "that"]}, {"sent": "water comes out of the hose .", "words": ["the", "of", "hose", "out", "water"]}, {"sent": "he has having a look at the hose pipe on the grass now isnt he ?", "words": ["the", "look", "hose", "he", "at", "on", "grass", "a"]}, {"sent": "yeah you can stick that hose into the little hole .", "words": ["can", "the", "stick", "into", "you", "little", "hose", "that"]}, {"sent": "hose him down hose him down .", "words": ["hose", "him", "down"]}, {"sent": "and what is usually a hose pipe for the fire engine .", "words": ["the", "what", "for", "is", "and", "hose", "a"]}, {"sent": "and then discovered that the fuel hose was short too short and couldnt reach his gas tank .", "words": ["the", "was", "and", "hose", "too", "his", "then", "that"]}, {"sent": "is it the hose ?", "words": ["the", "hose", "it", "is"]}, {"sent": "is Spot coming with the water hose ?", "words": ["the", "is", "with", "hose", "water"]}, {"sent": "it makes this hose look huge .", "words": ["this", "hose", "it", "look"]}, {"sent": "because the water was very cold wasnt it when Gary sprayed you with the hose pipe .", "words": ["the", "was", "with", "you", "hose", "it", "water", "when", "cold", "because"]}, {"sent": "the hose just clips on there .", "words": ["the", "hose", "on", "there"]}, {"sent": "thats a hose .", "words": ["hose", "a"]}, {"sent": "well he needs a hose .", "words": ["hose", "he", "a"]}, {"sent": "you get the hose out .", "words": ["the", "you", "get", "hose", "out"]}, {"sent": "whatever hose she can get thats a mike .", "words": ["can", "get", "hose", "she", "a"]}, {"sent": "like the hose that Daddy pumps gas with .", "words": ["the", "with", "like", "hose", "that"]}, {"sent": "he needs some arms to hold his hose with .", "words": ["hold", "to", "with", "hose", "he", "his", "some"]}, {"sent": "rabbit cant tell which hose is fixed to the tap .", "words": ["the", "is", "to", "hose", "which"]}, {"sent": "is that the hose ?", "words": ["the", "hose", "that", "is"]}, {"sent": "we could pretend this is a long hose .", "words": ["is", "we", "long", "this", "hose", "pretend", "a", "could"]}, {"sent": "do you think they would have had trouble putting the fire out with a short hose ?", "words": ["the", "do", "think", "with", "you", "have", "they", "hose", "out", "would", "a"]}, {"sent": "are you trying to do a knot with the hose pipe ?", "words": ["the", "do", "to", "with", "you", "hose", "are", "a"]}, {"sent": "because Lolly turn on the water hydrant before the fire fighters attached the hose .", "words": ["the", "hose", "water", "on", "because"]}, {"sent": "thats a hose for the water squirt the water out .", "words": ["the", "for", "hose", "out", "water", "a"]}, {"sent": "on the end of the hose thing I think .", "words": ["the", "think", "of", "hose", "on"]}, {"sent": "and what comes out of the hose ?", "words": ["the", "what", "of", "and", "hose", "out"]}, {"sent": "I wanted to put my sock in the washing machine said Susie with all the other clothes and .", "words": ["the", "all", "put", "my", "to", "with", "other", "and", "washing machine", "in", "sock"]}, {"sent": "its the washing machine .", "words": ["the", "washing machine"]}, {"sent": "its soap and water in the washing machine .", "words": ["the", "soap", "and", "washing machine", "in", "water"]}, {"sent": "washing machine to do some laundry .", "words": ["to", "do", "washing machine", "some"]}, {"sent": "that must be from the washing machine .", "words": ["the", "be", "that", "washing machine"]}, {"sent": "its in the washing machine .", "words": ["the", "in", "washing machine"]}, {"sent": "youre going to help Mummy load the washing machine now arent you ?", "words": ["the", "help", "to", "washing machine", "you"]}, {"sent": "I went down xxx shirts in the washing machine xxx .", "words": ["the", "in", "down", "washing machine"]}, {"sent": "washing machine .", "words": ["washing machine"]}, {"sent": "are you going to help Mummy load the washing machine now ?", "words": ["the", "help", "to", "you", "washing machine", "are"]}, {"sent": "what shall we put inside the washing machine ?", "words": ["the", "what", "put", "we", "inside", "washing machine"]}, {"sent": "take a bath in the washing machine ?", "words": ["the", "washing machine", "in", "take", "a"]}, {"sent": "well this part of the house is peaceful because you cant hear the washing machine and you can just hear some insects outside and thats about all .", "words": ["the", "can", "all", "is", "of", "outside", "you", "hear", "this", "and", "washing machine", "house", "about", "some", "because"]}, {"sent": "I think that must be the washing machine .", "words": ["the", "think", "be", "washing machine", "that"]}, {"sent": "down the washing machine .", "words": ["the", "down", "washing machine"]}, {"sent": "no I hafta put the laundry in the laundry basket and then carry it to the washing machine .", "words": ["the", "put", "to", "and", "carry", "washing machine", "in", "it", "basket", "then"]}, {"sent": "hes pushing the button on the washing machine .", "words": ["the", "on", "button", "washing machine"]}, {"sent": "xxx youve put your Teletubbies on top of the washing machine and theyve all gone underneath .", "words": ["the", "all", "put", "all gone", "your", "of", "and", "washing machine", "on"]}, {"sent": "youre thinking that the washing machine is a fire engine .", "words": ["the", "is", "washing machine", "that", "a"]}, {"sent": "the door of the washing machine .", "words": ["the", "of", "washing machine", "door"]}, {"sent": "its the washing machine .", "words": ["the", "washing machine"]}, {"sent": "its in the washing machine because .", "words": ["the", "in", "because", "washing machine"]}, {"sent": "put your socks in the washing machine .", "words": ["the", "put", "your", "washing machine", "in"]}, {"sent": "yesterday I was telling you that the washing machine caught fire and there was a pussy cat in the house .", "words": ["the", "was", "and", "you", "there", "cat", "in", "house", "washing machine", "that", "a"]}, {"sent": "Naima Im gonna go over to the washing machine now .", "words": ["the", "go", "to", "washing machine", "over"]}, {"sent": "you throw them in the washing machine .", "words": ["the", "throw", "them", "you", "washing machine", "in"]}, {"sent": "and while youre doing that Im going to load the washing machine .", "words": ["the", "to", "washing machine", "and", "that"]}, {"sent": "put it in the washing machine .", "words": ["the", "put", "washing machine", "in", "it"]}, {"sent": "is panda in the washing machine ?", "words": ["the", "washing machine", "in", "is"]}, {"sent": "were gonna look at the washing machine .", "words": ["the", "look", "washing machine", "at", "were"]}, {"sent": "I cant do anything until Ive got the washing machine out the way and the and the washing out .", "words": ["the", "do", "washing machine", "and", "out"]}, {"sent": "please dont go in the in the washing machine .", "words": ["the", "go", "washing machine", "dont", "in"]}, {"sent": "xxx its probably the washing machine .", "words": ["the", "washing machine"]}, {"sent": "in the washing machine ?", "words": ["the", "in", "washing machine"]}, {"sent": "I stuck them in the washing machine .", "words": ["the", "them", "washing machine", "in", "stuck"]}, {"sent": "in the dryer and the washing machine .", "words": ["the", "washing machine", "and", "in", "dryer"]}, {"sent": "weve got to put the washing machine on in a minute havent we ?", "words": ["the", "put", "we", "to", "washing machine", "in", "on", "a"]}, {"sent": "is this the washing machine ?", "words": ["the", "this", "washing machine", "is"]}, {"sent": "washing machine is ready Freddy ready Freddy .", "words": ["washing machine", "is"]}, {"sent": "its the washing machine .", "words": ["the", "washing machine"]}, {"sent": "put the washing basket by the washing machine .", "words": ["the", "put", "washing machine", "by", "basket"]}, {"sent": "are you going to put it in the washing machine ?", "words": ["the", "put", "to", "you", "washing machine", "in", "are", "it"]}, {"sent": "you throw them in the washing machine and they come out really clean .", "words": ["the", "clean", "throw", "them", "you", "and", "they", "in", "washing machine", "out"]}, {"sent": "when you go to bed tonight Im going to put your clothes in the washing machine .", "words": ["the", "put", "your", "go", "to", "bed", "you", "washing machine", "in", "when"]}, {"sent": "because the washing machine rips the clothes up .", "words": ["the", "up", "because", "washing machine"]}, {"sent": "because of the washing machine ?", "words": ["of", "the", "because", "washing machine"]}, {"sent": "your hair looks like something I could take off and stick in the washing machine .", "words": ["off", "the", "stick", "your", "like", "and", "washing machine", "in", "could", "hair", "take"]}, {"sent": "washing machine xxx .", "words": ["washing machine"]}, {"sent": "thats a washing machine .", "words": ["washing machine", "a"]}, {"sent": "in Grandmas washing machine .", "words": ["in", "washing machine"]}, {"sent": "just put your sleep suit that youve thrown around in the washing machine .", "words": ["the", "put", "your", "sleep", "washing machine", "in", "around", "that"]}, {"sent": "and Mommy washed him last night in the washing machine .", "words": ["the", "last", "and", "washing machine", "in", "him"]}, {"sent": "did you put them in the washing machine and put the washing machine on ?", "words": ["the", "put", "them", "you", "and", "washing machine", "in", "did", "on"]}, {"sent": "do you think if Mummy put some washing in the washing machine Thomas that when the machine has finished you and Grandma could peg the washing out ?", "words": ["the", "do", "think", "put", "if", "when", "you", "and", "washing machine", "in", "out", "some", "could", "that"]}, {"sent": "it was like the time we went with the washing machine .", "words": ["the", "we", "was", "with", "like", "washing machine", "it"]}, {"sent": "just the washing machine .", "words": ["the", "washing machine"]}, {"sent": "some day well throw them all in the washing machine .", "words": ["the", "all", "throw", "them", "washing machine", "in", "some"]}, {"sent": "now we needta give the washing machine a drink dont we ?", "words": ["the", "drink", "we", "washing machine", "dont", "give", "a"]}, {"sent": "in the house though wheres the washing machine ?", "words": ["the", "in", "house", "washing machine"]}, {"sent": "you were right Nathan theyre going to put it in the washing machine uhn isnt she ?", "words": ["the", "put", "to", "you", "washing machine", "in", "it", "were", "she"]}, {"sent": "where is the washing machine ?", "words": ["where", "the", "washing machine", "is"]}, {"sent": "theyve been through the washing machine .", "words": ["the", "washing machine"]}, {"sent": "the buttons on the washing machine ?", "words": ["the", "on", "washing machine"]}, {"sent": "I think Im going to put the clothes that are in the washing machine into the dryer and dry them .", "words": ["the", "think", "put", "into", "to", "dry", "them", "and", "washing machine", "in", "are", "that", "dryer"]}, {"sent": "hes gonna be in the washing machine ?", "words": ["the", "be", "in", "washing machine"]}, {"sent": "we must remember to take Paf out before we put your trousers in the washing machine mustnt we ?", "words": ["the", "put", "your", "we", "to", "washing machine", "in", "out", "take"]}, {"sent": "oh no we dont want those to go in the washing machine sweetheart .", "words": ["the", "those", "go", "we", "to", "washing machine", "dont", "in"]}, {"sent": "and her dress needs put in the washing machine doesnt she ?", "words": ["the", "put", "her", "and", "washing machine", "in", "dress", "she"]}, {"sent": "I hafta get the clothes outof the washing machine first .", "words": ["get", "the", "first", "washing machine"]}, {"sent": "because we needta get the washing machine turned on otherwise we wont have any clean clothes will we ?", "words": ["the", "clean", "we", "will", "any", "washing machine", "get", "have", "on", "because"]}, {"sent": "any time Ive that washing machine on Cathy theres a damp smell .", "words": ["any", "washing machine", "on", "that", "a"]}, {"sent": "heres a washing machine .", "words": ["washing machine", "a"]}, {"sent": "and Mummy can forget about it and put it through the washing machine .", "words": ["can", "the", "put", "and", "washing machine", "it", "about"]}, {"sent": "and one day when I was waiting at the station a lady came and said shed had a pussy cat that looked just like Purdie and one day shed gone out and left the pussy cat in the house and their washing machine caught fire yes darling the washing machine caught fire .", "words": ["the", "their", "was", "like", "and", "washing machine", "cat", "that", "in", "house", "out", "at", "when", "a"]}, {"sent": "put a jumper in the washing machine and then we must go .", "words": ["the", "put", "we", "go", "and", "washing machine", "in", "then", "a"]}, {"sent": "dont drop it behind the washing machine .", "words": ["the", "behind", "washing machine", "dont", "it", "drop"]}, {"sent": "heres the washing machine .", "words": ["the", "washing machine"]}, {"sent": "its the washing machine .", "words": ["the", "washing machine"]}, {"sent": "now so theres the washing machine right ?", "words": ["the", "so", "washing machine"]}, {"sent": "why is it in the washing machine ?", "words": ["the", "is", "washing machine", "in", "it", "why"]}, {"sent": "I think that goes in the washing utility room with the washing machine .", "words": ["the", "think", "with", "washing machine", "in", "room", "that"]}, {"sent": "isnt that great getting them in the washing machine ?", "words": ["the", "them", "washing machine", "in", "that"]}, {"sent": "xxx youre putting it in the washing machine ?", "words": ["the", "in", "it", "washing machine"]}, {"sent": "just to the washing machine .", "words": ["the", "to", "washing machine"]}, {"sent": "he has it in the washing machine doesnt he ?", "words": ["the", "washing machine", "in", "it", "he"]}, {"sent": "its in the washing machine .", "words": ["the", "in", "washing machine"]}, {"sent": "but it has all come out of the washing machine .", "words": ["the", "all", "of", "washing machine", "it", "out", "but"]}, {"sent": "washing machine .", "words": ["washing machine"]}, {"sent": "just the washing machine .", "words": ["the", "washing machine"]}, {"sent": "that you can put in the washing machine .", "words": ["can", "the", "put", "you", "washing machine", "in", "that"]}, {"sent": "well they are going in the washing machine .", "words": ["the", "washing machine", "they", "in", "are"]}, {"sent": "they needta be in the washing machine dont they ?", "words": ["the", "be", "washing machine", "they", "in", "dont"]}, {"sent": "well wheres the washing machine ?", "words": ["the", "washing machine"]}, {"sent": "did you put them in the washing machine ?", "words": ["the", "put", "them", "you", "washing machine", "in", "did"]}, {"sent": "I think thats the washing machine and whats that ?", "words": ["the", "think", "washing machine", "and", "that"]}, {"sent": "are you going to come and turn this washing machine on please ?", "words": ["to", "and", "you", "this", "washing machine", "are", "on"]}, {"sent": "Dipsys in the washing machine .", "words": ["the", "in", "washing machine"]}, {"sent": "glasses .", "words": ["glasses"]}, {"sent": "what are you gonna put the glasses on ?", "words": ["the", "what", "put", "you", "glasses", "are", "on"]}, {"sent": "oh the glasses on Jaafar .", "words": ["the", "on", "glasses"]}, {"sent": "you should have brought your glasses huh ?", "words": ["have", "your", "glasses", "you"]}, {"sent": "you dont wear glasses either do you ?", "words": ["dont", "do", "glasses", "you"]}, {"sent": "just leave them up if theyve got glasses on .", "words": ["up", "if", "them", "glasses", "on"]}, {"sent": "alright how about your glasses do you want your glasses ?", "words": ["do", "how", "your", "glasses", "you", "about"]}, {"sent": "oh you dope you dont wanna take your glasses to bed .", "words": ["your", "to", "bed", "you", "glasses", "dont", "wanna", "take"]}, {"sent": "and once a little girl was trying to help put away the glasses and accidently broke ten glasses while she was trying to help her Mummy put them away .", "words": ["the", "help", "put", "to", "was", "them", "little", "and", "glasses", "away", "her", "she", "a"]}, {"sent": "oh gimme those other eyes please and the glasses if youre not going to use them .", "words": ["the", "those", "if", "to", "them", "glasses", "and", "other", "not"]}, {"sent": "I did not put your glasses on you .", "words": ["put", "your", "glasses", "you", "did", "on", "not"]}, {"sent": "these two glasses are fatter than this glass .", "words": ["these", "glasses", "this", "are", "glass"]}, {"sent": "now were gonna comb your hair and were gonna put your glasses on and were gonna go .", "words": ["put", "your", "go", "comb", "and", "glasses", "on", "hair", "were"]}, {"sent": "perhaps if I wore my glasses more often .", "words": ["my", "more", "if", "glasses"]}, {"sent": "glasses .", "words": ["glasses"]}, {"sent": "thats right you had glasses from Disneyland .", "words": ["glasses", "you"]}, {"sent": "Mama doesnt wear glasses .", "words": ["glasses"]}, {"sent": "these glasses I guess make everything look tilted to me .", "words": ["these", "to", "glasses", "look", "me", "make"]}, {"sent": "where do you wear glasses ?", "words": ["where", "do", "glasses", "you"]}, {"sent": "a man with glasses .", "words": ["with", "glasses", "a"]}, {"sent": "just tell that lady who has the glasses on listen .", "words": ["the", "listen", "glasses", "who", "on", "that"]}, {"sent": "has she got glasses on ?", "words": ["on", "she", "glasses"]}, {"sent": "those your glasses ?", "words": ["your", "those", "glasses"]}, {"sent": "are those glasses for you ?", "words": ["for", "those", "you", "glasses", "are"]}, {"sent": "is that your new glasses ?", "words": ["your", "is", "glasses", "new", "that"]}, {"sent": "Mummys glasses are balancing on her ears arent they ?", "words": ["glasses", "they", "are", "on", "her"]}, {"sent": "his glasses .", "words": ["his", "glasses"]}, {"sent": "hes got glasses on oh alright .", "words": ["on", "glasses"]}, {"sent": "give me my glasses back .", "words": ["my", "back", "glasses", "me", "give"]}, {"sent": "sometimes Grandma has glasses .", "words": ["glasses"]}, {"sent": "so we think you should have two pairs of glasses .", "words": ["think", "we", "of", "so", "you", "have", "glasses"]}, {"sent": "Mommys glasses .", "words": ["glasses"]}, {"sent": "theyre not gonna look like glasses anymore .", "words": ["glasses", "not", "like", "look"]}, {"sent": "xxx my glasses .", "words": ["my", "glasses"]}, {"sent": "shes got glasses as well hasnt she ?", "words": ["she", "glasses"]}, {"sent": "with the glasses .", "words": ["the", "with", "glasses"]}, {"sent": "herere your glasses .", "words": ["your", "glasses"]}, {"sent": "do you wear glasses ?", "words": ["do", "glasses", "you"]}, {"sent": "look all the kittys are wearing glasses .", "words": ["the", "all", "glasses", "are", "look"]}, {"sent": "glasses .", "words": ["glasses"]}, {"sent": "ywan put the glasses on ?", "words": ["the", "put", "on", "glasses"]}, {"sent": "theyre my glasses .", "words": ["my", "glasses"]}, {"sent": "my glasses are falling .", "words": ["my", "are", "glasses"]}, {"sent": "didnt recognize you without your glasses .", "words": ["your", "glasses", "you"]}, {"sent": "thank you for letting me shop at the shop and buying a pair of glasses .", "words": ["the", "for", "of", "you", "and", "glasses", "at", "me", "a"]}, {"sent": "I wanna buy these glasses .", "words": ["these", "buy", "wanna", "glasses"]}, {"sent": "was it John with the glasses ?", "words": ["the", "was", "with", "glasses", "it"]}, {"sent": "where are my glasses ?", "words": ["where", "my", "are", "glasses"]}, {"sent": "theres your glasses .", "words": ["your", "glasses"]}, {"sent": "in Reginas glasses .", "words": ["in", "glasses"]}, {"sent": "wash our glasses .", "words": ["wash", "glasses", "our"]}, {"sent": "the glasses .", "words": ["the", "glasses"]}, {"sent": "theyre glasses .", "words": ["glasses"]}, {"sent": "glasses .", "words": ["glasses"]}, {"sent": "shes got grey hair and glasses .", "words": ["hair", "glasses", "and"]}, {"sent": "otherwise youre desperately in need of glasses .", "words": ["of", "in", "need", "glasses"]}, {"sent": "dont dont play with the glasses .", "words": ["the", "play", "with", "glasses", "dont"]}, {"sent": "is this like glasses or something ?", "words": ["this", "glasses", "like", "is"]}, {"sent": "oh hes got some glasses hasnt he ?", "words": ["some", "he", "glasses"]}, {"sent": "you can see them in my glasses ?", "words": ["can", "my", "them", "you", "glasses", "in", "see"]}, {"sent": "whose glasses those ?", "words": ["those", "glasses"]}, {"sent": "no glasses .", "words": ["glasses"]}, {"sent": "we got this one for Brian so maybe hed start wearing his glasses better .", "words": ["better", "for", "we", "so", "glasses", "this", "his"]}, {"sent": "the snowmans wearing glasses ?", "words": ["the", "glasses"]}, {"sent": "I needta figure something out about these glasses of water .", "words": ["water", "these", "of", "glasses", "about", "out"]}, {"sent": "put on the glasses .", "words": ["on", "the", "put", "glasses"]}, {"sent": "he mustnt have any glasses .", "words": ["have", "any", "he", "glasses"]}, {"sent": "who else wears glasses .", "words": ["glasses", "who"]}, {"sent": "no dont play with glasses .", "words": ["dont", "play", "with", "glasses"]}, {"sent": "do they go around your neck or do the glasses go on your eyes ?", "words": ["the", "do", "your", "go", "glasses", "they", "around", "on"]}, {"sent": "your space glasses ?", "words": ["your", "glasses"]}, {"sent": "yes you are gonna wreck those glasses .", "words": ["are", "those", "glasses", "you"]}, {"sent": "Johnny wears glasses .", "words": ["glasses"]}, {"sent": "my glasses .", "words": ["my", "glasses"]}, {"sent": "you like getting my glasses .", "words": ["glasses", "my", "like", "you"]}, {"sent": "okay take off your glasses .", "words": ["off", "take", "glasses", "your"]}, {"sent": "glasses .", "words": ["glasses"]}, {"sent": "mommy has glasses .", "words": ["glasses"]}, {"sent": "and you had sunsuits and sunhats and glasses .", "words": ["glasses", "you", "and"]}, {"sent": "glasses ?", "words": ["glasses"]}, {"sent": "he needs his glasses to read ?", "words": ["to", "glasses", "he", "read", "his"]}, {"sent": "Eve do you have some glasses .", "words": ["do", "glasses", "have", "you", "some"]}, {"sent": "sun glasses .", "words": ["sun", "glasses"]}, {"sent": "some pretty glasses .", "words": ["some", "pretty", "glasses"]}, {"sent": "what a nifty pair of glasses you have .", "words": ["what", "of", "glasses", "you", "have", "a"]}, {"sent": "Dan where do his glasses go ?", "words": ["where", "do", "go", "glasses", "his"]}, {"sent": "he doesnt have his glasses on ?", "words": ["glasses", "have", "on", "he", "his"]}, {"sent": "what happened to his other glasses ?", "words": ["what", "to", "other", "glasses", "his"]}, {"sent": "me and Erics da only ones that have glasses on .", "words": ["glasses", "have", "and", "on", "me", "that"]}, {"sent": "what kind of glasses ?", "words": ["of", "what", "glasses"]}, {"sent": "had got long hair and glasses on hadnt he ?", "words": ["long", "glasses", "and", "he", "on", "hair"]}, {"sent": "and when you need glasses .", "words": ["need", "glasses", "you", "and", "when"]}, {"sent": "glasses .", "words": ["glasses"]}, {"sent": "did the pink one have some glasses ?", "words": ["the", "glasses", "have", "did", "some"]}, {"sent": "take my glasses off .", "words": ["off", "my", "take", "glasses"]}, {"sent": "wow nice glasses .", "words": ["nice", "glasses"]}, {"sent": "so when I am in class and I hafta see the board I hafta wear my glasses .", "words": ["the", "my", "so", "and", "glasses", "in", "am", "see", "when"]}, {"sent": "no we dont put glasses in the milk .", "words": ["the", "put", "we", "glasses", "dont", "in", "milk"]}, {"sent": "youll break her leg .", "words": ["her", "break", "leg"]}, {"sent": "I dont think youre playing a trick when you pretend to break your arm .", "words": ["think", "break", "your", "to", "you", "dont", "arm", "pretend", "when", "a"]}, {"sent": "dont force it you might break it .", "words": ["dont", "it", "break", "you"]}, {"sent": "break .", "words": ["break"]}, {"sent": "how did you break it ?", "words": ["how", "you", "it", "did", "break"]}, {"sent": "did you break his rattle ?", "words": ["his", "break", "did", "you"]}, {"sent": "dont break it though .", "words": ["dont", "it", "break"]}, {"sent": "you cant treat these like that theyll break .", "words": ["these", "like", "you", "that", "break"]}, {"sent": "because youll break it .", "words": ["it", "break", "because"]}, {"sent": "did you break it ?", "words": ["it", "break", "did", "you"]}, {"sent": "yeah Doctor Weist will be upset if I if I break it .", "words": ["if", "be", "it", "will", "break"]}, {"sent": "poor Ursula what will she do for a pencil if you break it ?", "words": ["what", "do", "for", "if", "pencil", "poor", "you", "it", "will", "break", "she", "a"]}, {"sent": "just break a bit off .", "words": ["off", "break", "a"]}, {"sent": "because itll break .", "words": ["break", "because"]}, {"sent": "no you cant break that bread apart .", "words": ["bread", "that", "break", "you"]}, {"sent": "thatll break the crayon .", "words": ["the", "break", "crayon"]}, {"sent": "do you think theyll break it ?", "words": ["do", "think", "you", "it", "break"]}, {"sent": "how did you break that ?", "words": ["how", "break", "you", "did", "that"]}, {"sent": "you didnt break them .", "words": ["them", "break", "you"]}, {"sent": "theyll break .", "words": ["break"]}, {"sent": "yeah wait a minute dont break it yet dont break it yet .", "words": ["wait", "dont", "it", "break", "a"]}, {"sent": "its easy to break it and they snap back on again uhhuh ?", "words": ["back", "to", "and", "they", "it", "on", "break"]}, {"sent": "just dont break them thats all because weve managed to keep them .", "words": ["all", "to", "them", "dont", "break", "because"]}, {"sent": "youll break the glass .", "words": ["the", "break", "glass"]}, {"sent": "dont break it .", "words": ["dont", "it", "break"]}, {"sent": "dont break it .", "words": ["dont", "it", "break"]}, {"sent": "ready for my tea break now .", "words": ["my", "for", "break"]}, {"sent": "please dont break anything Darling .", "words": ["dont", "break"]}, {"sent": "I thought somebody was going to break in .", "words": ["to", "was", "in", "break"]}, {"sent": "dont break it yet .", "words": ["dont", "it", "break"]}, {"sent": "Fraser you be gentle with that otherwise youre gonna break it .", "words": ["break", "be", "with", "you", "it", "gentle", "that"]}, {"sent": "youll break it though if you dont stay off of it .", "words": ["off", "if", "of", "you", "dont", "it", "stay", "break"]}, {"sent": "thin branches can break .", "words": ["can", "break"]}, {"sent": "what wont break ?", "words": ["what", "break"]}, {"sent": "dont break it .", "words": ["dont", "it", "break"]}, {"sent": "its gonna break .", "words": ["break"]}, {"sent": "oops careful with the bricks because youll break the train .", "words": ["the", "train", "with", "careful", "break", "because"]}, {"sent": "youll break it mind .", "words": ["it", "break"]}, {"sent": "because youll break it .", "words": ["it", "break", "because"]}, {"sent": "it will break .", "words": ["will", "it", "break"]}, {"sent": "dont touch dont touch Pilchard because hell break .", "words": ["dont", "break", "because", "touch"]}, {"sent": "its spring break .", "words": ["break"]}, {"sent": "you might break it .", "words": ["it", "break", "you"]}, {"sent": "because they break and make a big mess .", "words": ["break", "and", "they", "big", "make", "because", "a"]}, {"sent": "do you know anybody who did the same thing my aunt break a leg or break an arm or something ?", "words": ["the", "do", "my", "you", "an", "same", "did", "leg", "who", "arm", "break", "a"]}, {"sent": "dont play with that y Thomas youll break it come on .", "words": ["play", "with", "dont", "that", "it", "on", "break"]}, {"sent": "do you think the men open their flask of tea and have a little break ?", "words": ["the", "do", "think", "their", "open", "of", "you", "and", "have", "little", "break", "a"]}, {"sent": "youll break it pulling it like that .", "words": ["that", "it", "break", "like"]}, {"sent": "we hafta throw them in the bin if you break them .", "words": ["the", "throw", "if", "we", "them", "you", "in", "break"]}, {"sent": "dont break it .", "words": ["dont", "it", "break"]}, {"sent": "uh oh youll look break that xxx .", "words": ["that", "break", "look"]}, {"sent": "Dimitra might break them ?", "words": ["break", "them"]}, {"sent": "you seem to break a lot of things dont you ?", "words": ["of", "to", "a lot", "you", "dont", "break", "a"]}, {"sent": "but wh why doesnt it break anything ?", "words": ["break", "it", "why", "but"]}, {"sent": "and did it break open ?", "words": ["open", "and", "it", "did", "break"]}, {"sent": "dont break it will you ?", "words": ["you", "dont", "it", "will", "break"]}, {"sent": "you want me to break it in half ?", "words": ["to", "you", "in", "it", "me", "break"]}, {"sent": "come one well well break them up into some are about days and some are about faces .", "words": ["up", "into", "them", "and", "are", "about", "some", "break"]}, {"sent": "youll break it .", "words": ["it", "break"]}, {"sent": "because like we dont want the movers to break it .", "words": ["the", "we", "to", "like", "dont", "it", "break", "because"]}, {"sent": "you might break it doing that sweetheart .", "words": ["that", "it", "break", "you"]}, {"sent": "youll break the door .", "words": ["the", "break", "door"]}, {"sent": "Mister Mann let me get the best a the boy then he break it up .", "words": ["the", "up", "get", "it", "he", "me", "then", "break", "a"]}, {"sent": "and they dont break like normal crayons .", "words": ["like", "and", "dont", "they", "break"]}, {"sent": "youll break that Thomas .", "words": ["that", "break"]}, {"sent": "youll break it .", "words": ["it", "break"]}, {"sent": "because if you drop it itll break .", "words": ["if", "you", "it", "drop", "break", "because"]}, {"sent": "dont break it .", "words": ["dont", "it", "break"]}, {"sent": "dont break it .", "words": ["dont", "it", "break"]}, {"sent": "so I think it might break again .", "words": ["so", "think", "it", "break"]}, {"sent": "dont let Joseph break that house thing .", "words": ["house", "dont", "that", "break"]}, {"sent": "you may break that .", "words": ["that", "break", "you"]}, {"sent": "youve gotta be very careful of these because they can break you know ?", "words": ["can", "these", "of", "be", "you", "they", "careful", "break", "because"]}, {"sent": "why dont you pick up Cromers pen and give it to him before you break it .", "words": ["up", "pick", "to", "you", "and", "dont", "him", "it", "pen", "give", "break", "why"]}, {"sent": "that break apart and go back together .", "words": ["back", "go", "and", "that", "break"]}, {"sent": "well then dont break it .", "words": ["dont", "then", "it", "break"]}, {"sent": "ohdear I cant break them either .", "words": ["break", "them"]}, {"sent": "shes in there shes taking a break .", "words": ["there", "in", "break", "a"]}, {"sent": "youll break Dadas new computer .", "words": ["new", "break"]}, {"sent": "he didnt break it .", "words": ["it", "break", "he"]}, {"sent": "youll break it .", "words": ["it", "break"]}, {"sent": "dont you are going to break that bus Thomas .", "words": ["to", "you", "dont", "that", "are", "break", "bus"]}, {"sent": "well just have a little break I think sweetheart .", "words": ["think", "little", "have", "break", "a"]}, {"sent": "you want to break you ?", "words": ["to", "break", "want to", "you"]}, {"sent": "you might break it .", "words": ["it", "break", "you"]}, {"sent": "sweetie lets not break that .", "words": ["that", "break", "not"]}, {"sent": "I think if we pulled an egg wed break it wouldnt we ?", "words": ["think", "egg", "if", "we", "an", "it", "break"]}, {"sent": "does china clay break easily ?", "words": ["does", "break"]}, {"sent": "because it might break .", "words": ["it", "break", "because"]}, {"sent": "heres my pipe and Im gonna break it .", "words": ["my", "it", "break", "and"]}, {"sent": "well I think we oughta have a little break now .", "words": ["think", "we", "little", "have", "break", "a"]}, {"sent": "you might break it .", "words": ["it", "break", "you"]}, {"sent": "youre going to break it .", "words": ["to", "it", "break"]}, {"sent": "break it ?", "words": ["it", "break"]}, {"sent": "dont wanna break it uhn do we ?", "words": ["do", "we", "dont", "it", "wanna", "break"]}, {"sent": "oh dont bite that youll break your teeth willnt you ?", "words": ["your", "you", "dont", "that", "bite", "break"]}, {"sent": "what else did Daddy break then ?", "words": ["what", "then", "break", "did"]}, {"sent": "careful dont break him .", "words": ["dont", "him", "break", "careful"]}, {"sent": "lookee can you button that ?", "words": ["can", "that", "button", "you"]}, {"sent": "what is that a button ?", "words": ["what", "is", "that", "button", "a"]}, {"sent": "press this square button .", "words": ["this", "button"]}, {"sent": "now would you like to stop it with the red button ?", "words": ["the", "red", "to", "stop", "like", "with", "you", "it", "would", "button"]}, {"sent": "one button .", "words": ["button"]}, {"sent": "where do you want the other button ?", "words": ["where", "do", "the", "other", "you", "button"]}, {"sent": "here press the button ?", "words": ["here", "the", "button"]}, {"sent": "push the button .", "words": ["the", "push", "button"]}, {"sent": "what happens if you press the button on your socks ?", "words": ["the", "what", "your", "if", "you", "on", "button"]}, {"sent": "press the button press the button .", "words": ["the", "button"]}, {"sent": "is that the babys button .", "words": ["the", "that", "button", "is"]}, {"sent": "does the lady pop up when you push that button ?", "words": ["the", "does", "up", "pop", "push", "button", "you", "that", "when"]}, {"sent": "press the silver button .", "words": ["the", "button"]}, {"sent": "how does a button work ?", "words": ["does", "how", "work", "button", "a"]}, {"sent": "then we take the button push it through .", "words": ["the", "push", "button", "we", "it", "then", "take"]}, {"sent": "okay now how many button how many beads do we have on each one here ?", "words": ["do", "how", "each", "we", "have", "here", "beads", "on", "button"]}, {"sent": "see if you can button these .", "words": ["can", "these", "if", "you", "see", "button"]}, {"sent": "think if you press the button itll ring a bell downstairs somewhere .", "words": ["the", "think", "if", "you", "button", "a"]}, {"sent": "the button turns the head around .", "words": ["the", "head", "button", "around"]}, {"sent": "when you press the button the trailer comes down on the back of your bicycle ?", "words": ["the", "your", "button", "back", "of", "bicycle", "you", "on", "down", "when"]}, {"sent": "wash your belly button .", "words": ["your", "belly button", "wash", "button"]}, {"sent": "help me button it .", "words": ["me", "help", "it", "button"]}, {"sent": "does does Mickey Mouse do anything when you push the button ?", "words": ["the", "does", "do", "push", "button", "you", "when"]}, {"sent": "that the man let you press the button .", "words": ["the", "that", "button", "you"]}, {"sent": "mash your button .", "words": ["your", "button"]}, {"sent": "see well put it up on the table and then you can push the button again .", "words": ["up", "the", "put", "can", "push", "table", "and", "you", "it", "on", "see", "then", "button"]}, {"sent": "when he press the button that means the clock stop and you know the time you took from here to there .", "words": ["the", "when", "button", "clock", "to", "stop", "and", "you", "here", "there", "he", "that"]}, {"sent": "I dont know we pushed the wrong button .", "words": ["the", "dont", "button", "we"]}, {"sent": "and the other thing I found out was your belly button .", "words": ["the", "your", "was", "other", "and", "belly button", "out", "button"]}, {"sent": "that is a button .", "words": ["button", "that", "a", "is"]}, {"sent": "couldnt you button them on ?", "words": ["on", "them", "button", "you"]}, {"sent": "button .", "words": ["button"]}, {"sent": "do the red button .", "words": ["the", "do", "red", "button"]}, {"sent": "trying to turn the button .", "words": ["the", "to", "button"]}, {"sent": "well Ill have a chocolate button in that case .", "words": ["chocolate", "have", "in", "that", "button", "a"]}, {"sent": "you wanna press the green button now ?", "words": ["the", "you", "green", "wanna", "button"]}, {"sent": "you push a button and the numbers appear ?", "words": ["the", "push", "and", "you", "button", "a"]}, {"sent": "press the second button .", "words": ["the", "button"]}, {"sent": "and what happens when you press the button ?", "words": ["the", "what", "button", "you", "and", "when"]}, {"sent": "do you want me to do that button for you ?", "words": ["do", "for", "button", "to", "you", "me", "that"]}, {"sent": "wheres your belly button ?", "words": ["belly button", "your", "button"]}, {"sent": "these are button things you can unbutton and shut .", "words": ["can", "these", "you", "and", "are", "button"]}, {"sent": "well push another button and thatll probably come up .", "words": ["up", "push", "and", "another", "button"]}, {"sent": "tummy button .", "words": ["button", "tummy"]}, {"sent": "would you like to press down on that button ?", "words": ["button", "to", "like", "you", "would", "on", "down", "that"]}, {"sent": "well thats not my button wailed Toad that button is square and my button is round .", "words": ["my", "button", "is", "and", "that", "not"]}, {"sent": "see the button ?", "words": ["the", "see", "button"]}, {"sent": "the top button does what ?", "words": ["the", "does", "what", "button"]}, {"sent": "because thats the button you pushed .", "words": ["the", "button", "because", "you"]}, {"sent": "because that ones got a that jackets got a button missing hasnt it ?", "words": ["that", "it", "button", "because", "a"]}, {"sent": "I see your belly button .", "words": ["see", "your", "belly button", "button"]}, {"sent": "heres the button .", "words": ["the", "button"]}, {"sent": "and theres the button unbutton it and button it .", "words": ["the", "it", "button", "and"]}, {"sent": "this button darling .", "words": ["this", "button"]}, {"sent": "you see from the little microphone there theres a button ?", "words": ["the", "little", "you", "there", "see", "button", "a"]}, {"sent": "use this button ?", "words": ["this", "button"]}, {"sent": "well let me loosen the button .", "words": ["the", "me", "button"]}, {"sent": "your stop and go button .", "words": ["your", "go", "stop", "and", "button"]}, {"sent": "press the button and what ?", "words": ["the", "what", "button", "and"]}, {"sent": "if you press a button like this and then you put the ball there .", "words": ["the", "put", "if", "like", "you", "and", "this", "ball", "there", "then", "button", "a"]}, {"sent": "thats Mommys button Francis .", "words": ["button"]}, {"sent": "now we press this button here okay ?", "words": ["here", "this", "button", "we"]}, {"sent": "the button alone .", "words": ["the", "button"]}, {"sent": "I think its the button on the microphone .", "words": ["the", "think", "button", "on"]}, {"sent": "there heres on there I think hes got a shirt like yours and and his button stays done up doesnt it the button doesnt keep coming undone does it ?", "words": ["up", "the", "think", "does", "shirt", "like", "and", "his", "there", "it", "on", "button", "a"]}, {"sent": "on the button every half hour he wakens up .", "words": ["the", "up", "he", "every", "on", "button"]}, {"sent": "she has got a button on her dress .", "words": ["button", "dress", "on", "her", "she", "a"]}, {"sent": "would you wanna go and let Chelsea show you that button ?", "words": ["button", "go", "show", "and", "you", "wanna", "would", "that"]}, {"sent": "button .", "words": ["button"]}, {"sent": "lets just press a button and then what we can do .", "words": ["can", "what", "do", "we", "and", "then", "button", "a"]}, {"sent": "here let me get this other button .", "words": ["other", "here", "get", "this", "me", "button"]}, {"sent": "thats another button .", "words": ["another", "button"]}, {"sent": "push the button .", "words": ["the", "push", "button"]}, {"sent": "you hafta press the green button darling .", "words": ["the", "button", "green", "you"]}, {"sent": "she say belly button .", "words": ["belly button", "say", "she", "button"]}, {"sent": "no I dont know it might be just xxx is it to turn it on Violet or just a button on his shirt ?", "words": ["is", "to", "be", "shirt", "his", "dont", "it", "on", "button", "a"]}, {"sent": "oh you didnt press the button properly did you ?", "words": ["the", "button", "did", "you"]}, {"sent": "you push the red button to make it stop there .", "words": ["the", "red", "push", "to", "stop", "you", "there", "make", "it", "button"]}, {"sent": "you havent pressed the button .", "words": ["the", "button", "you"]}, {"sent": "you push the button on the side and she laughs .", "words": ["the", "push", "you", "and", "on", "button", "she"]}, {"sent": "you press the button dont you ?", "words": ["the", "dont", "button", "you"]}, {"sent": "the button yes I think it has holes .", "words": ["the", "think", "it", "button"]}, {"sent": "well stop it with the red button .", "words": ["the", "red", "stop", "with", "it", "button"]}, {"sent": "when when when hes finished we willnt press another button .", "words": ["another", "when", "button", "we"]}, {"sent": "oh hes gotta press that button dont you ?", "words": ["dont", "that", "button", "you"]}, {"sent": "he could zip zippers and button buttons .", "words": ["could", "he", "button", "and"]}, {"sent": "you take the little button and you find the hole and you push it through .", "words": ["the", "push", "little", "you", "and", "it", "find", "take", "button"]}, {"sent": "dont be pressing the button .", "words": ["the", "dont", "be", "button"]}, {"sent": "well youve not pressed the button hard enough have you ?", "words": ["the", "you", "have", "hard", "button", "not"]}, {"sent": "no you hafta push the button .", "words": ["the", "push", "button", "you"]}, {"sent": "and his belly button .", "words": ["his", "belly button", "button", "and"]}, {"sent": "shes got a button look .", "words": ["look", "button", "a"]}, {"sent": "thats a button thats right .", "words": ["button", "a"]}, {"sent": "he lost his button yeah .", "words": ["his", "he", "button"]}, {"sent": "well you press the button first the big one .", "words": ["the", "button", "first", "you", "big"]}, {"sent": "xxx push the other button xxx show Gloria .", "words": ["the", "push", "show", "other", "button"]}, {"sent": "just press the button on the tv there .", "words": ["the", "there", "tv", "on", "button"]}, {"sent": "press the button and the back of the lorry comes down .", "words": ["the", "back", "of", "and", "down", "button"]}, {"sent": "yeah little belly button .", "words": ["belly button", "button", "little"]}, {"sent": "well you can share it .", "words": ["can", "it", "share", "you"]}, {"sent": "okay you can share them .", "words": ["can", "them", "share", "you"]}, {"sent": "youre nice to share with me .", "words": ["share", "to", "nice", "with", "me"]}, {"sent": "no we can share them .", "words": ["can", "them", "share", "we"]}, {"sent": "because there are two snakes now to share your breakfast .", "words": ["share", "your", "to", "there", "are", "because"]}, {"sent": "you didnt share toys with me ?", "words": ["me", "share", "with", "you"]}, {"sent": "can I share your pizza ?", "words": ["can", "share", "pizza", "your"]}, {"sent": "you gonna share one with mommy ?", "words": ["share", "with", "you"]}, {"sent": "because its nice to share isnt it ?", "words": ["share", "to", "nice", "it", "because"]}, {"sent": "thats nice of you to share .", "words": ["share", "of", "to", "nice", "you"]}, {"sent": "well you can share some of this .", "words": ["can", "share", "of", "you", "this", "some"]}, {"sent": "you share it .", "words": ["it", "share", "you"]}, {"sent": "can you share that with Mommy ?", "words": ["can", "share", "with", "you", "that"]}, {"sent": "share please .", "words": ["share"]}, {"sent": "no you dont wanna share today ?", "words": ["dont", "share", "wanna", "you"]}, {"sent": "tell her share cute .", "words": ["share", "cute", "her"]}, {"sent": "share the reef with blue .", "words": ["the", "share", "with", "blue"]}, {"sent": "I thought we were gonna share it .", "words": ["share", "it", "were", "we"]}, {"sent": "did you share at school today Matty ?", "words": ["share", "school", "you", "did", "at"]}, {"sent": "am I gonna share the cupcake ?", "words": ["the", "share", "am"]}, {"sent": "me and Dad can share one and you and Viley can have your own .", "words": ["can", "share", "your", "you", "and", "have", "me"]}, {"sent": "everybody do your share clean up whoa you okay there pumpkin ?", "words": ["up", "do", "clean", "share", "your", "you", "there", "pumpkin"]}, {"sent": "should he get to eat all those trees or should he share them ?", "words": ["all", "those", "share", "to", "them", "get", "eat", "he"]}, {"sent": "I share it back .", "words": ["it", "share", "back"]}, {"sent": "you share it with him .", "words": ["share", "with", "you", "him", "it"]}, {"sent": "you wanna share you wanna share the icecream with her ?", "words": ["the", "share", "with", "you", "wanna", "her"]}, {"sent": "so you hafta let me share them if I take them if I want them .", "words": ["share", "if", "so", "them", "you", "me", "take"]}, {"sent": "now are you going to share with me ?", "words": ["share", "to", "with", "you", "are", "me"]}, {"sent": "I dont wanna share .", "words": ["dont", "share", "wanna"]}, {"sent": "to share for you to eat it all .", "words": ["all", "for", "share", "to", "you", "eat", "it"]}, {"sent": "did you wanna share my breakfast ?", "words": ["my", "share", "you", "did", "wanna"]}, {"sent": "oh ice cream so we can share it ?", "words": ["can", "ice cream", "share", "we", "so", "it", "ice"]}, {"sent": "oh youre coming to share .", "words": ["to", "share"]}, {"sent": "I thought you were sposta share .", "words": ["share", "were", "you"]}, {"sent": "you tell her to share it .", "words": ["share", "to", "you", "it", "her"]}, {"sent": "you hafta you know when Awww comes you must share your toys Awww okay ?", "words": ["share", "when", "your", "you"]}, {"sent": "and for us to share .", "words": ["for", "share", "to", "and", "us"]}, {"sent": "are you gonna be a good girl and share things ?", "words": ["share", "be", "you", "and", "are", "good", "a"]}, {"sent": "maybe Mommyll share one with you .", "words": ["share", "with", "you"]}, {"sent": "youre sposta share with me .", "words": ["me", "share", "with"]}, {"sent": "dont we share with you ?", "words": ["share", "we", "with", "you", "dont"]}, {"sent": "cant you share it ?", "words": ["it", "share", "you"]}, {"sent": "you wanna share that one no ?", "words": ["that", "share", "wanna", "you"]}, {"sent": "because Mrsjwww wants to share it too .", "words": ["share", "to", "it", "too", "because"]}, {"sent": "thank you for letting me share them .", "words": ["for", "share", "them", "you", "me"]}, {"sent": "and this is a book that Denny was very sweet to bring in to share with us .", "words": ["share", "is", "was", "to", "with", "bring", "and", "this", "in", "us", "book", "that", "a"]}, {"sent": "we gotta share these toys .", "words": ["these", "share", "we"]}, {"sent": "nice of you to share it .", "words": ["share", "of", "to", "nice", "you", "it"]}, {"sent": "Brittany share now .", "words": ["share"]}, {"sent": "we we share everything .", "words": ["share", "we"]}, {"sent": "its very nice when youve got somebody you can share things with isnt it and talk to and play with .", "words": ["can", "share", "to", "nice", "with", "play", "you", "and", "it", "talk", "when"]}, {"sent": "give me share .", "words": ["me", "give", "share"]}, {"sent": "share a plate ?", "words": ["share", "plate", "a"]}, {"sent": "hey hey hey you gotta share dont drink so much .", "words": ["share", "drink", "so", "you", "dont", "much"]}, {"sent": "to share too much .", "words": ["to", "share", "much", "too"]}, {"sent": "youre not going to share ?", "words": ["to", "share", "not"]}, {"sent": "you have to share .", "words": ["have", "to", "share", "you"]}, {"sent": "oh but we like it when you share information .", "words": ["share", "we", "like", "you", "it", "when", "but"]}, {"sent": "I wanna share them .", "words": ["share", "wanna", "them"]}, {"sent": "did she share nicely too ?", "words": ["too", "share", "did", "she"]}, {"sent": "no Paul let him share .", "words": ["him", "share"]}, {"sent": "is Amy gonna try and nick it or are you gonna share it ?", "words": ["share", "is", "and", "you", "it", "are", "try"]}, {"sent": "share your toybox .", "words": ["your", "share"]}, {"sent": "and we hafta share the work dont we ?", "words": ["the", "work", "share", "we", "and", "dont"]}, {"sent": "you have to share .", "words": ["have", "to", "share", "you"]}, {"sent": "because we always share everything dont we ?", "words": ["dont", "share", "because", "we"]}, {"sent": "share with the baby ?", "words": ["the", "share", "with"]}, {"sent": "xxx share it .", "words": ["it", "share"]}, {"sent": "are they going to share Annie and Clarabel ?", "words": ["share", "to", "and", "they", "are"]}, {"sent": "can you share it with me ?", "words": ["can", "share", "with", "you", "it", "me"]}, {"sent": "have you got another share pair a shoes you can take to school with you ?", "words": ["can", "share", "to", "school", "with", "you", "have", "another", "take", "a"]}, {"sent": "do you share with your friends there ?", "words": ["do", "your", "share", "with", "you", "there"]}, {"sent": "and then Ill share some with Laura .", "words": ["share", "with", "and", "some", "then"]}, {"sent": "you share darling .", "words": ["share", "you"]}, {"sent": "oh thats very nice its very nice to share books because books are so great .", "words": ["share", "to", "so", "nice", "are", "because"]}, {"sent": "now are you going to share those chocolates ?", "words": ["share", "those", "to", "you", "are"]}, {"sent": "now you gotta share .", "words": ["share", "you"]}, {"sent": "Incys going to share with .", "words": ["to", "share", "with"]}, {"sent": "does this horse share with me ?", "words": ["does", "share", "with", "this", "horse", "me"]}, {"sent": "a share blanket .", "words": ["blanket", "share", "a"]}, {"sent": "whod you who else did you share these with ?", "words": ["these", "share", "with", "you", "did", "who"]}, {"sent": "share okay .", "words": ["share"]}, {"sent": "Bro you hafta share it .", "words": ["it", "share", "you"]}, {"sent": "wont you share your tapioca with Sarah ?", "words": ["your", "share", "with", "you"]}, {"sent": "well if youd like to share my stone soup you can bring something small to put into the soup .", "words": ["can", "the", "my", "put", "share", "if", "into", "to", "like", "bring", "you", "stone", "soup"]}, {"sent": "Joanna youll hafta share it .", "words": ["it", "share"]}, {"sent": "do you think shed share it ?", "words": ["do", "think", "share", "you", "it"]}, {"sent": "would you share one with me ?", "words": ["share", "with", "you", "would", "me"]}, {"sent": "they like to share things .", "words": ["to", "they", "share", "like"]}, {"sent": "shes going to share these sandwiches .", "words": ["to", "share", "these"]}, {"sent": "share well share them .", "words": ["share", "them"]}, {"sent": "youve got to share .", "words": ["to", "share"]}, {"sent": "to share cookies with him .", "words": ["to", "him", "share", "with"]}, {"sent": "hes not gonna share all .", "words": ["all", "share", "not"]}, {"sent": "we can share .", "words": ["can", "share", "we"]}, {"sent": "do you want to share a soda ?", "words": ["do", "share", "soda", "to", "you", "want to", "a"]}, {"sent": "because really when youre with lotsof other people youve got to share things .", "words": ["share", "to", "with", "other", "when", "because"]}, {"sent": "there is the bathroom door .", "words": ["the", "is", "there", "bathroom", "door"]}, {"sent": "just a second let me use the bathroom .", "words": ["the", "me", "bathroom", "a"]}, {"sent": "Ive put the wrapper in the bathroom bin .", "words": ["the", "in", "put", "bathroom"]}, {"sent": "in the bathroom and xxx room ?", "words": ["the", "and", "in", "room", "bathroom"]}, {"sent": "gotta go to the bathroom Jilly Ill be right back .", "words": ["the", "back", "go", "to", "be", "bathroom"]}, {"sent": "what else goes in the bathroom ?", "words": ["the", "what", "in", "bathroom"]}, {"sent": "so Mummy has a clock in the bathroom so she can see .", "words": ["the", "can", "clock", "see", "so", "in", "bathroom", "she", "a"]}, {"sent": "do you hafta go to the bathroom ?", "words": ["the", "do", "go", "to", "you", "bathroom"]}, {"sent": "okay lets come outof the bathroom and shut the door .", "words": ["the", "bathroom", "door", "and"]}, {"sent": "well you hafta wait until they finish unless you wanna go in the other bathroom .", "words": ["the", "wait", "go", "finish", "you", "other", "they", "in", "wanna", "bathroom"]}, {"sent": "in the bathroom .", "words": ["the", "bathroom", "in"]}, {"sent": "ah go to the bathroom .", "words": ["the", "to", "bathroom", "go"]}, {"sent": "Bwww likes to go to the bathroom by herself .", "words": ["the", "go", "to", "by", "bathroom"]}, {"sent": "its not in the bathroom .", "words": ["the", "bathroom", "in", "not"]}, {"sent": "the first thing I do when I go to take a bath is I walk into the bathroom and then what happens ?", "words": ["the", "do", "what", "into", "go", "is", "to", "first", "then", "and", "walk", "bathroom", "take", "when", "a"]}, {"sent": "I gotta go to the bathroom .", "words": ["the", "to", "bathroom", "go"]}, {"sent": "and ones in the bathroom .", "words": ["the", "bathroom", "in", "and"]}, {"sent": "or maybe you can can you lay the bathroom rug for me ?", "words": ["can", "the", "for", "me", "you", "bathroom"]}, {"sent": "go in the bathroom .", "words": ["the", "bathroom", "in", "go"]}, {"sent": "is that really the bathroom ?", "words": ["the", "bathroom", "that", "is"]}, {"sent": "get tissue from the um bathroom .", "words": ["get", "the", "bathroom", "tissue"]}, {"sent": "switch out the bathroom light ?", "words": ["the", "bathroom", "out", "light"]}, {"sent": "is this the bathroom ?", "words": ["the", "this", "bathroom", "is"]}, {"sent": "toothbrush away in the bathroom Fraser .", "words": ["the", "toothbrush", "in", "away", "bathroom"]}, {"sent": "well it was on the bathroom floor wasnt it ?", "words": ["the", "was", "it", "on", "bathroom"]}, {"sent": "can I go to the bathroom ?", "words": ["can", "the", "go", "to", "bathroom"]}, {"sent": "well well do that upstairs in the bathroom .", "words": ["the", "do", "in", "bathroom", "that"]}, {"sent": "wheres the bathroom ?", "words": ["the", "bathroom"]}, {"sent": "outof the bathroom .", "words": ["the", "bathroom"]}, {"sent": "I think that must be the bathroom .", "words": ["the", "think", "be", "bathroom", "that"]}, {"sent": "whats the rabbit doing in the bathroom ?", "words": ["the", "bathroom", "in"]}, {"sent": "do you know where the bathroom is here ?", "words": ["where", "do", "the", "is", "you", "here", "bathroom"]}, {"sent": "go to the bathroom and Ill polish it up for you .", "words": ["the", "up", "for", "go", "to", "and", "you", "it", "bathroom"]}, {"sent": "did you put it in the bathroom ?", "words": ["the", "put", "you", "in", "it", "did", "bathroom"]}, {"sent": "oh this is a bathroom .", "words": ["this", "bathroom", "a", "is"]}, {"sent": "you needta stop or tell me you needta go to the bathroom .", "words": ["the", "go", "to", "stop", "you", "bathroom", "me"]}, {"sent": "Andy crawling in in the bathroom .", "words": ["the", "bathroom", "in"]}, {"sent": "heres the bathroom .", "words": ["the", "bathroom"]}, {"sent": "excuse me I think they hafta go to the bathroom .", "words": ["the", "think", "go", "to", "they", "bathroom", "me"]}, {"sent": "a record player a television bathroom livingroom .", "words": ["bathroom", "a"]}, {"sent": "lets leave this bathroom .", "words": ["this", "bathroom"]}, {"sent": "but you went to the bathroom like a big girl right ?", "words": ["the", "to", "like", "you", "bathroom", "big", "but", "a"]}, {"sent": "if you needta spit up you go to the bathroom okay ?", "words": ["up", "the", "if", "go", "to", "you", "bathroom"]}, {"sent": "I know the first thing you hafta do is walk in to the bathroom and then what happens ?", "words": ["the", "do", "what", "is", "to", "first", "you", "and", "in", "bathroom", "then", "walk"]}, {"sent": "lets go in the bathroom .", "words": ["the", "bathroom", "in", "go"]}, {"sent": "thats for the bathroom .", "words": ["the", "bathroom", "for"]}, {"sent": "because they needta go upstairs to the bathroom .", "words": ["the", "go", "to", "they", "bathroom", "because"]}, {"sent": "if you keep using the bathroom like you have you can have apple juice .", "words": ["the", "can", "if", "like", "you", "have", "apple", "bathroom", "juice"]}, {"sent": "in my bathroom .", "words": ["bathroom", "in", "my"]}, {"sent": "when your baby goes to the bathroom you know xxx .", "words": ["the", "your", "to", "you", "bathroom", "when"]}, {"sent": "we have a sink in the bathroom there .", "words": ["the", "we", "have", "there", "in", "sink", "bathroom", "a"]}, {"sent": "hey do you hafta go to the bathroom ?", "words": ["the", "do", "go", "to", "you", "bathroom"]}, {"sent": "in the bathroom ?", "words": ["the", "bathroom", "in"]}, {"sent": "rob bring me the lotion out the bathroom please .", "words": ["the", "bring", "out", "bathroom", "me"]}, {"sent": "and I know the people live behind us are decorating their bathroom green arent they ?", "words": ["the", "their", "behind", "and", "us", "they", "are", "green", "bathroom"]}, {"sent": "this is the bathroom ?", "words": ["the", "this", "bathroom", "is"]}, {"sent": "he hadta go in the bathroom to wash his hands .", "words": ["the", "go", "to", "in", "he", "his", "bathroom", "wash"]}, {"sent": "could she just use your bathroom ?", "words": ["bathroom", "could", "your", "she"]}, {"sent": "and things didnt go well in the bathroom .", "words": ["the", "go", "and", "in", "bathroom"]}, {"sent": "Nina if you wanna use water you let me get it because you got the whole bathroom wet before .", "words": ["the", "wet", "if", "you", "get", "it", "wanna", "water", "me", "bathroom", "because"]}, {"sent": "see if you can find a bath and then we can start the bathroom .", "words": ["can", "the", "if", "we", "you", "and", "find", "see", "then", "bathroom", "a"]}, {"sent": "you think I should scream at my baby because she doesnt go to the bathroom in the toilet ?", "words": ["the", "think", "my", "go", "to", "you", "in", "at", "bathroom", "she", "because"]}, {"sent": "we can put all these fish in the bathroom .", "words": ["can", "all", "put", "these", "the", "we", "in", "fish", "bathroom"]}, {"sent": "you hafta go to the bathroom ?", "words": ["the", "go", "to", "you", "bathroom"]}, {"sent": "I keep telling you if you flood the bathroom the water will come all downstairs wont it ?", "words": ["the", "all", "if", "you", "it", "water", "will", "bathroom"]}, {"sent": "Pete lets go take a little trip to the bathroom ?", "words": ["the", "go", "to", "little", "bathroom", "take", "a"]}, {"sent": "you have any aspirin in your bathroom ?", "words": ["your", "any", "you", "have", "in", "bathroom"]}, {"sent": "dont go in the bathroom dear if somebodys in it .", "words": ["the", "if", "go", "dont", "in", "it", "bathroom"]}, {"sent": "theres not really a bathroom in this house .", "words": ["this", "in", "house", "bathroom", "not", "a"]}, {"sent": "if you wanna hear Patricia talking all ya gotta do is xxx go to the bathroom .", "words": ["the", "all", "do", "if", "is", "go", "to", "hear", "you", "wanna", "bathroom"]}, {"sent": "mommy I want you to bring me to the bathroom .", "words": ["the", "to", "bring", "you", "bathroom", "me"]}, {"sent": "why dont put that thing let me get that kid outof that bathroom .", "words": ["put", "get", "dont", "bathroom", "me", "that", "why"]}, {"sent": "oh I love having grand central station when I go to the bathroom .", "words": ["the", "go", "to", "love", "bathroom", "when"]}, {"sent": "lets go to the bathroom .", "words": ["the", "to", "bathroom", "go"]}, {"sent": "thats for the bathroom .", "words": ["the", "bathroom", "for"]}, {"sent": "but the scales are up in the bathroom .", "words": ["the", "up", "in", "are", "bathroom", "but"]}, {"sent": "did you push it in the bathroom ?", "words": ["the", "push", "you", "in", "it", "did", "bathroom"]}, {"sent": "oh you can go to the bathroom yourself .", "words": ["can", "the", "yourself", "go", "to", "you", "bathroom"]}, {"sent": "were in the bathroom now arent we ?", "words": ["the", "we", "in", "bathroom", "were"]}, {"sent": "maybe you oughta try to go in the bathroom .", "words": ["the", "go", "to", "you", "in", "bathroom", "try", "try to"]}, {"sent": "the bathroom what ?", "words": ["the", "bathroom", "what"]}, {"sent": "and go up to the bathroom .", "words": ["the", "up", "go", "to", "and", "bathroom"]}, {"sent": "xxx just a little okay now dont take that outside the bathroom let me put it back .", "words": ["the", "put", "back", "outside", "me", "little", "dont", "it", "bathroom", "take", "that", "a"]}, {"sent": "go in the udder bathroom .", "words": ["the", "bathroom", "in", "go"]}, {"sent": "you usually watch Pippin when Mummy goes up to the bathroom .", "words": ["up", "the", "watch", "to", "you", "bathroom", "when"]}, {"sent": "look this can be the bathroom .", "words": ["can", "the", "be", "this", "look", "bathroom"]}, {"sent": "anybody in the bathroom ?", "words": ["the", "bathroom", "in"]}, {"sent": "wanna use the bathroom .", "words": ["the", "bathroom", "wanna"]}, {"sent": "in the bathroom .", "words": ["the", "bathroom", "in"]}, {"sent": "back in the bathroom so you can .", "words": ["the", "can", "back", "so", "you", "in", "bathroom"]}, {"sent": "Im Im gonna look in the oven an see if Kips in the oven getting brown no hes not in the oven getting brown lemme see Im a look in the bathroom again behind the door .", "words": ["the", "door", "if", "brown", "not", "behind", "in", "an", "look", "see", "bathroom", "oven", "a"]}, {"sent": "and then um I was in the bathroom and he was going poopoo and he said .", "words": ["the", "was", "and", "in", "he", "bathroom", "then"]}, {"sent": "bring it back there in the bathroom .", "words": ["the", "back", "bring", "there", "in", "it", "bathroom"]}, {"sent": "bring toys into the bathroom ?", "words": ["the", "bathroom", "bring", "into"]}, {"sent": "now he must clean the bathroom .", "words": ["clean", "the", "he", "bathroom"]}, {"sent": "can you xxx ask me if you wanna use the secret bathroom .", "words": ["can", "the", "if", "you", "wanna", "bathroom", "me"]}, {"sent": "do you think you should make a slide down for this bathroom too ?", "words": ["do", "think", "slide", "for", "you", "this", "too", "bathroom", "down", "make", "a"]}, {"sent": "okay you go up to the bathroom Ill wait for you down here .", "words": ["the", "up", "wait", "for", "go", "to", "you", "here", "bathroom", "down"]}, {"sent": "I think there are some Kix in your high chair .", "words": ["think", "your", "chair", "high chair", "there", "in", "are", "high", "some"]}, {"sent": "Mommy is wiping and then well put you out of your high chair .", "words": ["put", "your", "is", "of", "chair", "you", "and", "high chair", "out", "high", "then"]}, {"sent": "neat high chair isnt it Maggiemoo ?", "words": ["high chair", "it", "chair", "high"]}, {"sent": "well put you in your high chair ?", "words": ["put", "your", "chair", "you", "high chair", "in", "high"]}, {"sent": "and put you in your high chair .", "words": ["put", "your", "chair", "you", "and", "high chair", "in", "high"]}, {"sent": "and then when you could sit up we useta have a high chair there .", "words": ["up", "we", "sit", "then", "chair", "you", "and", "have", "there", "high chair", "high", "could", "when", "a"]}, {"sent": "Ive no high chair Cathy .", "words": ["high chair", "chair", "high"]}, {"sent": "so I got her that and the high chair .", "words": ["the", "so", "her", "chair", "and", "high chair", "high", "that"]}, {"sent": "Naima could have porridge sitting in her high chair just like Goldilocks has .", "words": ["like", "chair", "have", "high chair", "in", "high", "could", "her"]}, {"sent": "your high chair ?", "words": ["high chair", "your", "chair", "high"]}, {"sent": "I tell you what you were xxx like this morning in her high chair if you had a mike around .", "words": ["what", "around", "if", "her", "like", "chair", "you", "high chair", "this", "in", "high", "were", "a"]}, {"sent": "ew the high chair ew the high chair .", "words": ["the", "high chair", "chair", "high"]}, {"sent": "what is that a high chair ?", "words": ["what", "is", "chair", "high chair", "high", "that", "a"]}, {"sent": "the high chair ?", "words": ["the", "high chair", "chair", "high"]}, {"sent": "shall we put the wooden baby on the high chair ?", "words": ["the", "put", "we", "chair", "high chair", "high", "on"]}, {"sent": "alright Remis in the high chair .", "words": ["the", "chair", "high chair", "in", "high"]}, {"sent": "youve got the high chair over there in the corner .", "words": ["the", "chair", "high chair", "there", "in", "high", "over"]}, {"sent": "lets put your high chair back in the spot .", "words": ["the", "put", "your", "back", "chair", "high chair", "in", "high"]}, {"sent": "youre banging it into the metal on the high chair .", "words": ["the", "into", "chair", "high chair", "it", "high", "on"]}, {"sent": "is there anything else that you wanna bring up into the high chair ?", "words": ["up", "the", "into", "is", "bring", "chair", "you", "high chair", "there", "wanna", "high", "that"]}, {"sent": "Ill pretend Im a high chair .", "words": ["chair", "high chair", "high", "pretend", "a"]}, {"sent": "sitting in my high chair .", "words": ["my", "chair", "high chair", "in", "high"]}, {"sent": "okay well lets go get in your high chair .", "words": ["your", "go", "chair", "get", "high chair", "in", "high"]}, {"sent": "she was sitting in her high chair and then she said she wanted to sit next to me so I put her in the booth next to me and she stood up and she stood up and I was like Lily you better sit down and she wah fell right down .", "words": ["the", "put", "like", "high chair", "high", "me", "chair", "then", "down", "she", "up", "was", "to", "so", "and", "you", "better", "in", "sit", "her"]}, {"sent": "are you setting up in the high chair ?", "words": ["up", "the", "chair", "you", "high chair", "in", "are", "high"]}, {"sent": "more mess on your high chair .", "words": ["your", "chair", "high chair", "more", "high", "on"]}, {"sent": "shell just drop what shes doing and run into the kitchen and shake her high chair .", "words": ["the", "what", "into", "shake", "chair", "and", "high chair", "run", "drop", "kitchen", "high", "her"]}, {"sent": "do you wanna sit in the high chair ?", "words": ["the", "do", "chair", "you", "high chair", "in", "wanna", "high", "sit"]}, {"sent": "see on the high chair .", "words": ["the", "chair", "high chair", "high", "on", "see"]}, {"sent": "sit in your high chair now .", "words": ["your", "chair", "high chair", "in", "high", "sit"]}, {"sent": "she has a little high chair for them so she sits on the high chair and feeds them and puts the bib on them and then .", "words": ["the", "for", "so", "them", "chair", "little", "and", "bib", "high chair", "high", "on", "then", "she", "a"]}, {"sent": "a red high chair ?", "words": ["red", "chair", "high chair", "high", "a"]}, {"sent": "theres more pieces there were a bunch in your high chair .", "words": ["your", "chair", "high chair", "there", "more", "in", "high", "were", "a"]}, {"sent": "heres a high chair for the baby .", "words": ["the", "for", "chair", "high chair", "high", "a"]}, {"sent": "I wanna sit in the high chair .", "words": ["the", "chair", "high chair", "in", "wanna", "high", "sit"]}, {"sent": "hes under the high chair .", "words": ["the", "chair", "high chair", "under", "high"]}, {"sent": "its a toy from your high chair isnt it ?", "words": ["your", "chair", "high chair", "toy", "it", "high", "a"]}, {"sent": "Tunde sat in the high chair this morning .", "words": ["the", "chair", "high chair", "this", "in", "high"]}, {"sent": "yes in the high chair .", "words": ["the", "chair", "high chair", "in", "high"]}, {"sent": "imagine me and my high chair given away too .", "words": ["my", "chair", "and", "high chair", "away", "high", "me", "too"]}, {"sent": "yeah um just even like youknow like the little tray in her um high chair .", "words": ["the", "her", "like", "chair", "little", "high chair", "in", "high", "tray"]}, {"sent": "actually the girl that lent me the high chair .", "words": ["the", "chair", "high chair", "high", "me", "that"]}, {"sent": "you wanna put Andy in his high chair ?", "words": ["put", "chair", "you", "high chair", "in", "wanna", "high", "his"]}, {"sent": "a high chair for a dog ?", "words": ["for", "chair", "high chair", "high", "dog", "a"]}, {"sent": "lets go sit in your high chair .", "words": ["your", "go", "chair", "high chair", "in", "high", "sit"]}, {"sent": "get in your high chair .", "words": ["your", "chair", "get", "high chair", "in", "high"]}, {"sent": "but if youd like a high chair my little boy Thomas .", "words": ["my", "if", "like", "chair", "little", "high chair", "high", "but", "a"]}, {"sent": "just uh it was a girl lent me a high chair the last time .", "words": ["the", "was", "last", "chair", "high chair", "it", "high", "me", "a"]}, {"sent": "are you sitting in the high chair ?", "words": ["the", "chair", "you", "high chair", "in", "are", "high"]}, {"sent": "heres the high chair for the baby .", "words": ["the", "for", "chair", "high chair", "high"]}, {"sent": "the kitty wants to go on the high chair .", "words": ["the", "go", "to", "chair", "high chair", "kitty", "high", "on"]}, {"sent": "go over to your high chair .", "words": ["your", "go", "to", "chair", "high chair", "high", "over"]}, {"sent": "xxx use the high chair ?", "words": ["the", "high chair", "chair", "high"]}, {"sent": "have you no high chair ?", "words": ["chair", "you", "have", "high chair", "high"]}, {"sent": "when you were in your high chair ?", "words": ["your", "chair", "you", "high chair", "in", "high", "when", "were"]}, {"sent": "let me slide your high chair out into the living room .", "words": ["the", "slide", "your", "into", "living room", "chair", "high chair", "room", "out", "high", "me"]}, {"sent": "so she puts them in the swing and she puts them in the high chair and she puts in the car seat .", "words": ["the", "swing", "so", "car", "them", "chair", "and", "high chair", "in", "high", "she"]}, {"sent": "sit in your high chair .", "words": ["your", "chair", "high chair", "in", "high", "sit"]}, {"sent": "I think they were on the floor under the high chair .", "words": ["the", "think", "chair", "high chair", "they", "under", "high", "on", "were"]}, {"sent": "lets get up in your high chair .", "words": ["up", "your", "chair", "get", "high chair", "in", "high"]}, {"sent": "lets go over here with your high chair .", "words": ["your", "go", "with", "chair", "here", "high chair", "high", "over"]}, {"sent": "thats the wipe clean mat that we useta pull down when you were a baby underneath your high chair .", "words": ["the", "clean", "your", "when", "we", "down", "chair", "pull", "you", "high chair", "high", "wipe", "that", "were", "a"]}, {"sent": "Remi mommy I wanna sit in the high chair .", "words": ["the", "chair", "high chair", "in", "wanna", "high", "sit"]}, {"sent": "it might be useful anyway to have the high chair .", "words": ["the", "to", "be", "chair", "have", "high chair", "it", "high"]}, {"sent": "youll get to eat your breakfast in your high chair .", "words": ["your", "to", "chair", "get", "eat", "in", "high chair", "high"]}, {"sent": "that came with her high chair .", "words": ["with", "chair", "high chair", "that", "high", "her"]}, {"sent": "and then when you got too big for you high chair we bought this little booster seat .", "words": ["for", "we", "chair", "you", "and", "high", "big", "this", "little", "high chair", "too", "then", "when"]}, {"sent": "okay Ill give you milk first and then you can get in your high chair .", "words": ["can", "your", "first", "then", "chair", "and", "you", "get", "in", "high chair", "high", "milk", "give"]}, {"sent": "shall I pretend Im the high chair ?", "words": ["the", "chair", "high chair", "high", "pretend"]}, {"sent": "he doesnt need his high chair anymore because hes a big boy now .", "words": ["need", "chair", "high chair", "he", "high", "his", "big", "because", "a"]}, {"sent": "well if you need a high chair for your restaurant theres one in in my loft .", "words": ["my", "for", "need", "if", "your", "chair", "you", "high chair", "in", "high", "a"]}, {"sent": "what do we use now instead of a high chair ?", "words": ["what", "do", "we", "of", "chair", "high chair", "high", "a"]}, {"sent": "are there crumbs in your high chair ?", "words": ["your", "chair", "high chair", "there", "in", "are", "high"]}, {"sent": "were in the high chair a long time .", "words": ["the", "chair", "long", "high chair", "in", "high", "were", "a"]}, {"sent": "here Andy you can sit in your high chair .", "words": ["can", "your", "chair", "you", "here", "high chair", "in", "high", "sit"]}, {"sent": "would you like to come and sit in your high chair ?", "words": ["your", "to", "like", "chair", "and", "you", "high", "in", "high chair", "would", "sit"]}, {"sent": "xxx the high chair takes up such a lot of room if you could uh xxx .", "words": ["the", "up", "if", "of", "a lot", "chair", "you", "high chair", "room", "high", "could", "a"]}, {"sent": "look at the daddy xxx xxx the high chair xxx .", "words": ["the", "look", "chair", "high", "high chair", "at"]}, {"sent": "if you get into your high chair well let Purdie in and give her some lunch .", "words": ["your", "if", "into", "chair", "you", "get", "and", "in", "high chair", "high", "some", "give", "her"]}, {"sent": "the high chair .", "words": ["the", "high chair", "chair", "high"]}, {"sent": "you useta have a high chair when you were a baby .", "words": ["chair", "you", "have", "high chair", "high", "when", "were", "a"]}, {"sent": "like the high chair .", "words": ["the", "like", "chair", "high chair", "high"]}, {"sent": "put one right there high chair .", "words": ["put", "chair", "high chair", "there", "high"]}, {"sent": "its you banging your fist on the high chair Thomas .", "words": ["the", "your", "chair", "you", "high chair", "high", "on"]}, {"sent": "yes right there is a blue high chair .", "words": ["is", "chair", "blue", "high chair", "there", "high", "a"]}, {"sent": "yeah thats Frasers high chair now .", "words": ["high chair", "chair", "high"]}, {"sent": "lets go over here with your high chair .", "words": ["your", "go", "with", "chair", "here", "high chair", "high", "over"]}, {"sent": "hi I man I just cant get over you sitting in that high chair .", "words": ["chair", "you", "get", "high chair", "in", "high", "over", "that"]}, {"sent": "lets sit in your high chair .", "words": ["your", "chair", "high chair", "in", "high", "sit"]}, {"sent": "high chair again ?", "words": ["high chair", "chair", "high"]}, {"sent": "no but you were in your high chair and you useta look up and look at them .", "words": ["up", "look", "your", "them", "chair", "you", "and", "high chair", "in", "at", "high", "were", "but"]}, {"sent": "high chair .", "words": ["high chair", "chair", "high"]}, {"sent": "BabyKangas in a high chair like you useta sit in when you were a baby .", "words": ["like", "chair", "you", "high chair", "in", "high", "sit", "when", "were", "a"]}, {"sent": "this is a big baking potato .", "words": ["is", "this", "big", "a", "potato"]}, {"sent": "xxx sweet potato .", "words": ["potato"]}, {"sent": "oh is it mashed potato ?", "words": ["it", "potato", "is"]}, {"sent": "okay then lets put the animals back in the bag and then we gone play with mister potato head .", "words": ["the", "put", "back", "we", "head", "play", "with", "and", "in", "then", "potato"]}, {"sent": "putting a potato chip on there ?", "words": ["there", "on", "potato chip", "a", "potato"]}, {"sent": "go get your potato head and Ill play potato head .", "words": ["your", "go", "head", "play", "and", "get", "potato"]}, {"sent": "isnt that a nice potato ?", "words": ["nice", "that", "potato", "a"]}, {"sent": "its a very its a potato peeler but its another very sharp knife Thomas .", "words": ["a", "knife", "another", "but", "potato"]}, {"sent": "so everybody has a potato .", "words": ["so", "potato", "a"]}, {"sent": "I need shrimp steak and potato .", "words": ["need", "potato", "and"]}, {"sent": "thats too big for for the little potato you hafta put this one on the big one .", "words": ["the", "put", "for", "little", "you", "this", "too", "on", "big", "potato"]}, {"sent": "potato you hafta get the bigger potato probably .", "words": ["get", "the", "you", "potato"]}, {"sent": "potato waffle ?", "words": ["potato"]}, {"sent": "mister and misses potato head .", "words": ["head", "potato", "and"]}, {"sent": "mash potato mash potato .", "words": ["potato"]}, {"sent": "who wants some more potato ?", "words": ["some", "more", "potato", "who"]}, {"sent": "mashed potato mashed potato .", "words": ["potato"]}, {"sent": "looks like ice cream or mashed potato .", "words": ["ice", "ice cream", "like", "potato"]}, {"sent": "a potato peeler .", "words": ["potato", "a"]}, {"sent": "potato salad .", "words": ["potato"]}, {"sent": "are you a potato ?", "words": ["potato", "are", "a", "you"]}, {"sent": "youre an old potato .", "words": ["old", "an", "potato"]}, {"sent": "he wants potato and fishfingers ?", "words": ["he", "and", "potato"]}, {"sent": "is that because you told him to have potato and fishfingers ?", "words": ["is", "to", "you", "have", "and", "him", "that", "because", "potato"]}, {"sent": "would you like potato leek soup ?", "words": ["like", "you", "would", "soup", "potato"]}, {"sent": "the little baby potato or the mommy potato ?", "words": ["the", "potato", "little"]}, {"sent": "thats sweet potato thats a different kind you you eat the baby food kind thats the real thing .", "words": ["the", "you", "eat", "food", "potato", "a"]}, {"sent": "yeah Im gonna change baby potato head right now .", "words": ["head", "potato"]}, {"sent": "make the potato for Mom too huh ?", "words": ["the", "for", "too", "make", "potato"]}, {"sent": "hes going over to the potato patch .", "words": ["the", "to", "over", "potato"]}, {"sent": "see this has to go on to baby potato .", "words": ["go", "to", "this", "on", "see", "potato"]}, {"sent": "and mashed potato .", "words": ["potato", "and"]}, {"sent": "and play potato head ?", "words": ["head", "play", "potato", "and"]}, {"sent": "you wanna open the back of mister potato head ?", "words": ["the", "back", "open", "of", "head", "you", "wanna", "potato"]}, {"sent": "Lara eats the potato .", "words": ["the", "potato"]}, {"sent": "chicken and potato .", "words": ["chicken", "potato", "and"]}, {"sent": "do you like potato waffles ?", "words": ["do", "like", "potato", "you"]}, {"sent": "well buy a sweet potato pie and say goodbye before we .", "words": ["we", "say", "and", "buy", "a", "potato"]}, {"sent": "hm my sweet potato pie .", "words": ["my", "potato"]}, {"sent": "the potato man .", "words": ["the", "potato"]}, {"sent": "potato .", "words": ["potato"]}, {"sent": "well give baby potato .", "words": ["give", "potato"]}, {"sent": "youre big enough to cut potato up surely ?", "words": ["up", "cut", "to", "big", "potato"]}, {"sent": "oh what about your potato head we havent played that in a long time .", "words": ["what", "your", "we", "head", "long", "in", "about", "that", "a", "potato"]}, {"sent": "wheres the potato ?", "words": ["the", "potato"]}, {"sent": "and they did potato prints didnt they ?", "words": ["they", "did", "potato", "and"]}, {"sent": "because you like fishfingers and potato dont you ?", "words": ["like", "and", "you", "dont", "because", "potato"]}, {"sent": "do you like potato peel ?", "words": ["do", "like", "potato", "you"]}, {"sent": "did you eat potato waffle ?", "words": ["eat", "did", "potato", "you"]}, {"sent": "he might have potato ?", "words": ["have", "he", "potato"]}, {"sent": "how many more potato chips are there left ?", "words": ["how", "there", "more", "are", "potato"]}, {"sent": "theres the potato masher .", "words": ["the", "potato"]}, {"sent": "and theyre made of fish and mashed potato .", "words": ["of", "fish", "potato", "and"]}, {"sent": "do and a big big potato a big baked potato ?", "words": ["do", "and", "big", "a", "potato"]}, {"sent": "oh this isnt mashed potato people often think it is but its cloud fluff .", "words": ["think", "is", "this", "it", "cloud", "but", "potato"]}, {"sent": "and I will bring in the sweet potato .", "words": ["the", "bring", "and", "in", "will", "potato"]}, {"sent": "mashed potato .", "words": ["potato"]}, {"sent": "some potato chips and some pickles and cheese .", "words": ["some", "cheese", "and", "potato"]}, {"sent": "I just made some potato chips you just slice the potato really thin and then you deepfry it .", "words": ["the", "and", "you", "it", "some", "then", "potato"]}, {"sent": "just finish all your potato as well .", "words": ["all", "your", "finish", "potato"]}, {"sent": "this is a sweet potato plant .", "words": ["is", "this", "plant", "a", "potato"]}, {"sent": "youve to get just eyes got just eyes for the big potato .", "words": ["the", "for", "to", "get", "big", "potato"]}, {"sent": "the potato ?", "words": ["the", "potato"]}, {"sent": "sweet potato fries look theyre right in the container for sale .", "words": ["the", "for", "in", "look", "potato"]}, {"sent": "so well put that in with the potato .", "words": ["the", "put", "so", "with", "in", "that", "potato"]}, {"sent": "potato .", "words": ["potato"]}, {"sent": "those are for baby potato I think .", "words": ["think", "for", "those", "are", "potato"]}, {"sent": "hot potato .", "words": ["hot", "potato"]}, {"sent": "potato .", "words": ["potato"]}, {"sent": "some people call it sweet potato .", "words": ["some", "it", "potato"]}, {"sent": "you would like Mommy to be the big potato head ?", "words": ["the", "to", "be", "head", "like", "you", "would", "big", "potato"]}, {"sent": "youve got last potato .", "words": ["last", "potato"]}, {"sent": "leek and potato ?", "words": ["potato", "and"]}, {"sent": "thats a piece of potato .", "words": ["of", "potato", "a"]}, {"sent": "make us some mashed potato .", "words": ["us", "make", "potato", "some"]}, {"sent": "do you want mashed potato ?", "words": ["do", "potato", "you"]}, {"sent": "mhm come on so eat some meat and potato now .", "words": ["so", "meat", "and", "eat", "on", "some", "potato"]}, {"sent": "okay you go put this on baby potato .", "words": ["put", "go", "you", "this", "on", "potato"]}, {"sent": "and I will play potato head with you .", "words": ["head", "play", "with", "and", "you", "will", "potato"]}, {"sent": "yeah after you eat some potato .", "words": ["eat", "some", "potato", "you"]}, {"sent": "one potato two potato three potato four .", "words": ["potato"]}, {"sent": "Shem the lion wants some pretend potato chips what should we do ?", "words": ["the", "what", "do", "we", "some", "pretend", "lion", "potato"]}, {"sent": "sweet potato squares .", "words": ["potato"]}, {"sent": "you could use a wooden spoon but a potato mashers a bit easier isnt it ?", "words": ["potato", "you", "it", "spoon", "could", "but", "a"]}, {"sent": "did you put any mash potato on the seat ?", "words": ["the", "put", "any", "you", "did", "on", "potato"]}, {"sent": "get mister potato head shoes .", "words": ["get", "head", "potato"]}, {"sent": "how about would you like to play potato head ?", "words": ["how", "to", "head", "play", "like", "you", "about", "would", "potato"]}, {"sent": "hi Alex wheres my baby potato head ?", "words": ["my", "head", "potato"]}, {"sent": "pass potato .", "words": ["potato"]}, {"sent": "everybody knows Badger likes mashed potato .", "words": ["potato"]}, {"sent": "half half a potato waffle ?", "words": ["potato", "a"]}, {"sent": "a little baby potato .", "words": ["potato", "little", "a"]}, {"sent": "you had potato smiley faces and beans .", "words": ["beans", "and", "potato", "you"]}, {"sent": "oh look it says here we can put bread or cheese or potato or bacon .", "words": ["can", "put", "bread", "we", "here", "it", "look", "cheese", "potato"]}, {"sent": "one potato .", "words": ["potato"]}, {"sent": "wanna do mashed potato ?", "words": ["do", "wanna", "potato"]}, {"sent": "did ya ?", "words": ["did ya", "did"]}, {"sent": "what did ya see ?", "words": ["see", "what", "did ya", "did"]}, {"sent": "down by the bay where the watermelons grow back to my home I dare not go for if I do my mother will say did ya ever see an apple wearing a bapple did ya ever see an orange eating a blouse did ya ever have a time when ya couldnt make a rhyme down by the bay .", "words": ["the", "go", "an", "did ya", "when", "not", "where", "by", "have", "down", "home", "for", "if", "to", "will", "apple", "see", "make", "a", "do", "my", "back", "say", "orange", "did"]}, {"sent": "how far did ya get ?", "words": ["get", "did ya", "how", "did"]}, {"sent": "did ya ?", "words": ["did ya", "did"]}, {"sent": "did ya tell Kent what the EasterBunny brought ?", "words": ["the", "what", "did ya", "did"]}, {"sent": "did ya get it ?", "words": ["get", "did ya", "it", "did"]}, {"sent": "what did ya find ?", "words": ["did ya", "what", "did", "find"]}, {"sent": "what did ya crash the cars for ?", "words": ["the", "what", "for", "did", "did ya"]}, {"sent": "did ya see that ?", "words": ["did ya", "see", "that", "did"]}, {"sent": "did ya get enough .", "words": ["get", "did ya", "did"]}, {"sent": "did ya learn that in school ?", "words": ["school", "in", "did", "did ya", "that"]}, {"sent": "well if I did ya think Id be standing here ?", "words": ["think", "if", "be", "here", "did", "did ya"]}, {"sent": "what did ya play with ?", "words": ["what", "play", "with", "did", "did ya"]}, {"sent": "did ya see Uncle Kenny ?", "words": ["did ya", "see", "did"]}, {"sent": "what did ya see ?", "words": ["see", "what", "did ya", "did"]}, {"sent": "you didnt even put a bandaid on it did ya ?", "words": ["put", "did ya", "you", "it", "did", "on", "a"]}, {"sent": "did ya hurt your hand ?", "words": ["your", "hand", "hurt", "did", "did ya"]}, {"sent": "did ya hear moms pen pen go click click ?", "words": ["go", "hear", "did", "pen", "did ya"]}, {"sent": "who did ya go to see ?", "words": ["go", "to", "did", "who", "did ya", "see"]}, {"sent": "did ya saw it ?", "words": ["did ya", "it", "did"]}, {"sent": "did ya spill a little on me ?", "words": ["did ya", "little", "spill", "did", "on", "me", "a"]}, {"sent": "did ya have any hot milk ?", "words": ["did ya", "any", "have", "hot", "did", "milk"]}, {"sent": "did ya hear me ?", "words": ["did ya", "me", "did", "hear"]}, {"sent": "no did ya find it ?", "words": ["did ya", "it", "did", "find"]}, {"sent": "did ya grab some mittens for him ?", "words": ["mittens", "for", "him", "did", "did ya", "some"]}, {"sent": "what did ya have for dinner ?", "words": ["what", "for", "have", "did", "did ya"]}, {"sent": "what did ya say ?", "words": ["did ya", "what", "say", "did"]}, {"sent": "yeah did ya hear the birdie ?", "words": ["the", "did ya", "did", "hear"]}, {"sent": "did ya ?", "words": ["did ya", "did"]}, {"sent": "where did ya go to see a fish ?", "words": ["where", "go", "to", "did", "fish", "did ya", "see", "a"]}, {"sent": "down by the bay where the watermelons grow back to my home I dare not go for if I do my mother will say did ya ever see llamas eating their pajamas down by the bay .", "words": ["the", "go", "pajamas", "did ya", "not", "where", "their", "by", "down", "home", "for", "if", "to", "will", "see", "do", "my", "back", "say", "did"]}, {"sent": "why did ya hook it up ?", "words": ["up", "it", "did", "did ya", "why"]}, {"sent": "how did ya get a xxx ?", "words": ["how", "get", "did", "did ya", "a"]}, {"sent": "what else did ya get ?", "words": ["get", "what", "did ya", "did"]}, {"sent": "what did ya find ?", "words": ["did ya", "what", "did", "find"]}, {"sent": "oh what did ya find ?", "words": ["did ya", "what", "did", "find"]}, {"sent": "did ya see that ?", "words": ["did ya", "see", "that", "did"]}, {"sent": "what did ya find ?", "words": ["did ya", "what", "did", "find"]}, {"sent": "did ya ?", "words": ["did ya", "did"]}, {"sent": "what did ya do ?", "words": ["did ya", "what", "do", "did"]}, {"sent": "where did ya stick it ?", "words": ["where", "stick", "it", "did", "did ya"]}, {"sent": "at whose house did ya see a clown ?", "words": ["house", "did", "at", "did ya", "see", "a"]}, {"sent": "did ya do it ?", "words": ["did ya", "do", "it", "did"]}, {"sent": "what did ya do today in school ?", "words": ["what", "do", "school", "in", "did", "did ya"]}, {"sent": "did ya hear that ?", "words": ["did ya", "that", "did", "hear"]}, {"sent": "did ya ?", "words": ["did ya", "did"]}, {"sent": "oo did ya get a booboo .", "words": ["get", "did ya", "did", "a"]}, {"sent": "why did ya say that on Christmas ?", "words": ["did ya", "say", "did", "on", "that", "why"]}, {"sent": "what else did ya see ?", "words": ["see", "what", "did ya", "did"]}, {"sent": "what did ya like the least ?", "words": ["the", "what", "like", "did", "did ya"]}, {"sent": "did you bump into her did ya ?", "words": ["bump", "into", "you", "did", "did ya", "her"]}, {"sent": "did ya want me to keep that door open ?", "words": ["open", "to", "did", "did ya", "me", "that", "door"]}, {"sent": "did ya want this one ?", "words": ["did ya", "this", "did"]}, {"sent": "what did ya think of that ?", "words": ["what", "think", "of", "did", "did ya", "that"]}, {"sent": "did ya ?", "words": ["did ya", "did"]}, {"sent": "did ya ?", "words": ["did ya", "did"]}, {"sent": "you didnt do that did ya ?", "words": ["do", "you", "did", "did ya", "that"]}, {"sent": "did ya tell Melissa where I just brought ya ?", "words": ["where", "did ya", "did"]}, {"sent": "what did ya think ?", "words": ["did ya", "what", "think", "did"]}, {"sent": "oh how did ya do that ?", "words": ["do", "how", "did", "did ya", "that"]}, {"sent": "did ya ever see a whale with a polkadot tail down by the bay .", "words": ["the", "with", "by", "did", "did ya", "see", "down", "a"]}, {"sent": "did ya ?", "words": ["did ya", "did"]}, {"sent": "did ya put it in my hair ?", "words": ["put", "my", "in", "it", "did", "did ya", "hair"]}, {"sent": "what did ya find ?", "words": ["did ya", "what", "did", "find"]}, {"sent": "what else did ya do ?", "words": ["did ya", "what", "do", "did"]}, {"sent": "what did ya do ?", "words": ["did ya", "what", "do", "did"]}, {"sent": "hey where did ya get that ?", "words": ["where", "get", "did", "did ya", "that"]}, {"sent": "down by the bay where the watermelons grow back to my home I dare not go for if I do my mother will say did ya ever see a bear combing his hair down by the bay .", "words": ["the", "go", "bear", "did ya", "not", "where", "by", "down", "home", "for", "if", "to", "will", "see", "hair", "a", "do", "my", "back", "say", "did", "his"]}, {"sent": "did ya wanna play ball William ?", "words": ["play", "ball", "did", "wanna", "did ya"]}, {"sent": "did ya do it ?", "words": ["did ya", "do", "it", "did"]}, {"sent": "did ya show him your stitches ?", "words": ["your", "show", "him", "did", "did ya"]}, {"sent": "you went to another house this morning did ya ?", "words": ["to", "you", "another", "this", "house", "did", "did ya"]}, {"sent": "there we go did ya see that ?", "words": ["go", "we", "there", "did", "did ya", "see", "that"]}, {"sent": "did ya ?", "words": ["did ya", "did"]}, {"sent": "did ya like it ?", "words": ["did ya", "it", "did", "like"]}, {"sent": "did ya xxx give me the slips ?", "words": ["the", "did", "did ya", "me", "give"]}, {"sent": "how much did ya make ?", "words": ["how", "much", "did", "did ya", "make"]}, {"sent": "ya didnt see me this morning did ya ?", "words": ["me", "this", "did", "did ya", "see"]}, {"sent": "what did ya do with Christopher today ?", "words": ["what", "do", "with", "did", "did ya"]}, {"sent": "did ya saw it ?", "words": ["did ya", "it", "did"]}, {"sent": "did ya hear that ?", "words": ["did ya", "that", "did", "hear"]}, {"sent": "did ya see that rocking chair ?", "words": ["rocking chair", "chair", "did", "did ya", "see", "that"]}, {"sent": "you were walking in a circle but you didnt fall down did ya ?", "words": ["fall", "you", "in", "did", "did ya", "down", "were", "but", "a"]}, {"sent": "you got it outof the door for us did ya ?", "words": ["the", "for", "you", "us", "it", "did", "did ya", "door"]}, {"sent": "you didnt cut it very good did ya ?", "words": ["cut", "you", "it", "did", "good", "did ya"]}, {"sent": "did ya see it ?", "words": ["did ya", "see", "it", "did"]}, {"sent": "oh what did ya get pie ?", "words": ["get", "what", "did ya", "did"]}, {"sent": "did ya get it ?", "words": ["get", "did ya", "it", "did"]}, {"sent": "did ya have fun yesterday ?", "words": ["have", "did ya", "did"]}, {"sent": "what did ya do you touched the ball ?", "words": ["the", "what", "do", "you", "ball", "did", "did ya"]}, {"sent": "did ya wanna have yogurt or do ya wanna color ?", "words": ["do", "yogurt", "have", "did", "wanna", "did ya"]}, {"sent": "did ya hear that ?", "words": ["did ya", "that", "did", "hear"]}, {"sent": "did ya put on her sockie ?", "words": ["put", "did ya", "did", "on", "her"]}, {"sent": "why did ya ask Daddy for some money ?", "words": ["money", "for", "did", "did ya", "some", "why"]}, {"sent": "what did ya find ?", "words": ["did ya", "what", "did", "find"]}, {"sent": "you didnt fall for that one did ya ?", "words": ["for", "fall", "you", "did", "did ya", "that"]}, {"sent": "and Im on vitamins .", "words": ["on", "vitamins", "and"]}, {"sent": "you want the the vitamins ?", "words": ["the", "vitamins", "you"]}, {"sent": "theyd rather have their vitamins ?", "words": ["have", "vitamins", "their"]}, {"sent": "the minute ya got into bed ya were yelling for hot milk and cocoa and vitamins and everything else .", "words": ["the", "for", "into", "bed", "and", "vitamins", "hot", "milk", "were"]}, {"sent": "I know I got my own vitamins .", "words": ["vitamins", "my"]}, {"sent": "Chocks are vitamins .", "words": ["vitamins", "are"]}, {"sent": "good vitamins .", "words": ["vitamins", "good"]}, {"sent": "okay did you eat both your vitamins ?", "words": ["your", "you", "eat", "vitamins", "did"]}, {"sent": "some vitamins and some xxx .", "words": ["vitamins", "some", "and"]}, {"sent": "did you have your vitamins this morning ?", "words": ["your", "you", "have", "vitamins", "this", "did"]}, {"sent": "because hes got no vitamins .", "words": ["vitamins", "because"]}, {"sent": "vitamins and minerals are very good for you .", "words": ["for", "and", "vitamins", "you", "are", "good"]}, {"sent": "the vitamins ?", "words": ["the", "vitamins"]}, {"sent": "and some vitamins .", "words": ["vitamins", "some", "and"]}, {"sent": "and her vitamins .", "words": ["vitamins", "her", "and"]}, {"sent": "no I know I just start eating the Flintstone vitamins yeah I know actually Lily has her xxx can I call you back at uh at at tenish ?", "words": ["the", "can", "back", "you", "vitamins", "at", "her"]}, {"sent": "you want some vitamins ?", "words": ["vitamins", "some", "you"]}, {"sent": "those are vitamins .", "words": ["vitamins", "are", "those"]}, {"sent": "and vitamins .", "words": ["vitamins", "and"]}, {"sent": "hey you forgot to take your vitamins today .", "words": ["your", "to", "you", "vitamins", "take"]}, {"sent": "does Mummy give you vitamins ?", "words": ["vitamins", "does", "give", "you"]}, {"sent": "say vitamins .", "words": ["vitamins", "say"]}, {"sent": "these are your vitamins .", "words": ["vitamins", "these", "are", "your"]}, {"sent": "dont worry I wont eat one of your vitamins .", "words": ["your", "of", "eat", "dont", "vitamins"]}, {"sent": "does Bugs take vitamins ?", "words": ["vitamins", "does", "take"]}, {"sent": "want your vitamins ?", "words": ["vitamins", "your"]}, {"sent": "what kinds of vitamins are those ?", "words": ["what", "those", "of", "vitamins", "are"]}, {"sent": "Adams vitamins .", "words": ["vitamins"]}, {"sent": "I forgot your vitamins this morning .", "words": ["vitamins", "this", "your"]}, {"sent": "I forgot your vitamins this morning .", "words": ["vitamins", "this", "your"]}, {"sent": "this taste like vitamins .", "words": ["vitamins", "this", "taste", "like"]}, {"sent": "juice has vitamins .", "words": ["vitamins", "juice"]}, {"sent": "Todd come and get your vitamins .", "words": ["get", "vitamins", "your", "and"]}, {"sent": "doh what about your vitamins .", "words": ["vitamins", "what", "your", "about"]}, {"sent": "Toddy you need you need your vitamins too right ?", "words": ["your", "need", "you", "vitamins", "too"]}, {"sent": "come on go get your vitamins quick .", "words": ["your", "go", "vitamins", "get", "on"]}, {"sent": "eat salads they have they have some good vitamins in them .", "words": ["them", "eat", "they", "have", "vitamins", "in", "good", "some"]}, {"sent": "and make sure you drink some orange juice and make sure you take your vitamins and your uh and your vitamin c .", "words": ["your", "drink", "you", "and", "vitamins", "orange", "some", "take", "juice", "make"]}, {"sent": "no vitamins .", "words": ["vitamins"]}, {"sent": "ei vitamins and iron .", "words": ["vitamins", "and"]}, {"sent": "and vitamins and iron .", "words": ["vitamins", "and"]}, {"sent": "and vitamins for you .", "words": ["vitamins", "for", "you", "and"]}, {"sent": "okay did you eat both your vitamins ?", "words": ["your", "you", "eat", "vitamins", "did"]}, {"sent": "Im just getting your your vitamins .", "words": ["vitamins", "your"]}, {"sent": "vitamins .", "words": ["vitamins"]}, {"sent": "nasty nasty vitamins .", "words": ["vitamins"]}, {"sent": "today today we are going to give you some vitamins .", "words": ["we", "to", "you", "vitamins", "are", "some", "give"]}, {"sent": "should we start you on vitamins today or should we wait until youre done with this medicine ?", "words": ["wait", "we", "medicine", "with", "you", "vitamins", "this", "on"]}, {"sent": "well well wait until you finish all your medicine then well give start giving you vitamins again .", "words": ["all", "wait", "give", "your", "medicine", "finish", "you", "vitamins", "then"]}, {"sent": "vitamins are a little hard to take .", "words": ["to", "little", "vitamins", "are", "hard", "take", "a"]}, {"sent": "Mandy lets take your vitamins dear .", "words": ["your", "vitamins", "take"]}, {"sent": "want some vitamins Nomi ?", "words": ["vitamins", "some"]}, {"sent": "vitamins .", "words": ["vitamins"]}, {"sent": "vitamins ?", "words": ["vitamins"]}, {"sent": "she put two vitamins together two purple ones and said .", "words": ["vitamins", "put", "she", "and"]}, {"sent": "Nomi you didnt have your vitamins this morning did you ?", "words": ["your", "you", "have", "this", "vitamins", "did"]}, {"sent": "and anyway youre not allowed vitamins like that .", "words": ["like", "and", "vitamins", "that", "not"]}, {"sent": "and your vitamins ?", "words": ["vitamins", "your", "and"]}, {"sent": "what kind of vitamins ?", "words": ["of", "what", "vitamins"]}, {"sent": "how many vitamins do you have ?", "words": ["do", "how", "you", "vitamins", "have"]}, {"sent": "do you like blue vitamins ?", "words": ["do", "like", "blue", "vitamins", "you"]}, {"sent": "he likes you taking those vitamins dont he ?", "words": ["those", "you", "vitamins", "dont", "he"]}, {"sent": "do you take vitamins ?", "words": ["vitamins", "do", "take", "you"]}, {"sent": "what vitamins do you take ?", "words": ["what", "do", "you", "vitamins", "take"]}, {"sent": "he needs some vitamins probably .", "words": ["vitamins", "some", "he"]}, {"sent": "does it got vitamins ?", "words": ["vitamins", "does", "it"]}, {"sent": "you shouldnt really xxx fruit and vegetables too much because a lot of the vitamins the goodness comes out .", "words": ["the", "of", "a lot", "and", "you", "vitamins", "out", "too", "much", "because", "a"]}, {"sent": "a jelly baby soft and chewy vitamins .", "words": ["jelly", "soft", "and", "vitamins", "a"]}, {"sent": "with vitamins A C D and E .", "words": ["vitamins", "with", "and"]}, {"sent": "theyre not vitamins .", "words": ["vitamins", "not"]}, {"sent": "tastes like vitamins and tastes like paracetamols ?", "words": ["vitamins", "like", "and"]}, {"sent": "vitamins .", "words": ["vitamins"]}, {"sent": "tomorrow morning you get some more vitamins .", "words": ["you", "get", "vitamins", "more", "some"]}, {"sent": "oh you want some vitamins .", "words": ["vitamins", "some", "you"]}, {"sent": "well lets finish the book and then you can have your breakfast and you can have vitamins breakfast .", "words": ["the", "can", "your", "finish", "and", "you", "have", "vitamins", "book", "then"]}, {"sent": "but you had your vitamins already today .", "words": ["vitamins", "your", "but", "you"]}, {"sent": "those are vitamins .", "words": ["vitamins", "are", "those"]}, {"sent": "you like vitamins dont ya ?", "words": ["vitamins", "dont", "like", "you"]}, {"sent": "those are vitamins over there .", "words": ["those", "vitamins", "there", "are", "over"]}, {"sent": "no vitamins .", "words": ["vitamins"]}, {"sent": "these are the health food store vitamins .", "words": ["the", "these", "store", "vitamins", "food", "are"]}, {"sent": "I thought you told your Mommy you had cereal that had lotsof good tasting vitamins in it .", "words": ["your", "you", "vitamins", "in", "it", "good", "that", "cereal"]}, {"sent": "some vitamins .", "words": ["vitamins", "some"]}, {"sent": "well this bottle of vitamins is almost gone .", "words": ["is", "of", "vitamins", "this", "bottle"]}, {"sent": "get your vitamins .", "words": ["get", "vitamins", "your"]}, {"sent": "you saw your dinosaur while you were flying in the sky ?", "words": ["the", "your", "sky", "you", "in", "were"]}, {"sent": "next minute Im in the sky .", "words": ["the", "in", "sky"]}, {"sent": "thats the suns big and round in the sky isnt it ?", "words": ["the", "sky", "and", "in", "it", "big"]}, {"sent": "to touch the sky .", "words": ["the", "to", "touch", "sky"]}, {"sent": "the sky filled with clouds and the rain began to fall .", "words": ["the", "rain", "sky", "to", "with", "fall", "and"]}, {"sent": "the sky ?", "words": ["the", "sky"]}, {"sent": "blue air above oh like sky you mean ?", "words": ["sky", "like", "you", "blue", "above"]}, {"sent": "xxx in the sky .", "words": ["the", "in", "sky"]}, {"sent": "what does he do up in the sky ?", "words": ["up", "what", "do", "does", "the", "sky", "in", "he"]}, {"sent": "the sky is falling .", "words": ["the", "is", "sky"]}, {"sent": "flying in the sky .", "words": ["the", "in", "sky"]}, {"sent": "because it really just kind of cuts it down to the simple youknow like up down up in the sky .", "words": ["the", "up", "sky", "of", "to", "like", "in", "it", "down", "because"]}, {"sent": "high up in the sky .", "words": ["up", "the", "sky", "in", "high"]}, {"sent": "thats the sky .", "words": ["the", "sky"]}, {"sent": "and thats the sky .", "words": ["the", "sky", "and"]}, {"sent": "the sky .", "words": ["the", "sky"]}, {"sent": "well try up here because thats the sky I think isnt it ?", "words": ["up", "the", "think", "sky", "here", "it", "try", "because"]}, {"sent": "say way high in the sky .", "words": ["the", "sky", "say", "in", "high"]}, {"sent": "and what happened when she looked up into the sky ?", "words": ["up", "what", "the", "into", "sky", "and", "when", "she"]}, {"sent": "hes a plane in the sky ?", "words": ["the", "in", "sky", "a"]}, {"sent": "and who lives in the sky ?", "words": ["the", "sky", "and", "in", "who"]}, {"sent": "its a big eye in the sky watching them .", "words": ["the", "sky", "them", "eye", "in", "big", "a"]}, {"sent": "but my sky writing smoke is white and so are the clouds so maybe white should be my favorite color .", "words": ["the", "my", "is", "sky", "so", "be", "and", "are", "white", "but"]}, {"sent": "falling from the sky .", "words": ["the", "sky"]}, {"sent": "its only a big noise in the sky isnt it ?", "words": ["the", "sky", "in", "it", "big", "a"]}, {"sent": "you see the blue sky ?", "words": ["the", "sky", "blue", "you", "see"]}, {"sent": "and now we need one with the sky .", "words": ["the", "need", "we", "sky", "with", "and"]}, {"sent": "can you see the clouds in the sky ?", "words": ["can", "the", "sky", "you", "in", "see"]}, {"sent": "can you see all the stars sparkling in the sky in this book ?", "words": ["can", "the", "all", "sky", "you", "this", "in", "book", "see"]}, {"sent": "whats this flying in the sky .", "words": ["the", "this", "in", "sky"]}, {"sent": "oh yes that was an aeroplane in the sky .", "words": ["the", "sky", "was", "an", "in", "that"]}, {"sent": "thatll look wonderful flying in the sky .", "words": ["the", "in", "sky", "look"]}, {"sent": "what flies in the sky ?", "words": ["the", "what", "in", "sky"]}, {"sent": "its coming down from the sky .", "words": ["the", "down", "sky"]}, {"sent": "theyre going up in the sky ?", "words": ["up", "the", "in", "sky"]}, {"sent": "in the sky right in the sky .", "words": ["the", "in", "sky"]}, {"sent": "you come and try and catch the money falling from the sky .", "words": ["the", "money", "sky", "catch", "and", "you", "try"]}, {"sent": "and the storybook shows him sitting there looking out of the window at the moon in the sky .", "words": ["the", "moon", "sky", "of", "and", "there", "him", "in", "out", "at", "window"]}, {"sent": "the sky is falling cheeped Chicken Little and we must tell the king .", "words": ["the", "we", "is", "sky", "and"]}, {"sent": "how many clouds there are in the sky ?", "words": ["the", "how", "sky", "there", "in", "are"]}, {"sent": "its going to use it to float him up into the sky .", "words": ["up", "the", "into", "sky", "to", "him", "it"]}, {"sent": "up in the sky .", "words": ["up", "the", "in", "sky"]}, {"sent": "we can see from the sky were looking down .", "words": ["can", "the", "sky", "we", "see", "down", "were"]}, {"sent": "thats right it goes in the sky .", "words": ["the", "in", "it", "sky"]}, {"sent": "a leg going out to the sky and a leg going down to the ground .", "words": ["the", "sky", "to", "and", "leg", "out", "down", "a"]}, {"sent": "do you think its gonna touch the sky .", "words": ["the", "do", "think", "sky", "touch", "you"]}, {"sent": "oh the sky ?", "words": ["the", "sky"]}, {"sent": "a boat in the sky .", "words": ["the", "sky", "in", "boat", "a"]}, {"sent": "when sometimes there was an owl in the sky with the stars and the moon and other times there was a cockerel crowing and the sun shining .", "words": ["the", "moon", "sky", "was", "sun", "with", "owl", "and", "there", "an", "in", "other", "when", "a"]}, {"sent": "and some clouds in the sky .", "words": ["the", "sky", "and", "in", "some"]}, {"sent": "the sky is falling cheeped Chicken Little and we must tell the king .", "words": ["the", "we", "is", "sky", "and"]}, {"sent": "whats a fish doing flying in the sky ?", "words": ["the", "sky", "in", "fish", "a"]}, {"sent": "up to the sky .", "words": ["up", "to", "sky", "the"]}, {"sent": "can do you think it looks like somebody has got a paintbrush and gone boomboomboom and drawn loads of stripes in the sky ?", "words": ["can", "do", "think", "the", "sky", "of", "like", "you", "and", "in", "it", "a"]}, {"sent": "thats the sky .", "words": ["the", "sky"]}, {"sent": "thats right sky .", "words": ["sky"]}, {"sent": "with what what else lives in the sky ?", "words": ["the", "what", "sky", "with", "in"]}, {"sent": "how many insects are up in the sky Fraser ?", "words": ["up", "the", "how", "sky", "in", "are"]}, {"sent": "whys he up in the sky ?", "words": ["up", "the", "sky", "in", "he"]}, {"sent": "like diamonds in the sky huh ?", "words": ["the", "in", "like", "sky"]}, {"sent": "what shall we put in the sky ?", "words": ["the", "what", "put", "we", "sky", "in"]}, {"sent": "sky .", "words": ["sky"]}, {"sent": "whats up in the sky ?", "words": ["up", "the", "in", "sky"]}, {"sent": "in the sky .", "words": ["the", "in", "sky"]}, {"sent": "whos flying in the sky here ?", "words": ["the", "in", "here", "sky"]}, {"sent": "the sky is up and we must xxx .", "words": ["the", "up", "is", "sky", "we", "and"]}, {"sent": "in the sky ?", "words": ["the", "in", "sky"]}, {"sent": "sitting high up in the sky .", "words": ["up", "the", "sky", "in", "high"]}, {"sent": "up in the sky .", "words": ["up", "the", "in", "sky"]}, {"sent": "whats up in the sky ?", "words": ["up", "the", "in", "sky"]}, {"sent": "you can make the sky in this park .", "words": ["can", "the", "sky", "you", "this", "in", "park", "make"]}, {"sent": "up in the sky .", "words": ["up", "the", "in", "sky"]}, {"sent": "in the sky outside .", "words": ["the", "outside", "in", "sky"]}, {"sent": "now weve got two pieces of sky left .", "words": ["of", "sky"]}, {"sent": "flying in the sky .", "words": ["the", "in", "sky"]}, {"sent": "flys up in the sky ?", "words": ["up", "the", "in", "sky"]}, {"sent": "and then this is us from the sky we were in the basket and that was the other balloon down below .", "words": ["the", "is", "sky", "we", "was", "and", "other", "us", "this", "in", "basket", "that", "balloon", "then", "down", "were"]}, {"sent": "xxx the sky is on him ?", "words": ["the", "is", "sky", "him", "on"]}, {"sent": "in the sky ?", "words": ["the", "in", "sky"]}, {"sent": "make it touch the sky .", "words": ["the", "sky", "touch", "it", "make"]}, {"sent": "an aeroplane in the sky .", "words": ["the", "in", "an", "sky"]}, {"sent": "she says anything like round like that in the sky or crescent shape shell do so anytime theres a sun she think its a moon .", "words": ["the", "do", "think", "moon", "sky", "so", "sun", "like", "in", "that", "she", "a"]}, {"sent": "like a diamond in the sky .", "words": ["the", "sky", "like", "in", "a"]}, {"sent": "fall sky ?", "words": ["fall", "sky"]}, {"sent": "oh Henny Penny the sky is falling .", "words": ["the", "is", "sky"]}, {"sent": "the post lorrys going up in the sky ?", "words": ["the", "in", "up", "sky"]}, {"sent": "high in the sky .", "words": ["the", "in", "sky", "high"]}, {"sent": "pink sky .", "words": ["sky"]}, {"sent": "the sun is setting in the sky .", "words": ["the", "sky", "is", "sun", "in"]}, {"sent": "have you ever seen the paths that aircraft make in the sky ?", "words": ["the", "sky", "you", "have", "that", "in", "make"]}, {"sent": "whoosh up into the sky with a big noise and it gave you a fright didnt it ?", "words": ["up", "the", "into", "sky", "with", "and", "you", "it", "big", "a"]}, {"sent": "into the sky .", "words": ["the", "into", "sky"]}, {"sent": "the guys are riding in the sky .", "words": ["the", "in", "are", "sky"]}, {"sent": "blue sky .", "words": ["sky", "blue"]}, {"sent": "how about we read the night sky ?", "words": ["the", "how", "we", "sky", "read", "about"]}, {"sent": "and aeroplanes go very high in the sky dont they ?", "words": ["the", "go", "sky", "and", "dont", "in", "they", "high"]}, {"sent": "he floats up into the sky on a blue balloon .", "words": ["up", "the", "into", "sky", "blue", "he", "balloon", "on", "a"]}, {"sent": "like a diamond in the sky .", "words": ["the", "sky", "like", "in", "a"]}, {"sent": "up in the sky .", "words": ["up", "the", "in", "sky"]}, {"sent": "the sky grew dark and it began to rain .", "words": ["the", "rain", "sky", "to", "and", "it", "dark"]}, {"sent": "was it a bird bath ?", "words": ["was", "it", "bird", "a"]}, {"sent": "is that a bird ?", "words": ["bird", "that", "a", "is"]}, {"sent": "yeah bird .", "words": ["bird"]}, {"sent": "is that the right kind of bird ?", "words": ["the", "is", "of", "bird", "that"]}, {"sent": "red bird red bird what do you see ?", "words": ["red", "what", "do", "bird", "you", "see"]}, {"sent": "we need to buy some bird food .", "words": ["need", "we", "need to", "to", "bird", "food", "some", "buy"]}, {"sent": "maybe this bird ?", "words": ["this", "bird"]}, {"sent": "no thats why shes a lazy lazy bird .", "words": ["bird", "why", "a"]}, {"sent": "the big bird .", "words": ["the", "bird", "big"]}, {"sent": "the bird flew up to its new home .", "words": ["the", "up", "to", "bird", "new", "home"]}, {"sent": "the lady bird might wanna sit down .", "words": ["the", "bird", "wanna", "sit", "down"]}, {"sent": "whats the bird called ?", "words": ["the", "bird"]}, {"sent": "thats um a bird of some sort .", "words": ["of", "some", "bird", "a"]}, {"sent": "can you see the bird there you put the bird in there .", "words": ["can", "the", "put", "bird", "you", "there", "in", "see"]}, {"sent": "because theres an egg in that nest and theres no mother bird to look after it .", "words": ["egg", "to", "bird", "and", "in", "an", "it", "look", "that", "because"]}, {"sent": "does the bird like to go to the water ?", "words": ["the", "does", "go", "to", "bird", "like", "water"]}, {"sent": "bird xxx .", "words": ["bird"]}, {"sent": "Sarah Im telling ya if that bird gets out you better run .", "words": ["better", "if", "bird", "you", "run", "out", "that"]}, {"sent": "see that bird ?", "words": ["see", "bird", "that"]}, {"sent": "xxx thats a bird what does the bird say ?", "words": ["the", "does", "what", "bird", "say", "a"]}, {"sent": "the big bird ?", "words": ["the", "bird", "big"]}, {"sent": "you see the magpies a pretty bird really xxx .", "words": ["the", "bird", "you", "pretty", "see", "a"]}, {"sent": "does that sound good to you little bird ?", "words": ["does", "to", "bird", "you", "little", "good", "that"]}, {"sent": "you were the early bird .", "words": ["the", "bird", "were", "you"]}, {"sent": "do you think its a mommy bird of a daddy bird or a baby bird ?", "words": ["do", "think", "of", "bird", "you", "a"]}, {"sent": "you think another big bird wants to go in a little tree ah .", "words": ["think", "tree", "go", "to", "bird", "you", "another", "little", "in", "big", "a"]}, {"sent": "a bird .", "words": ["bird", "a"]}, {"sent": "see the bird on his head ?", "words": ["the", "head", "bird", "his", "on", "see"]}, {"sent": "red bird red bird what do you see ?", "words": ["red", "what", "do", "bird", "you", "see"]}, {"sent": "a little bird he sees .", "words": ["bird", "he", "little", "a"]}, {"sent": "hello little bird .", "words": ["bird", "little"]}, {"sent": "bird .", "words": ["bird"]}, {"sent": "thats a bird .", "words": ["bird", "a"]}, {"sent": "do you think Gwww might like a bird ?", "words": ["do", "think", "bird", "like", "you", "a"]}, {"sent": "a bird .", "words": ["bird", "a"]}, {"sent": "what is a friendly little bird with a .", "words": ["what", "is", "bird", "with", "little", "a"]}, {"sent": "thats just a bird .", "words": ["bird", "a"]}, {"sent": "Im as a free little bird as I can be .", "words": ["can", "be", "bird", "little", "a"]}, {"sent": "whats the bird doing ?", "words": ["the", "bird"]}, {"sent": "right you have a bird .", "words": ["have", "bird", "a", "you"]}, {"sent": "bird .", "words": ["bird"]}, {"sent": "what does the bird say ?", "words": ["the", "what", "does", "bird", "say"]}, {"sent": "mm hm bird .", "words": ["bird"]}, {"sent": "this is the bird library .", "words": ["the", "this", "bird", "is"]}, {"sent": "wheres that white little bird ?", "words": ["white", "that", "bird", "little"]}, {"sent": "there is the bird .", "words": ["the", "there", "bird", "is"]}, {"sent": "is this the bird ?", "words": ["the", "this", "bird", "is"]}, {"sent": "but Daddy he didnt kill that bird Laura .", "words": ["that", "bird", "he", "but"]}, {"sent": "oh big bird .", "words": ["bird", "big"]}, {"sent": "whats baby bird doing ?", "words": ["bird"]}, {"sent": "but a bird or lets see some other kind of animal with strong teeth could probably do it .", "words": ["do", "a", "of", "bird", "with", "other", "animal", "it", "see", "could", "but", "some"]}, {"sent": "the bird is watching .", "words": ["the", "bird", "is"]}, {"sent": "he wanted to go in and her a bird and a goldfish and a turtle .", "words": ["go", "to", "bird", "and", "in", "he", "turtle", "her", "a"]}, {"sent": "remember what type of special bird they are near the ocean ?", "words": ["the", "what", "of", "bird", "they", "are"]}, {"sent": "its a little yellow bird .", "words": ["yellow", "bird", "little", "a"]}, {"sent": "the early bird catches the ?", "words": ["the", "bird"]}, {"sent": "bird .", "words": ["bird"]}, {"sent": "its the bird .", "words": ["the", "bird"]}, {"sent": "bird .", "words": ["bird"]}, {"sent": "bird bird .", "words": ["bird"]}, {"sent": "bird .", "words": ["bird"]}, {"sent": "did you tell Daddy about the guy with the bird in his pocket ?", "words": ["the", "bird", "with", "you", "in", "did", "about", "his"]}, {"sent": "a bird okay .", "words": ["bird", "a"]}, {"sent": "I think the cat is watching the bird .", "words": ["the", "think", "is", "bird", "cat"]}, {"sent": "thats a funny bird .", "words": ["bird", "a"]}, {"sent": "shh Kim keeps quiet and still theres a pretty bird in the yard theres the bird .", "words": ["the", "bird", "and", "in", "pretty", "quiet", "a"]}, {"sent": "here I made a little bird for you .", "words": ["for", "bird", "little", "here", "you", "a"]}, {"sent": "number ten has a bird on it .", "words": ["on", "it", "bird", "a"]}, {"sent": "a bird going east .", "words": ["bird", "a"]}, {"sent": "what kind of bird are you ?", "words": ["what", "of", "bird", "you", "are"]}, {"sent": "a toucan bird .", "words": ["bird", "a"]}, {"sent": "another bird .", "words": ["another", "bird"]}, {"sent": "that bird has my kite .", "words": ["my", "bird", "that"]}, {"sent": "whats this part of the bird called ?", "words": ["of", "this", "bird", "the"]}, {"sent": "Im gonna throw the bird seed away in the garbage okay ?", "words": ["the", "throw", "bird", "in", "away", "garbage"]}, {"sent": "thats a big bird .", "words": ["bird", "big", "a"]}, {"sent": "Big bird is back in .", "words": ["in", "bird", "back", "is"]}, {"sent": "whats the bird say ?", "words": ["the", "bird", "say"]}, {"sent": "theres the bird sweetheart .", "words": ["the", "bird"]}, {"sent": "the bird .", "words": ["the", "bird"]}, {"sent": "pterodactyl its kind of like a dinosaur bird .", "words": ["of", "bird", "like", "a"]}, {"sent": "please may I cut round the the bird ?", "words": ["the", "bird", "cut"]}, {"sent": "we have a bird nest yeah but where do they eat ?", "words": ["where", "do", "we", "bird", "have", "they", "eat", "but", "a"]}, {"sent": "and the bird is taming .", "words": ["the", "bird", "is", "and"]}, {"sent": "oh now youre eating the girls they came to see that big bird .", "words": ["the", "to", "bird", "they", "big", "see", "that"]}, {"sent": "I can still hear the bird tweeting .", "words": ["can", "the", "bird", "hear"]}, {"sent": "can you draw a bird ?", "words": ["can", "bird", "draw", "you", "a"]}, {"sent": "she swallowed the bird to catch .", "words": ["the", "to", "bird", "catch", "she"]}, {"sent": "is that the little bird stuck in the snow ?", "words": ["the", "is", "bird", "snow", "little", "in", "stuck", "that"]}, {"sent": "what kind of bird is that one ?", "words": ["what", "is", "of", "bird", "that"]}, {"sent": "bird .", "words": ["bird"]}, {"sent": "lion the bird goes on the tree .", "words": ["the", "tree", "bird", "on", "lion"]}, {"sent": "the bird is in a house .", "words": ["the", "is", "bird", "in", "house", "a"]}, {"sent": "thats a bird .", "words": ["bird", "a"]}, {"sent": "the early bird gets the worm .", "words": ["the", "bird"]}, {"sent": "can you see that bird sitting ?", "words": ["can", "bird", "you", "see", "that"]}, {"sent": "big bird .", "words": ["bird", "big"]}, {"sent": "theres that bird again .", "words": ["bird", "that"]}, {"sent": "that says bird .", "words": ["bird", "that"]}, {"sent": "inside .", "words": ["inside"]}, {"sent": "a spider and whats inside there ?", "words": ["there", "inside", "and", "a"]}, {"sent": "I dont want a polarbear inside my tshirt .", "words": ["dont", "inside", "my", "a"]}, {"sent": "inside Jack could hear a thumping .", "words": ["inside", "hear", "a", "could"]}, {"sent": "it has got a little cut out in the box so that you can see whats inside .", "words": ["the", "can", "box", "cut", "so", "inside", "little", "you", "in", "it", "out", "see", "that", "a"]}, {"sent": "whats inside ?", "words": ["inside"]}, {"sent": "Ill just have a look inside the books .", "words": ["the", "inside", "have", "look", "a"]}, {"sent": "put it inside .", "words": ["put", "it", "inside"]}, {"sent": "its a blanket that has coils inside and they plug in the wall to keep you warm .", "words": ["the", "to", "inside", "and", "you", "they", "in", "blanket", "that", "a"]}, {"sent": "is that Scoop inside that truck ?", "words": ["truck", "inside", "that", "is"]}, {"sent": "I think he has come from inside the church hasnt he ?", "words": ["the", "think", "church", "inside", "he"]}, {"sent": "Dick can wait inside the school bus too ?", "words": ["can", "the", "wait", "school", "inside", "too", "bus"]}, {"sent": "her teeth are inside her mouth arent they ?", "words": ["inside", "mouth", "they", "are", "her"]}, {"sent": "I think were gonna hafta go inside Fraser .", "words": ["think", "inside", "were", "go"]}, {"sent": "no more crackers inside its empty .", "words": ["more", "inside", "empty"]}, {"sent": "those stay inside .", "words": ["stay", "inside", "those"]}, {"sent": "and in between the outside and the inside of the beaker its filled with water .", "words": ["the", "of", "outside", "inside", "with", "and", "in", "water"]}, {"sent": "and guess what inside it says .", "words": ["what", "inside", "it", "and"]}, {"sent": "its inside out .", "words": ["inside", "out"]}, {"sent": "whats inside of the doghouse ?", "words": ["of", "the", "inside"]}, {"sent": "thatll be inside out if you do it like that .", "words": ["do", "if", "be", "inside", "like", "you", "it", "out", "that"]}, {"sent": "inside xxx huh ?", "words": ["inside"]}, {"sent": "and what about inside the church ?", "words": ["the", "what", "church", "inside", "and", "about"]}, {"sent": "theres real money inside too .", "words": ["money", "inside", "too"]}, {"sent": "blue inside rabbit ?", "words": ["inside", "blue"]}, {"sent": "you hafta scoop the ball inside there .", "words": ["the", "inside", "you", "ball", "there"]}, {"sent": "alright stay inside ?", "words": ["stay", "inside"]}, {"sent": "we go inside .", "words": ["go", "inside", "we"]}, {"sent": "he hid this block inside of here .", "words": ["block", "of", "inside", "here", "this", "he"]}, {"sent": "alright then lets go inside come on .", "words": ["on", "then", "inside", "go"]}, {"sent": "uhhuh lets see whats inside this book look .", "words": ["inside", "this", "look", "book", "see"]}, {"sent": "whats inside this tin ?", "words": ["this", "inside"]}, {"sent": "ow Deidre which way you like Remis pants inside the boot or outside ?", "words": ["the", "outside", "inside", "like", "you", "pants", "which"]}, {"sent": "wanna go inside ?", "words": ["inside", "wanna", "go"]}, {"sent": "ah inside the space between the cushion and the edge of the sofa .", "words": ["the", "of", "inside", "and", "sofa"]}, {"sent": "there was one inside your packet .", "words": ["was", "there", "inside", "your"]}, {"sent": "until Dad blew some air inside and made it very big .", "words": ["inside", "and", "it", "some", "big"]}, {"sent": "no no listen sweethuh you you hafta be responsible now to see to it that that doesnt happen to you again because you can turn that light off from inside sothat nobody can turn it on accidentally from the outside .", "words": ["can", "off", "the", "listen", "light", "to", "be", "inside", "outside", "you", "it", "on", "see", "that", "because"]}, {"sent": "what are the teddy bears doing inside the playbus ?", "words": ["the", "what", "inside", "are"]}, {"sent": "whats inside the truck ?", "words": ["the", "inside", "truck"]}, {"sent": "daddy put water inside so it would be heavy .", "words": ["put", "so", "inside", "be", "would", "it", "water", "heavy"]}, {"sent": "you go inside and have a look dont you ?", "words": ["go", "inside", "and", "have", "you", "dont", "look", "a"]}, {"sent": "maybe just the drinks inside the cup .", "words": ["the", "inside", "cup"]}, {"sent": "right this keys inside but when youre here on the inside you just have like a little thing that you turn that little thing on the knob and thats how you lock it is this her hairbrush ?", "words": ["the", "like", "when", "have", "it", "on", "that", "how", "is", "you", "here", "little", "and", "keys", "a", "inside", "this", "her", "but"]}, {"sent": "it hasta have money inside .", "words": ["have", "inside", "it", "money"]}, {"sent": "so I can bring it inside .", "words": ["can", "so", "inside", "bring", "it"]}, {"sent": "look inside yourself Simba you are more than what you have become .", "words": ["what", "yourself", "inside", "you", "have", "more", "are", "look"]}, {"sent": "whats inside there ?", "words": ["there", "inside"]}, {"sent": "there she was inside xxx in the corner .", "words": ["the", "was", "inside", "there", "in", "she"]}, {"sent": "its something inside that box .", "words": ["inside", "box", "that"]}, {"sent": "fishes inside ?", "words": ["inside"]}, {"sent": "its red inside and green outside .", "words": ["red", "outside", "inside", "and", "green"]}, {"sent": "inside the house ?", "words": ["the", "inside", "house"]}, {"sent": "would you like to look inside this refrigerated cabinet and see if theres a cabinet ?", "words": ["if", "to", "inside", "like", "would", "you", "and", "this", "look", "see", "a"]}, {"sent": "and its not nice inside .", "words": ["inside", "nice", "not", "and"]}, {"sent": "ahhah whats inside that one ?", "words": ["inside", "that"]}, {"sent": "theyve got a present inside .", "words": ["inside", "present", "a"]}, {"sent": "is the little engine inside ?", "words": ["the", "inside", "little", "is"]}, {"sent": "just inside the door .", "words": ["the", "inside", "door"]}, {"sent": "come now said Mrs Pots gently all you hafta do is show her whats inside your heart you must act like a gentleman Lumiere said .", "words": ["all", "do", "your", "is", "show", "inside", "like", "you", "her", "a"]}, {"sent": "dont put your pear inside the lorry will you ?", "words": ["the", "put", "your", "inside", "you", "dont", "will"]}, {"sent": "not too bad inside .", "words": ["inside", "not", "bad", "too"]}, {"sent": "inside their cages .", "words": ["inside", "their"]}, {"sent": "look inside .", "words": ["inside", "look"]}, {"sent": "inside of this .", "words": ["of", "this", "inside"]}, {"sent": "lets go inside the tool shed .", "words": ["the", "inside", "go"]}, {"sent": "it was so cold that we hadta have our picnic inside the truck .", "words": ["the", "we", "was", "so", "inside", "truck", "our", "have", "it", "that", "cold"]}, {"sent": "put the little girl inside .", "words": ["the", "put", "inside", "little"]}, {"sent": "inside voices .", "words": ["inside"]}, {"sent": "and I know you do still use your bicycle occasionally inside but I cant remember the last time I saw you using it .", "words": ["the", "do", "your", "inside", "last", "bicycle", "you", "and", "it", "but"]}, {"sent": "youre inside .", "words": ["inside"]}, {"sent": "bottles inside .", "words": ["inside"]}, {"sent": "theyre going inside that pot .", "words": ["inside", "that"]}, {"sent": "oh actually you hafta color the whole the whole shape thats inside .", "words": ["the", "inside", "you"]}, {"sent": "you wanna open up the castle and see whats inside ?", "words": ["the", "up", "open", "inside", "you", "and", "wanna", "see"]}, {"sent": "see whats inside the castle .", "words": ["the", "see", "inside"]}, {"sent": "now lets put the pigs inside it .", "words": ["the", "it", "put", "inside"]}, {"sent": "theres nothing inside just a tube .", "words": ["inside", "a"]}, {"sent": "what do you think is inside ?", "words": ["what", "do", "think", "is", "inside", "you"]}, {"sent": "wanna go inside if youre tired ?", "words": ["if", "go", "inside", "tired", "wanna"]}, {"sent": "because its getting winter soon and theyll needta come inside willnt they ?", "words": ["they", "inside", "because", "and"]}, {"sent": "so she slowly stepped inside .", "words": ["so", "inside", "she"]}, {"sent": "but this ones being used inside .", "words": ["this", "inside", "but"]}, {"sent": "here lets go inside and he can do it .", "words": ["can", "do", "go", "inside", "and", "here", "it", "he"]}, {"sent": "what inside Grandads house ?", "words": ["what", "inside", "house"]}, {"sent": "feel what is inside of here .", "words": ["what", "is", "of", "inside", "here"]}, {"sent": "was he a toy inside an easter egg ?", "words": ["egg", "was", "inside", "toy", "an", "he", "a"]}, {"sent": "can anybody thats felt the bag already think what might be inside it ?", "words": ["can", "the", "think", "what", "be", "inside", "it"]}, {"sent": "I can we we can make the flower inside cant we ?", "words": ["can", "the", "we", "flower", "inside", "make"]}, {"sent": "can you put them inside ?", "words": ["can", "put", "inside", "them", "you"]}, {"sent": "oh youre inside the zoo .", "words": ["the", "zoo", "inside"]}, {"sent": "theres a little teensie shell inside of here .", "words": ["of", "inside", "little", "here", "a"]}, {"sent": "and were inside .", "words": ["inside", "were", "and"]}, {"sent": "and she got inside of you ?", "words": ["of", "inside", "you", "and", "she"]}, {"sent": "not inside .", "words": ["inside", "not"]}, {"sent": "thats inside inside .", "words": ["inside"]}, {"sent": "you see with that thing when hes still in that and then hes in the walker and all no bother .", "words": ["the", "all", "walker", "when", "with", "you", "and", "in", "see", "then", "that"]}, {"sent": "shes a better talker than walker .", "words": ["better", "walker", "a"]}, {"sent": "Julie walker did that .", "words": ["walker", "that", "did"]}, {"sent": "I will be a tightrope walker and walk across the air to you said his mother .", "words": ["the", "walker", "be", "to", "and", "you", "his", "will", "walk", "a"]}, {"sent": "how bout walker ?", "words": ["how", "walker"]}, {"sent": "he skipped lightly across and finding it fun to be a tightrope walker high about the rest of the circus .", "words": ["the", "walker", "of", "to", "be", "and", "it", "he", "about", "high", "a"]}, {"sent": "you were in the baby walker .", "words": ["the", "walker", "you", "in", "were"]}, {"sent": "like little Rickys walker ?", "words": ["walker", "like", "little"]}, {"sent": "are you gonna be the tight rope walker today ?", "words": ["the", "walker", "be", "you", "are"]}, {"sent": "I thought you were gonna be tight rope gonna be the balloon lady then you were gonna be the tightrope walke taip rope walker .", "words": ["the", "walker", "be", "you", "balloon", "then", "were"]}, {"sent": "xxx Johnny walker red .", "words": ["red", "walker"]}, {"sent": "walk with your walker ?", "words": ["your", "walker", "walk", "with"]}, {"sent": "put you in your walker sweetiepie .", "words": ["put", "walker", "your", "you", "in"]}, {"sent": "youre such a good walker .", "words": ["walker", "good", "a"]}, {"sent": "do you wanna fetch your baby walker over ?", "words": ["do", "walker", "your", "you", "wanna", "over"]}, {"sent": "youve just taken the wrapper from baby walker .", "words": ["the", "walker"]}, {"sent": "come here little walker .", "words": ["here", "walker", "little"]}, {"sent": "he loves in that walker .", "words": ["that", "in", "walker", "he"]}, {"sent": "she wasnt a quick walker .", "words": ["walker", "she", "a"]}, {"sent": "hehe and Olivia the tightrope walker .", "words": ["the", "walker", "and"]}, {"sent": "last summer I was Gaspar the dog walker .", "words": ["the", "walker", "was", "last", "dog"]}, {"sent": "youre a good walker sweetie .", "words": ["walker", "good", "a"]}, {"sent": "oh school yes and thats Gospal the dog walker .", "words": ["the", "walker", "school", "and", "dog"]}, {"sent": "last year I was Gospal the dog walker .", "words": ["the", "walker", "was", "last", "dog"]}, {"sent": "last summer I was Gospal the dog dog walker when our neighbors went on vacation I walked their dogs .", "words": ["the", "walker", "their", "was", "last", "our", "on", "when", "dog"]}, {"sent": "then I will be a tightrope walker said the mother bunny and I will walk to where you are .", "words": ["the", "where", "walker", "bunny", "be", "to", "and", "you", "are", "will", "then", "walk", "a"]}, {"sent": "baby walker .", "words": ["walker"]}, {"sent": "and hes sitting on the baby walker with you isnt he ?", "words": ["the", "walker", "with", "and", "you", "he", "on"]}, {"sent": "slow walker too .", "words": ["walker", "slow", "too"]}, {"sent": "I Im laughing because we have a lit she has a walker thing thats a stroller .", "words": ["walker", "we", "have", "stroller", "she", "because", "a"]}, {"sent": "shes a fast walker .", "words": ["fast", "walker", "a"]}, {"sent": "walker ?", "words": ["walker"]}, {"sent": "maybe thats a baby walker .", "words": ["walker", "a"]}, {"sent": "I love being the tight rope walker .", "words": ["the", "walker", "love"]}, {"sent": "well come over here and play with the little walker .", "words": ["the", "walker", "play", "with", "and", "here", "little", "over"]}, {"sent": "you gonna push that walker across the room ?", "words": ["the", "walker", "push", "you", "room", "that"]}, {"sent": "gotta move the walker .", "words": ["the", "walker"]}, {"sent": "walk with your walker ?", "words": ["your", "walker", "walk", "with"]}, {"sent": "you dont never like the walker .", "words": ["the", "walker", "like", "you", "dont"]}, {"sent": "you wanna go in your walker ?", "words": ["walker", "your", "go", "you", "in", "wanna"]}, {"sent": "you wanna go in your walker ?", "words": ["walker", "your", "go", "you", "in", "wanna"]}, {"sent": "you can go in your walker if you wanna .", "words": ["can", "walker", "your", "if", "go", "you", "in", "wanna"]}, {"sent": "xxx put you in the walker then .", "words": ["the", "put", "walker", "you", "in", "then"]}, {"sent": "do you wanna go in your walker ?", "words": ["do", "walker", "your", "go", "you", "in", "wanna"]}, {"sent": "you little walker boy .", "words": ["walker", "little", "you"]}, {"sent": "your walker blocks it .", "words": ["walker", "it", "your"]}, {"sent": "lets get outof your walker .", "words": ["get", "walker", "your"]}, {"sent": "did you like that walker ?", "words": ["walker", "like", "you", "did", "that"]}, {"sent": "I think you liked that walker she had .", "words": ["think", "walker", "you", "that", "she"]}, {"sent": "are you walking around with the baby walker ?", "words": ["the", "walker", "with", "you", "are", "around"]}, {"sent": "its the baby walker .", "words": ["the", "walker"]}, {"sent": "youve put the baby walker away and youve brought out the green truck with the building bricks have you ?", "words": ["the", "put", "walker", "truck", "with", "and", "have", "you", "away", "out", "green"]}, {"sent": "well you did a poo and then you took the blue triangle plastic shape from your baby walker and you started making shapes in your poo didnt you ?", "words": ["the", "walker", "your", "and", "you", "blue", "in", "did", "then", "a"]}, {"sent": "the wheel on the baby walker is round .", "words": ["the", "on", "walker", "is"]}, {"sent": "is there anything on the baby walker with a star ?", "words": ["the", "walker", "is", "star", "with", "there", "on", "a"]}, {"sent": "shall we put the green triangle from the baby walker on top of the green triangle in the book ?", "words": ["the", "put", "walker", "we", "of", "in", "green", "on", "book"]}, {"sent": "oh no you mean your baby walker dont you ?", "words": ["dont", "walker", "your", "you"]}, {"sent": "because you dont need a baby walker now do you ?", "words": ["do", "walker", "need", "you", "dont", "because", "a"]}, {"sent": "and your baby walker .", "words": ["walker", "your", "and"]}, {"sent": "uhuh that is his walker like Amy has .", "words": ["walker", "is", "like", "his", "that"]}, {"sent": "shall we go and get you a tissue ?", "words": ["go", "we", "tissue", "you", "get", "and", "a"]}, {"sent": "let me go and get some tissue .", "words": ["go", "tissue", "and", "get", "me", "some"]}, {"sent": "Kevin give mommy a tissue please .", "words": ["give", "tissue", "a"]}, {"sent": "Im just going to get a tissue and well deal with that .", "words": ["tissue", "to", "with", "and", "get", "that", "a"]}, {"sent": "you need a tissue honey ?", "words": ["tissue", "need", "a", "you"]}, {"sent": "xxx tissue .", "words": ["tissue"]}, {"sent": "a tissue miss you .", "words": ["tissue", "you", "a"]}, {"sent": "okay Im gonna go get a tissue for William .", "words": ["for", "tissue", "go", "get", "a"]}, {"sent": "you need a tissue I know .", "words": ["tissue", "need", "a", "you"]}, {"sent": "shall I get an tissue for you ?", "words": ["for", "tissue", "you", "get", "an"]}, {"sent": "have you got a tissue ?", "words": ["have", "tissue", "a", "you"]}, {"sent": "get a tissue .", "words": ["get", "tissue", "a"]}, {"sent": "xxx get a tissue ?", "words": ["get", "tissue", "a"]}, {"sent": "xxx some tissue for that nose .", "words": ["for", "tissue", "some", "that", "nose"]}, {"sent": "tissue .", "words": ["tissue"]}, {"sent": "go and get a tissue please .", "words": ["tissue", "go", "and", "get", "a"]}, {"sent": "a tissue dont you ?", "words": ["dont", "tissue", "you", "a"]}, {"sent": "loo roll tissue .", "words": ["tissue"]}, {"sent": "mind the tissue .", "words": ["the", "tissue"]}, {"sent": "a tissue ?", "words": ["tissue", "a"]}, {"sent": "you need a tissue when you sneezed .", "words": ["need", "tissue", "you", "when", "a"]}, {"sent": "tissue .", "words": ["tissue"]}, {"sent": "we need some tissue .", "words": ["tissue", "some", "need", "we"]}, {"sent": "okay lets take the tissue thats right down there .", "words": ["the", "tissue", "there", "down", "take"]}, {"sent": "its like a tissue .", "words": ["tissue", "like", "a"]}, {"sent": "okay mommys gotta get a tissue .", "words": ["get", "tissue", "a"]}, {"sent": "come here get a tissue .", "words": ["here", "get", "tissue", "a"]}, {"sent": "let Mommy wipe your nose with a tissue okay ?", "words": ["your", "tissue", "nose", "with", "wipe", "a"]}, {"sent": "would you like a tissue ?", "words": ["tissue", "like", "you", "would", "a"]}, {"sent": "yeah well Ill get you a tissue for that .", "words": ["for", "tissue", "you", "get", "that", "a"]}, {"sent": "wheres your tissue ?", "words": ["your", "tissue"]}, {"sent": "Ill just get a tissue .", "words": ["get", "tissue", "a"]}, {"sent": "you need a tissue for you nose and a wipe for your face dont you ?", "words": ["for", "need", "your", "face", "tissue", "nose", "you", "and", "dont", "wipe", "a"]}, {"sent": "could you grab him a tissue love ?", "words": ["tissue", "you", "him", "love", "could", "a"]}, {"sent": "and while youre eating your cake Mummys going to get some cellotape and the scissors and were going to repair the tissue box .", "words": ["the", "your", "box", "cake", "tissue", "to", "scissors", "and", "get", "some", "were"]}, {"sent": "er shall we get a tissue ?", "words": ["get", "tissue", "a", "we"]}, {"sent": "Ill go and get a tissue .", "words": ["tissue", "go", "and", "get", "a"]}, {"sent": "youre sposta wait til the tissue comes before you blow .", "words": ["the", "wait", "tissue", "you", "blow"]}, {"sent": "do you want a tissue Nomi ?", "words": ["do", "tissue", "a", "you"]}, {"sent": "would you like a tissue ?", "words": ["tissue", "like", "you", "would", "a"]}, {"sent": "that tissue .", "words": ["that", "tissue"]}, {"sent": "do you want a tissue ?", "words": ["do", "tissue", "a", "you"]}, {"sent": "go and find that tissue and blow your nose .", "words": ["your", "tissue", "go", "and", "find", "that", "blow", "nose"]}, {"sent": "huhh give me a tissue Deidre .", "words": ["me", "give", "tissue", "a"]}, {"sent": "let me get a tissue for your nose .", "words": ["for", "your", "tissue", "nose", "get", "me", "a"]}, {"sent": "you need a tissue honey .", "words": ["tissue", "need", "a", "you"]}, {"sent": "is that your tissue ?", "words": ["your", "tissue", "that", "is"]}, {"sent": "Mummys going to hafta go and get a cloth now let alone a tissue .", "words": ["go", "tissue", "to", "and", "get", "a"]}, {"sent": "Mummy has just gone to reach a tissue down from the box and she has discovered Daddys work keys so I bet he has been in a mess today without his keys .", "words": ["the", "keys", "box", "work", "tissue", "to", "so", "and", "in", "he", "his", "down", "she", "a"]}, {"sent": "let me get you a different tissue then .", "words": ["tissue", "you", "get", "me", "then", "a"]}, {"sent": "envelopes and tissue paper .", "words": ["paper", "tissue", "and"]}, {"sent": "get a tissue Lara .", "words": ["get", "tissue", "a"]}, {"sent": "would you please pick up the tissue paper and put it all in the garbage ?", "words": ["the", "up", "put", "all", "tissue", "pick", "paper", "you", "and", "in", "it", "would", "garbage"]}, {"sent": "oh tissue .", "words": ["tissue"]}, {"sent": "mom would you gimme a tissue ?", "words": ["a", "tissue", "you", "would"]}, {"sent": "because I cant find a tissue .", "words": ["a", "tissue", "because", "find"]}, {"sent": "theres a tissue for you .", "words": ["for", "tissue", "you", "a"]}, {"sent": "add screwed up tissue paper and bottle tops and paint it brightly .", "words": ["up", "paint", "tissue", "paper", "and", "it", "bottle"]}, {"sent": "whered we put our tissue at ?", "words": ["put", "tissue", "we", "our", "at"]}, {"sent": "theres bits of tissue lying on the floor .", "words": ["of", "the", "on", "tissue"]}, {"sent": "do you want a tissue Lara ?", "words": ["do", "tissue", "a", "you"]}, {"sent": "tissue .", "words": ["tissue"]}, {"sent": "do you want a tissue ?", "words": ["do", "tissue", "a", "you"]}, {"sent": "do you want a tissue ?", "words": ["do", "tissue", "a", "you"]}, {"sent": "need a tissue ?", "words": ["tissue", "need", "a"]}, {"sent": "are you taking a tissue from the box ?", "words": ["the", "box", "tissue", "you", "are", "a"]}, {"sent": "go and get tissue .", "words": ["get", "tissue", "and", "go"]}, {"sent": "this is a clean tissue .", "words": ["clean", "tissue", "is", "this", "a"]}, {"sent": "have you got Ill just get a tissue then .", "words": ["tissue", "you", "have", "get", "then", "a"]}, {"sent": "here Ill get a tissue for you if I can reach .", "words": ["can", "for", "if", "tissue", "you", "here", "get", "a"]}, {"sent": "so when you need your tissue .", "words": ["your", "need", "tissue", "so", "you", "when"]}, {"sent": "with the tissue ?", "words": ["the", "with", "tissue"]}, {"sent": "youre not eating the tissue .", "words": ["the", "not", "tissue"]}, {"sent": "Ill put the tissue up my sleeve for your nose .", "words": ["the", "up", "put", "my", "for", "your", "tissue", "nose"]}, {"sent": "well do it with a tissue .", "words": ["do", "tissue", "with", "it", "a"]}, {"sent": "I hafta get a tissue actually Lara .", "words": ["get", "tissue", "a"]}, {"sent": "dyou want a tissue ?", "words": ["tissue", "a"]}, {"sent": "tissue .", "words": ["tissue"]}, {"sent": "why dont ya ask mommy for a tissue ?", "words": ["for", "tissue", "dont", "why", "a"]}, {"sent": "Mamas just going to get a tissue from the other room .", "words": ["the", "tissue", "to", "other", "get", "room", "a"]}, {"sent": "tissue ?", "words": ["tissue"]}, {"sent": "lets see if I can get some more tissue paper to put up there .", "words": ["can", "up", "put", "if", "tissue", "to", "paper", "get", "there", "more", "see", "some"]}, {"sent": "tissue .", "words": ["tissue"]}, {"sent": "Im gonna get the tissue out of the bathroom .", "words": ["the", "tissue", "of", "get", "out", "bathroom"]}, {"sent": "Mum put the tissue here so when your nose runs you can you can pick up the tissue .", "words": ["the", "can", "put", "up", "your", "tissue", "pick", "so", "you", "here", "when", "nose"]}, {"sent": "hey you know what mommas gonna go get a tissue for your nose .", "words": ["what", "for", "your", "go", "tissue", "nose", "you", "get", "a"]}, {"sent": "you wanna throw a tissue ?", "words": ["throw", "tissue", "you", "wanna", "a"]}, {"sent": "oh we need a tissue wait a minute .", "words": ["wait", "need", "tissue", "we", "a"]}, {"sent": "Xavier did you hide Mommys tissue ?", "words": ["tissue", "did", "hide", "you"]}, {"sent": "here sweetie heres a tissue for you .", "words": ["for", "tissue", "you", "here", "a"]}, {"sent": "a nice tissue .", "words": ["nice", "tissue", "a"]}, {"sent": "youve a tissue in this pocket as well .", "words": ["this", "in", "tissue", "a"]}, {"sent": "wheres that tissue gone ?", "words": ["that", "tissue"]}, {"sent": "Im just gonna find a tissue for my nose .", "words": ["my", "for", "tissue", "nose", "find", "a"]}, {"sent": "yeah I need to get you a tissue .", "words": ["need", "tissue", "need to", "to", "you", "get", "a"]}, {"sent": "shall we get a tissue and wipe your nose ?", "words": ["your", "tissue", "we", "nose", "and", "get", "wipe", "a"]}, {"sent": "whoopsie heres a tissue for you .", "words": ["for", "tissue", "you", "a"]}, {"sent": "Ill get a tissue then .", "words": ["get", "then", "tissue", "a"]}, {"sent": "I took this pillow .", "words": ["this", "pillow"]}, {"sent": "butterfly pillow xxx .", "words": ["butterfly", "pillow"]}, {"sent": "he pooped on his pillow ?", "words": ["on", "his", "he", "pillow"]}, {"sent": "heres the car going on the pillow .", "words": ["the", "car", "on", "pillow"]}, {"sent": "this pillow with a .", "words": ["this", "with", "a", "pillow"]}, {"sent": "where is your pillow .", "words": ["where", "your", "pillow", "is"]}, {"sent": "pillow snuffer .", "words": ["pillow"]}, {"sent": "ninight on Daddys pillow .", "words": ["on", "pillow"]}, {"sent": "you can have it for your pillow if you like .", "words": ["can", "for", "your", "if", "like", "you", "have", "it", "pillow"]}, {"sent": "her pillow ?", "words": ["her", "pillow"]}, {"sent": "not that pillow .", "words": ["that", "not", "pillow"]}, {"sent": "xxx the butterfly moved behind this pillow .", "words": ["the", "butterfly", "behind", "this", "pillow"]}, {"sent": "he took a pillow and went outside .", "words": ["outside", "and", "he", "a", "pillow"]}, {"sent": "their heads on the pillow right ?", "words": ["on", "the", "their", "pillow"]}, {"sent": "and you do the um pillow and the letter one .", "words": ["the", "do", "you", "and", "pillow"]}, {"sent": "do you want this pillow ?", "words": ["do", "pillow", "this", "you"]}, {"sent": "do you know someone who has a pillow and covers ?", "words": ["do", "you", "and", "who", "a", "pillow"]}, {"sent": "and do you relax and lie on the pillow ?", "words": ["the", "do", "you", "and", "on", "pillow"]}, {"sent": "go throw your pillow .", "words": ["your", "throw", "pillow", "go"]}, {"sent": "wheres the mushy pillow ?", "words": ["the", "pillow"]}, {"sent": "and that is another kind of pillow .", "words": ["is", "of", "and", "another", "that", "pillow"]}, {"sent": "on a pillow .", "words": ["on", "pillow", "a"]}, {"sent": "feathers theyre having a pillow fight uhn arent they ?", "words": ["they", "pillow", "a"]}, {"sent": "can you see under the pillow ?", "words": ["can", "the", "you", "under", "see", "pillow"]}, {"sent": "you crawled back and touched your pillow when you said that .", "words": ["your", "back", "and", "you", "that", "when", "pillow"]}, {"sent": "you dont want the pillow ?", "words": ["the", "dont", "pillow", "you"]}, {"sent": "put your head on your pillow .", "words": ["put", "your", "head", "on", "pillow"]}, {"sent": "you wanna lie on your pillow ?", "words": ["your", "you", "wanna", "on", "pillow"]}, {"sent": "whos on the pillow ?", "words": ["on", "the", "pillow"]}, {"sent": "come here whats on Grandmas pillow ?", "words": ["here", "on", "pillow"]}, {"sent": "ninight you gonna put your head on your pillow .", "words": ["put", "your", "head", "you", "on", "pillow"]}, {"sent": "its the pillow case .", "words": ["the", "pillow"]}, {"sent": "pillow to go nighnigh on .", "words": ["go", "to", "on", "pillow"]}, {"sent": "I put it like that and I use it as a pillow .", "words": ["put", "pillow", "like", "and", "it", "that", "a"]}, {"sent": "I hadta wash your pillow this morning .", "words": ["your", "this", "wash", "pillow"]}, {"sent": "pillow pillow pillow .", "words": ["pillow"]}, {"sent": "theres a little pillow aswell .", "words": ["pillow", "little", "a"]}, {"sent": "one pillow .", "words": ["pillow"]}, {"sent": "did you have a pillow .", "words": ["you", "have", "did", "a", "pillow"]}, {"sent": "do you want your pillow at the back darling so its more comfortable ?", "words": ["the", "do", "your", "back", "so", "you", "more", "at", "pillow"]}, {"sent": "you got that one for your pillow .", "words": ["for", "your", "you", "that", "pillow"]}, {"sent": "and what did the lady leave on our pillow every night ?", "words": ["the", "what", "and", "our", "did", "every", "on", "pillow"]}, {"sent": "a pillow .", "words": ["pillow", "a"]}, {"sent": "you mean a blue pillow ?", "words": ["blue", "pillow", "a", "you"]}, {"sent": "you found your pillow .", "words": ["your", "pillow", "you"]}, {"sent": "you dont need your pillow at school .", "words": ["your", "need", "school", "you", "dont", "at", "pillow"]}, {"sent": "it was on but it was rubbing against the pillow .", "words": ["the", "was", "it", "on", "but", "pillow"]}, {"sent": "there is your pillow .", "words": ["there", "your", "pillow", "is"]}, {"sent": "throw your pillow .", "words": ["your", "throw", "pillow"]}, {"sent": "arent you a pillow ?", "words": ["pillow", "a", "you"]}, {"sent": "throw that pillow .", "words": ["that", "throw", "pillow"]}, {"sent": "where is your pillow .", "words": ["where", "your", "pillow", "is"]}, {"sent": "can you put the pillow underneath the mouses head ?", "words": ["can", "the", "put", "head", "you", "pillow"]}, {"sent": "pillow .", "words": ["pillow"]}, {"sent": "Jojos pillow .", "words": ["pillow"]}, {"sent": "wheres the babys pillow ?", "words": ["the", "pillow"]}, {"sent": "it could be it looks like a pillow .", "words": ["pillow", "be", "like", "it", "could", "a"]}, {"sent": "sit on the big pillow .", "words": ["the", "on", "sit", "big", "pillow"]}, {"sent": "does the big pillow have a zipper ?", "words": ["the", "does", "have", "zipper", "big", "a", "pillow"]}, {"sent": "oh here comes the pillow .", "words": ["here", "the", "pillow"]}, {"sent": "is that Awwws pillow ?", "words": ["that", "pillow", "is"]}, {"sent": "its a pillow case really .", "words": ["pillow", "a"]}, {"sent": "thats Mummys pillow .", "words": ["pillow"]}, {"sent": "you want a blanket or a pillow ?", "words": ["blanket", "pillow", "a", "you"]}, {"sent": "fancy swinging a pillow at her Mummy .", "words": ["at", "her", "pillow", "a"]}, {"sent": "whos on your pillow ?", "words": ["on", "your", "pillow"]}, {"sent": "where is your pillow .", "words": ["where", "your", "pillow", "is"]}, {"sent": "go find your pillow .", "words": ["your", "pillow", "find", "go"]}, {"sent": "see how the goose was gonna give a pillow and Danny didnt need one .", "words": ["the", "goose", "how", "need", "pillow", "was", "and", "see", "give", "a"]}, {"sent": "do you want a pillow for the mouse too ?", "words": ["the", "do", "for", "you", "mouse", "too", "a", "pillow"]}, {"sent": "theres a pillow on the bus ?", "words": ["the", "on", "bus", "a", "pillow"]}, {"sent": "yeah you did pee pee on your pillow .", "words": ["your", "you", "did", "on", "pillow"]}, {"sent": "and youre not gonna pull the pillow off of my chair .", "words": ["the", "off", "my", "of", "chair", "pull", "and", "not", "pillow"]}, {"sent": "you want the pillow ?", "words": ["the", "pillow", "you"]}, {"sent": "does Pat the bunny like uh the squishy pillow ?", "words": ["the", "does", "bunny", "like", "pillow"]}, {"sent": "can you go ninight on your pillow ?", "words": ["can", "your", "go", "you", "on", "pillow"]}, {"sent": "good you crawled over to your pillow to pick it up and threw it .", "words": ["up", "your", "pick", "to", "you", "and", "it", "good", "over", "pillow"]}, {"sent": "shes got a pillow .", "words": ["pillow", "a"]}, {"sent": "and where did you have the pillow fights ?", "words": ["where", "the", "you", "have", "and", "did", "pillow"]}, {"sent": "pillow ?", "words": ["pillow"]}, {"sent": "pillow .", "words": ["pillow"]}, {"sent": "is it called the pillow hospital ?", "words": ["the", "it", "pillow", "is"]}, {"sent": "youre a wriggly pillow .", "words": ["pillow", "a"]}, {"sent": "that doll slept on the pillow beside him .", "words": ["the", "beside", "doll", "him", "on", "that", "pillow"]}, {"sent": "where is your pillow .", "words": ["where", "your", "pillow", "is"]}, {"sent": "pillow .", "words": ["pillow"]}, {"sent": "no bother assoonas his head hits the pillow thats him .", "words": ["the", "head", "him", "his", "pillow"]}, {"sent": "do you wanna lay your head on that nice pillow ?", "words": ["do", "your", "head", "nice", "you", "wanna", "on", "that", "pillow"]}, {"sent": "your Mommy did needlepoint on that pillow .", "words": ["your", "did", "on", "that", "pillow"]}, {"sent": "pillow .", "words": ["pillow"]}, {"sent": "did Daddy try and swipe your squishy pillow to sleep with last night ?", "words": ["your", "sleep", "to", "last", "with", "and", "did", "try", "pillow"]}, {"sent": "and whose pillow is that ?", "words": ["that", "pillow", "is", "and"]}, {"sent": "theres a cloud on that pillow .", "words": ["cloud", "on", "that", "a", "pillow"]}, {"sent": "come on back over here and lets play with your pillow .", "words": ["your", "back", "play", "with", "and", "here", "on", "over", "pillow"]}, {"sent": "are you going to come and have a look under this pillow and see if the lions there ?", "words": ["the", "pillow", "if", "to", "and", "have", "you", "under", "are", "this", "there", "look", "see", "a"]}, {"sent": "Ive got a lumpy pillow .", "words": ["pillow", "a"]}, {"sent": "because that was Jwwws pillow wasnt it ?", "words": ["was", "it", "that", "because", "pillow"]}, {"sent": "mommy it wont draw .", "words": ["it", "draw"]}, {"sent": "you could have your own room with a big chalkboard to draw on .", "words": ["your", "to", "draw", "with", "you", "have", "room", "on", "could", "big", "a"]}, {"sent": "what are you gonna draw ?", "words": ["what", "are", "draw", "you"]}, {"sent": "draw big big black clouds like this .", "words": ["draw", "like", "black", "this", "big"]}, {"sent": "do you know what youre going to draw ?", "words": ["do", "what", "to", "draw", "you"]}, {"sent": "everyone likes to draw and paint .", "words": ["to", "draw", "paint", "and"]}, {"sent": "it wont roll if I draw it .", "words": ["it", "draw", "if"]}, {"sent": "you draw mummy you draw a flower .", "words": ["flower", "draw", "a", "you"]}, {"sent": "can you draw some more ?", "words": ["can", "draw", "you", "more", "some"]}, {"sent": "mhm were gonna draw purple .", "words": ["draw", "were"]}, {"sent": "do you think that Eleanor can draw draw it with her feet ?", "words": ["can", "do", "think", "draw", "with", "her", "you", "it", "that"]}, {"sent": "will you draw Mommy a brown picture ?", "words": ["brown", "draw", "picture", "you", "will", "a"]}, {"sent": "uhuh you draw on yours .", "words": ["on", "draw", "you"]}, {"sent": "draw a big circle there .", "words": ["there", "draw", "big", "a"]}, {"sent": "you draw one .", "words": ["draw", "you"]}, {"sent": "draw some sand ?", "words": ["some", "draw"]}, {"sent": "are you going to draw something else ?", "words": ["to", "are", "draw", "you"]}, {"sent": "should we sit down here and well draw a picture ?", "words": ["we", "draw", "picture", "and", "here", "sit", "down", "a"]}, {"sent": "you draw the neck .", "words": ["the", "draw", "you"]}, {"sent": "draw over here on the paper .", "words": ["the", "paper", "draw", "here", "on", "over"]}, {"sent": "are you gonna draw a happy one then ?", "words": ["happy", "draw", "you", "are", "then", "a"]}, {"sent": "go like that with your hands and then Ill draw eyes there .", "words": ["your", "go", "draw", "with", "like", "and", "there", "then", "that"]}, {"sent": "can you turn it over and draw on the other side then please .", "words": ["can", "the", "draw", "and", "you", "other", "it", "on", "then", "over"]}, {"sent": "draw a picnic ?", "words": ["draw", "a"]}, {"sent": "what are you gonna draw now for me ?", "words": ["what", "for", "draw", "you", "are", "me"]}, {"sent": "can you draw a .", "words": ["can", "draw", "a", "you"]}, {"sent": "you wanna draw ?", "words": ["draw", "wanna", "you"]}, {"sent": "did you draw a b c ?", "words": ["draw", "did", "a", "you"]}, {"sent": "what you draw ?", "words": ["what", "draw", "you"]}, {"sent": "what shall we draw now ?", "words": ["what", "draw", "we"]}, {"sent": "because she wanted you to draw a picture .", "words": ["to", "draw", "picture", "you", "she", "because", "a"]}, {"sent": "you draw raindrops here .", "words": ["here", "draw", "you"]}, {"sent": "you just draw with it .", "words": ["it", "draw", "with", "you"]}, {"sent": "okay you draw one you draw one and Ill draw one okay ?", "words": ["draw", "and", "you"]}, {"sent": "lets see you draw a circle .", "words": ["see", "draw", "a", "you"]}, {"sent": "shall we draw some pliers with brown ?", "words": ["brown", "we", "draw", "with", "some"]}, {"sent": "you want me to draw a teddy ?", "words": ["to", "draw", "you", "me", "a"]}, {"sent": "are you going to sit down and draw your toys ?", "words": ["your", "to", "draw", "you", "and", "are", "sit", "down"]}, {"sent": "whats the pencil gonna draw ?", "words": ["the", "draw", "pencil"]}, {"sent": "do you want me to draw a cat ?", "words": ["do", "to", "draw", "you", "cat", "me", "a"]}, {"sent": "draw a line from each rod to show who will catch what .", "words": ["what", "each", "show", "to", "draw", "catch", "who", "will", "a"]}, {"sent": "Mummys going to draw .", "words": ["to", "draw"]}, {"sent": "man what you draw ?", "words": ["what", "draw", "you"]}, {"sent": "can you draw an a b c ?", "words": ["can", "draw", "you", "an", "a"]}, {"sent": "wanna draw a picture ?", "words": ["draw", "picture", "wanna", "a"]}, {"sent": "what do you want me to draw ?", "words": ["what", "do", "to", "draw", "you", "me"]}, {"sent": "draw draw draw .", "words": ["draw"]}, {"sent": "comeon draw a draw a square .", "words": ["draw", "a"]}, {"sent": "but you can still draw a snake .", "words": ["can", "draw", "you", "but", "a"]}, {"sent": "I want you do draw a dot thats over the line .", "words": ["the", "do", "draw", "you", "over", "a"]}, {"sent": "do you want me to draw .", "words": ["do", "to", "draw", "you", "me"]}, {"sent": "he couldnt draw .", "words": ["draw", "he"]}, {"sent": "wanna draw well draw a brick .", "words": ["draw", "wanna", "a"]}, {"sent": "can you draw that now ?", "words": ["can", "draw", "that", "you"]}, {"sent": "you dont draw on your feet .", "words": ["your", "draw", "you", "dont", "on"]}, {"sent": "can you draw some crosses on them ?", "words": ["can", "draw", "them", "you", "on", "some"]}, {"sent": "Im gonna draw a cat for you if I can do it .", "words": ["can", "do", "for", "if", "draw", "you", "cat", "it", "a"]}, {"sent": "you gotta draw one there right and we fold it in half and we do another one yeah ?", "words": ["do", "we", "draw", "and", "you", "there", "in", "it", "another"]}, {"sent": "can you color in my circle if I draw a circle ?", "words": ["can", "my", "if", "draw", "you", "in", "a"]}, {"sent": "okay well just draw one circle and another dark circle in the middle and it can be an eye .", "words": ["the", "can", "be", "draw", "eye", "and", "another", "in", "it", "an", "dark"]}, {"sent": "yeah but I wanna see if you can draw that .", "words": ["can", "if", "draw", "you", "wanna", "see", "that", "but"]}, {"sent": "you gonna draw a truck ?", "words": ["draw", "truck", "a", "you"]}, {"sent": "shall we draw what shall we do now ?", "words": ["what", "do", "draw", "we"]}, {"sent": "go on then draw Sammy snake .", "words": ["on", "then", "draw", "go"]}, {"sent": "can you draw a kitty ?", "words": ["can", "draw", "you", "kitty", "a"]}, {"sent": "all right what should we draw ?", "words": ["all", "what", "draw", "we"]}, {"sent": "want me to draw a swing ?", "words": ["swing", "to", "draw", "me", "a"]}, {"sent": "what shall I draw first ?", "words": ["what", "draw", "first"]}, {"sent": "oh no you cant draw on the block sweetie only on the paper .", "words": ["the", "block", "paper", "draw", "you", "on"]}, {"sent": "draw the fish .", "words": ["the", "draw", "fish"]}, {"sent": "draw a little red nose .", "words": ["red", "draw", "little", "a", "nose"]}, {"sent": "wanna draw a pumpkin ?", "words": ["pumpkin", "draw", "wanna", "a"]}, {"sent": "you gonna draw a picture of McDonalds restaurant ?", "words": ["of", "draw", "picture", "you", "a"]}, {"sent": "draw a wheel ?", "words": ["draw", "a"]}, {"sent": "draw with the crayons okay .", "words": ["the", "draw", "with"]}, {"sent": "go ahead and draw me while I draw this .", "words": ["go", "draw", "and", "this", "me"]}, {"sent": "oh did you draw the stem on that one ?", "words": ["the", "draw", "you", "did", "on", "that"]}, {"sent": "what else shall we draw ?", "words": ["what", "draw", "we"]}, {"sent": "here Ill draw well draw a house .", "words": ["here", "draw", "house", "a"]}, {"sent": "comeon draw mommy a page pretty picture .", "words": ["draw", "pretty", "picture", "a"]}, {"sent": "I dont know if I can draw a better dog .", "words": ["can", "better", "if", "draw", "dont", "dog", "a"]}, {"sent": "can you draw me a picture ?", "words": ["can", "draw", "picture", "you", "me", "a"]}, {"sent": "draw that one ?", "words": ["draw", "that"]}, {"sent": "she has to draw a staircase she cant draw a staircase .", "words": ["to", "draw", "she", "a"]}, {"sent": "come here Mamall draw you a pig come here .", "words": ["draw", "pig", "here", "you", "a"]}, {"sent": "youre gonna draw Daddy a deer ?", "words": ["deer", "draw", "a"]}, {"sent": "how do you draw a baby one ?", "words": ["do", "how", "draw", "you", "a"]}, {"sent": "what do you wanna draw on ?", "words": ["what", "do", "draw", "you", "wanna", "on"]}, {"sent": "draw on the other side huh ?", "words": ["on", "the", "draw", "other"]}, {"sent": "youd like to draw it ?", "words": ["to", "draw", "like", "it"]}, {"sent": "that is it draw around your fingers .", "words": ["your", "is", "draw", "it", "around", "that"]}, {"sent": "oh theres your painting and draw .", "words": ["draw", "your", "and"]}, {"sent": "on top of your draw on top of your toy box .", "words": ["your", "box", "of", "draw", "toy", "on"]}, {"sent": "can you draw mommy a firetruck ?", "words": ["can", "draw", "you", "firetruck", "a"]}, {"sent": "mummie draw that for me .", "words": ["me", "draw", "that", "for"]}, {"sent": "can Mummy draw on this piece of paper ?", "words": ["can", "of", "paper", "draw", "this", "on"]}, {"sent": "oh I cant draw bugs .", "words": ["draw"]}, {"sent": "are those your pajamas ?", "words": ["your", "are", "those", "pajamas"]}, {"sent": "oh okay so you think he would wear pajamas ?", "words": ["think", "so", "you", "he", "pajamas", "would"]}, {"sent": "no its nighttime silly here lets button up your pajamas .", "words": ["up", "your", "here", "pajamas", "button"]}, {"sent": "show me do you have a teddy bear on your pajamas ?", "words": ["do", "your", "show", "you", "have", "bear", "pajamas", "on", "me", "a"]}, {"sent": "or is it five minutes to nappie change pajamas on dawdle time delay ?", "words": ["is", "to", "it", "pajamas", "on"]}, {"sent": "or would you like to put on pajamas ?", "words": ["put", "to", "like", "you", "pajamas", "would", "on"]}, {"sent": "get his pajamas on ?", "words": ["get", "his", "on", "pajamas"]}, {"sent": "what are you doing wearing pajamas in the middle of the day ?", "words": ["the", "what", "of", "you", "in", "are", "pajamas"]}, {"sent": "wart have you got on your pajamas ?", "words": ["your", "you", "have", "pajamas", "on"]}, {"sent": "is it WinniethePooh on your pajamas ?", "words": ["your", "is", "it", "pajamas", "on"]}, {"sent": "wait first gotta get pajamas on so youll have something to cover your feet when you go on the floor see its got feetsie in them .", "words": ["the", "wait", "your", "go", "so", "to", "first", "them", "you", "get", "have", "in", "pajamas", "cover", "on", "see", "when"]}, {"sent": "there are your pajamas on the stairs .", "words": ["the", "your", "there", "are", "pajamas", "on", "stairs"]}, {"sent": "wet pajamas .", "words": ["wet", "pajamas"]}, {"sent": "is he wearing pajamas ?", "words": ["he", "pajamas", "is"]}, {"sent": "you know when I came to pick you up you didnt have pajamas on .", "words": ["up", "pick", "to", "you", "have", "pajamas", "on", "when"]}, {"sent": "Brett do you wanna xxx with pajamas on ?", "words": ["do", "with", "you", "pajamas", "wanna", "on"]}, {"sent": "youve still got your pajamas on .", "words": ["on", "your", "pajamas"]}, {"sent": "these are pajamas .", "words": ["these", "are", "pajamas"]}, {"sent": "Thomass blue pajamas .", "words": ["pajamas", "blue"]}, {"sent": "oh these are my favorite pajamas .", "words": ["these", "my", "are", "pajamas"]}, {"sent": "hang on lets put your pajamas on please .", "words": ["on", "put", "your", "pajamas"]}, {"sent": "whose pajamas are they ?", "words": ["they", "are", "pajamas"]}, {"sent": "yeah Katy put on her pajamas but Kit couldnt find his pajamas anywhere .", "words": ["put", "his", "pajamas", "find", "on", "her", "but"]}, {"sent": "we were trying to get your pajamas on .", "words": ["your", "we", "to", "get", "pajamas", "on", "were"]}, {"sent": "put on your pajamas Brett .", "words": ["on", "put", "your", "pajamas"]}, {"sent": "and pajamas .", "words": ["pajamas", "and"]}, {"sent": "he came very early and we were still in our pajamas werent we ?", "words": ["we", "and", "our", "in", "he", "pajamas", "were"]}, {"sent": "oh are they your pajamas ?", "words": ["they", "are", "your", "pajamas"]}, {"sent": "lets get your pajamas and put them in here so theyll be ready for us .", "words": ["put", "for", "your", "so", "be", "them", "and", "get", "here", "in", "us", "pajamas"]}, {"sent": "pajamas by yourself .", "words": ["yourself", "pajamas", "by"]}, {"sent": "thats pajamas .", "words": ["pajamas"]}, {"sent": "what color are your pajamas ?", "words": ["what", "are", "your", "pajamas"]}, {"sent": "oh look she has pajamas on .", "words": ["on", "pajamas", "she", "look"]}, {"sent": "Eleanors not got pajamas .", "words": ["pajamas", "not"]}, {"sent": "or do you wanna put your pajamas on first ?", "words": ["do", "put", "your", "first", "you", "pajamas", "wanna", "on"]}, {"sent": "I bet those are pretty pajamas .", "words": ["pretty", "are", "those", "pajamas"]}, {"sent": "you wanna wear your pajamas today ?", "words": ["your", "pajamas", "wanna", "you"]}, {"sent": "those are your pajamas there .", "words": ["your", "those", "there", "are", "pajamas"]}, {"sent": "she still had her pajamas on last week didnt she when you went round ?", "words": ["when", "last", "you", "pajamas", "on", "her", "she"]}, {"sent": "youve got your pajamas on have you ?", "words": ["your", "you", "have", "pajamas", "on"]}, {"sent": "xxx I like your pajamas .", "words": ["pajamas", "your", "like"]}, {"sent": "does she want her pajamas ?", "words": ["does", "pajamas", "her", "she"]}, {"sent": "wheres your pajamas ?", "words": ["your", "pajamas"]}, {"sent": "wheres the hole in your pajamas ?", "words": ["the", "in", "your", "pajamas"]}, {"sent": "whats this on your pajamas ?", "words": ["on", "this", "your", "pajamas"]}, {"sent": "and then well put clean pajamas on .", "words": ["clean", "put", "and", "pajamas", "on", "then"]}, {"sent": "your pajamas .", "words": ["your", "pajamas"]}, {"sent": "put pajamas on and we can ride the car .", "words": ["can", "the", "put", "we", "car", "and", "pajamas", "on", "ride"]}, {"sent": "shall we put the pajamas on now ?", "words": ["the", "put", "we", "pajamas", "on"]}, {"sent": "lets put your pajamas on please .", "words": ["on", "put", "your", "pajamas"]}, {"sent": "cause were not gonna settle down and read nurse until you get pajamas on .", "words": ["and", "you", "get", "read", "pajamas", "on", "down", "were", "not"]}, {"sent": "taking you pajamas off ?", "words": ["off", "pajamas", "you"]}, {"sent": "hes got pajamas with feet in them .", "words": ["in", "with", "pajamas", "them"]}, {"sent": "whats on your pajamas ?", "words": ["on", "your", "pajamas"]}, {"sent": "take your pajamas off please .", "words": ["your", "take", "pajamas", "off"]}, {"sent": "more pajamas .", "words": ["more", "pajamas"]}, {"sent": "thesell be your pajamas .", "words": ["be", "your", "pajamas"]}, {"sent": "come on jump in your pajamas .", "words": ["your", "jump", "in", "pajamas", "on"]}, {"sent": "because you had your pajamas on last night didnt you ?", "words": ["your", "last", "you", "pajamas", "on", "because"]}, {"sent": "theyre pajamas .", "words": ["pajamas"]}, {"sent": "put your pajamas on and then do you want a story ?", "words": ["do", "put", "story", "your", "and", "you", "pajamas", "on", "then", "a"]}, {"sent": "PoPat pajamas ?", "words": ["pajamas"]}, {"sent": "shall we take her outof her pajamas ?", "words": ["her", "take", "pajamas", "we"]}, {"sent": "he took off his pajamas and left them on the floor .", "words": ["off", "the", "them", "and", "on", "he", "pajamas", "his"]}, {"sent": "Daddys just gonna go get Jwwws pajamas on .", "words": ["get", "on", "pajamas", "go"]}, {"sent": "now Mommy is putting up Morgies pajamas .", "words": ["up", "pajamas", "is"]}, {"sent": "pajamas .", "words": ["pajamas"]}, {"sent": "pajamas .", "words": ["pajamas"]}, {"sent": "the pair that Mummy has hung already are light blue trousers pajamas .", "words": ["the", "light", "blue", "are", "pajamas", "that"]}, {"sent": "shall we get pajamas now ?", "words": ["get", "pajamas", "we"]}, {"sent": "and oh we just got these new pajamas for xxx from Sam .", "words": ["these", "for", "we", "and", "pajamas", "new"]}, {"sent": "uh yes I have pajamas and everything in Catherines room .", "words": ["and", "have", "in", "room", "pajamas"]}, {"sent": "no lets put your pajamas on now .", "words": ["on", "put", "your", "pajamas"]}, {"sent": "do you know how to put your pajamas on ?", "words": ["do", "how", "put", "your", "to", "you", "pajamas", "on"]}, {"sent": "see if you can find your cuddly pajamas on your bed somewhere .", "words": ["can", "your", "if", "bed", "you", "pajamas", "find", "on", "see"]}, {"sent": "youre not gonna wear pajamas to watch the game is he ?", "words": ["the", "game", "watch", "is", "to", "he", "pajamas", "not"]}, {"sent": "why would you be wearing Jwwws pajamas ?", "words": ["be", "you", "pajamas", "would", "why"]}, {"sent": "and Ive got some pajamas .", "words": ["some", "pajamas", "and"]}, {"sent": "these can be your pajamas .", "words": ["can", "these", "your", "be", "pajamas"]}, {"sent": "bananas in pajamas .", "words": ["in", "pajamas"]}, {"sent": "theres no lions on your pajamas .", "words": ["on", "your", "pajamas"]}, {"sent": "come on put on your pajamas .", "words": ["on", "put", "your", "pajamas"]}, {"sent": "yeah but youre gonna put on your pajamas first .", "words": ["put", "your", "first", "pajamas", "on", "but"]}, {"sent": "get get your pajamas on .", "words": ["get", "on", "your", "pajamas"]}, {"sent": "and Mommy really wants to go to the potty and put on her nightgown while youre putting on your pajamas .", "words": ["the", "put", "your", "go", "to", "and", "potty", "pajamas", "on", "her"]}, {"sent": "do you have cowboys on your pajamas William ?", "words": ["do", "your", "you", "have", "pajamas", "on"]}, {"sent": "have you still got them trousers on pajamas ?", "words": ["them", "you", "have", "pajamas", "on"]}, {"sent": "oh is that her pajamas .", "words": ["her", "that", "pajamas", "is"]}, {"sent": "you were in your pajamas all day yesterday .", "words": ["all", "your", "you", "in", "pajamas", "were"]}, {"sent": "Mummy doesnt go in her pajamas does she ?", "words": ["does", "go", "in", "pajamas", "her", "she"]}, {"sent": "what have you got in your pajamas Fraser ?", "words": ["what", "your", "you", "have", "in", "pajamas"]}, {"sent": "those pajamas dont match .", "words": ["dont", "those", "pajamas"]}, {"sent": "do you wanna find dollys pajamas ?", "words": ["do", "you", "pajamas", "wanna", "find"]}, {"sent": "dollys got her pajamas on ready for bed .", "words": ["for", "bed", "pajamas", "on", "her"]}, {"sent": "I dont know about any xxx pajamas .", "words": ["dont", "any", "pajamas", "about"]}, {"sent": "okay you wanna put your pajamas on ?", "words": ["put", "your", "you", "pajamas", "wanna", "on"]}, {"sent": "how about these pajamas ?", "words": ["these", "how", "pajamas", "about"]}, {"sent": "er because we put our pajamas on at nighttime .", "words": ["put", "we", "our", "pajamas", "at", "on", "because"]}, {"sent": "did you needta take your medicine ?", "words": ["your", "medicine", "you", "did", "take"]}, {"sent": "the tractor can drink the medicine from the bottle cant it ?", "words": ["the", "can", "drink", "medicine", "it", "tractor", "bottle"]}, {"sent": "come and take your white medicine then .", "words": ["your", "medicine", "and", "then", "white", "take"]}, {"sent": "eat that medicine .", "words": ["eat", "medicine", "that"]}, {"sent": "theres spoon to give him his medicine .", "words": ["to", "medicine", "him", "spoon", "his", "give"]}, {"sent": "tell her to have to eat all her medicine .", "words": ["all", "to", "medicine", "have", "eat", "her"]}, {"sent": "so do you feel better now that you took some medicine ?", "words": ["better", "do", "so", "medicine", "you", "some", "that"]}, {"sent": "my my medicine .", "words": ["my", "medicine"]}, {"sent": "give her some more medicine before she goes to bed ?", "words": ["to", "medicine", "bed", "more", "some", "give", "her", "she"]}, {"sent": "just I got medicine on me .", "words": ["on", "me", "medicine"]}, {"sent": "so she has some special medicine off the doctor see ?", "words": ["off", "the", "see", "so", "medicine", "some", "she"]}, {"sent": "do you want some white medicine ?", "words": ["do", "medicine", "you", "some", "white"]}, {"sent": "so I can give you your medicine .", "words": ["can", "your", "so", "medicine", "you", "give"]}, {"sent": "take your medicine .", "words": ["your", "medicine", "take"]}, {"sent": "what color is this medicine ?", "words": ["what", "medicine", "this", "is"]}, {"sent": "he likes honey medicine .", "words": ["medicine", "he"]}, {"sent": "Ill give you some medicine outof this big cup .", "words": ["cup", "medicine", "you", "this", "some", "give", "big"]}, {"sent": "a medicine ?", "words": ["medicine", "a"]}, {"sent": "medicine ?", "words": ["medicine"]}, {"sent": "and well have medicine .", "words": ["have", "medicine", "and"]}, {"sent": "both of you need medicine dont you ?", "words": ["need", "of", "medicine", "you", "dont"]}, {"sent": "now youre getting a taste of your own medicine .", "words": ["your", "of", "medicine", "taste", "a"]}, {"sent": "oh that one needs medicine ?", "words": ["medicine", "that"]}, {"sent": "yay because you took your good medicine .", "words": ["your", "medicine", "you", "good", "because"]}, {"sent": "you hafta have your medicine as well .", "words": ["have", "medicine", "your", "you"]}, {"sent": "new medicine ?", "words": ["medicine", "new"]}, {"sent": "but I think shes holding some cough medicine .", "words": ["some", "think", "but", "medicine"]}, {"sent": "its only Mommys medicine .", "words": ["medicine"]}, {"sent": "shall we put some medicine on your spot ?", "words": ["put", "your", "we", "medicine", "on", "some"]}, {"sent": "I put the medicine there because I hafta xxx .", "words": ["the", "put", "medicine", "there", "because"]}, {"sent": "you feel better with pink medicine dont you ?", "words": ["better", "medicine", "with", "you", "dont"]}, {"sent": "yes put medicine on your bottom .", "words": ["put", "bottom", "your", "medicine", "on"]}, {"sent": "medicine ?", "words": ["medicine"]}, {"sent": "magic medicine .", "words": ["medicine"]}, {"sent": "xxx she have her medicine this afternoon ?", "words": ["medicine", "have", "this", "her", "she"]}, {"sent": "xxx medicine now .", "words": ["medicine"]}, {"sent": "do you think the farmers got any medicine for the cows ?", "words": ["the", "do", "think", "for", "medicine", "any", "you"]}, {"sent": "you like horrible medicine dont you ?", "words": ["dont", "medicine", "like", "you"]}, {"sent": "xxx give you your medicine .", "words": ["give", "medicine", "your", "you"]}, {"sent": "oh youre gonna get some medicine for me ?", "words": ["for", "medicine", "me", "get", "some"]}, {"sent": "you hafta have yellow medicine everyone has a lot of children have yellow medicine .", "words": ["of", "a lot", "medicine", "you", "have", "yellow", "a"]}, {"sent": "have you had your medicine as well ?", "words": ["have", "medicine", "your", "you"]}, {"sent": "no no medicine .", "words": ["medicine"]}, {"sent": "Ive got some more medicine for you .", "words": ["for", "medicine", "you", "more", "some"]}, {"sent": "some medicine .", "words": ["some", "medicine"]}, {"sent": "how about taking this medicine now .", "words": ["this", "how", "about", "medicine"]}, {"sent": "because little babies arent sposta get into medicine .", "words": ["into", "medicine", "little", "get", "because"]}, {"sent": "you hafta have your own medicine because xxx proper one .", "words": ["your", "medicine", "you", "have", "because"]}, {"sent": "but you know its just medicine right ?", "words": ["medicine", "but", "you"]}, {"sent": "you want cough medicine ?", "words": ["medicine", "you"]}, {"sent": "give some money and then you can have some medicine .", "words": ["money", "can", "medicine", "then", "and", "you", "have", "some", "give"]}, {"sent": "thankyou for taking that good medicine so well ?", "words": ["for", "so", "medicine", "good", "that"]}, {"sent": "and do we take that and get him some pills or some medicine ?", "words": ["do", "we", "medicine", "and", "get", "him", "some", "take", "that"]}, {"sent": "well I havent even done my I havent done my given them any medicine yet .", "words": ["any", "my", "them", "medicine"]}, {"sent": "you get medicine from .", "words": ["get", "medicine", "you"]}, {"sent": "some medicine .", "words": ["some", "medicine"]}, {"sent": "that looks like sponge medicine for my leg .", "words": ["my", "for", "medicine", "like", "leg", "that"]}, {"sent": "medicine .", "words": ["medicine"]}, {"sent": "thats got medicine in it still .", "words": ["in", "medicine", "it"]}, {"sent": "have some medicine .", "words": ["have", "some", "medicine"]}, {"sent": "now let us go get medicine .", "words": ["get", "us", "medicine", "go"]}, {"sent": "do you need some magic medicine ?", "words": ["do", "need", "medicine", "you", "some"]}, {"sent": "how well you take that medicine .", "words": ["how", "medicine", "you", "take", "that"]}, {"sent": "here comes your medicine .", "words": ["here", "medicine", "your"]}, {"sent": "now you hafta take medicine dont you think so ?", "words": ["think", "so", "medicine", "you", "dont", "take"]}, {"sent": "oh okay when when should she get the medicine ?", "words": ["the", "medicine", "get", "when", "she"]}, {"sent": "and we got some more medicine for you .", "words": ["for", "we", "medicine", "and", "you", "more", "some"]}, {"sent": "Im just going to get your white medicine medicine .", "words": ["your", "to", "medicine", "get", "white"]}, {"sent": "shall we give him some medicine then ?", "words": ["we", "medicine", "then", "him", "some", "give"]}, {"sent": "they look like chocolate medicine huh ?", "words": ["medicine", "chocolate", "like", "they", "look"]}, {"sent": "can I have some nice medicine ?", "words": ["can", "medicine", "nice", "have", "some"]}, {"sent": "his medicine .", "words": ["his", "medicine"]}, {"sent": "did you take medicine ?", "words": ["medicine", "take", "did", "you"]}, {"sent": "you had some medicine this morning didnt you ?", "words": ["some", "medicine", "this", "you"]}, {"sent": "heres your medicine .", "words": ["medicine", "your"]}, {"sent": "cough medicine .", "words": ["medicine"]}, {"sent": "xxx mommys gotta take her medicine .", "words": ["her", "take", "medicine"]}, {"sent": "can you go to the chemist and get this medicine ?", "words": ["can", "the", "go", "to", "medicine", "you", "and", "get", "this"]}, {"sent": "Ill get some medicine .", "words": ["get", "some", "medicine"]}, {"sent": "we can give him his medicine ?", "words": ["can", "we", "medicine", "him", "his", "give"]}, {"sent": "now lets try xxx medicine .", "words": ["medicine", "try"]}, {"sent": "he tells my mother what medicine to give me .", "words": ["what", "my", "to", "medicine", "he", "me", "give"]}, {"sent": "now shall we give her some medicine ?", "words": ["we", "medicine", "some", "give", "her"]}, {"sent": "you need some medicine soon dont you ?", "words": ["need", "medicine", "you", "dont", "some"]}, {"sent": "well youve just had your medicine sweetheart .", "words": ["medicine", "your"]}, {"sent": "and medicine .", "words": ["medicine", "and"]}, {"sent": "nono that is the medicine .", "words": ["the", "medicine", "that", "is"]}, {"sent": "you hafta have your medicine darling .", "words": ["have", "medicine", "your", "you"]}, {"sent": "they gave him some medicine ?", "words": ["they", "him", "medicine", "some"]}, {"sent": "would you like some medicine ?", "words": ["medicine", "like", "you", "would", "some"]}, {"sent": "cold medicine .", "words": ["medicine", "cold"]}, {"sent": "Im going to give her some medicine as well .", "words": ["to", "medicine", "some", "give", "her"]}, {"sent": "a little bit of medicine is good when you dont feel well but a lot of medicine is not very good for you .", "words": ["for", "is", "of", "a lot", "medicine", "little", "you", "dont", "good", "when", "not", "but", "a"]}, {"sent": "gonna have some medicine darling ?", "words": ["have", "some", "medicine"]}, {"sent": "feel better after you have some medicine .", "words": ["better", "medicine", "you", "have", "some"]}, {"sent": "medicine .", "words": ["medicine"]}, {"sent": "well bring the medicine over for the tractor .", "words": ["the", "for", "medicine", "bring", "tractor", "over"]}, {"sent": "you like horrible medicine dont you Becky ?", "words": ["dont", "medicine", "like", "you"]}, {"sent": "a pony .", "words": ["pony", "a"]}, {"sent": "I want my pony .", "words": ["pony", "my"]}, {"sent": "is that a pony ?", "words": ["pony", "that", "a", "is"]}, {"sent": "Im going to give the pony some fire there .", "words": ["the", "to", "pony", "there", "some", "give"]}, {"sent": "Mama I want a pony like that .", "words": ["pony", "that", "like", "a"]}, {"sent": "oh okay a great big pony .", "words": ["pony", "big", "a"]}, {"sent": "this is my pony .", "words": ["this", "my", "pony", "is"]}, {"sent": "are you hugging the pony ?", "words": ["the", "pony", "are", "you"]}, {"sent": "how can ShadowGay ride on a pony .", "words": ["can", "how", "pony", "on", "ride", "a"]}, {"sent": "oh a pretty pony .", "words": ["pony", "pretty", "a"]}, {"sent": "he is a pony ?", "words": ["pony", "he", "a", "is"]}, {"sent": "youre squeezing the pony .", "words": ["the", "pony"]}, {"sent": "no put the pony in it .", "words": ["the", "put", "pony", "in", "it"]}, {"sent": "and I have a pony .", "words": ["have", "pony", "a", "and"]}, {"sent": "if you ask Swww nicely shell take you on a pony .", "words": ["if", "you", "pony", "on", "take", "a"]}, {"sent": "thats a pony isnt it ?", "words": ["pony", "it", "a"]}, {"sent": "thats a pretty pony Melissa that pink one .", "words": ["pony", "that", "pretty", "a"]}, {"sent": "lets get in the boat and go look for the great big orange pony .", "words": ["the", "for", "go", "and", "get", "pony", "in", "big", "orange", "look", "boat"]}, {"sent": "did you write on Tony the pony ?", "words": ["the", "write", "you", "pony", "did", "on"]}, {"sent": "I wanna keep this pony .", "words": ["this", "pony", "wanna"]}, {"sent": "pony .", "words": ["pony"]}, {"sent": "who else wears pony tails ?", "words": ["pony", "who"]}, {"sent": "the pony .", "words": ["the", "pony"]}, {"sent": "pony .", "words": ["pony"]}, {"sent": "Phoebe likes to wear pony tails .", "words": ["to", "pony"]}, {"sent": "riding the pony .", "words": ["the", "pony"]}, {"sent": "youre brushing your hair with the pony ?", "words": ["the", "your", "with", "pony", "hair"]}, {"sent": "I want a pony .", "words": ["pony", "a"]}, {"sent": "well thats not really a pony .", "words": ["pony", "not", "a"]}, {"sent": "no youll break the pony hell cry .", "words": ["the", "pony", "cry", "break"]}, {"sent": "to take me to this place and ponies were there and xxx riding a little pony .", "words": ["to", "and", "little", "this", "there", "pony", "me", "take", "were", "a"]}, {"sent": "Yankee Doodle went to town riding on a pony stuck a funger .", "words": ["to", "pony", "stuck", "on", "a"]}, {"sent": "Thunder useta be a Kennywood pony .", "words": ["be", "pony", "a"]}, {"sent": "maybe our pony can watch our horse can watch the baby .", "words": ["can", "the", "watch", "our", "pony", "horse"]}, {"sent": "yeah pony .", "words": ["pony"]}, {"sent": "hers got one in a pony tail .", "words": ["pony", "in", "a", "hers"]}, {"sent": "or maybe this pony is .", "words": ["this", "pony", "is"]}, {"sent": "wow who else has pony tails ?", "words": ["pony", "who"]}, {"sent": "thats a pony .", "words": ["pony", "a"]}, {"sent": "do you ride a your pony ?", "words": ["do", "your", "you", "pony", "ride", "a"]}, {"sent": "well Ill just put this pony in my hair Ill just put it in mine .", "words": ["put", "my", "this", "pony", "in", "it", "mine", "hair"]}, {"sent": "thats a ruler see the ruler on there measure your pony there you go .", "words": ["the", "your", "go", "you", "there", "pony", "on", "see", "a"]}, {"sent": "oh okay lets put a pony in her hair .", "words": ["put", "pony", "in", "hair", "her", "a"]}, {"sent": "did you go for a ride on a pony when you were at the zoo ?", "words": ["the", "zoo", "for", "when", "go", "you", "pony", "did", "at", "on", "ride", "were", "a"]}, {"sent": "did you go riding on a pony yesterday ?", "words": ["go", "you", "pony", "did", "on", "a"]}, {"sent": "you wanna build your pony a what ?", "words": ["what", "your", "you", "pony", "build", "wanna", "a"]}, {"sent": "a pony .", "words": ["pony", "a"]}, {"sent": "look at this pony hes got green hair .", "words": ["look", "this", "pony", "green", "at", "hair"]}, {"sent": "thats a little pony tail on top of her head .", "words": ["of", "head", "little", "pony", "on", "her", "a"]}, {"sent": "Ive never seen a pony that looked like that .", "words": ["pony", "that", "like", "a"]}, {"sent": "lets go look for the great big orange pony .", "words": ["the", "for", "go", "pony", "orange", "look", "big"]}, {"sent": "its called a my little pony isnt it ?", "words": ["my", "little", "pony", "it", "a"]}, {"sent": "Yankee Doodle went to town riding on a pony .", "words": ["on", "to", "pony", "a"]}, {"sent": "this is my little pony .", "words": ["my", "is", "little", "this", "pony"]}, {"sent": "whats the pony for ?", "words": ["the", "pony", "for"]}, {"sent": "he learned how to ride the pony .", "words": ["the", "how", "to", "pony", "he", "ride"]}, {"sent": "maybe that that pony wants something to eat .", "words": ["to", "pony", "that", "eat"]}, {"sent": "the girl downstairs calls her the pony express because she keeps going back and forth and back .", "words": ["the", "back", "and", "pony", "her", "she", "because"]}, {"sent": "and a beige pony and a brown pony .", "words": ["pony", "brown", "a", "and"]}, {"sent": "well she is a very good wee pony youknow ?", "words": ["is", "pony", "good", "she", "a"]}, {"sent": "is it a pony ?", "words": ["pony", "it", "a", "is"]}, {"sent": "Mama she can have a little pony tail .", "words": ["can", "little", "have", "pony", "she", "a"]}, {"sent": "why is Naima mad at the pony ?", "words": ["the", "is", "mad", "pony", "at", "why"]}, {"sent": "youre too big for that little pony .", "words": ["for", "little", "pony", "big", "too", "that"]}, {"sent": "I know youre a big strong boy but Dobbins a big pony isnt she ?", "words": ["pony", "big", "she", "but", "a"]}, {"sent": "my little pony yeah .", "words": ["pony", "my", "little"]}, {"sent": "Pheobe wears pony tails right .", "words": ["pony"]}, {"sent": "look here Megan has a pony .", "words": ["here", "pony", "a", "look"]}, {"sent": "does papas pony have white feet ?", "words": ["have", "does", "white", "pony"]}, {"sent": "it flattens out her pony tail .", "words": ["pony", "it", "her", "out"]}, {"sent": "you fell off your pony ?", "words": ["off", "pony", "your", "you"]}, {"sent": "hes a nice pony hes gonna give you ah .", "words": ["nice", "you", "pony", "give", "a"]}, {"sent": "who else wears pony tails ?", "words": ["pony", "who"]}, {"sent": "is that a pony ?", "words": ["pony", "that", "a", "is"]}, {"sent": "oh look at these my little pony .", "words": ["these", "my", "look", "little", "pony", "at"]}, {"sent": "oh well a pony is a baby horse .", "words": ["horse", "pony", "is", "a"]}, {"sent": "are you mad at that pony ?", "words": ["mad", "you", "pony", "are", "at", "that"]}, {"sent": "are you mad at the pony ?", "words": ["the", "mad", "you", "pony", "are", "at"]}, {"sent": "isnt that a pretty pony ?", "words": ["pretty", "pony", "that", "a"]}, {"sent": "just Joanna rode the pony .", "words": ["the", "pony"]}, {"sent": "or a pony .", "words": ["pony", "a"]}, {"sent": "heres a my little pony .", "words": ["pony", "my", "little", "a"]}, {"sent": "oh Mama look at this pony .", "words": ["this", "at", "pony", "look"]}, {"sent": "no she wanted it all on the little pony tail .", "words": ["the", "all", "little", "pony", "it", "on", "she"]}, {"sent": "we stayed we stayed and queued and queued for to get her riding a pony .", "words": ["for", "we", "to", "and", "get", "pony", "her", "a"]}, {"sent": "do you want pony tails or braids ?", "words": ["do", "pony", "you"]}, {"sent": "oh a little pony .", "words": ["pony", "little", "a"]}, {"sent": "oh a little pony .", "words": ["pony", "little", "a"]}, {"sent": "you do like to step on your pony why do you like to step on your pony ?", "words": ["do", "your", "to", "like", "you", "pony", "on", "why"]}, {"sent": "did you ride the pony ?", "words": ["the", "you", "pony", "did", "ride"]}, {"sent": "a dog riding on a pony .", "words": ["on", "pony", "dog", "a"]}, {"sent": "oh God somebody grab that pony or else Im running round after it .", "words": ["pony", "it", "that"]}, {"sent": "you think thats a pony then do you ?", "words": ["do", "think", "you", "pony", "then", "a"]}, {"sent": "get me a big pony .", "words": ["get", "pony", "me", "big", "a"]}, {"sent": "you dont ride Tony the pony very much uhhuh .", "words": ["the", "much", "you", "dont", "pony", "ride"]}, {"sent": "give some pony pony some of your yoghurts .", "words": ["your", "of", "pony", "some", "give"]}, {"sent": "theres another pony .", "words": ["another", "pony"]}, {"sent": "hows the pony ?", "words": ["the", "pony"]}, {"sent": "that pony cant go up there ?", "words": ["up", "go", "pony", "there", "that"]}, {"sent": "what does the um owl say ?", "words": ["the", "what", "does", "say", "owl"]}, {"sent": "asked the owl .", "words": ["the", "owl"]}, {"sent": "is the owl standing on the tree ?", "words": ["the", "tree", "is", "owl", "on"]}, {"sent": "whats all the fuss their owl mother asked you knew I would come back the baby owls thought I knew it said Sarah .", "words": ["the", "all", "their", "back", "you", "owl", "it", "would"]}, {"sent": "is that an owl ?", "words": ["owl", "an", "that", "is"]}, {"sent": "owl \u7684 \u4e2d\u6587 \u662f \u4ec0\u4e48 ?", "words": ["owl"]}, {"sent": "it is an owl .", "words": ["owl", "an", "it", "is"]}, {"sent": "owl what ?", "words": ["what", "owl"]}, {"sent": "I think thats an owl .", "words": ["owl", "think", "an"]}, {"sent": "owl ?", "words": ["owl"]}, {"sent": "what does the owl say ?", "words": ["the", "what", "does", "say", "owl"]}, {"sent": "what does the owl say ?", "words": ["the", "what", "does", "say", "owl"]}, {"sent": "he can sound like an owl .", "words": ["can", "like", "owl", "an", "he"]}, {"sent": "wont be owl .", "words": ["be", "owl"]}, {"sent": "owl .", "words": ["owl"]}, {"sent": "owl .", "words": ["owl"]}, {"sent": "the springs wildflowers bloom bright as mother owl sits patiently on the nest .", "words": ["the", "on", "owl"]}, {"sent": "the owl is there thats the owl .", "words": ["the", "there", "is", "owl"]}, {"sent": "theyre all gonna visit owl in the treehouse ?", "words": ["the", "all", "in", "owl"]}, {"sent": "owl .", "words": ["owl"]}, {"sent": "\u54ce\u5440 \u771f \u68d2 owl .", "words": ["owl"]}, {"sent": "and wheres the owl ?", "words": ["the", "owl", "and"]}, {"sent": "o owl o w l ?", "words": ["owl"]}, {"sent": "say birdie owl .", "words": ["say", "owl"]}, {"sent": "theres an owl .", "words": ["an", "owl"]}, {"sent": "look at the owl .", "words": ["the", "owl", "at", "look"]}, {"sent": "thats a bird yes but its an owl isnt it ?", "words": ["bird", "owl", "an", "it", "but", "a"]}, {"sent": "is owl hiding ?", "words": ["owl", "is"]}, {"sent": "its like an owl looking backwards .", "words": ["an", "like", "owl"]}, {"sent": "thats an owl oh the blocks fell over .", "words": ["the", "an", "over", "owl"]}, {"sent": "an owl good boy didnt think youd remember that .", "words": ["think", "owl", "an", "good", "that"]}, {"sent": "wheres that owl ?", "words": ["that", "owl"]}, {"sent": "and we made owl because the paint was wet .", "words": ["the", "wet", "paint", "we", "was", "owl", "and", "because"]}, {"sent": "an owl ?", "words": ["an", "owl"]}, {"sent": "she had an owl ?", "words": ["an", "she", "owl"]}, {"sent": "as the long spring days pass father owl makes many flights to hunt for lemmings and hares .", "words": ["the", "for", "to", "long", "owl", "and"]}, {"sent": "Im going to read the story about the owl and the pussycat again .", "words": ["the", "to", "owl", "and", "read", "about", "story"]}, {"sent": "an owl in a towel ?", "words": ["owl", "in", "an", "towel", "a"]}, {"sent": "look at the owl .", "words": ["the", "owl", "at", "look"]}, {"sent": "an owl .", "words": ["an", "owl"]}, {"sent": "what does the owl say coo ?", "words": ["the", "what", "does", "say", "owl"]}, {"sent": "oh the owl and the moon .", "words": ["the", "moon", "and", "owl"]}, {"sent": "owl what does the owl say ?", "words": ["the", "what", "does", "say", "owl"]}, {"sent": "theres an owl do you see the owl ?", "words": ["the", "do", "you", "owl", "an", "see"]}, {"sent": "he can sound like an owl .", "words": ["can", "like", "owl", "an", "he"]}, {"sent": "and owl flies .", "words": ["owl", "and"]}, {"sent": "a caribou wanders too close to the nest mother and father owl shrill xxx to drive him off .", "words": ["the", "off", "to", "and", "owl", "drive", "him", "too", "close", "a"]}, {"sent": "owl ?", "words": ["owl"]}, {"sent": "whats the owl say ?", "words": ["the", "say", "owl"]}, {"sent": "what happened to the owl ?", "words": ["to", "what", "owl", "the"]}, {"sent": "look at the owl Noah ?", "words": ["the", "owl", "at", "look"]}, {"sent": "an owl .", "words": ["an", "owl"]}, {"sent": "an owl ?", "words": ["an", "owl"]}, {"sent": "thats owl standing on the grass .", "words": ["on", "the", "grass", "owl"]}, {"sent": "what is that owl doing at the top of that ladder ?", "words": ["the", "what", "is", "of", "owl", "ladder", "at", "that"]}, {"sent": "the owl and the pussycat .", "words": ["the", "and", "owl"]}, {"sent": "more owl ?", "words": ["more", "owl"]}, {"sent": "where is the birdie owl ?", "words": ["where", "the", "owl", "is"]}, {"sent": "thats an owl .", "words": ["an", "owl"]}, {"sent": "father owl calls out a loud hoo hoo .", "words": ["a", "loud", "out", "owl"]}, {"sent": "where is birdie owl ?", "words": ["where", "owl", "is"]}, {"sent": "on her branch on Sarahs branch Sarahs a girl owl they all went to sit on Sarahs branch .", "words": ["all", "to", "owl", "they", "on", "sit", "her", "a"]}, {"sent": "an owl .", "words": ["an", "owl"]}, {"sent": "said the owl .", "words": ["the", "owl"]}, {"sent": "owl .", "words": ["owl"]}, {"sent": "a big brown owl .", "words": ["owl", "brown", "big", "a"]}, {"sent": "look there is a owl up here .", "words": ["up", "is", "owl", "here", "there", "look", "a"]}, {"sent": "yeah and theres owl coming out to see to see whos knocking at his door .", "words": ["to", "owl", "and", "out", "at", "his", "see", "door"]}, {"sent": "thats owl .", "words": ["owl"]}, {"sent": "owl .", "words": ["owl"]}, {"sent": "theres an owl .", "words": ["an", "owl"]}, {"sent": "see that owl he lives way up there in that tree in that door .", "words": ["up", "tree", "owl", "there", "in", "he", "see", "that", "door"]}, {"sent": "an owl ?", "words": ["an", "owl"]}, {"sent": "and what is that owl doing ?", "words": ["what", "is", "owl", "and", "that"]}, {"sent": "Maisy closes her bedroom curtains xxx hoots the owl .", "words": ["the", "owl", "her", "bedroom"]}, {"sent": "thats the owl what does the owl say ?", "words": ["the", "what", "does", "say", "owl"]}, {"sent": "an owl .", "words": ["an", "owl"]}, {"sent": "he can sound like an owl .", "words": ["can", "like", "owl", "an", "he"]}, {"sent": "oh the toucan is under the owl .", "words": ["the", "under", "owl", "is"]}, {"sent": "all of the birds and animals heard Bambi even the owl who had settled for a long days sleep .", "words": ["the", "all", "for", "of", "sleep", "long", "owl", "and", "who", "a"]}, {"sent": "what does an owl say ?", "words": ["what", "does", "say", "owl", "an"]}, {"sent": "what does the owl say ?", "words": ["the", "what", "does", "say", "owl"]}, {"sent": "can you see the owl ?", "words": ["can", "the", "owl", "you", "see"]}, {"sent": "is that an owl ?", "words": ["owl", "an", "that", "is"]}, {"sent": "you do the owl brown and Ill do the leaves green .", "words": ["the", "do", "brown", "owl", "you", "and", "green"]}, {"sent": "where is the that is an owl .", "words": ["where", "the", "is", "owl", "an", "that"]}, {"sent": "hello owl .", "words": ["owl"]}, {"sent": "he can sound like an owl hoo hoo hoo .", "words": ["can", "like", "owl", "an", "he"]}, {"sent": "what does an owl say ?", "words": ["what", "does", "say", "owl", "an"]}, {"sent": "owl .", "words": ["owl"]}, {"sent": "wheres the owl ?", "words": ["the", "owl"]}, {"sent": "and the baby owls closed their owl eyes and wished their owl mother would come .", "words": ["the", "their", "owl", "and", "would"]}, {"sent": "their friend owl has come is waiting for the two brothers to come out to play .", "words": ["the", "for", "their", "is", "to", "play", "owl", "out"]}, {"sent": "xxx owl looked at rabbit .", "words": ["at", "owl"]}, {"sent": "what color shall we do the owl ?", "words": ["the", "what", "do", "we", "owl"]}, {"sent": "whats happened to the owl ?", "words": ["the", "to", "owl"]}, {"sent": "is that birdie owl ?", "words": ["owl", "that", "is"]}, {"sent": "owl doesnt eat honey .", "words": ["eat", "owl"]}, {"sent": "a bee might try to get the nectar from the flower yes .", "words": ["the", "to", "flower", "bee", "get", "try", "try to", "a"]}, {"sent": "um well every time I tried to give her a piece she would look at it for s youknow she would hold it but then she would put it down and go right back to the books or to the the bee she had over here .", "words": ["the", "put", "hold", "go", "it", "then", "down", "she", "for", "to", "and", "here", "every", "would", "a", "back", "bee", "at", "look", "over", "give", "her", "but"]}, {"sent": "was there a bee bee was in Eleanors bucket .", "words": ["bucket", "was", "bee", "there", "in", "a"]}, {"sent": "it looks like he likes that bee .", "words": ["bee", "like", "it", "he", "that"]}, {"sent": "bee .", "words": ["bee"]}, {"sent": "but whos holding the bee ?", "words": ["the", "bee", "but"]}, {"sent": "right now you like that bee .", "words": ["bee", "that", "like", "you"]}, {"sent": "that would even go in the one with the bee hive truck wouldnt it ?", "words": ["the", "go", "bee", "with", "truck", "in", "it", "would", "that"]}, {"sent": "and maybe a bee .", "words": ["bee", "a", "and"]}, {"sent": "bee butt .", "words": ["bee"]}, {"sent": "thats a bee .", "words": ["bee", "a"]}, {"sent": "the only thing he really played with was with the bee word .", "words": ["the", "was", "bee", "with", "he"]}, {"sent": "why are you upset about the bee ?", "words": ["the", "bee", "you", "are", "about", "why"]}, {"sent": "a bee .", "words": ["bee", "a"]}, {"sent": "think its a bee .", "words": ["think", "bee", "a"]}, {"sent": "its a bee .", "words": ["bee", "a"]}, {"sent": "oh you wanna show the busy bee ?", "words": ["the", "show", "bee", "you", "wanna"]}, {"sent": "its a bee .", "words": ["bee", "a"]}, {"sent": "theres a bee .", "words": ["bee", "a"]}, {"sent": "look heres a bee .", "words": ["bee", "a", "look"]}, {"sent": "look this is a bee .", "words": ["is", "bee", "this", "look", "a"]}, {"sent": "oh now weve got a bee .", "words": ["bee", "a"]}, {"sent": "bee would like some shoes .", "words": ["some", "bee", "like", "would"]}, {"sent": "hah oh thats a bee .", "words": ["bee", "a"]}, {"sent": "crazy bumble bee ?", "words": ["bee"]}, {"sent": "you wanna draw a picture of a bee ?", "words": ["of", "picture", "draw", "bee", "you", "wanna", "a"]}, {"sent": "a bee .", "words": ["bee", "a"]}, {"sent": "when you were with Daddy and there was a bee in your hair .", "words": ["your", "was", "bee", "with", "and", "you", "there", "in", "hair", "when", "were", "a"]}, {"sent": "hello bee .", "words": ["bee"]}, {"sent": "theres the bee .", "words": ["the", "bee"]}, {"sent": "well we talk about bees but I dont know if she would associate that with a bee .", "words": ["if", "we", "bee", "with", "would", "dont", "about", "talk", "that", "she", "but", "a"]}, {"sent": "do you know a rhyme about a bee ?", "words": ["do", "bee", "you", "about", "a"]}, {"sent": "the bee looked at the caterpillar .", "words": ["the", "bee", "at"]}, {"sent": "bee bee bee bee bee bee bee bee bee bee bee .", "words": ["bee"]}, {"sent": "if ever there is a bee in your mouth you must get it out quickly and tell Mummy .", "words": ["your", "if", "is", "bee", "mouth", "you", "get", "there", "in", "it", "and", "out", "a"]}, {"sent": "oh you like the bee better ?", "words": ["the", "better", "bee", "like", "you"]}, {"sent": "say bee bee black sheep .", "words": ["black", "sheep", "bee", "say"]}, {"sent": "the big bee is the buddy of the little bear .", "words": ["the", "is", "of", "bee", "little", "bear", "big"]}, {"sent": "this is a bee and a this is a bee .", "words": ["is", "bee", "and", "this", "a"]}, {"sent": "a bee is in xxx .", "words": ["in", "bee", "is", "a"]}, {"sent": "bless you bee .", "words": ["bee", "you"]}, {"sent": "bee bee black sheep .", "words": ["black", "bee", "sheep"]}, {"sent": "and this ones about a bee .", "words": ["bee", "and", "this", "about", "a"]}, {"sent": "a bee sat on my nose .", "words": ["my", "bee", "on", "a", "nose"]}, {"sent": "say bee bee bee .", "words": ["bee", "say"]}, {"sent": "Sally look at this little bee .", "words": ["look", "bee", "little", "this", "at"]}, {"sent": "Nwww was stung by a bee wasnt he ?", "words": ["was", "bee", "by", "he", "a"]}, {"sent": "you got the bee .", "words": ["the", "bee", "you"]}, {"sent": "he can sound like a bee MisterBrown can buzz .", "words": ["can", "bee", "like", "he", "a"]}, {"sent": "wheres the bee ?", "words": ["the", "bee"]}, {"sent": "thats the queen bee .", "words": ["the", "bee"]}, {"sent": "mkay lets go play with your bee .", "words": ["your", "go", "play", "with", "bee"]}, {"sent": "do you like the bee ?", "words": ["the", "do", "bee", "like", "you"]}, {"sent": "xxx the bee on your head .", "words": ["the", "your", "head", "bee", "on"]}, {"sent": "you like the bee ?", "words": ["the", "bee", "like", "you"]}, {"sent": "yes you were a bee .", "words": ["bee", "were", "a", "you"]}, {"sent": "a er well what can bee have from the shop ?", "words": ["can", "what", "the", "bee", "have", "a"]}, {"sent": "busy bee .", "words": ["bee"]}, {"sent": "bee .", "words": ["bee"]}, {"sent": "antenna so I can be a bee .", "words": ["can", "so", "be", "bee", "a"]}, {"sent": "wheres the bee ?", "words": ["the", "bee"]}, {"sent": "theres the bee look .", "words": ["the", "bee", "look"]}, {"sent": "oh you want the bee again ?", "words": ["the", "bee", "you"]}, {"sent": "he can sound like a bee .", "words": ["can", "bee", "like", "he", "a"]}, {"sent": "do you think hes a tickle bee ?", "words": ["do", "think", "bee", "tickle", "you", "a"]}, {"sent": "bumble bee ?", "words": ["bee"]}, {"sent": "can you see on those pink flowers there can you see that bee ?", "words": ["can", "those", "bee", "you", "there", "on", "see", "that"]}, {"sent": "is it a buzzy bee ?", "words": ["it", "bee", "a", "is"]}, {"sent": "you got the bee ?", "words": ["the", "bee", "you"]}, {"sent": "Thomas looks like a blue and green bee this morning .", "words": ["bee", "like", "blue", "and", "this", "green", "a"]}, {"sent": "no thats not a bee .", "words": ["bee", "not", "a"]}, {"sent": "that is a bee .", "words": ["bee", "that", "a", "is"]}, {"sent": "wheres that bee ?", "words": ["bee", "that"]}, {"sent": "you see a bee .", "words": ["see", "bee", "a", "you"]}, {"sent": "bee bee ?", "words": ["bee"]}, {"sent": "this is a bee .", "words": ["this", "bee", "a", "is"]}, {"sent": "if you see a bee outside though Thomas .", "words": ["if", "outside", "bee", "you", "see", "a"]}, {"sent": "go get the bee .", "words": ["get", "the", "bee", "go"]}, {"sent": "she really likes the bee .", "words": ["the", "bee", "she"]}, {"sent": "its a bee .", "words": ["bee", "a"]}, {"sent": "bee .", "words": ["bee"]}, {"sent": "bee bee bee .", "words": ["bee"]}, {"sent": "wanna play with your bee ?", "words": ["your", "play", "with", "bee", "wanna"]}, {"sent": "a bee and what noise does a bee make ?", "words": ["does", "what", "bee", "and", "make", "a"]}, {"sent": "here comes the busy bee .", "words": ["here", "the", "bee"]}, {"sent": "bee .", "words": ["bee"]}, {"sent": "but um your friend the bee is still here .", "words": ["the", "your", "is", "bee", "here", "but"]}, {"sent": "chopping the bee up in jail ?", "words": ["the", "in", "bee", "up"]}, {"sent": "bee bee black sheep .", "words": ["black", "bee", "sheep"]}, {"sent": "theres mister bee one two three oh .", "words": ["bee"]}, {"sent": "he can sound like a bee .", "words": ["can", "bee", "like", "he", "a"]}, {"sent": "he can sounds like a bee .", "words": ["can", "bee", "like", "he", "a"]}, {"sent": "and the bee goes buzz .", "words": ["the", "bee", "and"]}, {"sent": "you giving Mommy the bee ?", "words": ["the", "bee", "you"]}, {"sent": "see like a bee there .", "words": ["bee", "like", "there", "see", "a"]}, {"sent": "what does the bee say ?", "words": ["the", "what", "does", "bee", "say"]}, {"sent": "buzz buzz says the bee .", "words": ["the", "bee"]}, {"sent": "hes a bee .", "words": ["bee", "a"]}, {"sent": "Im not gonna wash all of these grapes .", "words": ["all", "these", "of", "grapes", "wash", "not"]}, {"sent": "Ive got green grapes .", "words": ["grapes", "green"]}, {"sent": "well wine is made out of the grapes they squish the grapes to make the wine .", "words": ["the", "is", "of", "to", "they", "grapes", "out", "make"]}, {"sent": "grapes .", "words": ["grapes"]}, {"sent": "want some grapes ?", "words": ["some", "grapes"]}, {"sent": "your asking for grapes and pears and we havent got any grapes or pears this morning .", "words": ["for", "your", "we", "any", "and", "this", "grapes"]}, {"sent": "grapes .", "words": ["grapes"]}, {"sent": "theyre not grapes .", "words": ["grapes", "not"]}, {"sent": "grapes .", "words": ["grapes"]}, {"sent": "is that fried grapes on your head ?", "words": ["your", "is", "head", "grapes", "on", "that"]}, {"sent": "grapes .", "words": ["grapes"]}, {"sent": "some grapes .", "words": ["some", "grapes"]}, {"sent": "and would you like a few grapes ?", "words": ["like", "you", "and", "grapes", "would", "a"]}, {"sent": "did you say mind the grapes or my grapes ?", "words": ["the", "my", "say", "you", "grapes", "did"]}, {"sent": "youve got the purple one so you want the grapes look .", "words": ["the", "so", "you", "grapes", "look"]}, {"sent": "or grapes isnt it ?", "words": ["it", "grapes"]}, {"sent": "so we have green grapes green pears and a banana in the basket .", "words": ["the", "we", "so", "and", "have", "banana", "in", "grapes", "basket", "green", "a"]}, {"sent": "are these grapes ?", "words": ["these", "are", "grapes"]}, {"sent": "grapes .", "words": ["grapes"]}, {"sent": "grapes .", "words": ["grapes"]}, {"sent": "and you might as well have the grapes because there arent many grapes .", "words": ["the", "you", "have", "and", "there", "grapes", "because"]}, {"sent": "you dont like the taste of pretend grapes .", "words": ["the", "of", "like", "taste", "you", "dont", "grapes", "pretend"]}, {"sent": "theyre grapes .", "words": ["grapes"]}, {"sent": "you like green grapes and black grapes dont you ?", "words": ["like", "and", "you", "black", "dont", "grapes", "green"]}, {"sent": "you had a lot of grapes yesterday didnt you ?", "words": ["of", "a lot", "you", "grapes", "a"]}, {"sent": "it was my grapes .", "words": ["was", "my", "it", "grapes"]}, {"sent": "green grapes .", "words": ["grapes", "green"]}, {"sent": "you want them grapes ?", "words": ["grapes", "them", "you"]}, {"sent": "I think youre doing a magic act with those grapes and cherries .", "words": ["think", "those", "with", "and", "grapes", "a"]}, {"sent": "what is that grapes ?", "words": ["what", "grapes", "that", "is"]}, {"sent": "grapes .", "words": ["grapes"]}, {"sent": "theyre green grapes arent they ?", "words": ["they", "grapes", "green"]}, {"sent": "oh what are you at grapes ?", "words": ["what", "you", "are", "grapes", "at"]}, {"sent": "its some grapes .", "words": ["some", "grapes"]}, {"sent": "a lot of the grapes we buy though are seedless .", "words": ["the", "we", "of", "a lot", "grapes", "are", "buy", "a"]}, {"sent": "I think I think thats all the grapes .", "words": ["the", "all", "think", "grapes"]}, {"sent": "little grapes .", "words": ["grapes", "little"]}, {"sent": "these are grapes .", "words": ["these", "are", "grapes"]}, {"sent": "berries and grapes and a banana .", "words": ["banana", "grapes", "a", "and"]}, {"sent": "this is juice from grapes .", "words": ["this", "juice", "grapes", "is"]}, {"sent": "grapes .", "words": ["grapes"]}, {"sent": "I dont think Ive got any grapes .", "words": ["dont", "think", "grapes", "any"]}, {"sent": "grapes .", "words": ["grapes"]}, {"sent": "all done grapes .", "words": ["all", "grapes"]}, {"sent": "no that grapes .", "words": ["grapes", "that"]}, {"sent": "cherries and grapes .", "words": ["grapes", "and"]}, {"sent": "I know you like grapes .", "words": ["grapes", "like", "you"]}, {"sent": "and other grapes you dont .", "words": ["other", "you", "dont", "and", "grapes"]}, {"sent": "grapes and what else ?", "words": ["what", "grapes", "and"]}, {"sent": "being last Zach saw just how the sidewalk artist draws a cow a speckled fish of grapes in a dish or anything that you might wish .", "words": ["the", "how", "sidewalk", "of", "last", "you", "wish", "dish", "in", "grapes", "fish", "that", "cow", "a"]}, {"sent": "I found some grapes .", "words": ["some", "grapes"]}, {"sent": "oranges grapes biscuits sandwiches orange juice .", "words": ["grapes", "juice", "orange"]}, {"sent": "grapes .", "words": ["grapes"]}, {"sent": "did you enjoy the grapes ?", "words": ["the", "grapes", "did", "you"]}, {"sent": "is that a bunch of grapes ?", "words": ["is", "of", "grapes", "that", "a"]}, {"sent": "grapes .", "words": ["grapes"]}, {"sent": "er heres some more grapes .", "words": ["some", "more", "grapes"]}, {"sent": "and put cheese and perhaps grapes and cheese and apple on sticks .", "words": ["put", "and", "grapes", "on", "apple", "cheese"]}, {"sent": "and grapes .", "words": ["grapes", "and"]}, {"sent": "grapes .", "words": ["grapes"]}, {"sent": "oh would Bob like to put the bag from the grapes in the bin for Mummy please .", "words": ["the", "put", "for", "to", "like", "in", "grapes", "would"]}, {"sent": "thats grapes .", "words": ["grapes"]}, {"sent": "Ive just washed some more grapes .", "words": ["some", "more", "grapes"]}, {"sent": "grapes are good you like grapes .", "words": ["like", "you", "grapes", "are", "good"]}, {"sent": "apple and grapes .", "words": ["apple", "grapes", "and"]}, {"sent": "means itll have grapes in it and what else ?", "words": ["what", "and", "have", "in", "it", "grapes"]}, {"sent": "those are grapes .", "words": ["grapes", "are", "those"]}, {"sent": "these are the grapes .", "words": ["the", "these", "are", "grapes"]}, {"sent": "would you like some more grapes ?", "words": ["like", "you", "more", "grapes", "would", "some"]}, {"sent": "and grapes .", "words": ["grapes", "and"]}, {"sent": "no didnt even buy any grapes this week .", "words": ["this", "grapes", "any", "buy"]}, {"sent": "and if we make some green balls a little bit bigger than the peas we could have some grapes couldnt we ?", "words": ["the", "if", "we", "peas", "and", "little", "have", "grapes", "green", "some", "could", "make", "a"]}, {"sent": "Nana and Grandad brought the grapes for you this morning didnt they ?", "words": ["the", "for", "you", "and", "this", "they", "grapes"]}, {"sent": "but no more grapes .", "words": ["more", "grapes", "but"]}, {"sent": "are they grapes ?", "words": ["they", "are", "grapes"]}, {"sent": "some shiny red grapes havent we ?", "words": ["red", "some", "grapes", "we"]}, {"sent": "it says we hafta find some grapes .", "words": ["we", "grapes", "it", "find", "some"]}, {"sent": "some cheese and some grapes ?", "words": ["some", "grapes", "cheese", "and"]}, {"sent": "can you see some grapes ?", "words": ["can", "you", "grapes", "see", "some"]}, {"sent": "are you going to pass me those grapes please .", "words": ["those", "to", "you", "are", "grapes", "me"]}, {"sent": "they do look like black grapes though dont they ?", "words": ["do", "like", "black", "they", "dont", "grapes", "look"]}, {"sent": "he could go and drive to the shops and buy my some grapes .", "words": ["the", "my", "go", "to", "and", "drive", "grapes", "he", "some", "could", "buy"]}, {"sent": "you just want grapes ?", "words": ["grapes", "you"]}, {"sent": "those grapes have been eaten .", "words": ["have", "grapes", "those"]}, {"sent": "and grapes yeah .", "words": ["grapes", "and"]}, {"sent": "theyre grapes .", "words": ["grapes"]}, {"sent": "and when theyre pressed together like this they look more like a bunch of green grapes than they do peas dont they ?", "words": ["do", "of", "peas", "like", "and", "this", "they", "more", "grapes", "dont", "green", "look", "when", "a"]}, {"sent": "Im washing the grapes .", "words": ["the", "grapes"]}, {"sent": "green and black grapes if you wash those and dry them .", "words": ["those", "if", "dry", "them", "and", "black", "you", "grapes", "green", "wash"]}, {"sent": "theres the grapes .", "words": ["the", "grapes"]}, {"sent": "would you like some grapes ?", "words": ["like", "you", "grapes", "would", "some"]}, {"sent": "have you any grapes please ?", "words": ["have", "grapes", "any", "you"]}, {"sent": "is it made with grapes from the castle ?", "words": ["the", "is", "with", "grapes", "it"]}, {"sent": "I think some things like the grapes are too large to go on top arent they ?", "words": ["the", "think", "go", "to", "like", "they", "grapes", "are", "too", "on", "some"]}, {"sent": "are they nice grapes ?", "words": ["nice", "they", "are", "grapes"]}, {"sent": "you dont have one of these grapes David .", "words": ["these", "of", "you", "have", "dont", "grapes"]}, {"sent": "its not my tummy youre sitting on though is it ?", "words": ["my", "is", "it", "tummy", "on", "not"]}, {"sent": "tummy .", "words": ["tummy"]}, {"sent": "let me listen to your tummy Mr snake .", "words": ["your", "listen", "to", "tummy", "me"]}, {"sent": "did you have a tummy ache last night ?", "words": ["last", "you", "have", "did", "tummy", "a"]}, {"sent": "thats a fat tummy .", "words": ["tummy", "a"]}, {"sent": "on his tummy .", "words": ["on", "his", "tummy"]}, {"sent": "yummy tummy ?", "words": ["tummy"]}, {"sent": "big tummy .", "words": ["big", "tummy"]}, {"sent": "is that going in your tummy ?", "words": ["your", "is", "in", "tummy", "that"]}, {"sent": "have you got big tummy and big lips ?", "words": ["and", "have", "you", "tummy", "big"]}, {"sent": "is that your tummy ?", "words": ["your", "tummy", "that", "is"]}, {"sent": "what a fat tummy .", "words": ["what", "tummy", "a"]}, {"sent": "on your tummy .", "words": ["on", "your", "tummy"]}, {"sent": "and no sooner had I put the plate down and whoop it all went in that tummy .", "words": ["the", "all", "put", "plate", "and", "in", "it", "tummy", "down", "that"]}, {"sent": "and your little tummy .", "words": ["tummy", "your", "little", "and"]}, {"sent": "on your tummy .", "words": ["on", "your", "tummy"]}, {"sent": "the baby is still in my tummy .", "words": ["the", "my", "is", "in", "tummy"]}, {"sent": "what do you wear on your tummy ?", "words": ["what", "do", "your", "you", "tummy", "on"]}, {"sent": "whose tummy ?", "words": ["tummy"]}, {"sent": "I think its Beckys tummy .", "words": ["think", "tummy"]}, {"sent": "are you showing me your tummy now ?", "words": ["your", "you", "are", "tummy", "me"]}, {"sent": "and wheres your tummy ?", "words": ["tummy", "your", "and"]}, {"sent": "I dont know why you started talking about tummy bugs .", "words": ["you", "dont", "about", "tummy", "why"]}, {"sent": "in your tummy ?", "words": ["in", "your", "tummy"]}, {"sent": "in your tummy you had .", "words": ["in", "your", "you", "tummy"]}, {"sent": "dry your other leg and dry your tummy .", "words": ["your", "dry", "and", "other", "leg", "tummy"]}, {"sent": "tummy ache .", "words": ["tummy"]}, {"sent": "because your tummy will be upset if you eat any more .", "words": ["your", "if", "be", "any", "you", "eat", "more", "tummy", "will", "because"]}, {"sent": "chubby little tummy all stuffed with fluff .", "words": ["all", "tummy", "with", "little"]}, {"sent": "oh are you brushing Floppys tummy ?", "words": ["are", "tummy", "you"]}, {"sent": "is that your tummy ?", "words": ["your", "tummy", "that", "is"]}, {"sent": "you cant stick this elbow in my tummy .", "words": ["stick", "my", "you", "this", "in", "tummy"]}, {"sent": "does your tummy have holes in it like that ?", "words": ["does", "your", "like", "have", "in", "it", "tummy", "that"]}, {"sent": "do you have a tummy under that shirt ?", "words": ["do", "shirt", "you", "have", "under", "tummy", "that", "a"]}, {"sent": "I was just patting my tummy .", "words": ["was", "my", "tummy"]}, {"sent": "does his tummy hurt ?", "words": ["his", "does", "hurt", "tummy"]}, {"sent": "Ill eat you all up in my tummy .", "words": ["up", "all", "my", "you", "eat", "in", "tummy"]}, {"sent": "you wanna see the names she has for this baby in my tummy ?", "words": ["the", "my", "for", "you", "this", "in", "wanna", "tummy", "see", "she"]}, {"sent": "I got strawberries in my tummy .", "words": ["in", "my", "tummy"]}, {"sent": "in that big tummy there .", "words": ["there", "in", "that", "tummy", "big"]}, {"sent": "have you really got tummy ache ?", "words": ["have", "tummy", "you"]}, {"sent": "yeah you were in my tummy and Will had come out my tummy .", "words": ["my", "you", "and", "in", "out", "tummy", "were"]}, {"sent": "oh your tummy made a noise ?", "words": ["your", "a", "tummy"]}, {"sent": "Grandad has got a poorly tummy .", "words": ["tummy", "a"]}, {"sent": "because your tummy hurts ?", "words": ["your", "because", "tummy"]}, {"sent": "and he carries little babies in his tummy .", "words": ["little", "and", "in", "he", "tummy", "his"]}, {"sent": "here in the tummy huh ?", "words": ["here", "the", "in", "tummy"]}, {"sent": "a tummy ?", "words": ["tummy", "a"]}, {"sent": "he has got a red tummy .", "words": ["red", "tummy", "he", "a"]}, {"sent": "because I didnt put them in that tummy .", "words": ["put", "them", "in", "tummy", "that", "because"]}, {"sent": "do you know why Ive got a big tummy ?", "words": ["do", "you", "tummy", "big", "why", "a"]}, {"sent": "that was Mummys tummy rumbling wasnt it ?", "words": ["was", "it", "that", "tummy"]}, {"sent": "whos this on your tummy ?", "words": ["on", "this", "your", "tummy"]}, {"sent": "do you think theres money in my tummy ?", "words": ["money", "do", "think", "my", "you", "in", "tummy"]}, {"sent": "ladys got a very big fat tummy .", "words": ["tummy", "big", "a"]}, {"sent": "in the tummy .", "words": ["the", "in", "tummy"]}, {"sent": "a tummy tummy tummy .", "words": ["tummy", "a"]}, {"sent": "in his tummy .", "words": ["his", "in", "tummy"]}, {"sent": "oh my tummy hurts .", "words": ["my", "tummy"]}, {"sent": "she hasnt got tummy ache .", "words": ["she", "tummy"]}, {"sent": "say yummy in my tummy .", "words": ["in", "say", "my", "tummy"]}, {"sent": "its squishing and squashing your tummy .", "words": ["tummy", "your", "and"]}, {"sent": "and you know the baby in Ronnys tummy ?", "words": ["the", "you", "and", "in", "tummy"]}, {"sent": "youve got a poorly tummy ?", "words": ["tummy", "a"]}, {"sent": "thats her tummy button .", "words": ["her", "button", "tummy"]}, {"sent": "big tummy ?", "words": ["big", "tummy"]}, {"sent": "but it helps me to hold all my children and tickle each one on the tummy .", "words": ["the", "all", "my", "hold", "each", "to", "tickle", "and", "it", "tummy", "on", "me", "but"]}, {"sent": "mm thats yummy in the babys tummy .", "words": ["the", "in", "tummy"]}, {"sent": "you can feel it go in your tummy ?", "words": ["can", "your", "go", "you", "in", "it", "tummy"]}, {"sent": "I think thats a leaf on his tummy .", "words": ["think", "his", "tummy", "on", "a"]}, {"sent": "he needs some food in his tummy .", "words": ["food", "in", "he", "tummy", "his", "some"]}, {"sent": "does tummy does liquorice give you tummy ache ?", "words": ["does", "give", "you", "tummy"]}, {"sent": "its just something that happens with the movement of the car makes your tummy feel poorly .", "words": ["the", "your", "of", "car", "with", "tummy", "that"]}, {"sent": "has your tummy has your tummy got a headache ?", "words": ["your", "a", "tummy"]}, {"sent": "soap doesnt go in your tummy .", "words": ["your", "go", "soap", "in", "tummy"]}, {"sent": "you dont chew them up and put them in your tummy .", "words": ["up", "put", "your", "them", "and", "you", "dont", "in", "tummy"]}, {"sent": "does Thomass tummy feel full ?", "words": ["does", "full", "tummy"]}, {"sent": "it went in that tummy so quickly .", "words": ["so", "in", "it", "tummy", "that"]}, {"sent": "bubbles on my tummy .", "words": ["on", "my", "tummy", "bubbles"]}, {"sent": "youre gonna xxx away your tummy against this ball .", "words": ["your", "ball", "this", "away", "tummy"]}, {"sent": "has she got a big tummy ?", "words": ["tummy", "big", "she", "a"]}, {"sent": "my tummy ?", "words": ["my", "tummy"]}, {"sent": "did you have a tummy ache ?", "words": ["you", "have", "did", "tummy", "a"]}, {"sent": "tummy tummy tummy .", "words": ["tummy"]}, {"sent": "big tummy .", "words": ["big", "tummy"]}, {"sent": "they wont hurt your tummy .", "words": ["hurt", "they", "your", "tummy"]}, {"sent": "full tummy .", "words": ["full", "tummy"]}, {"sent": "pressing Lalas tummy and shes talking .", "words": ["and", "tummy"]}, {"sent": "gonna put some food in your tummy .", "words": ["put", "your", "food", "in", "tummy", "some"]}, {"sent": "your tummy hurts ?", "words": ["your", "tummy"]}, {"sent": "into your big tummy .", "words": ["big", "your", "into", "tummy"]}, {"sent": "after the crocodile has bitten your tummy ?", "words": ["the", "your", "tummy"]}, {"sent": "youre going to have a very full tummy arent you ?", "words": ["full", "to", "you", "have", "tummy", "a"]}, {"sent": "Rosies going to tickle your tummy .", "words": ["to", "your", "tickle", "tummy"]}, {"sent": "ow thats my tummy .", "words": ["my", "tummy"]}, {"sent": "hes showing his tummy isnt he ?", "words": ["his", "he", "tummy"]}, {"sent": "have you got a big fat tummy ?", "words": ["you", "have", "tummy", "big", "a"]}, {"sent": "wheres your tummy ?", "words": ["your", "tummy"]}, {"sent": "yeah Mummys tummy .", "words": ["tummy"]}, {"sent": "yummy in her tummy ?", "words": ["in", "her", "tummy"]}, {"sent": "ladder .", "words": ["ladder"]}, {"sent": "the hook and ladder truck yeah .", "words": ["the", "truck", "ladder", "and"]}, {"sent": "thats a ladder yes .", "words": ["ladder", "a"]}, {"sent": "a ladder .", "words": ["ladder", "a"]}, {"sent": "and is there a ladder on it as well ?", "words": ["is", "and", "there", "it", "ladder", "on", "a"]}, {"sent": "no this ladder develops into a circus .", "words": ["this", "into", "ladder", "a"]}, {"sent": "its not a ladder .", "words": ["not", "ladder", "a"]}, {"sent": "it could be a ladder for the policeman the fire man to climb up .", "words": ["the", "up", "for", "be", "to", "it", "ladder", "could", "climb", "a"]}, {"sent": "where are you gonna put up the ladder ?", "words": ["where", "up", "put", "the", "you", "are", "ladder"]}, {"sent": "up goes the ladder .", "words": ["up", "the", "ladder"]}, {"sent": "where is the ladder ?", "words": ["where", "the", "ladder", "is"]}, {"sent": "can you go up the ladder ?", "words": ["can", "the", "up", "go", "you", "ladder"]}, {"sent": "what one on ladder ?", "words": ["on", "what", "ladder"]}, {"sent": "a ladder ?", "words": ["ladder", "a"]}, {"sent": "he was on the ladder and now hes climbing onto the tree isnt he ?", "words": ["the", "tree", "was", "and", "he", "ladder", "on"]}, {"sent": "the biggest thats the hook and ladder truck .", "words": ["the", "truck", "ladder", "and"]}, {"sent": "theres a fire up the ladder ?", "words": ["up", "the", "ladder", "a"]}, {"sent": "he has got a ladder on the back of the van .", "words": ["the", "back", "of", "he", "ladder", "on", "a"]}, {"sent": "I might leave ya way up on a ladder .", "words": ["up", "on", "ladder", "a"]}, {"sent": "but the ladder wasnt long enough to reach Huckle up in the play room .", "words": ["the", "up", "to", "play", "long", "in", "room", "ladder", "but"]}, {"sent": "the yellow ladder .", "words": ["the", "yellow", "ladder"]}, {"sent": "I want you to go back with Matthew when he comes down off the ladder so call him off .", "words": ["off", "the", "back", "go", "to", "so", "with", "you", "him", "he", "ladder", "down", "when"]}, {"sent": "maybe they need a longer ladder .", "words": ["they", "need", "ladder", "a"]}, {"sent": "oh Spots climbing up the ladder now isnt he ?", "words": ["up", "the", "he", "ladder"]}, {"sent": "policemans up the ladder .", "words": ["up", "the", "ladder"]}, {"sent": "ladder .", "words": ["ladder"]}, {"sent": "and youve been going up and down the slide pretending its a ladder .", "words": ["up", "the", "slide", "and", "ladder", "down", "a"]}, {"sent": "a ladder .", "words": ["ladder", "a"]}, {"sent": "you were pretending your steps on your slide were a ladder werent you ?", "words": ["slide", "your", "you", "ladder", "on", "were", "a"]}, {"sent": "didnt he climb down the ladder ?", "words": ["the", "climb", "he", "ladder", "down"]}, {"sent": "climbing a ladder .", "words": ["ladder", "a"]}, {"sent": "did he hafta come down the ladder and get it ?", "words": ["the", "and", "get", "it", "he", "did", "ladder", "down"]}, {"sent": "has it not got a ladder for it ?", "words": ["for", "it", "ladder", "not", "a"]}, {"sent": "you need a ladder do you ?", "words": ["do", "need", "you", "ladder", "a"]}, {"sent": "down the ladder .", "words": ["the", "down", "ladder"]}, {"sent": "it hasnt got a ladder on see .", "words": ["it", "ladder", "on", "see", "a"]}, {"sent": "a ladder ?", "words": ["ladder", "a"]}, {"sent": "Spot knocked the ladder over .", "words": ["the", "over", "ladder"]}, {"sent": "thats no ladder for that .", "words": ["for", "that", "ladder"]}, {"sent": "yeah but you have a ladder across the top that they dont have .", "words": ["the", "you", "have", "they", "dont", "ladder", "that", "but", "a"]}, {"sent": "you landed on the biggest ladder didnt you ?", "words": ["on", "the", "ladder", "you"]}, {"sent": "up the ladder down the ladder up the ladder down the ladder .", "words": ["up", "the", "down", "ladder"]}, {"sent": "no Im up on the ladder .", "words": ["up", "the", "on", "ladder"]}, {"sent": "hes climbing the ladder ?", "words": ["the", "ladder"]}, {"sent": "youre going to climb the ladder ?", "words": ["the", "to", "climb", "ladder"]}, {"sent": "theres a man with a ladder outside Joel .", "words": ["outside", "with", "ladder", "a"]}, {"sent": "dont fall off the ladder .", "words": ["off", "the", "fall", "dont", "ladder"]}, {"sent": "look there some people in there that we can put on the ladder .", "words": ["can", "the", "put", "we", "there", "in", "ladder", "look", "on", "some", "that"]}, {"sent": "or on top of a ladder ?", "words": ["on", "of", "ladder", "a"]}, {"sent": "one man in the ladder and the other firemen are all standing on the fire engine .", "words": ["the", "all", "other", "and", "in", "are", "ladder", "on"]}, {"sent": "yeah ladder .", "words": ["ladder"]}, {"sent": "I wonder is there a ladder or something ?", "words": ["there", "ladder", "a", "is"]}, {"sent": "and a fireman and a little fire engine er and a ladder .", "words": ["ladder", "little", "a", "and"]}, {"sent": "the right master is up on the ladder ?", "words": ["the", "up", "is", "ladder", "on"]}, {"sent": "a ladder from the fire engine look .", "words": ["the", "look", "ladder", "a"]}, {"sent": "thats right ladder .", "words": ["ladder"]}, {"sent": "Ill hold the ladder and you let Jimmy climb up .", "words": ["the", "up", "hold", "you", "and", "ladder", "climb"]}, {"sent": "my big ladder .", "words": ["my", "big", "ladder"]}, {"sent": "you see we have got this truck with a ladder on havent we ?", "words": ["we", "truck", "with", "you", "have", "this", "ladder", "on", "see", "a"]}, {"sent": "theres a little yellow piece that connects the two parts of the ladder .", "words": ["the", "of", "little", "ladder", "yellow", "that", "a"]}, {"sent": "but we cant get to the ladder as well .", "words": ["the", "we", "to", "get", "ladder", "but"]}, {"sent": "up the ladder as well .", "words": ["up", "the", "ladder"]}, {"sent": "when the ladder gets up there ?", "words": ["the", "up", "there", "ladder", "when"]}, {"sent": "but you have a ladder that goes across the top .", "words": ["the", "you", "have", "ladder", "that", "but", "a"]}, {"sent": "with his ladder .", "words": ["his", "with", "ladder"]}, {"sent": "what do you want me to do with the ladder ?", "words": ["the", "what", "do", "to", "with", "you", "ladder", "me"]}, {"sent": "the firemans up the ladder and what he doesnt know is who has got into the fire engine .", "words": ["the", "what", "up", "into", "is", "and", "he", "ladder", "who"]}, {"sent": "maybe you can fit the ladder in there .", "words": ["can", "the", "fit", "you", "there", "in", "ladder"]}, {"sent": "its got a ladder .", "words": ["ladder", "a"]}, {"sent": "see him up the ladder ?", "words": ["up", "the", "him", "ladder", "see"]}, {"sent": "make a ladder ?", "words": ["make", "ladder", "a"]}, {"sent": "the ladder came down again .", "words": ["the", "down", "ladder"]}, {"sent": "theres a ladder .", "words": ["ladder", "a"]}, {"sent": "alright he went up the ladder and lifted the puppets off their place on the rung and lifted them onto Stromboli .", "words": ["up", "the", "off", "their", "them", "and", "he", "ladder", "on"]}, {"sent": "hes up the ladder isnt he ?", "words": ["up", "the", "he", "ladder"]}, {"sent": "what about this big ladder here ?", "words": ["what", "here", "this", "ladder", "about", "big"]}, {"sent": "is that the ladder ?", "words": ["the", "that", "ladder", "is"]}, {"sent": "is somebody gonna climb the ladder ?", "words": ["the", "climb", "ladder", "is"]}, {"sent": "the man on the ladder ?", "words": ["the", "on", "ladder"]}, {"sent": "xxx some people dont use these dont use the hand rails they use the top rung on the ladder .", "words": ["the", "these", "hand", "dont", "they", "ladder", "on", "some"]}, {"sent": "she went on the ladder and then she stepped on the moon .", "words": ["the", "moon", "and", "ladder", "on", "then", "she"]}, {"sent": "thats Granny coming up the ladder .", "words": ["up", "the", "ladder"]}, {"sent": "no you only go on the ladder .", "words": ["the", "go", "you", "ladder", "on"]}, {"sent": "she shes moved the ladder now .", "words": ["the", "she", "ladder"]}, {"sent": "you can reach up high with that ladder cant you ?", "words": ["can", "up", "with", "you", "ladder", "high", "that"]}, {"sent": "oh this ladder is stuck on here .", "words": ["is", "here", "this", "stuck", "ladder", "on"]}, {"sent": "what if he hes standing on this big tall ladder .", "words": ["what", "if", "this", "he", "ladder", "on", "big"]}, {"sent": "he going up ladder or down ladder ?", "words": ["up", "down", "he", "ladder"]}, {"sent": "how come that girl gets to go up the ladder ?", "words": ["up", "the", "how", "go", "to", "ladder", "that"]}, {"sent": "theyre not gonna make a tall ladder .", "words": ["make", "not", "ladder", "a"]}, {"sent": "you tell him hes a naughty man for climbing up on the ladder isnt he ?", "words": ["up", "the", "for", "you", "him", "he", "ladder", "on", "naughty", "a"]}, {"sent": "theres Roberto hes on a ladder .", "words": ["on", "ladder", "a"]}, {"sent": "can you make him climb the ladder ?", "words": ["can", "the", "you", "him", "ladder", "climb", "make"]}, {"sent": "a ladder .", "words": ["ladder", "a"]}, {"sent": "the ladder went up like a telescope to the top of the house .", "words": ["the", "up", "of", "to", "like", "house", "ladder", "a"]}, {"sent": "ah let me see where is the ladder .", "words": ["the", "where", "is", "me", "ladder", "see"]}, {"sent": "a ladder .", "words": ["ladder", "a"]}, {"sent": "down the ladder ?", "words": ["the", "down", "ladder"]}, {"sent": "got the ladder .", "words": ["the", "ladder"]}, {"sent": "yeah he climbed down the ladder .", "words": ["the", "down", "he", "ladder"]}, {"sent": "one little boy one little boy clap clap clap .", "words": ["clap", "little"]}, {"sent": "wanna clap a rhythm ?", "words": ["clap", "wanna", "a"]}, {"sent": "say clap clap clap clap clap clap clap .", "words": ["say", "clap"]}, {"sent": "clap clap clap .", "words": ["clap"]}, {"sent": "but and he just learned how to clap so its its thats uh patty cakes pretty fun for him .", "words": ["how", "for", "clap", "to", "so", "and", "him", "he", "pretty", "but"]}, {"sent": "and clap clap clap .", "words": ["clap", "and"]}, {"sent": "clap your hands .", "words": ["your", "clap"]}, {"sent": "and clap .", "words": ["clap", "and"]}, {"sent": "clap clap clap clap clap clap clap clap clap clap .", "words": ["clap"]}, {"sent": "clap clap clap clap clap .", "words": ["clap"]}, {"sent": "give a little clap .", "words": ["give", "clap", "little", "a"]}, {"sent": "can you clap your hands ?", "words": ["can", "your", "clap", "you"]}, {"sent": "a little clap for the cake .", "words": ["the", "for", "clap", "cake", "little", "a"]}, {"sent": "clap clap clap your hands .", "words": ["your", "clap"]}, {"sent": "if youre happy and ya know it and you really want to show it if youre happy and ya know it clap yer hands .", "words": ["if", "happy", "clap", "show", "to", "you", "and", "it", "want to"]}, {"sent": "clap clap clap .", "words": ["clap"]}, {"sent": "clap hands .", "words": ["clap"]}, {"sent": "clap clap clap .", "words": ["clap"]}, {"sent": "how do you clap ?", "words": ["do", "how", "clap", "you"]}, {"sent": "look she can clap her hands .", "words": ["can", "clap", "look", "her", "she"]}, {"sent": "clap clap clap clappity clap clap clap .", "words": ["clap"]}, {"sent": "xxx clap clap xxx xxx .", "words": ["clap"]}, {"sent": "can you clap ?", "words": ["can", "clap", "you"]}, {"sent": "if youre happy and you know it clap your hands .", "words": ["your", "if", "happy", "clap", "you", "and", "it"]}, {"sent": "clap clap clap .", "words": ["clap"]}, {"sent": "will we give ourselves a clap ?", "words": ["clap", "we", "will", "give", "a"]}, {"sent": "clap clap clap clap your hands together .", "words": ["your", "clap"]}, {"sent": "one little girl one little girl clap clap clap .", "words": ["clap", "little"]}, {"sent": "clap hands .", "words": ["clap"]}, {"sent": "open shut and give a little clap clap clap .", "words": ["clap", "open", "little", "and", "give", "a"]}, {"sent": "clap your feet .", "words": ["your", "clap"]}, {"sent": "give a little clap clap clap .", "words": ["give", "clap", "little", "a"]}, {"sent": "yay look at you clap your hands .", "words": ["look", "your", "clap", "you", "at"]}, {"sent": "clap clap clap .", "words": ["clap"]}, {"sent": "clap ?", "words": ["clap"]}, {"sent": "clap your hands .", "words": ["your", "clap"]}, {"sent": "clap Mamas hand .", "words": ["clap", "hand"]}, {"sent": "yeah can you clap ?", "words": ["can", "clap", "you"]}, {"sent": "clap Manman .", "words": ["clap"]}, {"sent": "clap clap clap .", "words": ["clap"]}, {"sent": "can you clap your hands ?", "words": ["can", "your", "clap", "you"]}, {"sent": "clap clap clap .", "words": ["clap"]}, {"sent": "if youre happy and you know it clap your hands .", "words": ["your", "if", "happy", "clap", "you", "and", "it"]}, {"sent": "clap clap clap .", "words": ["clap"]}, {"sent": "clap .", "words": ["clap"]}, {"sent": "clap your hands together .", "words": ["your", "clap"]}, {"sent": "one little finger one little finger clap clap clap .", "words": ["finger", "clap", "little"]}, {"sent": "watch him clap .", "words": ["him", "watch", "clap"]}, {"sent": "can you clap for me ?", "words": ["can", "for", "clap", "you", "me"]}, {"sent": "clap hands .", "words": ["clap"]}, {"sent": "clap .", "words": ["clap"]}, {"sent": "clap hands clap hands clap hands .", "words": ["clap"]}, {"sent": "clap your hands .", "words": ["your", "clap"]}, {"sent": "clap hands .", "words": ["clap"]}, {"sent": "then I clap .", "words": ["then", "clap"]}, {"sent": "clap clap clap clap clap .", "words": ["clap"]}, {"sent": "you gonna clap ?", "words": ["clap", "you"]}, {"sent": "yay let me hear you clap .", "words": ["me", "clap", "you", "hear"]}, {"sent": "can you clap ?", "words": ["can", "clap", "you"]}, {"sent": "clap .", "words": ["clap"]}, {"sent": "can you clap ?", "words": ["can", "clap", "you"]}, {"sent": "clap clap clap .", "words": ["clap"]}, {"sent": "clap your little feet feet feet .", "words": ["your", "clap", "little"]}, {"sent": "do you want me to clap again ?", "words": ["do", "clap", "to", "you", "me"]}, {"sent": "clap hands .", "words": ["clap"]}, {"sent": "you clap it ?", "words": ["it", "clap", "you"]}, {"sent": "clap hands clap hands clap hands .", "words": ["clap"]}, {"sent": "clap clap clap .", "words": ["clap"]}, {"sent": "four little finger four little fingers clap clap clap point to the ceiling point to the floor and lay them on your p .", "words": ["the", "your", "clap", "to", "them", "little", "and", "finger", "on"]}, {"sent": "wanna make me clap hands ?", "words": ["me", "make", "clap", "wanna"]}, {"sent": "xxx clap clap clap .", "words": ["clap"]}, {"sent": "say if youre happy and you know it clap your hands .", "words": ["your", "if", "happy", "clap", "say", "you", "and", "it"]}, {"sent": "clap clap clap .", "words": ["clap"]}, {"sent": "clap clap hands ?", "words": ["clap"]}, {"sent": "clap your hands .", "words": ["your", "clap"]}, {"sent": "yay clap clap clap .", "words": ["clap"]}, {"sent": "clap clap clap .", "words": ["clap"]}, {"sent": "clap clap clap .", "words": ["clap"]}, {"sent": "clap clap clap clap .", "words": ["clap"]}, {"sent": "clap clap clap clap clap clap clap .", "words": ["clap"]}, {"sent": "clap clap clap .", "words": ["clap"]}, {"sent": "if youre happy and you know it clap your hands .", "words": ["your", "if", "happy", "clap", "you", "and", "it"]}, {"sent": "can you clap ?", "words": ["can", "clap", "you"]}, {"sent": "three little fingers three little fingers clap clap clap .", "words": ["clap", "little"]}, {"sent": "wanna clap look .", "words": ["clap", "wanna", "look"]}, {"sent": "yay let me see you clap .", "words": ["see", "me", "clap", "you"]}, {"sent": "clap clap clap .", "words": ["clap"]}, {"sent": "wanna clap the babys hands ?", "words": ["the", "clap", "wanna"]}, {"sent": "clap .", "words": ["clap"]}, {"sent": "clap your hands .", "words": ["your", "clap"]}, {"sent": "one clap .", "words": ["clap"]}, {"sent": "Morgan clap your feet .", "words": ["your", "clap"]}, {"sent": "put the rhythm in your hands and go clap clap clap .", "words": ["the", "put", "your", "clap", "go", "and", "in"]}, {"sent": "clap ?", "words": ["clap"]}, {"sent": "would you like an apple ?", "words": ["like", "you", "an", "would", "apple"]}, {"sent": "apple chew .", "words": ["apple"]}, {"sent": "an apple .", "words": ["apple", "an"]}, {"sent": "its come from the apple tree .", "words": ["the", "apple", "tree"]}, {"sent": "apple juice .", "words": ["apple", "juice"]}, {"sent": "apple .", "words": ["apple"]}, {"sent": "do you think theyre having a minimilk or an apple one ?", "words": ["do", "think", "you", "an", "apple", "a"]}, {"sent": "eat some apple please .", "words": ["eat", "some", "apple"]}, {"sent": "it needs a wash because youve been putting pieces of apple and bits of toast in it .", "words": ["toast", "of", "and", "in", "it", "apple", "wash", "because", "a"]}, {"sent": "starts with an Annie apple ?", "words": ["apple", "an", "with"]}, {"sent": "would you like some more apple ?", "words": ["like", "you", "more", "would", "apple", "some"]}, {"sent": "you said you wanted cheese sandwiches and cheese and crisps and apple .", "words": ["apple", "cheese", "and", "you"]}, {"sent": "throw an apple ?", "words": ["apple", "an", "throw"]}, {"sent": "its a toffee apple .", "words": ["apple", "a"]}, {"sent": "oh is this the toffee apple on a stick .", "words": ["the", "stick", "is", "this", "on", "apple", "a"]}, {"sent": "wheres the a for apple ?", "words": ["the", "apple", "for", "a"]}, {"sent": "apple juice .", "words": ["apple", "juice"]}, {"sent": "apple juice ?", "words": ["apple", "juice"]}, {"sent": "Purdie might like an apple .", "words": ["apple", "an", "like"]}, {"sent": "apple .", "words": ["apple"]}, {"sent": "would ya like some apple ?", "words": ["apple", "some", "like", "would"]}, {"sent": "the only thing I can hear is Thomas eating an apple peel .", "words": ["the", "can", "is", "hear", "an", "apple"]}, {"sent": "candy apple red .", "words": ["red", "apple", "candy"]}, {"sent": "you do like apple .", "words": ["apple", "do", "like", "you"]}, {"sent": "heres an apple and a brush and comb flowers wanna smell the flowers ?", "words": ["the", "brush", "comb", "and", "an", "wanna", "apple", "a"]}, {"sent": "apple juice .", "words": ["apple", "juice"]}, {"sent": "good apple .", "words": ["apple", "good"]}, {"sent": "and an apple .", "words": ["apple", "an", "and"]}, {"sent": "apple juice .", "words": ["apple", "juice"]}, {"sent": "an apple tree or orange tree .", "words": ["tree", "apple", "an", "orange"]}, {"sent": "is this the apple ?", "words": ["the", "this", "apple", "is"]}, {"sent": "apple juice ?", "words": ["apple", "juice"]}, {"sent": "did you drop your apple on the floor ?", "words": ["the", "your", "you", "drop", "did", "on", "apple"]}, {"sent": "apple man .", "words": ["apple"]}, {"sent": "apple pie .", "words": ["apple"]}, {"sent": "wanna draw an apple wanna draw an apple ?", "words": ["apple", "an", "draw", "wanna"]}, {"sent": "oh apple juice .", "words": ["apple", "juice"]}, {"sent": "well you can try another one if you cant find the apple .", "words": ["can", "the", "if", "you", "another", "find", "apple", "try"]}, {"sent": "its a very little apple .", "words": ["apple", "little", "a"]}, {"sent": "can you color the apple red ?", "words": ["can", "the", "red", "you", "apple"]}, {"sent": "sounds a nice crisp apple Thomas .", "words": ["apple", "nice", "a"]}, {"sent": "a fake apple .", "words": ["apple", "a"]}, {"sent": "youve got an apple there yes .", "words": ["apple", "an", "there"]}, {"sent": "thats not an apple .", "words": ["apple", "an", "not"]}, {"sent": "heres your apple .", "words": ["apple", "your"]}, {"sent": "they had no apple juice .", "words": ["apple", "they", "juice"]}, {"sent": "she likes apple juice ?", "words": ["apple", "juice", "she"]}, {"sent": "apple ?", "words": ["apple"]}, {"sent": "oh apple .", "words": ["apple"]}, {"sent": "apple juice .", "words": ["apple", "juice"]}, {"sent": "oh you want apple okay .", "words": ["apple", "you"]}, {"sent": "oh youve finished the apple .", "words": ["the", "apple"]}, {"sent": "he likes that apple .", "words": ["apple", "that", "he"]}, {"sent": "this is an apple not a pear .", "words": ["is", "this", "an", "apple", "not", "a"]}, {"sent": "Mummy wouldnt eat your apple .", "words": ["eat", "apple", "your"]}, {"sent": "make an apple .", "words": ["apple", "an", "make"]}, {"sent": "mind your apple .", "words": ["apple", "your"]}, {"sent": "this is apple .", "words": ["apple", "this", "is"]}, {"sent": "theres some apple .", "words": ["apple", "some"]}, {"sent": "here look at the apple .", "words": ["the", "look", "here", "at", "apple"]}, {"sent": "apple Jack .", "words": ["apple"]}, {"sent": "a is for apple and alligator .", "words": ["for", "is", "and", "alligator", "apple", "a"]}, {"sent": "apple .", "words": ["apple"]}, {"sent": "do you wanna drink apple juice ?", "words": ["do", "drink", "you", "wanna", "apple", "juice"]}, {"sent": "six pieces of apple .", "words": ["of", "apple"]}, {"sent": "theres one apple .", "words": ["apple"]}, {"sent": "can you say apple ?", "words": ["can", "apple", "say", "you"]}, {"sent": "youre eating an apple .", "words": ["apple", "an"]}, {"sent": "apple juice .", "words": ["apple", "juice"]}, {"sent": "another apple .", "words": ["another", "apple"]}, {"sent": "oo apple juice .", "words": ["apple", "juice"]}, {"sent": "oh apple juice .", "words": ["apple", "juice"]}, {"sent": "would you give me the apple ?", "words": ["the", "you", "would", "apple", "me", "give"]}, {"sent": "apple pie .", "words": ["apple"]}, {"sent": "Lowly Worm and apple car .", "words": ["apple", "car", "and"]}, {"sent": "wheres the apple ?", "words": ["the", "apple"]}, {"sent": "apple .", "words": ["apple"]}, {"sent": "some apple ?", "words": ["apple", "some"]}, {"sent": "no no apple .", "words": ["apple"]}, {"sent": "em would you like this hot cross bun or an apple ?", "words": ["like", "you", "this", "hot", "an", "would", "apple"]}, {"sent": "do you like apple jacks ?", "words": ["apple", "do", "like", "you"]}, {"sent": "its a for apple .", "words": ["apple", "for", "a"]}, {"sent": "hm lets get an apple well peel an apple .", "words": ["get", "apple", "an"]}, {"sent": "she wants you to read the apple story first .", "words": ["the", "to", "first", "you", "read", "apple", "story", "she"]}, {"sent": "oh apple .", "words": ["apple"]}, {"sent": "how about an apple or a banana ?", "words": ["how", "banana", "an", "about", "apple", "a"]}, {"sent": "apple .", "words": ["apple"]}, {"sent": "sometimes she gives you apple juice ?", "words": ["apple", "juice", "she", "you"]}, {"sent": "more apple ?", "words": ["apple", "more"]}, {"sent": "an apple .", "words": ["apple", "an"]}, {"sent": "the red apple or the green apple ?", "words": ["the", "apple", "red", "green"]}, {"sent": "but its an apple .", "words": ["apple", "an", "but"]}, {"sent": "dried apple ?", "words": ["apple"]}, {"sent": "and you were asking for an apple earlier on werent you ?", "words": ["for", "you", "and", "an", "on", "apple", "were"]}, {"sent": "an apple .", "words": ["apple", "an"]}, {"sent": "some apple juice .", "words": ["apple", "some", "juice"]}, {"sent": "what that is thats Ruldolph and Father Christmas are pulling a Christmas cracker in front of the snowman .", "words": ["the", "what", "is", "of", "cracker", "and", "in", "are", "snowman", "that", "a"]}, {"sent": "and usually a piece of coal for the mouth but these tubbies Teletubbies have made a beautiful big smile on the snowman .", "words": ["the", "these", "for", "of", "mouth", "and", "have", "on", "snowman", "smile", "big", "but", "a"]}, {"sent": "did you ever see a s snowman .", "words": ["you", "snowman", "did", "see", "a"]}, {"sent": "of course hes the big snowman .", "words": ["of", "the", "snowman", "big"]}, {"sent": "is this mister snowman ?", "words": ["this", "snowman", "is"]}, {"sent": "you know what a snowman is ?", "words": ["what", "is", "you", "snowman", "a"]}, {"sent": "the snowman has a hat .", "words": ["the", "snowman", "hat", "a"]}, {"sent": "a snowman ?", "words": ["snowman", "a"]}, {"sent": "and finding two children with lotsof cans of spray snow and theyre making the snowman in the middle of the floor .", "words": ["the", "of", "snow", "with", "and", "in", "snowman"]}, {"sent": "the snowman one ?", "words": ["the", "snowman"]}, {"sent": "lets make a snowman Sam said Stella where does a snowman sleep in a soft fluffy snow bank answered Stella what does a snowman eat asked Sam snowballs snow peas and snow suits .", "words": ["where", "does", "what", "soft", "sleep", "snow", "peas", "and", "eat", "in", "snowman", "make", "a"]}, {"sent": "daddys gonna help you build a snowman this year right daddy ?", "words": ["help", "you", "this", "snowman", "build", "a"]}, {"sent": "do you wanna give me a bit more and then I can make a snowman ?", "words": ["can", "do", "then", "you", "and", "more", "snowman", "wanna", "me", "give", "make", "a"]}, {"sent": "lets make a snowman Sam said Stella .", "words": ["snowman", "make", "a"]}, {"sent": "you watched a bit of snowman .", "words": ["of", "snowman", "a", "you"]}, {"sent": "hes got a snowman jumper .", "words": ["snowman", "a"]}, {"sent": "heres a snowman .", "words": ["snowman", "a"]}, {"sent": "and a snowman .", "words": ["snowman", "a", "and"]}, {"sent": "is it a soldier Christmas er a soldier snowman that is marching .", "words": ["is", "it", "snowman", "that", "a"]}, {"sent": "I see some kids building a snowman .", "words": ["see", "snowman", "a", "some"]}, {"sent": "but snowman is my best friend .", "words": ["my", "snowman", "but", "is"]}, {"sent": "wheres the snowman ?", "words": ["the", "snowman"]}, {"sent": "huh you go sledding snowshoeing make a snowman .", "words": ["go", "you", "snowman", "make", "a"]}, {"sent": "okay heres your snowman .", "words": ["snowman", "your"]}, {"sent": "is it like a snowman ?", "words": ["is", "like", "it", "snowman", "a"]}, {"sent": "see the snowman .", "words": ["the", "see", "snowman"]}, {"sent": "is there a snowman there ?", "words": ["there", "snowman", "a", "is"]}, {"sent": "and that one says snowman snowman snowman snowman .", "words": ["snowman", "that", "and"]}, {"sent": "a snowman .", "words": ["snowman", "a"]}, {"sent": "mm youre making a snowman thats good .", "words": ["snowman", "good", "a"]}, {"sent": "did we make a snowman ?", "words": ["we", "snowman", "did", "make", "a"]}, {"sent": "can you see the snowman on there ?", "words": ["can", "the", "you", "there", "snowman", "on", "see"]}, {"sent": "theres the snowman .", "words": ["the", "snowman"]}, {"sent": "to build a little snowman ?", "words": ["to", "little", "snowman", "build", "a"]}, {"sent": "Im making a snowman .", "words": ["snowman", "a"]}, {"sent": "you cant see the snowman now Thomas .", "words": ["the", "see", "snowman", "you"]}, {"sent": "you got that snow man I got that snowman book .", "words": ["snow", "you", "snowman", "book", "that"]}, {"sent": "but youre sposta make a snowman outside .", "words": ["outside", "snowman", "make", "but", "a"]}, {"sent": "yeah thats a snowman in the picture isnt it ?", "words": ["the", "picture", "in", "snowman", "it", "a"]}, {"sent": "did you make a snowman too ?", "words": ["you", "snowman", "did", "too", "make", "a"]}, {"sent": "and whats the snowman wearing ?", "words": ["the", "snowman", "and"]}, {"sent": "how do you build a snowman ?", "words": ["do", "how", "you", "snowman", "build", "a"]}, {"sent": "abominable snowman .", "words": ["snowman"]}, {"sent": "but theyve built arms for this snowman .", "words": ["this", "for", "snowman", "but"]}, {"sent": "hi mister snowman .", "words": ["snowman"]}, {"sent": "a snowman .", "words": ["snowman", "a"]}, {"sent": "wheres the snowman ?", "words": ["the", "snowman"]}, {"sent": "three elves made a magical snowman who sang with a squeaky sound .", "words": ["who", "snowman", "with", "a"]}, {"sent": "and oh my gosh theres snowman .", "words": ["my", "snowman", "and"]}, {"sent": "a snowman ?", "words": ["snowman", "a"]}, {"sent": "theyre building a snowman .", "words": ["snowman", "a"]}, {"sent": "thats a snowman .", "words": ["snowman", "a"]}, {"sent": "wanna see my snowman ?", "words": ["see", "my", "snowman", "wanna"]}, {"sent": "and Daddy built that snowman .", "words": ["snowman", "that", "and"]}, {"sent": "snowman wheres youre snowman ?", "words": ["snowman"]}, {"sent": "look snowman and theres a little boy .", "words": ["little", "and", "snowman", "look", "a"]}, {"sent": "you knocked down my snowman .", "words": ["my", "down", "snowman", "you"]}, {"sent": "think well well be able to make a snowman ?", "words": ["think", "to", "be", "snowman", "make", "a"]}, {"sent": "wheres snowman in the house ?", "words": ["the", "in", "snowman", "house"]}, {"sent": "and some buttons and they dressed him as if he was a real snowman and then when the snow fell they covered the no snowman and he looked like a real snowman anyway didnt he ?", "words": ["the", "if", "was", "snow", "like", "and", "they", "him", "snowman", "he", "some", "then", "when", "a"]}, {"sent": "weve got a snowman .", "words": ["snowman", "a"]}, {"sent": "thats the bottom of the snowman .", "words": ["the", "of", "bottom", "snowman"]}, {"sent": "look what look at the look at the snowman he made .", "words": ["the", "what", "look", "snowman", "he", "at"]}, {"sent": "theyre gonna build a snowman .", "words": ["snowman", "build", "a"]}, {"sent": "could make a bigger snowman couldnt we ?", "words": ["we", "snowman", "could", "make", "a"]}, {"sent": "well theres a snowman bubble .", "words": ["snowman", "a"]}, {"sent": "did you make a snowman ?", "words": ["you", "snowman", "did", "make", "a"]}, {"sent": "Im gonna put your sunglasses on your snowman .", "words": ["on", "put", "snowman", "your"]}, {"sent": "so James brought the snowman inside .", "words": ["the", "so", "inside", "snowman"]}, {"sent": "and find the snowman .", "words": ["the", "snowman", "find", "and"]}, {"sent": "Im a snowman I like ice said the snowman .", "words": ["the", "like", "snowman", "ice", "a"]}, {"sent": "happy christmas snowman .", "words": ["snowman", "happy"]}, {"sent": "a snowman .", "words": ["snowman", "a"]}, {"sent": "wow you made a snowman and a reindeer .", "words": ["snowman", "and", "a", "you"]}, {"sent": "the snowman .", "words": ["the", "snowman"]}, {"sent": "now what do you wanna make to go with the snowman ?", "words": ["the", "what", "do", "go", "to", "with", "you", "snowman", "wanna", "make"]}, {"sent": "see if we can make a really big snowman .", "words": ["can", "if", "we", "big", "snowman", "see", "make", "a"]}, {"sent": "you draw your own snowman .", "words": ["draw", "your", "snowman", "you"]}, {"sent": "doo snowman xxx .", "words": ["snowman"]}, {"sent": "thats a snowman .", "words": ["snowman", "a"]}, {"sent": "FatherChristmas wears white uh wears red and a snowman is white .", "words": ["red", "is", "and", "snowman", "white", "a"]}, {"sent": "Eleanor wintertime you make a snowman .", "words": ["snowman", "make", "a", "you"]}, {"sent": "yeah snowman .", "words": ["snowman"]}, {"sent": "thats a snowman isnt it ?", "words": ["it", "snowman", "a"]}, {"sent": "poor snowman .", "words": ["snowman", "poor"]}, {"sent": "to spray snow onto the snowman he has built inside the house .", "words": ["the", "to", "inside", "snow", "snowman", "he", "house"]}, {"sent": "do you not wanna read the frog prince or the princess and the snowman ?", "words": ["the", "do", "you", "frog", "and", "snowman", "read", "wanna", "not"]}, {"sent": "youre squishing the snowman .", "words": ["the", "snowman"]}, {"sent": "the seal has thrown a snowball and its hiding behind the snowman .", "words": ["the", "behind", "and", "snowman", "a"]}, {"sent": "oh the snowman broke wheres his hat he must be cold .", "words": ["the", "be", "snowman", "he", "hat", "his", "cold"]}, {"sent": "another snowman .", "words": ["another", "snowman"]}, {"sent": "Frosty the snowman .", "words": ["the", "snowman"]}, {"sent": "that has a snowman on the front huh ?", "words": ["the", "snowman", "on", "that", "a"]}, {"sent": "the snowman .", "words": ["the", "snowman"]}, {"sent": "what else can we give a snowman ?", "words": ["can", "what", "we", "snowman", "give", "a"]}, {"sent": "you threw the snowman off .", "words": ["the", "snowman", "off", "you"]}, {"sent": "theres a little puppy .", "words": ["puppy", "little", "a"]}, {"sent": "are you gonna get that puppy those puppys ears ?", "words": ["those", "you", "get", "are", "puppy", "that"]}, {"sent": "dont needta yell at the puppy hi puppy .", "words": ["the", "dont", "puppy", "at"]}, {"sent": "he is a nice puppy thats right .", "words": ["is", "nice", "puppy", "he", "a"]}, {"sent": "pat the puppy .", "words": ["the", "puppy"]}, {"sent": "whats that puppy chewing on ?", "words": ["on", "that", "puppy"]}, {"sent": "puppy ?", "words": ["puppy"]}, {"sent": "puppy in the watermelon puppy in the watermelon .", "words": ["the", "in", "puppy"]}, {"sent": "wheres the puppy ?", "words": ["the", "puppy"]}, {"sent": "puppy goes down the slide .", "words": ["the", "down", "puppy", "slide"]}, {"sent": "lets go see if theres a puppy .", "words": ["if", "go", "puppy", "see", "a"]}, {"sent": "once upon a time the puppy went through the watermelon .", "words": ["the", "puppy", "a"]}, {"sent": "no puppy .", "words": ["puppy"]}, {"sent": "its a puppy .", "words": ["puppy", "a"]}, {"sent": "I saw her when she was a puppy but not since then .", "words": ["when", "was", "not", "puppy", "then", "her", "she", "but", "a"]}, {"sent": "a puppy a puppy dog .", "words": ["puppy", "dog", "a"]}, {"sent": "Michael and baby both snuggled close to mother boat and dolly and puppy were in Michaels arms the reindeer blanket wrapped around them all back and forth back and forth they rocked .", "words": ["the", "all", "back", "to", "them", "and", "they", "in", "puppy", "around", "blanket", "close", "boat", "were"]}, {"sent": "now I also havent seen you playing with the puppy very much should we give the puppy away ?", "words": ["the", "we", "with", "you", "away", "puppy", "give", "much"]}, {"sent": "show the puppy to Mary .", "words": ["show", "the", "to", "puppy"]}, {"sent": "yeah its a puppy .", "words": ["puppy", "a"]}, {"sent": "does puppy like raisins oh listen sweetie its the moon song should I turn it up for you ?", "words": ["the", "does", "up", "for", "moon", "listen", "like", "you", "it", "puppy"]}, {"sent": "as bunny looks behind a bush puppy jumps up .", "words": ["up", "bunny", "behind", "puppy", "a"]}, {"sent": "puppy .", "words": ["puppy"]}, {"sent": "no his name isnt puppy .", "words": ["his", "puppy"]}, {"sent": "and theres a wee puppy .", "words": ["puppy", "a", "and"]}, {"sent": "its a puppy .", "words": ["puppy", "a"]}, {"sent": "where did you get that puppy ?", "words": ["where", "you", "get", "puppy", "did", "that"]}, {"sent": "puppy dog ?", "words": ["puppy", "dog"]}, {"sent": "is there a puppy ?", "words": ["there", "puppy", "a", "is"]}, {"sent": "puppy ?", "words": ["puppy"]}, {"sent": "we were talking about how there was a puppy inside but we never saw that puppy .", "words": ["how", "we", "was", "inside", "there", "that", "puppy", "about", "were", "but", "a"]}, {"sent": "ah puppy .", "words": ["puppy"]}, {"sent": "or a present for puppy ?", "words": ["for", "puppy", "present", "a"]}, {"sent": "I have a lady puppy .", "words": ["have", "puppy", "a"]}, {"sent": "a puppy .", "words": ["puppy", "a"]}, {"sent": "wheres the puppy ?", "words": ["the", "puppy"]}, {"sent": "that lil puppy ?", "words": ["that", "puppy"]}, {"sent": "what are you doing to puppy ?", "words": ["what", "to", "you", "are", "puppy"]}, {"sent": "and theres the puppy .", "words": ["the", "puppy", "and"]}, {"sent": "wheres the puppy dog ?", "words": ["the", "puppy", "dog"]}, {"sent": "whats puppy say ?", "words": ["puppy", "say"]}, {"sent": "puppy is on the garbage .", "words": ["the", "is", "puppy", "on", "garbage"]}, {"sent": "is that a lil puppy ?", "words": ["puppy", "that", "a", "is"]}, {"sent": "the puppy blows the candle out .", "words": ["the", "puppy", "out"]}, {"sent": "puppy thats Zoe the puppy .", "words": ["the", "puppy"]}, {"sent": "the puppy .", "words": ["the", "puppy"]}, {"sent": "he can ride the puppy ?", "words": ["can", "the", "puppy", "he", "ride"]}, {"sent": "looka there a little puppy dog and a mouse .", "words": ["little", "and", "there", "puppy", "dog", "mouse", "a"]}, {"sent": "no its not the puppy dog .", "words": ["the", "dog", "puppy", "not"]}, {"sent": "is she a puppy dog ?", "words": ["is", "puppy", "she", "dog", "a"]}, {"sent": "make some trees theres a little puppy .", "words": ["little", "puppy", "some", "make", "a"]}, {"sent": "puppy ?", "words": ["puppy"]}, {"sent": "hi puppy .", "words": ["puppy"]}, {"sent": "isnt the puppy dirty too ?", "words": ["the", "puppy", "dirty", "too"]}, {"sent": "not wanna go to grandmas house and see the puppy ?", "words": ["the", "go", "to", "and", "house", "puppy", "wanna", "see", "not"]}, {"sent": "the puppy looks startled .", "words": ["the", "puppy"]}, {"sent": "ha theres a puppy .", "words": ["puppy", "a"]}, {"sent": "a puppy dog ?", "words": ["puppy", "dog", "a"]}, {"sent": "can you find a puppy book ?", "words": ["can", "you", "puppy", "find", "book", "a"]}, {"sent": "puppy .", "words": ["puppy"]}, {"sent": "say hi puppy .", "words": ["say", "puppy"]}, {"sent": "be quiet puppy .", "words": ["be", "puppy", "quiet"]}, {"sent": "the little puppy see him sleeping .", "words": ["the", "little", "him", "puppy", "see"]}, {"sent": "is that a puppy ?", "words": ["puppy", "that", "a", "is"]}, {"sent": "you wanna see picture of you and the puppy ?", "words": ["the", "of", "picture", "you", "and", "puppy", "wanna", "see"]}, {"sent": "puppy puppy puppy .", "words": ["puppy"]}, {"sent": "puppy puppy puppy .", "words": ["puppy"]}, {"sent": "sit right here and Ill do it easy did the puppy do it ?", "words": ["the", "do", "and", "here", "it", "puppy", "did", "sit"]}, {"sent": "hes a happy little puppy huh .", "words": ["puppy", "little", "happy", "a"]}, {"sent": "puppy doesnt mind water in his eye right ?", "words": ["eye", "in", "puppy", "water", "his"]}, {"sent": "yeah to see Grandmas new puppy .", "words": ["see", "to", "new", "puppy"]}, {"sent": "puppy whats a puppy say ?", "words": ["puppy", "say", "a"]}, {"sent": "puppy .", "words": ["puppy"]}, {"sent": "theres the puppy .", "words": ["the", "puppy"]}, {"sent": "where does the puppy go ?", "words": ["where", "does", "the", "go", "puppy"]}, {"sent": "oh youre a mucky puppy arent you ?", "words": ["puppy", "you", "a"]}, {"sent": "what else should we make for the puppy maybe the puppy needs a sweater .", "words": ["the", "what", "for", "sweater", "we", "puppy", "make", "a"]}, {"sent": "wheres the puppy ?", "words": ["the", "puppy"]}, {"sent": "puppy .", "words": ["puppy"]}, {"sent": "Chris go get the puppy .", "words": ["get", "the", "puppy", "go"]}, {"sent": "the puppy .", "words": ["the", "puppy"]}, {"sent": "its not a cat it looks like a cat cause its so little but its a puppy its a baby dog .", "words": ["dog", "so", "like", "little", "cat", "it", "puppy", "not", "but", "a"]}, {"sent": "puppy .", "words": ["puppy"]}, {"sent": "puppy wants to come too said Michael .", "words": ["to", "puppy", "too"]}, {"sent": "oh its the puppy dog .", "words": ["the", "puppy", "dog"]}, {"sent": "can you go get puppy honey ?", "words": ["can", "go", "you", "get", "puppy"]}, {"sent": "puppy .", "words": ["puppy"]}, {"sent": "puppy .", "words": ["puppy"]}, {"sent": "yep and the puppy did so we got red jaws .", "words": ["the", "red", "we", "so", "and", "puppy", "did"]}, {"sent": "this is a puppy .", "words": ["this", "puppy", "a", "is"]}, {"sent": "hes only a puppy .", "words": ["puppy", "a"]}, {"sent": "what should the puppy .", "words": ["the", "what", "puppy"]}, {"sent": "we saw a puppy in the pet shop that had spots around his eyes .", "words": ["the", "we", "in", "puppy", "around", "his", "that", "a"]}, {"sent": "puppy dog puppy dog .", "words": ["puppy", "dog"]}, {"sent": "with a puppy a kitten a goldfish a bunny young people old people anyone funny .", "words": ["old", "bunny", "with", "puppy", "a"]}, {"sent": "puppy dog .", "words": ["puppy", "dog"]}, {"sent": "sit down how about the puppy ?", "words": ["the", "how", "puppy", "about", "sit", "down"]}, {"sent": "Im sure youll also notice Jessica that even though almost all her words are like xxx dirty puppy the only one thats totally different is moon .", "words": ["the", "all", "moon", "is", "like", "that", "are", "puppy", "dirty", "her"]}, {"sent": "do you think that this puppy is Grandmas or Moms puppy ?", "words": ["do", "think", "is", "you", "this", "puppy", "that"]}, {"sent": "is that a pink puppy ?", "words": ["puppy", "that", "a", "is"]}, {"sent": "old boots ?", "words": ["old", "boots"]}, {"sent": "youve got some Wellington boots havent you ?", "words": ["some", "boots", "you"]}, {"sent": "take off her boots ?", "words": ["off", "boots", "her", "take"]}, {"sent": "or they can go in boots .", "words": ["can", "go", "they", "in", "boots"]}, {"sent": "look at your lovely boots .", "words": ["boots", "at", "your", "look"]}, {"sent": "see boots ?", "words": ["see", "boots"]}, {"sent": "you like boots ?", "words": ["boots", "like", "you"]}, {"sent": "you want the other shoes you want the work boots these are boots .", "words": ["the", "these", "work", "other", "you", "boots", "are"]}, {"sent": "theyre Carolines boots arent they ?", "words": ["boots", "they"]}, {"sent": "they are funny boots .", "words": ["boots", "they", "are"]}, {"sent": "and a factory where they made pencils and where they made Wellington boots .", "words": ["where", "and", "they", "boots", "a"]}, {"sent": "and he has got red boots .", "words": ["red", "boots", "he", "and"]}, {"sent": "not his boots red pants .", "words": ["red", "boots", "pants", "his", "not"]}, {"sent": "those are their moon boots and their shiny moon boots and moon gloves .", "words": ["those", "their", "moon", "and", "boots", "are", "gloves"]}, {"sent": "do you think theyre making Wellington boots ?", "words": ["do", "think", "boots", "you"]}, {"sent": "she should have the cowboy boots on .", "words": ["the", "have", "boots", "on", "she"]}, {"sent": "and red boots .", "words": ["red", "boots", "and"]}, {"sent": "putting his boots on ?", "words": ["his", "boots", "on"]}, {"sent": "you have got Barney boots .", "words": ["have", "boots", "you"]}, {"sent": "a little bit like Santas boots .", "words": ["boots", "like", "little", "a"]}, {"sent": "he has got green Wellington boots .", "words": ["boots", "he", "green"]}, {"sent": "his boots ?", "words": ["his", "boots"]}, {"sent": "can you draw his boots ?", "words": ["can", "draw", "you", "boots", "his"]}, {"sent": "they must be lovely boots .", "words": ["boots", "they", "be"]}, {"sent": "these are his boots .", "words": ["his", "these", "are", "boots"]}, {"sent": "and boots especially .", "words": ["boots", "and"]}, {"sent": "oh theyre not very small but theyre smaller than yours and they just fit those boots perfectly .", "words": ["those", "fit", "and", "they", "boots", "not", "but"]}, {"sent": "Caroline seen your new boots ?", "words": ["boots", "new", "your"]}, {"sent": "we dont want the welly boots in the way then do we ?", "words": ["the", "do", "we", "dont", "in", "boots", "then"]}, {"sent": "theres theyre theyre old old boots and you use them for the last time .", "words": ["old", "the", "for", "last", "them", "you", "and", "boots"]}, {"sent": "you are a bossy boots .", "words": ["boots", "are", "a", "you"]}, {"sent": "let us put his boots down in there .", "words": ["put", "us", "boots", "in", "there", "his", "down"]}, {"sent": "you want Dorian boots off ?", "words": ["off", "boots", "you"]}, {"sent": "those are lovely boots .", "words": ["boots", "are", "those"]}, {"sent": "because youre getting it down in your boots .", "words": ["your", "boots", "in", "it", "down", "because"]}, {"sent": "you havent got any boots ?", "words": ["boots", "any", "you"]}, {"sent": "these toys those should have boots on .", "words": ["these", "those", "have", "boots", "on"]}, {"sent": "you dont put welly boots in the washing machine do you ?", "words": ["the", "do", "put", "you", "washing machine", "dont", "boots", "in"]}, {"sent": "boots on .", "words": ["on", "boots"]}, {"sent": "pair of boots .", "words": ["of", "boots"]}, {"sent": "boots on .", "words": ["on", "boots"]}, {"sent": "youd better ask Caroline if shes got her welly boots .", "words": ["better", "boots", "her", "if"]}, {"sent": "I I have boots and I have an umbrella I got boots Mary Mary Mary Poppins .", "words": ["have", "boots", "an", "and"]}, {"sent": "ya new boots .", "words": ["boots", "new"]}, {"sent": "get on your boots .", "words": ["get", "boots", "on", "your"]}, {"sent": "right I shall take my boots off now .", "words": ["off", "boots", "my", "take"]}, {"sent": "there are the boots .", "words": ["the", "there", "are", "boots"]}, {"sent": "is she has she got orange boots ?", "words": ["boots", "orange", "she", "is"]}, {"sent": "how many boots Fraser ?", "words": ["boots", "how"]}, {"sent": "hes got his boots on .", "words": ["his", "boots", "on"]}, {"sent": "mister bossy boots .", "words": ["boots"]}, {"sent": "has everybody got their welly boots ?", "words": ["boots", "their"]}, {"sent": "oh look what that little pussycat underneath with his Wellie boots is doing .", "words": ["what", "is", "with", "little", "boots", "look", "his", "that"]}, {"sent": "shall we go and buy you some boots ?", "words": ["go", "we", "you", "and", "boots", "some", "buy"]}, {"sent": "you help daddy put his laces in his boots huh ?", "words": ["help", "put", "you", "boots", "in", "his"]}, {"sent": "theyre lovely boots as well arent they ?", "words": ["boots", "they"]}, {"sent": "no alright Ill do it with my boots on .", "words": ["do", "my", "with", "boots", "it", "on"]}, {"sent": "Im putting my boots on .", "words": ["on", "boots", "my"]}, {"sent": "youve got the LoneRangers boots on right ?", "words": ["the", "boots", "on"]}, {"sent": "and some boots .", "words": ["some", "boots", "and"]}, {"sent": "has she got some roller boots on now ?", "words": ["on", "some", "boots", "she"]}, {"sent": "because he wants to get his boots wet .", "words": ["wet", "to", "get", "boots", "he", "his", "because"]}, {"sent": "she needs to have the boots on ?", "words": ["the", "to", "have", "boots", "on", "she"]}, {"sent": "I think well hafta buy you some boots .", "words": ["think", "you", "boots", "some", "buy"]}, {"sent": "green boots .", "words": ["boots", "green"]}, {"sent": "and he has got big sturdy boots on .", "words": ["and", "boots", "he", "on", "big"]}, {"sent": "Adam doesnt have cowboy boots like that .", "words": ["have", "boots", "that", "like"]}, {"sent": "do you see boots ?", "words": ["see", "do", "boots", "you"]}, {"sent": "because I dont want you to scuff them good wee boots .", "words": ["to", "them", "you", "dont", "boots", "good", "because"]}, {"sent": "is Miffy gonna wear boots ?", "words": ["boots", "is"]}, {"sent": "shes not got roller boots .", "words": ["boots", "not"]}, {"sent": "boots .", "words": ["boots"]}, {"sent": "them boots ?", "words": ["boots", "them"]}, {"sent": "hm Mummy had better go and put her boots on .", "words": ["better", "put", "go", "and", "boots", "on", "her"]}, {"sent": "okay she has boots on .", "words": ["on", "boots", "she"]}, {"sent": "them boots would you wear them ?", "words": ["boots", "them", "you", "would"]}, {"sent": "off the chair with your boots Fraser .", "words": ["off", "the", "your", "with", "chair", "boots"]}, {"sent": "are you doing these boots ?", "words": ["boots", "these", "are", "you"]}, {"sent": "you think we can fit her pants on over her boots ?", "words": ["can", "over", "think", "we", "fit", "you", "boots", "pants", "on", "her"]}, {"sent": "a rope lets pull boots out .", "words": ["boots", "pull", "out", "a"]}, {"sent": "do you know I got a pair of black boots outof Barrets xxx .", "words": ["do", "of", "you", "black", "boots", "a"]}, {"sent": "are they cowboy rain boots ?", "words": ["boots", "they", "are", "rain"]}, {"sent": "what did you do have my boots on ?", "words": ["what", "do", "my", "you", "have", "boots", "did", "on"]}, {"sent": "do you think he needs his boots on ?", "words": ["do", "think", "you", "boots", "on", "he", "his"]}, {"sent": "he needs those boots on .", "words": ["on", "boots", "he", "those"]}, {"sent": "where are your red boots ?", "words": ["where", "red", "your", "boots", "are"]}, {"sent": "youve got your Barny boots on .", "words": ["on", "boots", "your"]}, {"sent": "you love your boots on .", "words": ["your", "you", "boots", "love", "on"]}, {"sent": "shall I do his boots purple ?", "words": ["his", "do", "boots"]}, {"sent": "and shes got her boots on .", "words": ["on", "boots", "her", "and"]}, {"sent": "youve got boots on that page .", "words": ["on", "boots", "that"]}, {"sent": "and going out with hats and scarves and gloves and coats and boots .", "words": ["with", "and", "boots", "gloves", "out"]}, {"sent": "you wanna take off the boots instead ?", "words": ["off", "the", "you", "boots", "wanna", "take"]}, {"sent": "your boots on .", "words": ["on", "boots", "your"]}, {"sent": "well then take your boots off and come in .", "words": ["off", "your", "and", "boots", "in", "then", "take"]}, {"sent": "theyre not boots .", "words": ["boots", "not"]}, {"sent": "Wellington boots .", "words": ["boots"]}, {"sent": "lets check backpack for something to help boots out of the sand .", "words": ["the", "help", "for", "of", "to", "boots", "out"]}, {"sent": "his red coat and hat and boots are on the chair ready for when he goes out on ChristmasEve delivering parcels and letters .", "words": ["red", "the", "for", "chair", "and", "boots", "on", "are", "he", "hat", "out", "his", "when", "coat"]}, {"sent": "was it a bowl ?", "words": ["bowl", "was", "it", "a"]}, {"sent": "you finish the other things in your bowl first .", "words": ["the", "bowl", "your", "first", "finish", "other", "you", "in"]}, {"sent": "what is in that bowl .", "words": ["bowl", "what", "is", "in", "that"]}, {"sent": "the bowl ?", "words": ["the", "bowl"]}, {"sent": "actually thats what she ate and Im I left some in the bowl that I might eat .", "words": ["the", "what", "bowl", "and", "eat", "in", "some", "that", "she"]}, {"sent": "dyou still have porridge in your bowl ?", "words": ["have", "in", "your", "bowl"]}, {"sent": "in a round bowl .", "words": ["bowl", "in", "a"]}, {"sent": "Im gonna get the bowl .", "words": ["get", "the", "bowl"]}, {"sent": "in a bowl ?", "words": ["bowl", "in", "a"]}, {"sent": "but what do you want in the bowl ?", "words": ["the", "what", "do", "bowl", "you", "in", "but"]}, {"sent": "a brown bowl .", "words": ["bowl", "brown", "a"]}, {"sent": "the bowl .", "words": ["the", "bowl"]}, {"sent": "oh you can put the bowl in .", "words": ["can", "the", "put", "bowl", "you", "in"]}, {"sent": "I just needta get that back in the bowl .", "words": ["the", "bowl", "back", "get", "in", "that"]}, {"sent": "Ive got a bowl of water here .", "words": ["bowl", "of", "here", "water", "a"]}, {"sent": "if youre tipping those Cornflakes between the bowl and the back of the train .", "words": ["the", "bowl", "those", "if", "back", "train", "of", "and"]}, {"sent": "xxx do a scoop and put it in the bowl .", "words": ["the", "do", "put", "bowl", "and", "in", "it", "a"]}, {"sent": "you want plain yogurt in a bowl .", "words": ["bowl", "yogurt", "you", "in", "a"]}, {"sent": "you needta take the lid off the sugar bowl .", "words": ["the", "bowl", "off", "you", "take"]}, {"sent": "fingers out youll have it in your bowl .", "words": ["bowl", "your", "have", "in", "it", "out"]}, {"sent": "you have a bowl here .", "words": ["bowl", "you", "have", "here", "a"]}, {"sent": "roll your sleeves up and give them a quick rinse in the bowl theres a good boy .", "words": ["up", "the", "bowl", "your", "them", "and", "in", "good", "give", "a"]}, {"sent": "Ill get you your own bowl .", "words": ["get", "your", "bowl", "you"]}, {"sent": "its not a bowl but it it kinda looks like a bowl .", "words": ["bowl", "like", "it", "not", "but", "a"]}, {"sent": "she said on Sunday breakfast they all had three knives and three forks and they were eating a bowl of RiceKrispies .", "words": ["bowl", "all", "of", "and", "they", "on", "were", "she", "a"]}, {"sent": "in a bowl ?", "words": ["bowl", "in", "a"]}, {"sent": "the pussy cat has gone in the fish bowl .", "words": ["the", "bowl", "cat", "in", "fish"]}, {"sent": "that bowl is okay but this bowl .", "words": ["bowl", "is", "this", "that", "but"]}, {"sent": "whats in the bowl ?", "words": ["the", "in", "bowl"]}, {"sent": "Ill just get a little bowl to put those in .", "words": ["bowl", "put", "those", "to", "little", "get", "in", "a"]}, {"sent": "and your bowl is .", "words": ["bowl", "your", "is", "and"]}, {"sent": "your putting your head in the bowl now .", "words": ["the", "bowl", "your", "head", "in"]}, {"sent": "you have it in your bowl .", "words": ["bowl", "your", "you", "have", "in", "it"]}, {"sent": "the bowl is chocolate .", "words": ["the", "chocolate", "bowl", "is"]}, {"sent": "Im gonna go make me a bowl of that soup .", "words": ["bowl", "go", "of", "that", "soup", "me", "make", "a"]}, {"sent": "now would you like to put that sugar in the bowl please .", "words": ["the", "bowl", "put", "to", "like", "you", "in", "would", "that"]}, {"sent": "heres your bowl .", "words": ["bowl", "your"]}, {"sent": "this can be a bowl to put it in .", "words": ["can", "bowl", "put", "be", "to", "this", "in", "it", "a"]}, {"sent": "Im just trying to post the Cornflakes in the cereal bowl and got them all over the table .", "words": ["the", "bowl", "all", "table", "to", "them", "and", "in", "over", "cereal"]}, {"sent": "do you want that to be your bowl ?", "words": ["bowl", "do", "your", "to", "be", "you", "that"]}, {"sent": "can you put it in a bowl for mamma ?", "words": ["can", "bowl", "put", "for", "you", "in", "it", "a"]}, {"sent": "can I put them in the bowl ?", "words": ["can", "the", "put", "bowl", "them", "in"]}, {"sent": "empty everything in the bags into that metal bowl there .", "words": ["the", "bowl", "into", "empty", "there", "in", "that"]}, {"sent": "a bowl ?", "words": ["bowl", "a"]}, {"sent": "hands in the bowl .", "words": ["the", "in", "bowl"]}, {"sent": "do you remember we put those in the fruit bowl the other day didnt we ?", "words": ["the", "do", "put", "bowl", "those", "we", "you", "other", "in"]}, {"sent": "his bowl yeah .", "words": ["his", "bowl"]}, {"sent": "another bowl .", "words": ["another", "bowl"]}, {"sent": "its a big bowl isnt it ?", "words": ["bowl", "it", "big", "a"]}, {"sent": "get a bowl for her too .", "words": ["bowl", "for", "get", "too", "her", "a"]}, {"sent": "would you like to go clean Dorothys bowl ?", "words": ["clean", "bowl", "go", "to", "like", "you", "would"]}, {"sent": "you done with your bowl ?", "words": ["your", "with", "bowl", "you"]}, {"sent": "she goes to the littlest bowl of porridge .", "words": ["the", "bowl", "of", "to", "she"]}, {"sent": "Im just looking for a bowl .", "words": ["bowl", "for", "a"]}, {"sent": "would you like to lick the bowl when were done ?", "words": ["the", "bowl", "when", "to", "like", "you", "would", "lick", "were"]}, {"sent": "and hes got a fish bowl that he puts the fish in and out of that he likes .", "words": ["bowl", "the", "of", "and", "in", "he", "out", "fish", "that", "a"]}, {"sent": "and move your bowl in front of you .", "words": ["bowl", "your", "of", "and", "you", "in"]}, {"sent": "you might wanna pull your bowl a little closer to your chin there .", "words": ["bowl", "your", "to", "chin", "pull", "you", "little", "there", "wanna", "a"]}, {"sent": "can you pour some in a bowl so I can eat it ?", "words": ["can", "bowl", "so", "pour", "you", "eat", "in", "it", "some", "a"]}, {"sent": "I have a bowl .", "words": ["have", "bowl", "a"]}, {"sent": "now we needta tip that margarine down to this white bowl .", "words": ["bowl", "we", "to", "white", "this", "down", "that"]}, {"sent": "that was her best bowl ?", "words": ["bowl", "was", "her", "that"]}, {"sent": "all right well put them back in the bowl if youre all done .", "words": ["the", "all", "put", "bowl", "if", "back", "them", "in"]}, {"sent": "shall I put it in the fruit bowl with the strawberries and grapes ?", "words": ["the", "bowl", "put", "with", "and", "in", "it", "grapes"]}, {"sent": "picture of a bowl of Cheerios .", "words": ["of", "picture", "bowl", "a"]}, {"sent": "its a fruit bowl .", "words": ["bowl", "a"]}, {"sent": "in the bowl ?", "words": ["the", "in", "bowl"]}, {"sent": "theyre out the fruit bowl .", "words": ["the", "bowl", "out"]}, {"sent": "okay give me your spoon then Ill scrape your bowl .", "words": ["bowl", "your", "then", "spoon", "me", "give"]}, {"sent": "no no dumping your bowl .", "words": ["bowl", "your"]}, {"sent": "go get a wee bowl outof your kitchen .", "words": ["bowl", "your", "go", "get", "kitchen", "a"]}, {"sent": "crows like their food moistened so the brainy bird used the frisbee to scoop water out of his bath pan and carry it to his food bowl where he poured it onto his chow .", "words": ["the", "bowl", "where", "their", "of", "so", "to", "bird", "like", "and", "carry", "food", "it", "he", "out", "water", "his"]}, {"sent": "okay put it in the bowl .", "words": ["the", "bowl", "put", "in", "it"]}, {"sent": "a bowl of sand is xxx .", "words": ["bowl", "of", "is", "a"]}, {"sent": "can you put all the bits back in your bowl then please ?", "words": ["can", "all", "put", "the", "your", "bowl", "back", "you", "in", "then"]}, {"sent": "the water in the white bowl .", "words": ["the", "bowl", "in", "water", "white"]}, {"sent": "does he get it in a bowl ?", "words": ["bowl", "does", "get", "in", "it", "he", "a"]}, {"sent": "dont they all go in the bowl ?", "words": ["the", "all", "bowl", "go", "dont", "in", "they"]}, {"sent": "its the bowl of water that MissusTwitchett has got on the floor with her towel and her sponge .", "words": ["the", "bowl", "of", "with", "her", "and", "towel", "water", "on", "that"]}, {"sent": "and theres not there wont be a boat in a fish bowl .", "words": ["bowl", "be", "and", "there", "in", "fish", "boat", "not", "a"]}, {"sent": "a bowl ya .", "words": ["bowl", "a"]}, {"sent": "whats in this bowl ?", "words": ["bowl", "this", "in"]}, {"sent": "thats the sugar bowl .", "words": ["the", "bowl"]}, {"sent": "lets put the crackers on here and the bowl goes here .", "words": ["the", "bowl", "put", "and", "here", "on"]}, {"sent": "see all that egg in your bowl ?", "words": ["bowl", "all", "your", "egg", "in", "see", "that"]}, {"sent": "bowl of water .", "words": ["bowl", "of", "water"]}, {"sent": "move it into the bowl .", "words": ["the", "it", "into", "bowl"]}, {"sent": "the bowl ?", "words": ["the", "bowl"]}, {"sent": "what color is the bowl ?", "words": ["the", "what", "bowl", "is"]}, {"sent": "swimming in the bowl Im swimming Im swimming in the bowl in my pretend pool .", "words": ["the", "bowl", "my", "pool", "in", "pretend"]}, {"sent": "oh this mixing bowl is very heavy .", "words": ["bowl", "this", "heavy", "is"]}, {"sent": "where does a bowl of fruit go ?", "words": ["where", "does", "bowl", "go", "of", "a"]}, {"sent": "a medium sized bowl for ?", "words": ["bowl", "for", "a"]}, {"sent": "you want me to put a few more in your bowl ?", "words": ["bowl", "put", "your", "to", "you", "more", "in", "me", "a"]}, {"sent": "let Mommy go and get your bowl .", "words": ["bowl", "your", "go", "and", "get"]}, {"sent": "and putting it back in your bowl ?", "words": ["bowl", "your", "back", "and", "in", "it"]}, {"sent": "all the whales are gonna go in one bowl okay ?", "words": ["the", "all", "bowl", "go", "in", "are"]}, {"sent": "look at your bowl .", "words": ["bowl", "at", "your", "look"]}, {"sent": "honey ya havta careful with mommys bowl okay because if something happens to this bowl mommy might cry .", "words": ["bowl", "if", "to", "cry", "with", "this", "careful", "because"]}, {"sent": "look at that donkey .", "words": ["donkey", "at", "that", "look"]}, {"sent": "donkey .", "words": ["donkey"]}, {"sent": "whats donkey doing ?", "words": ["donkey"]}, {"sent": "how do you put the donkey in ?", "words": ["the", "do", "how", "put", "you", "in", "donkey"]}, {"sent": "and a baby donkey on the farm .", "words": ["the", "and", "donkey", "on", "a"]}, {"sent": "thats a donkey .", "words": ["donkey", "a"]}, {"sent": "where did you go on the donkey ?", "words": ["where", "the", "go", "you", "did", "donkey", "on"]}, {"sent": "a donkey .", "words": ["donkey", "a"]}, {"sent": "a cat and a baby donkey are growing up on the farm .", "words": ["up", "the", "and", "cat", "are", "donkey", "on", "a"]}, {"sent": "theres the donkey .", "words": ["the", "donkey"]}, {"sent": "thats not a donkey .", "words": ["donkey", "not", "a"]}, {"sent": "donkey actually .", "words": ["donkey"]}, {"sent": "and a donkey .", "words": ["donkey", "a", "and"]}, {"sent": "and donkey fall .", "words": ["donkey", "fall", "and"]}, {"sent": "is it a donkey ?", "words": ["donkey", "it", "a", "is"]}, {"sent": "a donkey .", "words": ["donkey", "a"]}, {"sent": "wheres the donkey cart ?", "words": ["the", "donkey"]}, {"sent": "its a Democrat donkey .", "words": ["donkey", "a"]}, {"sent": "thats a nice donkey isnt it .", "words": ["it", "donkey", "nice", "a"]}, {"sent": "a donkey ?", "words": ["donkey", "a"]}, {"sent": "the donkey and the groundhog yeah .", "words": ["the", "and", "donkey"]}, {"sent": "wheres the donkey go ?", "words": ["the", "go", "donkey"]}, {"sent": "on a donkey .", "words": ["on", "donkey", "a"]}, {"sent": "donkey ?", "words": ["donkey"]}, {"sent": "and as the hat on the donkey or eagle or the thing .", "words": ["the", "and", "hat", "donkey", "on"]}, {"sent": "a calf and a baby donkey are growing up on a farm .", "words": ["up", "and", "are", "donkey", "on", "a"]}, {"sent": "thats the donkey Peter .", "words": ["the", "donkey"]}, {"sent": "well but you know what remember last time mommys glasses broke so I dont think I want donkey to climb on my glasses .", "words": ["what", "think", "my", "so", "last", "to", "glasses", "you", "dont", "donkey", "on", "climb", "but"]}, {"sent": "no thats a calf and a baby donkey are growing up on a farm .", "words": ["up", "and", "are", "donkey", "on", "a"]}, {"sent": "and a donkey .", "words": ["donkey", "a", "and"]}, {"sent": "donkey .", "words": ["donkey"]}, {"sent": "whats a donkey say ?", "words": ["donkey", "say", "a"]}, {"sent": "donkey .", "words": ["donkey"]}, {"sent": "no thats a donkey .", "words": ["donkey", "a"]}, {"sent": "what e a donkey .", "words": ["what", "donkey", "a"]}, {"sent": "youre a donkey ?", "words": ["donkey", "a"]}, {"sent": "flower crocodile cow donkey baby .", "words": ["flower", "cow", "donkey"]}, {"sent": "donkey heehaw goes the donkey .", "words": ["the", "donkey"]}, {"sent": "how does the donkey go ?", "words": ["the", "does", "how", "go", "donkey"]}, {"sent": "I think its a donkey .", "words": ["donkey", "think", "a"]}, {"sent": "oh I dont think Purdie has ever seen a donkey Thomas .", "words": ["donkey", "dont", "think", "a"]}, {"sent": "donkey and the cow .", "words": ["the", "cow", "and", "donkey"]}, {"sent": "a donkey .", "words": ["donkey", "a"]}, {"sent": "oops the donkey fell off and now the mouse is riding now the donkey is riding there .", "words": ["the", "off", "is", "and", "there", "donkey", "mouse"]}, {"sent": "a real donkey ?", "words": ["donkey", "a"]}, {"sent": "where is the donkey ?", "words": ["where", "the", "donkey", "is"]}, {"sent": "whats the donkey say ?", "words": ["the", "say", "donkey"]}, {"sent": "uhhuh its a donkey .", "words": ["donkey", "a"]}, {"sent": "is does this belong to the donkey ?", "words": ["the", "does", "is", "to", "this", "donkey"]}, {"sent": "was John Shrek and you were the donkey ?", "words": ["the", "was", "you", "and", "donkey", "were"]}, {"sent": "the donkey is dancing dancing dancing .", "words": ["the", "is", "donkey"]}, {"sent": "you know what the donkey says ?", "words": ["the", "what", "donkey", "you"]}, {"sent": "the dog and the donkey .", "words": ["the", "donkey", "dog", "and"]}, {"sent": "little donkey .", "words": ["donkey", "little"]}, {"sent": "yeah theyre playing pin the tail on the donkey .", "words": ["the", "on", "donkey"]}, {"sent": "oh goodness me I dont think theyll bother having a donkey ride then .", "words": ["think", "dont", "donkey", "me", "then", "ride", "a"]}, {"sent": "now you know what shes from Iceland so she doesnt even probably know how to play pin the tail on the donkey can we tell her how to play the game ?", "words": ["the", "what", "how", "can", "game", "we", "so", "to", "play", "you", "donkey", "on", "her", "she"]}, {"sent": "Eeyores a donkey .", "words": ["donkey", "a"]}, {"sent": "and thats a donkey thats a donkey and thats the mouse you have the mouse says .", "words": ["the", "and", "you", "have", "donkey", "mouse", "a"]}, {"sent": "yeah its a baby donkey .", "words": ["donkey", "a"]}, {"sent": "thats a donkey like you couldve ridden on at the zoo with Abi but you didnt .", "words": ["the", "zoo", "with", "like", "you", "at", "donkey", "on", "but", "a"]}, {"sent": "have that donkey .", "words": ["have", "that", "donkey"]}, {"sent": "a calf and a baby donkey are growing up on a farm .", "words": ["up", "and", "are", "donkey", "on", "a"]}, {"sent": "pin the tail on the donkey suggested a monkey .", "words": ["the", "monkey", "donkey", "on", "a"]}, {"sent": "what does a donkey make ?", "words": ["what", "does", "donkey", "make", "a"]}, {"sent": "no thats a donkey .", "words": ["donkey", "a"]}, {"sent": "a donkey about like a horse arent they ?", "words": ["like", "they", "about", "donkey", "horse", "a"]}, {"sent": "and a baby donkey heehaw heehaw .", "words": ["donkey", "a", "and"]}, {"sent": "a donkey .", "words": ["donkey", "a"]}, {"sent": "are you a donkey ?", "words": ["donkey", "are", "a", "you"]}, {"sent": "donkey .", "words": ["donkey"]}, {"sent": "how about the donkey what does the donkey say ?", "words": ["the", "what", "how", "does", "say", "about", "donkey"]}, {"sent": "Eeyore the donkey ?", "words": ["the", "donkey"]}, {"sent": "donkey gonna ride the car ?", "words": ["the", "car", "ride", "donkey"]}, {"sent": "xxx I think a donkey huh ?", "words": ["donkey", "think", "a"]}, {"sent": "shall I put the donkey away .", "words": ["the", "put", "away", "donkey"]}, {"sent": "hello donkey here he is .", "words": ["here", "he", "is", "donkey"]}, {"sent": "thats a donkey .", "words": ["donkey", "a"]}, {"sent": "a donkey .", "words": ["donkey", "a"]}, {"sent": "sheep and the donkey the geese and the goat .", "words": ["the", "donkey", "sheep", "and"]}, {"sent": "whatd the donkey say ?", "words": ["the", "say", "donkey"]}, {"sent": "did you throw the donkey ?", "words": ["the", "throw", "you", "did", "donkey"]}, {"sent": "thats a donkey .", "words": ["donkey", "a"]}, {"sent": "thats the little boy and his mummie and the donkey .", "words": ["the", "and", "little", "donkey", "his"]}, {"sent": "now the camel wants the alligator to kiss the donkey .", "words": ["the", "to", "alligator", "donkey", "kiss"]}, {"sent": "then pin the tail on the donkey and pass the parcel ah .", "words": ["the", "and", "donkey", "on", "then"]}, {"sent": "hiya donkey .", "words": ["donkey"]}, {"sent": "well I think Eeyore the donkey from WinniethePooh probably is the carrot person .", "words": ["the", "think", "is", "donkey"]}, {"sent": "or the donkey the patriotic donkey we dont have that either .", "words": ["the", "we", "have", "dont", "donkey", "that"]}, {"sent": "that sounds like a donkey .", "words": ["donkey", "that", "like", "a"]}, {"sent": "the old gray donkey Eeyore stood by himself in a thistly corner of the forest .", "words": ["the", "old", "of", "by", "in", "donkey", "a"]}, {"sent": "thats a donkey .", "words": ["donkey", "a"]}, {"sent": "is that a donkey ?", "words": ["donkey", "that", "a", "is"]}, {"sent": "theres donkey .", "words": ["donkey"]}, {"sent": "you dont like the donkey ?", "words": ["the", "like", "you", "dont", "donkey"]}, {"sent": "well whats a donkey say ?", "words": ["donkey", "say", "a"]}, {"sent": "would you like a donkey ?", "words": ["like", "would", "you", "donkey", "a"]}, {"sent": "oh I thought it was a donkey .", "words": ["was", "donkey", "it", "a"]}, {"sent": "wheres the donkey ?", "words": ["the", "donkey"]}, {"sent": "thats a donkey .", "words": ["donkey", "a"]}, {"sent": "wheres your project pencil ?", "words": ["your", "pencil"]}, {"sent": "you see and it makes it like a pencil .", "words": ["like", "pencil", "and", "you", "it", "see", "a"]}, {"sent": "theres the pencil car .", "words": ["the", "car", "pencil"]}, {"sent": "do you wanna get a pencil ?", "words": ["do", "pencil", "you", "get", "wanna", "a"]}, {"sent": "it has a pencil and a point and it helps you make circles .", "words": ["pencil", "you", "and", "it", "make", "a"]}, {"sent": "I dont know where you pencil case has gone .", "words": ["where", "dont", "pencil", "you"]}, {"sent": "when did you put the pencil in there ?", "words": ["the", "put", "pencil", "you", "there", "in", "did", "when"]}, {"sent": "you hold the pencil properly .", "words": ["the", "hold", "pencil", "you"]}, {"sent": "remember you hafta have your pencil .", "words": ["have", "your", "pencil", "you"]}, {"sent": "orange find an orange pencil .", "words": ["an", "orange", "pencil", "find"]}, {"sent": "this pencil writes beautifully .", "words": ["this", "pencil"]}, {"sent": "its a pencil pen kind of thing .", "words": ["of", "pen", "pencil", "a"]}, {"sent": "no thats Eves pencil .", "words": ["pencil"]}, {"sent": "and a free yellow pencil ?", "words": ["pencil", "yellow", "a", "and"]}, {"sent": "is that two pencil ?", "words": ["that", "pencil", "is"]}, {"sent": "now would you believe this pencil .", "words": ["pencil", "this", "you", "would"]}, {"sent": "okay Ill write with the pencil and you write with the pen .", "words": ["the", "with", "write", "and", "pencil", "you", "pen"]}, {"sent": "pencil .", "words": ["pencil"]}, {"sent": "hold the pencil .", "words": ["the", "hold", "pencil"]}, {"sent": "could I have the pencil ?", "words": ["have", "the", "could", "pencil"]}, {"sent": "what did we do with that pencil ?", "words": ["what", "do", "we", "with", "pencil", "did", "that"]}, {"sent": "no put that pencil down David a wee second .", "words": ["put", "pencil", "down", "that", "a"]}, {"sent": "would you please take the pencil off the floor ?", "words": ["the", "off", "pencil", "you", "would", "take"]}, {"sent": "you hafta put your pen in your pencil case .", "words": ["put", "your", "pencil", "you", "in", "pen"]}, {"sent": "pencil ?", "words": ["pencil"]}, {"sent": "hey Lily can you put these pencils in your pencil box ?", "words": ["can", "these", "put", "your", "box", "pencil", "you", "in"]}, {"sent": "let me just do this pencil .", "words": ["me", "do", "this", "pencil"]}, {"sent": "pencil .", "words": ["pencil"]}, {"sent": "I might hafta use the yellow pencil Thomas to use the blue pencil because the .", "words": ["the", "to", "pencil", "blue", "yellow", "because"]}, {"sent": "oh my I need a pencil .", "words": ["my", "need", "pencil", "a"]}, {"sent": "or at least the the case of the pencil .", "words": ["the", "of", "pencil", "at"]}, {"sent": "its showing a picture of a yellow pencil .", "words": ["of", "picture", "pencil", "yellow", "a"]}, {"sent": "theres no pen or pencil in that Peter .", "words": ["in", "that", "pencil", "pen"]}, {"sent": "oh dear weve dropped a pencil as well .", "words": ["pencil", "a"]}, {"sent": "bring the pencil and the pad on the table here and Ill draw our house .", "words": ["the", "table", "draw", "bring", "pencil", "and", "here", "our", "house", "on"]}, {"sent": "Eve has the pencil .", "words": ["the", "pencil"]}, {"sent": "wanna borrow this pencil ?", "words": ["this", "wanna", "pencil"]}, {"sent": "wheres my pencil ?", "words": ["my", "pencil"]}, {"sent": "could you pass me a pencil out of that ?", "words": ["of", "pencil", "you", "out", "me", "could", "that", "a"]}, {"sent": "well put them in your pencil case then .", "words": ["put", "your", "them", "pencil", "in", "then"]}, {"sent": "and theres a pencil on the floor look .", "words": ["the", "pencil", "and", "look", "on", "a"]}, {"sent": "you do your work with a pencil .", "words": ["do", "your", "work", "with", "pencil", "you", "a"]}, {"sent": "Ma you got a pen or a pencil or .", "words": ["pencil", "pen", "a", "you"]}, {"sent": "and a pencil and pencil sharpener and some chocolate didnt we ?", "words": ["we", "chocolate", "pencil", "and", "some", "a"]}, {"sent": "does it hafta be in pencil it said xxx in pencil .", "words": ["does", "be", "pencil", "in", "it"]}, {"sent": "xxx pencil or I need ?", "words": ["need", "pencil"]}, {"sent": "xxx my pencil broke .", "words": ["my", "pencil"]}, {"sent": "you write with a pencil .", "words": ["with", "pencil", "write", "you", "a"]}, {"sent": "oh I need my pencil .", "words": ["my", "need", "pencil"]}, {"sent": "yeah I just took the pencil away so Jenny wouldnt get it .", "words": ["the", "so", "pencil", "get", "away", "it"]}, {"sent": "xxx pencil .", "words": ["pencil"]}, {"sent": "not with the pencil Eve .", "words": ["the", "with", "not", "pencil"]}, {"sent": "I dont think you write with pencil on that Adam .", "words": ["think", "with", "write", "you", "pencil", "dont", "on", "that"]}, {"sent": "you ask Papa please and hell get you your pencil .", "words": ["your", "pencil", "and", "get", "you"]}, {"sent": "you take your pencil out .", "words": ["your", "pencil", "you", "out", "take"]}, {"sent": "wheres the black pencil ?", "words": ["the", "black", "pencil"]}, {"sent": "xxx when you go to school you need a pencil and ruler .", "words": ["need", "go", "to", "school", "pencil", "you", "and", "when", "a"]}, {"sent": "pencil case ?", "words": ["pencil"]}, {"sent": "thats a pencil in the pot .", "words": ["the", "in", "pencil", "a"]}, {"sent": "Im putting my pencil away .", "words": ["my", "away", "pencil"]}, {"sent": "Peter wheres my pencil ?", "words": ["my", "pencil"]}, {"sent": "do you think thats your pencil now ?", "words": ["do", "think", "your", "pencil", "you"]}, {"sent": "what kind of a pencil is this ?", "words": ["what", "is", "of", "pencil", "this", "a"]}, {"sent": "hold the pencil properly darling .", "words": ["the", "hold", "pencil"]}, {"sent": "what color is this asked Mr owl holding up a colored pencil .", "words": ["up", "what", "is", "pencil", "owl", "this", "a"]}, {"sent": "on the pencil .", "words": ["on", "the", "pencil"]}, {"sent": "you threw the pencil away ?", "words": ["the", "away", "pencil", "you"]}, {"sent": "I need a I need a pencil .", "words": ["need", "pencil", "a"]}, {"sent": "do you wanna put your animals on your pencil ?", "words": ["do", "put", "your", "pencil", "you", "wanna", "on"]}, {"sent": "wheres your pencil case gone ?", "words": ["your", "pencil"]}, {"sent": "well soon have a nice point on that pencil wont we ?", "words": ["we", "nice", "pencil", "have", "on", "that", "a"]}, {"sent": "give me two pencil .", "words": ["me", "give", "pencil"]}, {"sent": "Adam pick the pencil up and put it on the table .", "words": ["the", "up", "put", "table", "pick", "pencil", "and", "it", "on"]}, {"sent": "no no dont take the pencil off the the paper .", "words": ["the", "off", "paper", "pencil", "dont", "take"]}, {"sent": "red pencil ?", "words": ["red", "pencil"]}, {"sent": "you can use the pencil and I pencil and I will use my lovely blue pen .", "words": ["can", "the", "my", "pencil", "and", "you", "blue", "pen", "will"]}, {"sent": "a pen no not a pencil a pen .", "words": ["pen", "not", "pencil", "a"]}, {"sent": "thats a marking pencil .", "words": ["pencil", "a"]}, {"sent": "if you like yellow youll like this pencil .", "words": ["if", "like", "pencil", "you", "this", "yellow"]}, {"sent": "does your pencil have numbers ?", "words": ["have", "does", "your", "pencil"]}, {"sent": "you write with a pencil .", "words": ["with", "pencil", "write", "you", "a"]}, {"sent": "uh that ones broken and you know what the pencil sharpener is .", "words": ["the", "what", "is", "pencil", "you", "and", "broken", "that"]}, {"sent": "a pencil ?", "words": ["pencil", "a"]}, {"sent": "you wanna sharpen the pencils in the pencil sharpener ?", "words": ["the", "pencil", "you", "in", "wanna"]}, {"sent": "but when youve finished your pencils you must put them all back in your pencil case .", "words": ["all", "put", "your", "back", "them", "pencil", "you", "in", "when", "but"]}, {"sent": "but we do xxx maybe we could find a pencil anyway .", "words": ["do", "we", "pencil", "find", "could", "but", "a"]}, {"sent": "theres a pencil right there .", "words": ["there", "pencil", "a"]}, {"sent": "you write with a pencil .", "words": ["with", "pencil", "write", "you", "a"]}, {"sent": "you dropped your pencil .", "words": ["your", "pencil", "you"]}, {"sent": "what happened to the pencil ?", "words": ["to", "what", "pencil", "the"]}, {"sent": "put pencil in here .", "words": ["here", "in", "put", "pencil"]}, {"sent": "I can use my pencil .", "words": ["can", "my", "pencil"]}, {"sent": "thats to color in with and thats a pencil .", "words": ["to", "with", "pencil", "and", "in", "a"]}, {"sent": "you see the way he can hold his pencil ?", "words": ["the", "can", "hold", "pencil", "you", "he", "his", "see"]}, {"sent": "that makes your pencil nice and sharp doesnt it ?", "words": ["your", "nice", "pencil", "and", "it", "that"]}, {"sent": "a pencil writes .", "words": ["pencil", "a"]}, {"sent": "I wanna go home to the big pencil box said the pencil but I cant find it .", "words": ["the", "box", "go", "to", "pencil", "it", "wanna", "find", "big", "but", "home"]}, {"sent": "move the pencil out now .", "words": ["the", "out", "pencil"]}, {"sent": "you write only in pencil in the book .", "words": ["the", "pencil", "write", "you", "in", "book"]}, {"sent": "gentle .", "words": ["gentle"]}, {"sent": "hes giving him a gentle push .", "words": ["gentle", "him", "push", "a"]}, {"sent": "be gentle .", "words": ["be", "gentle"]}, {"sent": "just gentle .", "words": ["gentle"]}, {"sent": "just do it gentle .", "words": ["do", "gentle", "it"]}, {"sent": "gentle with that too .", "words": ["that", "gentle", "with", "too"]}, {"sent": "gentle .", "words": ["gentle"]}, {"sent": "well be gentle .", "words": ["be", "gentle"]}, {"sent": "you hafta be nice with the toys and gentle okay ?", "words": ["the", "be", "nice", "with", "you", "and", "gentle"]}, {"sent": "be gentle please .", "words": ["be", "gentle"]}, {"sent": "be very very gentle though please .", "words": ["be", "gentle"]}, {"sent": "somebodys not gentle .", "words": ["gentle", "not"]}, {"sent": "be gentle with it Thomas .", "words": ["be", "gentle", "it", "with"]}, {"sent": "gentle sweetie .", "words": ["gentle"]}, {"sent": "gentle gentle gentle William okay ?", "words": ["gentle"]}, {"sent": "gentle .", "words": ["gentle"]}, {"sent": "gentle gentle .", "words": ["gentle"]}, {"sent": "I told you to be gentle .", "words": ["to", "gentle", "be", "you"]}, {"sent": "we hafta be gentle with this milk .", "words": ["we", "be", "with", "this", "milk", "gentle"]}, {"sent": "now are you going to give her a gentle stroke ?", "words": ["to", "gentle", "you", "are", "give", "her", "a"]}, {"sent": "be very gentle with him .", "words": ["be", "gentle", "with", "him"]}, {"sent": "gentle .", "words": ["gentle"]}, {"sent": "oh gentle .", "words": ["gentle"]}, {"sent": "youve gotta be gentle .", "words": ["be", "gentle"]}, {"sent": "do it very gentle very gentle .", "words": ["do", "gentle", "it"]}, {"sent": "gentle gentle gentle .", "words": ["gentle"]}, {"sent": "gentle gentle gentle .", "words": ["gentle"]}, {"sent": "be gentle with baby .", "words": ["be", "gentle", "with"]}, {"sent": "gentle gentle .", "words": ["gentle"]}, {"sent": "Fraser be gentle with those .", "words": ["be", "gentle", "with", "those"]}, {"sent": "gentle right ?", "words": ["gentle"]}, {"sent": "very nice and gentle .", "words": ["gentle", "nice", "and"]}, {"sent": "gentle .", "words": ["gentle"]}, {"sent": "be very gentle .", "words": ["be", "gentle"]}, {"sent": "oh gentle .", "words": ["gentle"]}, {"sent": "uhoh ow gentle .", "words": ["gentle"]}, {"sent": "can you be gentle ?", "words": ["can", "be", "gentle", "you"]}, {"sent": "gentle thats nice .", "words": ["gentle", "nice"]}, {"sent": "be gentle with the baby .", "words": ["the", "be", "gentle", "with"]}, {"sent": "gentle gentle sweetie .", "words": ["gentle"]}, {"sent": "just a little gentle one .", "words": ["gentle", "little", "a"]}, {"sent": "gentle .", "words": ["gentle"]}, {"sent": "you be very gentle .", "words": ["be", "gentle", "you"]}, {"sent": "were you very gentle ?", "words": ["gentle", "were", "you"]}, {"sent": "but youve got to be very very gentle .", "words": ["to", "gentle", "be", "but"]}, {"sent": "Lily gentle with the microphone .", "words": ["the", "gentle", "with"]}, {"sent": "gentle .", "words": ["gentle"]}, {"sent": "be gentle .", "words": ["be", "gentle"]}, {"sent": "gentle yeah .", "words": ["gentle"]}, {"sent": "oh well be gentle with the ambulance driver .", "words": ["the", "be", "gentle", "with"]}, {"sent": "youre being very gentle with these teddys at the moment .", "words": ["the", "these", "with", "at", "gentle"]}, {"sent": "be gentle with the baby .", "words": ["the", "be", "gentle", "with"]}, {"sent": "gentle no throwing .", "words": ["gentle"]}, {"sent": "you hafta be gentle .", "words": ["be", "gentle", "you"]}, {"sent": "thats lovely and gentle .", "words": ["gentle", "and"]}, {"sent": "is it a gentle place ?", "words": ["gentle", "it", "a", "is"]}, {"sent": "be gentle okay ?", "words": ["be", "gentle"]}, {"sent": "gentle gentle .", "words": ["gentle"]}, {"sent": "be gentle please .", "words": ["be", "gentle"]}, {"sent": "you be gentle pretty girl .", "words": ["be", "gentle", "pretty", "you"]}, {"sent": "oops be gentle .", "words": ["be", "gentle"]}, {"sent": "gentle .", "words": ["gentle"]}, {"sent": "Thomas just be gentle because you mustnt break the side of the bowl .", "words": ["the", "bowl", "of", "be", "you", "gentle", "break", "because"]}, {"sent": "be gentle with mommy .", "words": ["be", "gentle", "with"]}, {"sent": "please be very very gentle with him .", "words": ["be", "gentle", "with", "him"]}, {"sent": "gentle .", "words": ["gentle"]}, {"sent": "are you going to be gentle ?", "words": ["to", "be", "you", "are", "gentle"]}, {"sent": "gotta be gentle .", "words": ["be", "gentle"]}, {"sent": "gentle .", "words": ["gentle"]}, {"sent": "gentle .", "words": ["gentle"]}, {"sent": "gentle .", "words": ["gentle"]}, {"sent": "gentle .", "words": ["gentle"]}, {"sent": "be gentle .", "words": ["be", "gentle"]}, {"sent": "being nice and gentle .", "words": ["gentle", "nice", "and"]}, {"sent": "oh be gentle .", "words": ["be", "gentle"]}, {"sent": "sometimes the wind is a gentle breeze sometimes it is strong and you feel it push you fall winds blow leaves from the trees and swirl them around and around into rustling piles .", "words": ["the", "push", "into", "is", "them", "fall", "wind", "and", "you", "it", "around", "gentle", "blow", "a"]}, {"sent": "but the cow wasnt gentle .", "words": ["the", "gentle", "cow", "but"]}, {"sent": "gentle you know xxx .", "words": ["gentle", "you"]}, {"sent": "gentle with the ball .", "words": ["the", "gentle", "with", "ball"]}, {"sent": "*SI1: I know but see youre not being that gentle with the racket see its all scraped up .", "words": ["the", "all", "up", "with", "see", "gentle", "that", "not", "but"]}, {"sent": "is Mommy gentle ?", "words": ["gentle", "is"]}, {"sent": "much more gentle isnt it ?", "words": ["gentle", "more", "it", "much"]}, {"sent": "be gentle .", "words": ["be", "gentle"]}, {"sent": "ah gentle .", "words": ["gentle"]}, {"sent": "be gentle .", "words": ["be", "gentle"]}, {"sent": "I theyre theyre quite gentle Thomas .", "words": ["gentle"]}, {"sent": "gentle .", "words": ["gentle"]}, {"sent": "you hafta be gentle with these toys .", "words": ["these", "be", "with", "you", "gentle"]}, {"sent": "gentle .", "words": ["gentle"]}, {"sent": "and he and hes very gentle with babies too like hes very good about like youknow touching gently and he helps feed the bottles like he loves to do that so Imean .", "words": ["the", "do", "feed", "to", "so", "with", "like", "and", "he", "good", "too", "about", "gentle", "that"]}, {"sent": "gentle baby .", "words": ["gentle"]}, {"sent": "and you just do gentle practice .", "words": ["do", "gentle", "you", "and"]}, {"sent": "he was very gentle opening the door wasnt he ?", "words": ["the", "was", "he", "gentle", "door"]}, {"sent": "gentle Ethan .", "words": ["gentle"]}, {"sent": "*SI1: be gentle with the racket okay ?", "words": ["the", "be", "gentle", "with"]}, {"sent": "be gentle .", "words": ["be", "gentle"]}, {"sent": "and the farmers gentle children .", "words": ["the", "gentle", "and"]}, {"sent": "gentle .", "words": ["gentle"]}, {"sent": "oh youre so gentle arent you ?", "words": ["so", "gentle", "you"]}, {"sent": "Amy has got bubbles has she ?", "words": ["she", "bubbles"]}, {"sent": "hey ya gonna feed the bear some bubbles ?", "words": ["the", "bear", "feed", "some", "bubbles"]}, {"sent": "where do bubbles go ?", "words": ["where", "do", "go", "bubbles"]}, {"sent": "yeah she kept turning it over and I think she was interested in the bubbles .", "words": ["the", "think", "was", "and", "in", "it", "over", "she", "bubbles"]}, {"sent": "I can blow some good bubbles .", "words": ["can", "good", "some", "blow", "bubbles"]}, {"sent": "she has got bubbles in the bath .", "words": ["the", "in", "she", "bubbles"]}, {"sent": "although I think we actually bought your bubbles in Woolworths didnt we ?", "words": ["think", "your", "we", "in", "bubbles"]}, {"sent": "no bubbles today .", "words": ["bubbles"]}, {"sent": "oh its those gas bubbles .", "words": ["those", "bubbles"]}, {"sent": "oh youre getting it all over the floor the soap bubbles .", "words": ["the", "all", "soap", "it", "over", "bubbles"]}, {"sent": "bubbles in my bath with lights in them like rainbows .", "words": ["my", "them", "with", "like", "in", "bubbles"]}, {"sent": "baby hippos blow bubbles under water .", "words": ["water", "under", "blow", "bubbles"]}, {"sent": "look at all those bubbles .", "words": ["all", "look", "those", "at", "bubbles"]}, {"sent": "shes covered in bubbles isnt she ?", "words": ["in", "she", "bubbles"]}, {"sent": "well well go to the shop and well get some bubbles .", "words": ["the", "go", "to", "and", "get", "some", "bubbles"]}, {"sent": "I dont understand how those bubbles get on the paper .", "words": ["the", "how", "those", "paper", "get", "dont", "on", "bubbles"]}, {"sent": "bubbles .", "words": ["bubbles"]}, {"sent": "and they put paper over the top of glass and all the bubbles made pretty pictures .", "words": ["the", "all", "put", "of", "paper", "and", "they", "pretty", "glass", "over", "bubbles"]}, {"sent": "you want your bubbles ?", "words": ["bubbles", "your", "you"]}, {"sent": "bubbles .", "words": ["bubbles"]}, {"sent": "do you have bubbles in your water ?", "words": ["do", "your", "you", "have", "in", "water", "bubbles"]}, {"sent": "shes just blowing bubbles .", "words": ["bubbles"]}, {"sent": "how come you wanna make bubbles in your apple juice ?", "words": ["how", "your", "you", "in", "wanna", "apple", "juice", "make", "bubbles"]}, {"sent": "bubbles bubbles .", "words": ["bubbles"]}, {"sent": "say bubbles .", "words": ["say", "bubbles"]}, {"sent": "the bubbles .", "words": ["the", "bubbles"]}, {"sent": "see the bubbles ?", "words": ["the", "see", "bubbles"]}, {"sent": "how about the bubbles ?", "words": ["the", "how", "about", "bubbles"]}, {"sent": "hey William whered you put your bubbles ?", "words": ["bubbles", "put", "your", "you"]}, {"sent": "the trouble with bubbles .", "words": ["the", "with", "bubbles"]}, {"sent": "Mommys making bubbles Naima .", "words": ["bubbles"]}, {"sent": "yeah Mommys gonna make more bubbles right now .", "words": ["more", "make", "bubbles"]}, {"sent": "theyre bubbles .", "words": ["bubbles"]}, {"sent": "mummy get new bubbles ?", "words": ["get", "new", "bubbles"]}, {"sent": "we play with bubbles in the tub .", "words": ["the", "we", "play", "with", "in", "bubbles"]}, {"sent": "was he blowing lots of bubbles ?", "words": ["of", "was", "he", "bubbles"]}, {"sent": "you wanna play with the bubbles ?", "words": ["the", "play", "with", "you", "wanna", "bubbles"]}, {"sent": "well bubbles do cheer us up .", "words": ["up", "do", "us", "bubbles"]}, {"sent": "were you blowing bubbles ?", "words": ["bubbles", "were", "you"]}, {"sent": "see the bubbles ?", "words": ["the", "see", "bubbles"]}, {"sent": "is she playing with some bubbles ?", "words": ["is", "with", "some", "she", "bubbles"]}, {"sent": "then theyre big big bubbles .", "words": ["then", "big", "bubbles"]}, {"sent": "what a lot of bubbles .", "words": ["what", "of", "a lot", "a", "bubbles"]}, {"sent": "look at all those bubbles .", "words": ["all", "look", "those", "at", "bubbles"]}, {"sent": "how do you blow bubbles ?", "words": ["do", "how", "you", "blow", "bubbles"]}, {"sent": "you made bubbles didnt you ?", "words": ["bubbles", "you"]}, {"sent": "usually we make millions of bubbles .", "words": ["of", "bubbles", "make", "we"]}, {"sent": "Im blowing all these bubbles and youre just ignoring them .", "words": ["all", "these", "them", "and", "bubbles"]}, {"sent": "bubbles .", "words": ["bubbles"]}, {"sent": "bubbles .", "words": ["bubbles"]}, {"sent": "bubbles yeah .", "words": ["bubbles"]}, {"sent": "I once took little bubbles in the the skin .", "words": ["bubbles", "the", "in", "little"]}, {"sent": "you could feed the bear some bubbles you could pretend to feed the bear some bubbles .", "words": ["the", "to", "bear", "you", "feed", "pretend", "some", "could", "bubbles"]}, {"sent": "*SI1: lemme blow bubbles when youre .", "words": ["when", "blow", "bubbles"]}, {"sent": "see the little bubbles in your pop ?", "words": ["the", "pop", "your", "little", "in", "see", "bubbles"]}, {"sent": "blow your bubbles ?", "words": ["your", "blow", "bubbles"]}, {"sent": "and thats got bubbles .", "words": ["bubbles", "and"]}, {"sent": "oh youre blowing bubbles at me ?", "words": ["me", "at", "bubbles"]}, {"sent": "bubbles Mommy bubbles .", "words": ["bubbles"]}, {"sent": "it was a tube of miracle bubbles .", "words": ["of", "was", "it", "a", "bubbles"]}, {"sent": "you do some bubbles on Lalas head .", "words": ["do", "head", "you", "on", "some", "bubbles"]}, {"sent": "Lalas covered in bubbles isnt she ?", "words": ["in", "she", "bubbles"]}, {"sent": "more bubbles ?", "words": ["more", "bubbles"]}, {"sent": "wanna would you like to play with the bubbles ?", "words": ["the", "to", "play", "with", "like", "you", "wanna", "would", "bubbles"]}, {"sent": "cant you blow bubbles ?", "words": ["bubbles", "blow", "you"]}, {"sent": "yeah bubbles .", "words": ["bubbles"]}, {"sent": "Chelsea can take the bubbles where theres room for bubbles .", "words": ["can", "the", "where", "for", "room", "take", "bubbles"]}, {"sent": "bubbles .", "words": ["bubbles"]}, {"sent": "blow bubbles .", "words": ["blow", "bubbles"]}, {"sent": "yeah theres a duck and bubbles in the bathtub .", "words": ["the", "and", "in", "bathtub", "duck", "a", "bubbles"]}, {"sent": "bubbles in my hair .", "words": ["in", "hair", "my", "bubbles"]}, {"sent": "do you know how to blow bubbles ?", "words": ["do", "how", "to", "you", "blow", "bubbles"]}, {"sent": "baby hippos blow bubbles under water .", "words": ["water", "under", "blow", "bubbles"]}, {"sent": "shall Mummy blow some bubbles and see if we can get some on Lalas head ?", "words": ["can", "if", "we", "bubbles", "head", "and", "get", "on", "see", "blow", "some"]}, {"sent": "barber baby bubbles and a bumblebee .", "words": ["a", "and", "bubbles"]}, {"sent": "bubbles .", "words": ["bubbles"]}, {"sent": "pop pop pop those bubbles .", "words": ["pop", "those", "bubbles"]}, {"sent": "go with the bubbles .", "words": ["the", "bubbles", "with", "go"]}, {"sent": "go and see bubbles .", "words": ["bubbles", "see", "and", "go"]}, {"sent": "*SI1: no more bubbles ?", "words": ["more", "bubbles"]}, {"sent": "where are those bubbles going ?", "words": ["where", "are", "those", "bubbles"]}, {"sent": "you dont blow bubbles with it .", "words": ["with", "you", "dont", "it", "blow", "bubbles"]}, {"sent": "does Noddy have bubbles ?", "words": ["have", "does", "bubbles"]}, {"sent": "did you play bubbles when you saw her ?", "words": ["play", "her", "you", "did", "when", "bubbles"]}, {"sent": "barber baby bubbles and a bumblebee .", "words": ["a", "and", "bubbles"]}, {"sent": "usually those are excellent bubbles .", "words": ["are", "those", "bubbles"]}, {"sent": "have you got lots of bubbles in your bath ?", "words": ["your", "of", "you", "have", "in", "bubbles"]}, {"sent": "lots of bubbles arent there ?", "words": ["of", "there", "bubbles"]}, {"sent": "barber baby bubbles and a bumblebee .", "words": ["a", "and", "bubbles"]}, {"sent": "oops bubbles .", "words": ["bubbles"]}, {"sent": "because its like if we get bubbles on your hair it doesnt matter does it ?", "words": ["does", "your", "if", "we", "like", "get", "it", "on", "hair", "because", "bubbles"]}, {"sent": "bubbles ?", "words": ["bubbles"]}, {"sent": "can you go with MissChelsea and play bubbles ?", "words": ["can", "go", "play", "with", "and", "you", "bubbles"]}, {"sent": "thats right in the bubbles .", "words": ["the", "in", "bubbles"]}, {"sent": "barber baby bubbles .", "words": ["bubbles"]}, {"sent": "more bubbles than water .", "words": ["water", "more", "bubbles"]}, {"sent": "the only foam weve got is when weve got lotsof bubbles .", "words": ["the", "bubbles", "when", "is"]}, {"sent": "bubbles .", "words": ["bubbles"]}, {"sent": "you have any bubbles to get out ?", "words": ["to", "any", "you", "have", "get", "out", "bubbles"]}, {"sent": "and Mummy blew bubbles for you .", "words": ["bubbles", "for", "you", "and"]}, {"sent": "Thomas just sit on the sofa .", "words": ["on", "the", "sit", "sofa"]}, {"sent": "whats that on the sofa ?", "words": ["on", "the", "that", "sofa"]}, {"sent": "Alex can you say sofa ?", "words": ["can", "sofa", "say", "you"]}, {"sent": "can you pop the cushion back on the sofa ?", "words": ["can", "the", "pop", "back", "you", "on", "sofa"]}, {"sent": "youve already falled off the sofa on Sunday doing that .", "words": ["off", "the", "on", "sofa", "that"]}, {"sent": "thats two tissues Mummy has found stuffed down the sofa .", "words": ["the", "sofa", "down"]}, {"sent": "hes probably stuffed in the sofa somewhere .", "words": ["the", "sofa", "in"]}, {"sent": "why dont you have a sleep on the sofa ?", "words": ["the", "sleep", "you", "have", "dont", "on", "sofa", "why", "a"]}, {"sent": "so Mummy kept telling you not to jump on the sofa .", "words": ["the", "to", "so", "you", "jump", "on", "sofa", "not"]}, {"sent": "when Daddy was telling a story she was sitting on the sofa and Daddy was in a chair but wheres she sitting now ?", "words": ["the", "was", "chair", "and", "in", "on", "sofa", "story", "when", "she", "but", "a"]}, {"sent": "if you dont want it well put it back on the sofa .", "words": ["the", "put", "if", "back", "you", "dont", "it", "on", "sofa"]}, {"sent": "sofa .", "words": ["sofa"]}, {"sent": "behind the sofa .", "words": ["the", "sofa", "behind"]}, {"sent": "not that sofa .", "words": ["sofa", "that", "not"]}, {"sent": "uhhuh okay the sofa goes there .", "words": ["the", "sofa", "there"]}, {"sent": "well Ive just popped it back on the sofa Thomas .", "words": ["the", "back", "it", "on", "sofa"]}, {"sent": "Im sitting on the sofa .", "words": ["on", "the", "sofa"]}, {"sent": "on that sofa ?", "words": ["on", "sofa", "that"]}, {"sent": "and probably if ever the sofa lasts long enough .", "words": ["the", "if", "long", "and", "sofa"]}, {"sent": "are you gonna sit on the sofa then ?", "words": ["the", "then", "you", "are", "on", "sofa", "sit"]}, {"sent": "if you wanna come here you come over the sofa because theres nails here .", "words": ["the", "if", "you", "here", "wanna", "sofa", "over", "because"]}, {"sent": "behind the sofa ?", "words": ["the", "sofa", "behind"]}, {"sent": "Thomas dont jump on the sofa .", "words": ["the", "dont", "jump", "on", "sofa"]}, {"sent": "oh sofa .", "words": ["sofa"]}, {"sent": "we can show people to Lars new sofa .", "words": ["can", "we", "show", "to", "sofa", "new"]}, {"sent": "and then you pushed one of them underneath the sofa ?", "words": ["the", "of", "them", "you", "and", "sofa", "then"]}, {"sent": "are they all sitting on that big sofa ?", "words": ["all", "they", "big", "are", "on", "sofa", "that"]}, {"sent": "oh thats that like our sofa isnt it ?", "words": ["like", "our", "it", "sofa", "that"]}, {"sent": "should we make the sofa into the store ?", "words": ["the", "into", "we", "store", "sofa", "make"]}, {"sent": "she pulled the sofa covers .", "words": ["the", "sofa", "she"]}, {"sent": "over by the sofa .", "words": ["the", "sofa", "over", "by"]}, {"sent": "so weve got baby Miles asleep in the sofa now havent we ?", "words": ["the", "we", "so", "asleep", "in", "sofa"]}, {"sent": "sofa sofa sofa .", "words": ["sofa"]}, {"sent": "mhm thats a sofa .", "words": ["sofa", "a"]}, {"sent": "oh youre moving them round and about on the sofa .", "words": ["the", "them", "and", "about", "on", "sofa"]}, {"sent": "look heres a sofa .", "words": ["sofa", "a", "look"]}, {"sent": "on the sofa .", "words": ["on", "the", "sofa"]}, {"sent": "get on the sofa .", "words": ["get", "the", "on", "sofa"]}, {"sent": "its dark there underneath the sofa ?", "words": ["the", "there", "sofa", "dark"]}, {"sent": "can you go back can you go back to the to the sofa ?", "words": ["can", "the", "back", "go", "to", "you", "sofa"]}, {"sent": "oh we dont bash dont bash bricks on the sofa .", "words": ["the", "we", "dont", "on", "sofa"]}, {"sent": "as youre jumping on the sofa its going back onto the china cabinet isnt it ?", "words": ["the", "back", "it", "on", "sofa"]}, {"sent": "you going to go and sit on the sofa ?", "words": ["the", "go", "to", "and", "you", "on", "sofa", "sit"]}, {"sent": "so you put them on the sofa .", "words": ["the", "put", "so", "them", "you", "on", "sofa"]}, {"sent": "on the sofa ?", "words": ["on", "the", "sofa"]}, {"sent": "you could take the jacket off and sit here on the sofa .", "words": ["the", "off", "sit", "you", "and", "here", "on", "sofa", "could", "take", "jacket"]}, {"sent": "and put him on his side on the sofa .", "words": ["the", "put", "and", "his", "him", "on", "sofa"]}, {"sent": "on the sofa .", "words": ["on", "the", "sofa"]}, {"sent": "watch it doesnt go on the sofa darling .", "words": ["the", "watch", "go", "it", "on", "sofa"]}, {"sent": "do you wanna come and sit on the sofa ?", "words": ["the", "do", "and", "you", "wanna", "on", "sofa", "sit"]}, {"sent": "normally you sleep on the sofa during the day if youre having a sleep .", "words": ["the", "if", "sleep", "you", "on", "sofa", "a"]}, {"sent": "xxx on the sofa .", "words": ["on", "the", "sofa"]}, {"sent": "right sit on the sofa .", "words": ["on", "the", "sit", "sofa"]}, {"sent": "the sofa .", "words": ["the", "sofa"]}, {"sent": "what are you doing behind the sofa ?", "words": ["the", "what", "behind", "you", "are", "sofa"]}, {"sent": "and they would sit on the sofa and watch television wouldnt they ?", "words": ["the", "watch", "and", "they", "would", "on", "sofa", "sit"]}, {"sent": "can we send a police car over to the sofa please ?", "words": ["can", "the", "we", "car", "to", "sofa", "over", "a"]}, {"sent": "do you like my sofa ?", "words": ["do", "my", "like", "you", "sofa"]}, {"sent": "move the sofa .", "words": ["the", "sofa"]}, {"sent": "youve put behind the sofa .", "words": ["the", "sofa", "put", "behind"]}, {"sent": "I think thats the sofa right there .", "words": ["the", "sofa", "think", "there"]}, {"sent": "you cant put them down the sofa .", "words": ["the", "put", "them", "you", "sofa", "down"]}, {"sent": "is there still that blue box round the corner of the sofa Tom ?", "words": ["the", "box", "is", "of", "blue", "there", "sofa", "that"]}, {"sent": "youre not lying on the sofa watching anything .", "words": ["on", "the", "not", "sofa"]}, {"sent": "whats that on the sofa ?", "words": ["on", "the", "that", "sofa"]}, {"sent": "the family library is behind the sofa .", "words": ["the", "sofa", "behind", "is"]}, {"sent": "ah standing on the sofa .", "words": ["on", "the", "sofa"]}, {"sent": "this looks like a sofa .", "words": ["this", "sofa", "like", "a"]}, {"sent": "theres another block under the sofa .", "words": ["the", "block", "another", "under", "sofa"]}, {"sent": "can Mummy sit up on the sofa ?", "words": ["can", "the", "up", "on", "sofa", "sit"]}, {"sent": "it sounded like you were saying something about the sofa but uh .", "words": ["the", "like", "you", "it", "about", "sofa", "were", "but"]}, {"sent": "two on the sofa and one in each of the two armchairs .", "words": ["the", "each", "of", "and", "in", "on", "sofa"]}, {"sent": "oh its funny what you find when you look under the sofa .", "words": ["the", "what", "find", "you", "under", "look", "sofa", "when"]}, {"sent": "they like it better on the sofa ?", "words": ["the", "better", "like", "they", "it", "on", "sofa"]}, {"sent": "Im pretending thats the part that you sit on and this is the back of the sofa .", "words": ["the", "back", "is", "of", "you", "and", "this", "on", "sofa", "sit", "that"]}, {"sent": "there should be one in each chair and two on the sofa .", "words": ["the", "each", "be", "chair", "and", "there", "in", "on", "sofa"]}, {"sent": "run round the sofa .", "words": ["the", "run", "sofa"]}, {"sent": "youre going jumping over the sofa ?", "words": ["the", "sofa", "over"]}, {"sent": "is she behind the sofa ?", "words": ["the", "is", "behind", "sofa", "she"]}, {"sent": "on the sofa .", "words": ["on", "the", "sofa"]}, {"sent": "youre on the sofa .", "words": ["on", "the", "sofa"]}, {"sent": "well look whos behind the sofa ?", "words": ["the", "sofa", "behind", "look"]}, {"sent": "and a sofa ohmygoodness look at all this stuff .", "words": ["all", "and", "this", "at", "look", "sofa", "a"]}, {"sent": "two from the sofa .", "words": ["the", "sofa"]}, {"sent": "the sofa ?", "words": ["the", "sofa"]}, {"sent": "threw it behind the sofa silly ?", "words": ["the", "sofa", "it", "behind"]}, {"sent": "before we lose bits down the sofa .", "words": ["the", "sofa", "down", "we"]}, {"sent": "you can sit on the sofa if you want ?", "words": ["can", "the", "if", "you", "on", "sofa", "sit"]}, {"sent": "well let me lean against the sofa then .", "words": ["the", "me", "then", "sofa"]}, {"sent": "on the sofa ?", "words": ["on", "the", "sofa"]}, {"sent": "if she has gone behind the sofa its because she wants a bit of peace and quiet .", "words": ["the", "if", "of", "behind", "and", "quiet", "sofa", "she", "because", "a"]}, {"sent": "and who was asleep on the sofa when all this was going on .", "words": ["the", "all", "was", "and", "asleep", "this", "who", "on", "sofa", "when"]}, {"sent": "you keep getting on the sofa .", "words": ["on", "the", "sofa", "you"]}, {"sent": "sofa sofa sofa .", "words": ["sofa"]}, {"sent": "oh well should we put the sofa in front of it ?", "words": ["the", "put", "we", "of", "in", "it", "sofa"]}, {"sent": "youre hiding behind the sofa .", "words": ["the", "sofa", "behind"]}, {"sent": "Chalk runs in front of the sofa .", "words": ["of", "the", "in", "sofa"]}, {"sent": "theres your blanket sleepy baby .", "words": ["blanket", "sleepy", "your"]}, {"sent": "hes got a blanket .", "words": ["blanket", "a"]}, {"sent": "a red blanket .", "words": ["red", "blanket", "a"]}, {"sent": "heres Georgies blanket .", "words": ["blanket"]}, {"sent": "try and say blanket for Mummy .", "words": ["for", "say", "and", "blanket", "try"]}, {"sent": "maybe a pretend blanket .", "words": ["blanket", "pretend", "a"]}, {"sent": "what are you sitting on xxx blanket ?", "words": ["what", "blanket", "you", "are", "on"]}, {"sent": "okay Annes blanket .", "words": ["blanket"]}, {"sent": "wants to lie down on Jwww Annas blanket .", "words": ["on", "to", "blanket", "down"]}, {"sent": "alright get off the blanket .", "words": ["get", "the", "blanket", "off"]}, {"sent": "how about if we put your napkin in front of you sothat you dont get your new jammie blanket all full of spaghetti .", "words": ["all", "how", "put", "your", "if", "full", "we", "of", "spaghetti", "you", "get", "dont", "in", "about", "blanket", "new", "napkin"]}, {"sent": "what do you do with the blanket ?", "words": ["the", "what", "do", "with", "you", "blanket"]}, {"sent": "okay lets throw on a blanket .", "words": ["on", "blanket", "throw", "a"]}, {"sent": "dont you have a nice blanket on your bed ?", "words": ["your", "nice", "bed", "you", "have", "dont", "on", "blanket", "a"]}, {"sent": "put your blanket away first .", "words": ["put", "your", "first", "away", "blanket"]}, {"sent": "dyou wanna use that white pillow behind you for his bed and this can be the blanket ?", "words": ["can", "the", "blanket", "for", "be", "behind", "bed", "you", "and", "this", "wanna", "his", "white", "that", "pillow"]}, {"sent": "this is your blanket .", "words": ["blanket", "this", "your", "is"]}, {"sent": "heres a blanket for the baby .", "words": ["blanket", "the", "for", "a"]}, {"sent": "theres a blanket up there .", "words": ["blanket", "there", "up", "a"]}, {"sent": "are you going to get up on the settee and I give you your blanket ?", "words": ["up", "the", "blanket", "your", "to", "you", "get", "and", "are", "on", "give"]}, {"sent": "why dont you put that blanket on down there ?", "words": ["put", "you", "dont", "on", "there", "blanket", "down", "that", "why"]}, {"sent": "wheres his blanket ?", "words": ["his", "blanket"]}, {"sent": "oh does Adam have a blanket ?", "words": ["have", "does", "blanket", "a"]}, {"sent": "that baby blankets my babys blanket .", "words": ["blanket", "my", "that"]}, {"sent": "that green blanket round your shoulders .", "words": ["blanket", "that", "green", "your"]}, {"sent": "okay now you bring a blanket for me okay ?", "words": ["for", "bring", "you", "blanket", "me", "a"]}, {"sent": "she sees the blanket on the ground and the cats gone .", "words": ["the", "and", "on", "blanket", "she"]}, {"sent": "oh whos he making the blanket for ?", "words": ["the", "blanket", "for", "he"]}, {"sent": "you sit on the sofa and share the blanket .", "words": ["the", "blanket", "share", "you", "and", "on", "sofa", "sit"]}, {"sent": "aye shes put a blanket over his head a couple of times .", "words": ["put", "of", "head", "his", "blanket", "over", "a"]}, {"sent": "put this blanket around her .", "words": ["put", "this", "around", "blanket", "her"]}, {"sent": "Sam put the blanket on his head .", "words": ["the", "put", "blanket", "head", "on", "his"]}, {"sent": "should we get a little blanket and sit them all down .", "words": ["all", "we", "them", "little", "get", "and", "blanket", "sit", "down", "a"]}, {"sent": "its a blanket .", "words": ["blanket", "a"]}, {"sent": "you like that blanket dont you ?", "words": ["like", "you", "dont", "blanket", "that"]}, {"sent": "you gonna put your blanket away now ?", "words": ["put", "your", "you", "away", "blanket"]}, {"sent": "xxx blanket ?", "words": ["blanket"]}, {"sent": "get his blanket so hes comfy .", "words": ["get", "his", "blanket", "so"]}, {"sent": "blanket ?", "words": ["blanket"]}, {"sent": "thats going to be the blanket ?", "words": ["the", "to", "blanket", "be"]}, {"sent": "but if you havent got a fire blanket you can use something like an ordinary towel or a tea towel .", "words": ["can", "if", "like", "you", "an", "towel", "blanket", "but", "a"]}, {"sent": "I was feeling a little hot under the blanket maybe you just wanna sit under the blanket .", "words": ["the", "was", "little", "you", "under", "hot", "wanna", "blanket", "sit", "a"]}, {"sent": "try and say blanket ?", "words": ["blanket", "try", "say", "and"]}, {"sent": "well she she does but mainly I dont know I think Im like her safety blanket .", "words": ["does", "think", "like", "dont", "blanket", "her", "she", "but"]}, {"sent": "can you put her under the blanket ?", "words": ["can", "the", "put", "you", "under", "blanket", "her"]}, {"sent": "xxx blanket ?", "words": ["blanket"]}, {"sent": "what was he doing with your blanket ?", "words": ["what", "your", "was", "with", "he", "blanket"]}, {"sent": "yes thats your blanket .", "words": ["blanket", "your"]}, {"sent": "we could use a piece of paper for a blanket .", "words": ["for", "we", "of", "paper", "blanket", "could", "a"]}, {"sent": "Ill have my Mamas blanket .", "words": ["have", "my", "blanket"]}, {"sent": "thats Jwwws blanket isnt it ?", "words": ["blanket", "it"]}, {"sent": "I had a big green blanket that I took everywhere I went with me .", "words": ["with", "big", "green", "blanket", "me", "that", "a"]}, {"sent": "hafta give Mommy a blanket .", "words": ["blanket", "give", "a"]}, {"sent": "theres blanket like ours outof the window .", "words": ["blanket", "the", "window", "like"]}, {"sent": "the big blue blanket ?", "words": ["the", "blanket", "big", "blue"]}, {"sent": "are you looking for a blanket for the dolly ?", "words": ["the", "for", "you", "are", "blanket", "a"]}, {"sent": "we hafta gets the girls blanket with the extra boys blanket .", "words": ["the", "blanket", "with", "we"]}, {"sent": "that blanket does not smell very good .", "words": ["does", "good", "blanket", "that", "not"]}, {"sent": "would you like your blanket ?", "words": ["your", "like", "you", "would", "blanket"]}, {"sent": "well xxx um I think maybe you could try using it like a blanket cause its gonna be .", "words": ["think", "be", "like", "you", "it", "blanket", "could", "try", "a"]}, {"sent": "thats not a blanket .", "words": ["blanket", "not", "a"]}, {"sent": "she wants a blanket ?", "words": ["blanket", "she", "a"]}, {"sent": "is she gonna go in the blanket ?", "words": ["the", "go", "is", "in", "blanket", "she"]}, {"sent": "well where is your blanket ?", "words": ["where", "blanket", "your", "is"]}, {"sent": "you mean when you go to the circus youre gonna get a blanket to hide your face ?", "words": ["the", "your", "face", "hide", "go", "to", "you", "get", "blanket", "when", "a"]}, {"sent": "the blanket got in the wet .", "words": ["the", "in", "blanket", "wet"]}, {"sent": "who put your blanket in Frasers livingroom ?", "words": ["put", "your", "in", "who", "blanket"]}, {"sent": "Mister blanket monster .", "words": ["blanket"]}, {"sent": "what have you got on your blanket ?", "words": ["what", "blanket", "your", "you", "have", "on"]}, {"sent": "your blanket ?", "words": ["blanket", "your"]}, {"sent": "you want that blanket ?", "words": ["blanket", "that", "you"]}, {"sent": "do you know what Johns got in his blanket ?", "words": ["do", "what", "blanket", "you", "in", "his"]}, {"sent": "Ill spread out the blanket and then .", "words": ["the", "and", "out", "blanket", "then"]}, {"sent": "a blanket .", "words": ["blanket", "a"]}, {"sent": "lets get your blanket xxx .", "words": ["get", "blanket", "your"]}, {"sent": "oh is she getting cozy under the blanket ?", "words": ["the", "is", "under", "blanket", "she"]}, {"sent": "its the extra blanket .", "words": ["the", "blanket"]}, {"sent": "what about the green blanket green blanket ?", "words": ["the", "what", "green", "about", "blanket"]}, {"sent": "when you bring the blanket okay ?", "words": ["the", "bring", "you", "blanket", "when"]}, {"sent": "whos blanket is this ?", "words": ["blanket", "this", "is"]}, {"sent": "he wants a blanket ?", "words": ["blanket", "he", "a"]}, {"sent": "heres your blanket .", "words": ["blanket", "your"]}, {"sent": "she can have that as a as a blanket .", "words": ["can", "have", "blanket", "that", "she", "a"]}, {"sent": "heres a blanket .", "words": ["blanket", "a"]}, {"sent": "on the blanket .", "words": ["on", "the", "blanket"]}, {"sent": "you drive it on your blanket .", "words": ["blanket", "your", "you", "drive", "it", "on"]}, {"sent": "blanket .", "words": ["blanket"]}, {"sent": "xxx the pink blanket .", "words": ["the", "blanket"]}, {"sent": "this is your blanket for your baby ?", "words": ["for", "your", "is", "this", "blanket"]}, {"sent": "he has got a he has got a nice checked blanket hasnt he ?", "words": ["blanket", "nice", "he", "a"]}, {"sent": "your blanket is wet .", "words": ["blanket", "wet", "your", "is"]}, {"sent": "its a blanket darling .", "words": ["blanket", "a"]}, {"sent": "you putting my the blanket under my face ?", "words": ["the", "my", "face", "you", "under", "blanket"]}, {"sent": "blanket .", "words": ["blanket"]}, {"sent": "and where is she wearing her mittens ?", "words": ["where", "mittens", "is", "and", "her", "she"]}, {"sent": "you do have lots and lotsof mittens havent you ?", "words": ["do", "mittens", "and", "have", "you"]}, {"sent": "Ive got a book here that is got things that zipper and button and fuzzy mittens in it .", "words": ["mittens", "button", "is", "and", "here", "zipper", "in", "it", "book", "that", "a"]}, {"sent": "wheres your mittens or your gloves ?", "words": ["mittens", "gloves", "your"]}, {"sent": "my mittens ?", "words": ["my", "mittens"]}, {"sent": "when it snows youll wear mittens in the snow .", "words": ["the", "mittens", "snow", "in", "it", "when"]}, {"sent": "goodnight kittens and good night mittens .", "words": ["mittens", "good", "and"]}, {"sent": "some mittens .", "words": ["some", "mittens"]}, {"sent": "you dropped your mittens ?", "words": ["mittens", "your", "you"]}, {"sent": "our mittens we have found .", "words": ["have", "mittens", "we", "our"]}, {"sent": "and two little kittens and a pair of mittens .", "words": ["mittens", "of", "little", "and", "a"]}, {"sent": "look there are mittens .", "words": ["there", "mittens", "are", "look"]}, {"sent": "so they talked about mittens and what were the other familiar things that you think regardless of the number .", "words": ["the", "what", "mittens", "think", "of", "so", "and", "other", "they", "that", "you", "about", "were"]}, {"sent": "and eight mittens oh look at those balloons .", "words": ["mittens", "those", "and", "at", "look"]}, {"sent": "mittens .", "words": ["mittens"]}, {"sent": "our mittens we have washed .", "words": ["have", "mittens", "we", "our"]}, {"sent": "eight mittens .", "words": ["mittens"]}, {"sent": "and you didnt even give me my mittens .", "words": ["my", "mittens", "you", "and", "me", "give"]}, {"sent": "mm seven socks eight mittens .", "words": ["mittens"]}, {"sent": "hes wearing mittens on a string .", "words": ["on", "mittens", "a"]}, {"sent": "who lost their mittens .", "words": ["mittens", "their", "who"]}, {"sent": "baby mittens .", "words": ["mittens"]}, {"sent": "are those his mittens ?", "words": ["his", "mittens", "are", "those"]}, {"sent": "oh mother dear we greatly fear our mittens we have soiled .", "words": ["our", "have", "mittens", "we"]}, {"sent": "eight mittens .", "words": ["mittens"]}, {"sent": "eight mittens .", "words": ["mittens"]}, {"sent": "and her mittens ?", "words": ["mittens", "her", "and"]}, {"sent": "I dont think you had any mittens on when you went out did you ?", "words": ["think", "mittens", "any", "you", "dont", "did", "out", "on", "when"]}, {"sent": "look are they her mittens ?", "words": ["mittens", "they", "are", "look", "her"]}, {"sent": "the three little kittens lost their mittens and they began to cry .", "words": ["the", "mittens", "their", "to", "cry", "and", "little", "they"]}, {"sent": "lost their mittens and they began to cry oh mother dear .", "words": ["mittens", "their", "to", "cry", "and", "they"]}, {"sent": "and two little kittens and a pair of mittens .", "words": ["mittens", "of", "little", "and", "a"]}, {"sent": "wheres the mittens ?", "words": ["the", "mittens"]}, {"sent": "out mittens we have lost .", "words": ["have", "mittens", "out", "we"]}, {"sent": "and the mittens .", "words": ["the", "mittens", "and"]}, {"sent": "the three little kittens lost their mittens and they began to cry .", "words": ["the", "mittens", "their", "to", "cry", "and", "little", "they"]}, {"sent": "thats mittens .", "words": ["mittens"]}, {"sent": "theyre not mittens .", "words": ["mittens", "not"]}, {"sent": "and two tiny mittens .", "words": ["tiny", "mittens", "and"]}, {"sent": "mittens ?", "words": ["mittens"]}, {"sent": "eight mittens .", "words": ["mittens"]}, {"sent": "you dont have any mittens ?", "words": ["mittens", "any", "you", "have", "dont"]}, {"sent": "see her red mittens ?", "words": ["red", "see", "mittens", "her"]}, {"sent": "oh they found their mittens .", "words": ["they", "mittens", "their"]}, {"sent": "four mittens five lollipops .", "words": ["mittens"]}, {"sent": "I wonder if we take Loiss mittens out we can find it .", "words": ["can", "mittens", "if", "we", "it", "out", "find", "take"]}, {"sent": "when you were a tiny baby we useta put some little cotton mittens on you called scratch mittens .", "words": ["tiny", "put", "mittens", "we", "you", "little", "on", "some", "when", "were", "a"]}, {"sent": "the three little kittens put on their mittens and soon ate up the pie .", "words": ["the", "up", "put", "mittens", "their", "little", "and", "on"]}, {"sent": "oh your mittens .", "words": ["mittens", "your"]}, {"sent": "mittens ?", "words": ["mittens"]}, {"sent": "our mittens we have lost .", "words": ["have", "mittens", "we", "our"]}, {"sent": "look at those cute mittens .", "words": ["mittens", "look", "those", "cute", "at"]}, {"sent": "it was your mittens on a string wasnt it ?", "words": ["mittens", "your", "was", "it", "on", "a"]}, {"sent": "the three little kittens they washed their mittens and hung them up to dry .", "words": ["the", "up", "mittens", "their", "to", "dry", "them", "little", "and", "they"]}, {"sent": "mittens .", "words": ["mittens"]}, {"sent": "theyre called mi mittens .", "words": ["mittens"]}, {"sent": "and two little kittens and a pair of mittens .", "words": ["mittens", "of", "little", "and", "a"]}, {"sent": "see here see here our mittens we have lost .", "words": ["mittens", "we", "our", "here", "have", "see"]}, {"sent": "and a pair of mittens .", "words": ["of", "mittens", "a", "and"]}, {"sent": "eight mittens .", "words": ["mittens"]}, {"sent": "I think hes crying because he lost his mittens isnt he ?", "words": ["think", "mittens", "he", "his", "because"]}, {"sent": "shall I show the where the mittens were hidden ?", "words": ["the", "where", "mittens", "show", "were"]}, {"sent": "the three little kittens put on their mittens and soon ate up the pie oh mother dear we greatly fear our mittens we have soiled .", "words": ["the", "up", "put", "mittens", "their", "we", "little", "and", "our", "have", "on"]}, {"sent": "what soiled your mittens ?", "words": ["what", "mittens", "your"]}, {"sent": "I xxx mittens for christmas .", "words": ["mittens", "for"]}, {"sent": "what washed your mittens ?", "words": ["what", "mittens", "your"]}, {"sent": "thats a pair of mittens belonging to TabbyKitten .", "words": ["of", "to", "mittens", "a"]}, {"sent": "xxx well theres another pair there a pair of mittens .", "words": ["mittens", "of", "another", "there", "a"]}, {"sent": "those are like your mittens .", "words": ["mittens", "your", "those", "like", "are"]}, {"sent": "they lost their mittens and they began to cry .", "words": ["mittens", "their", "to", "cry", "and", "they"]}, {"sent": "they lost their mittens three little kittens .", "words": ["they", "mittens", "their", "little"]}, {"sent": "your mittens are super wet .", "words": ["wet", "mittens", "are", "your"]}, {"sent": "no mittens .", "words": ["mittens"]}, {"sent": "we bought her a little pair of mittens and we bought her this book as well .", "words": ["mittens", "we", "of", "little", "and", "this", "book", "her", "a"]}, {"sent": "eight mittens .", "words": ["mittens"]}, {"sent": "mittens on .", "words": ["on", "mittens"]}, {"sent": "gloves mittens .", "words": ["mittens", "gloves"]}, {"sent": "Ill take your mittens off .", "words": ["your", "mittens", "take", "off"]}, {"sent": "they said they lost their mittens ?", "words": ["they", "mittens", "their"]}, {"sent": "I think hes crying because he lost his mittens isnt he ?", "words": ["think", "mittens", "he", "his", "because"]}, {"sent": "do you have any mittens ?", "words": ["do", "mittens", "any", "you", "have"]}, {"sent": "our mittens we have found .", "words": ["have", "mittens", "we", "our"]}, {"sent": "eight mittens .", "words": ["mittens"]}, {"sent": "three little kittens they lost their mittens and they began to cry .", "words": ["mittens", "their", "to", "cry", "and", "little", "they"]}, {"sent": "eight mittens .", "words": ["mittens"]}, {"sent": "wear mittens in the snow ?", "words": ["the", "in", "mittens", "snow"]}, {"sent": "xxx if you prefer to wear your mittens okay fine .", "words": ["mittens", "your", "if", "fine", "to", "you"]}, {"sent": "theyre like mittens arent they ?", "words": ["they", "mittens", "like"]}, {"sent": "three little kittens had lost their mittens .", "words": ["mittens", "their", "little"]}, {"sent": "we found our mittens we have lost ?", "words": ["our", "have", "mittens", "we"]}, {"sent": "what did they do with their mittens ?", "words": ["what", "do", "mittens", "their", "with", "they", "did"]}, {"sent": "and our snowsuits well get our snowsuits on and our boots and our mittens and our hats .", "words": ["mittens", "our", "get", "boots", "and", "on"]}, {"sent": "the blue ones for staying in the blue coat and the green ones for staying in the brown coat and then when you go out youve always got a pair of mittens .", "words": ["the", "mittens", "for", "brown", "go", "of", "blue", "and", "you", "in", "green", "out", "then", "when", "coat", "a"]}, {"sent": "and theres the kittens and their mittens .", "words": ["the", "mittens", "their", "and"]}, {"sent": "Ill hafta start putting some mittens on you .", "words": ["on", "some", "mittens", "you"]}, {"sent": "lost your mittens .", "words": ["mittens", "your"]}, {"sent": "where are you going when you get the mittens ?", "words": ["where", "the", "mittens", "you", "get", "are", "when"]}, {"sent": "lost their mittens .", "words": ["mittens", "their"]}, {"sent": "I think weve got to cut that nail .", "words": ["think", "cut", "to", "nail", "that"]}, {"sent": "has the nail broken ?", "words": ["the", "nail", "broken"]}, {"sent": "a nail clipper and clip clip that nail honey because I think its a little bit xxx .", "words": ["think", "nail", "and", "little", "that", "because", "a"]}, {"sent": "oh let me take these nail clippers .", "words": ["nail", "me", "take", "these"]}, {"sent": "theres a little nail to fix it with .", "words": ["to", "fix", "nail", "with", "little", "it", "a"]}, {"sent": "its nail week ?", "words": ["nail"]}, {"sent": "it had cologne in it and powder and lipstick and everything in it nail polish .", "words": ["in", "it", "nail", "and"]}, {"sent": "because this oh my gosh look how long that nail is .", "words": ["my", "how", "is", "nail", "long", "this", "look", "that", "because"]}, {"sent": "heres a nail would you like to try ?", "words": ["to", "nail", "like", "you", "would", "try", "a"]}, {"sent": "nail .", "words": ["nail"]}, {"sent": "and this is a big big nail .", "words": ["is", "nail", "and", "this", "big", "a"]}, {"sent": "you can scrub your nails with the nail brush .", "words": ["can", "the", "your", "brush", "with", "nail", "you"]}, {"sent": "heres a nail and theres a xxx xxx .", "words": ["nail", "and", "a"]}, {"sent": "where is the nail varnish ?", "words": ["where", "the", "nail", "is"]}, {"sent": "look at this nail .", "words": ["this", "at", "nail", "look"]}, {"sent": "your toe nail going to xxx .", "words": ["toe", "to", "your", "nail"]}, {"sent": "nail clippers .", "words": ["nail"]}, {"sent": "nail cutting .", "words": ["nail"]}, {"sent": "let me have the other part of the nail .", "words": ["the", "of", "nail", "other", "have", "me"]}, {"sent": "got a nail ?", "words": ["nail", "a"]}, {"sent": "do you mean a finger nail ?", "words": ["do", "nail", "you", "finger", "a"]}, {"sent": "look at your thumb nail .", "words": ["nail", "at", "your", "look"]}, {"sent": "oh is there a nail there ?", "words": ["there", "nail", "a", "is"]}, {"sent": "its just my finger nail honey .", "words": ["finger", "my", "nail"]}, {"sent": "so Im sure later on it will come off in the bath if not Ill hafta get nail polish remover and remove the whole of the polish from that big toe and paint it pink so it matches the others .", "words": ["off", "the", "paint", "if", "of", "so", "toe", "will", "nail", "and", "get", "in", "it", "big", "on", "that", "not"]}, {"sent": "hammering a nail with a red hammer .", "words": ["red", "hammer", "nail", "with", "a"]}, {"sent": "what color nail varnish ?", "words": ["what", "nail"]}, {"sent": "thats a nail .", "words": ["nail", "a"]}, {"sent": "oh can I have that other nail ?", "words": ["can", "nail", "other", "have", "that"]}, {"sent": "Im getting the nail clippers .", "words": ["the", "nail"]}, {"sent": "no you dont have a nail in there .", "words": ["nail", "you", "have", "dont", "in", "there", "a"]}, {"sent": "nail clippers .", "words": ["nail"]}, {"sent": "thats a little nail brush that you had in your hand Thomas .", "words": ["your", "hand", "brush", "nail", "little", "you", "in", "that", "a"]}, {"sent": "Mummyll cut your nail later .", "words": ["your", "cut", "nail"]}, {"sent": "do you like putting nail polish on ?", "words": ["do", "nail", "like", "you", "on"]}, {"sent": "when it hit a nail in the road ?", "words": ["the", "nail", "in", "it", "hit", "when", "a"]}, {"sent": "poor Bertie hits a nail on the road and gets a flat tire .", "words": ["the", "nail", "poor", "and", "on", "a"]}, {"sent": "you put it on with a nail .", "words": ["put", "with", "nail", "you", "it", "on", "a"]}, {"sent": "I think thats supposed to be a nail .", "words": ["think", "to", "be", "nail", "a"]}, {"sent": "you hurt your nail ?", "words": ["hurt", "nail", "your", "you"]}, {"sent": "oh a big nail .", "words": ["nail", "big", "a"]}, {"sent": "thats called a nail brush .", "words": ["brush", "nail", "a"]}, {"sent": "yeah its a little nail .", "words": ["nail", "little", "a"]}, {"sent": "you cant put a nail nail inside the screwdriver .", "words": ["the", "put", "inside", "nail", "you", "a"]}, {"sent": "can I have the nail ?", "words": ["can", "the", "have", "nail"]}, {"sent": "you just take this nail off .", "words": ["off", "nail", "you", "this", "take"]}, {"sent": "nail varnish .", "words": ["nail"]}, {"sent": "nearly knocked my nail off then .", "words": ["off", "my", "nail", "then"]}, {"sent": "a nail in .", "words": ["in", "nail", "a"]}, {"sent": "I dont usually wear nail polish .", "words": ["dont", "nail"]}, {"sent": "shall I cut that finger nail ?", "words": ["finger", "that", "cut", "nail"]}, {"sent": "youve got more than one nail .", "words": ["more", "nail"]}, {"sent": "well I wont cut your nail off .", "words": ["off", "your", "cut", "nail"]}, {"sent": "nail .", "words": ["nail"]}, {"sent": "nail varnish ?", "words": ["nail"]}, {"sent": "isnt that dangerous Sarah that nail sticking out there .", "words": ["there", "that", "out", "nail"]}, {"sent": "well weve all got nails on our fingers Aran but what you mean is that youve got a nail that wants cutting .", "words": ["all", "what", "is", "nail", "our", "you", "on", "that", "but", "a"]}, {"sent": "what weve got to do now is get a cloth to wipe these little bits of finger nail up .", "words": ["up", "what", "do", "these", "is", "of", "to", "nail", "little", "get", "finger", "wipe", "a"]}, {"sent": "are you going to show Rose your nail polish ?", "words": ["your", "show", "to", "nail", "you", "are"]}, {"sent": "remember what youre not allowed to wear nail varnish are you ?", "words": ["what", "to", "nail", "you", "are", "not"]}, {"sent": "are you gonna put your nail polish away so it doesnt get lost ?", "words": ["put", "your", "so", "nail", "you", "get", "are", "away", "it"]}, {"sent": "nail varnish .", "words": ["nail"]}, {"sent": "have you got a short nail there ?", "words": ["nail", "you", "have", "there", "a"]}, {"sent": "see my nail ?", "words": ["see", "my", "nail"]}, {"sent": "nail nail .", "words": ["nail"]}, {"sent": "does one nail fit inside the other nail ?", "words": ["the", "does", "fit", "inside", "nail", "other"]}, {"sent": "the pinkie toe has got a big nail .", "words": ["the", "toe", "nail", "big", "a"]}, {"sent": "Ill go upstairs and get some scissors and Ill cut your nail for you .", "words": ["for", "your", "cut", "go", "scissors", "nail", "and", "get", "you", "some"]}, {"sent": "and youve caught your nail havent you ?", "words": ["nail", "your", "you", "and"]}, {"sent": "did your mummie put your nail polish on for you ?", "words": ["put", "for", "your", "nail", "you", "did", "on"]}, {"sent": "oh nail polish .", "words": ["nail"]}, {"sent": "thats the little piggies using the nail isnt he ?", "words": ["the", "he", "nail", "little"]}, {"sent": "oh your nail broke .", "words": ["your", "nail"]}, {"sent": "you got a nail ?", "words": ["nail", "a", "you"]}, {"sent": "NinNins just doing her nails with pink nail varnish .", "words": ["her", "with", "nail"]}, {"sent": "theyre for taking your nail varnish off .", "words": ["off", "for", "your", "nail"]}, {"sent": "you had a nail didnt you ?", "words": ["nail", "a", "you"]}, {"sent": "well he hammers nails and he turns the screws with the screwdriver hammers nail with a hammer .", "words": ["the", "hammer", "nail", "with", "and", "he", "a"]}, {"sent": "youve got a nail on your track .", "words": ["on", "nail", "your", "a"]}, {"sent": "is this your nail polish ?", "words": ["nail", "this", "your", "is"]}, {"sent": "nail clippers ?", "words": ["nail"]}, {"sent": "you have nail ?", "words": ["have", "nail", "you"]}, {"sent": "lets see if the nail clippers down here .", "words": ["the", "if", "nail", "here", "see", "down"]}, {"sent": "oh its got a big nail on it .", "words": ["nail", "it", "on", "big", "a"]}, {"sent": "a nail ?", "words": ["nail", "a"]}, {"sent": "can I have a look at your nail ?", "words": ["can", "your", "nail", "have", "at", "look", "a"]}, {"sent": "right well just have a little snip of that nail there .", "words": ["of", "nail", "little", "have", "there", "that", "a"]}, {"sent": "its hardly worth getting the nail clippers down for that little amount .", "words": ["the", "for", "nail", "little", "down", "that"]}, {"sent": "here lets put it in there and can you screw in the nail ?", "words": ["can", "the", "put", "nail", "and", "here", "there", "in", "it", "you"]}, {"sent": "and a hammer and a nail .", "words": ["hammer", "nail", "a", "and"]}, {"sent": "what color nail varnish does Mama put on ?", "words": ["what", "put", "does", "nail", "on"]}, {"sent": "now nail the pieces together very carefully .", "words": ["the", "nail"]}, {"sent": "look at that nail .", "words": ["nail", "at", "that", "look"]}, {"sent": "I shall put some nail varnish on .", "words": ["on", "some", "put", "nail"]}, {"sent": "a nail was sticking up and he stepped on it ?", "words": ["up", "was", "nail", "and", "it", "he", "on", "a"]}, {"sent": "a nail ?", "words": ["nail", "a"]}, {"sent": "that is a nail .", "words": ["nail", "that", "a", "is"]}, {"sent": "you got a nail through it .", "words": ["it", "nail", "a", "you"]}, {"sent": "who was playing with the play dough with you the other day ?", "words": ["the", "was", "play", "with", "you", "other", "who", "play dough"]}, {"sent": "this is called clay and the pink ones called play dough okay ?", "words": ["the", "is", "play", "and", "this", "play dough"]}, {"sent": "Jeremy I havent seen your blue play dough in a long time .", "words": ["your", "play", "long", "blue", "in", "play dough", "a"]}, {"sent": "whats play dough ?", "words": ["play", "play dough"]}, {"sent": "the little boxes of play dough ?", "words": ["the", "of", "play", "little", "play dough"]}, {"sent": "do you know where your play dough is ?", "words": ["where", "do", "your", "is", "play", "you", "play dough"]}, {"sent": "I thought the teachers called it play dough .", "words": ["the", "it", "play dough", "play"]}, {"sent": "youve got play dough in it .", "words": ["it", "in", "play", "play dough"]}, {"sent": "this is play dough .", "words": ["this", "play", "play dough", "is"]}, {"sent": "do you have play dough ?", "words": ["do", "play", "you", "have", "play dough"]}, {"sent": "your play dough fell down .", "words": ["down", "play", "your", "play dough"]}, {"sent": "careful honey make it on here because youre making the play dough go all over the rug .", "words": ["the", "all", "over", "go", "play", "here", "it", "careful", "on", "make", "play dough", "because"]}, {"sent": "this is play dough and this is clay .", "words": ["is", "play", "and", "this", "play dough"]}, {"sent": "I wanna play with play dough .", "words": ["play", "with", "play dough", "wanna"]}, {"sent": "shall we play with the play dough ?", "words": ["the", "we", "play", "with", "play dough"]}, {"sent": "play dough .", "words": ["play", "play dough"]}, {"sent": "this is the play dough darling .", "words": ["the", "is", "play", "this", "play dough"]}, {"sent": "you open the pink play dough darling .", "words": ["the", "open", "play", "you", "play dough"]}, {"sent": "uh oh play dough head .", "words": ["head", "play", "play dough"]}, {"sent": "papers coming off on the play dough .", "words": ["off", "the", "play", "on", "play dough"]}, {"sent": "whats fun to make with play dough ?", "words": ["to", "play", "with", "make", "play dough"]}, {"sent": "play dough .", "words": ["play", "play dough"]}, {"sent": "that probably feels a bit like play dough or plasticine .", "words": ["play", "like", "that", "play dough", "a"]}, {"sent": "get lots of play dough together .", "words": ["get", "of", "play", "play dough"]}, {"sent": "I wanna play with play dough .", "words": ["play", "with", "play dough", "wanna"]}, {"sent": "this is play dough .", "words": ["this", "play", "play dough", "is"]}, {"sent": "give me some play dough .", "words": ["play", "me", "give", "play dough", "some"]}, {"sent": "come down and do some play dough and make a puppet .", "words": ["do", "play", "and", "some", "down", "make", "play dough", "a"]}, {"sent": "because its covered with play dough ?", "words": ["play", "with", "play dough", "because"]}, {"sent": "come over here wheres your play wheres your play dough ?", "words": ["your", "play", "here", "over", "play dough"]}, {"sent": "whatre you going to make with play dough ?", "words": ["to", "play", "with", "you", "make", "play dough"]}, {"sent": "because I wouldnt just run out to the garage and bring her play dough in .", "words": ["the", "garage", "to", "play", "bring", "and", "run", "in", "out", "her", "play dough", "because"]}, {"sent": "they probably have play dough there .", "words": ["play", "have", "they", "there", "play dough"]}, {"sent": "I bought cha new play dough I bought cha new play dough .", "words": ["new", "play dough", "play"]}, {"sent": "you found more play dough .", "words": ["more", "play", "play dough", "you"]}, {"sent": "yeah I made a little kitty cat with that play dough .", "words": ["play", "with", "little", "cat", "kitty", "that", "play dough", "a"]}, {"sent": "xxx we should put the play dough away if youre not gonna play with it hm ?", "words": ["the", "put", "if", "we", "play", "with", "play dough", "away", "it", "not"]}, {"sent": "oh thats for the uh play dough .", "words": ["the", "for", "play dough", "play"]}, {"sent": "okay now what do I sposta do with this play dough ?", "words": ["what", "do", "play", "with", "this", "play dough"]}, {"sent": "you want the play dough ?", "words": ["the", "play", "play dough", "you"]}, {"sent": "I dont want this play dough ball .", "words": ["play", "ball", "dont", "this", "play dough"]}, {"sent": "make some nice new play dough .", "words": ["nice", "play", "some", "new", "make", "play dough"]}, {"sent": "thats my play dough bracelet oh dont you dare xxx .", "words": ["my", "play", "you", "dont", "play dough"]}, {"sent": "Debbie the kids dont wanna share their play dough with Gabriel .", "words": ["the", "share", "their", "play", "with", "dont", "wanna", "play dough"]}, {"sent": "pink ones play dough .", "words": ["play", "play dough"]}, {"sent": "you like this play dough .", "words": ["play", "like", "you", "this", "play dough"]}, {"sent": "I dont know what happened to your blue play dough .", "words": ["what", "your", "to", "play", "blue", "dont", "play dough"]}, {"sent": "this is clay and thats play dough .", "words": ["is", "play", "and", "this", "play dough"]}, {"sent": "what about play dough ?", "words": ["what", "play", "play dough", "about"]}, {"sent": "oh I dont have any play dough right now .", "words": ["play", "any", "have", "dont", "play dough"]}, {"sent": "thats called play dough .", "words": ["play", "play dough"]}, {"sent": "a play dough road .", "words": ["play", "play dough", "a"]}, {"sent": "Steven you hafta share play dough with everyone .", "words": ["share", "play", "with", "you", "play dough"]}, {"sent": "me like cookies better but me me like play dough for snack .", "words": ["better", "for", "play", "like", "me", "play dough", "but"]}, {"sent": "where is the play dough sticking ?", "words": ["where", "the", "is", "play", "play dough"]}, {"sent": "and do you have lotsof different colors of play dough ?", "words": ["do", "of", "play", "you", "have", "and", "play dough"]}, {"sent": "shall Mama make you some biscuits out of the play dough ?", "words": ["the", "of", "play", "you", "out", "some", "make", "play dough"]}, {"sent": "mm make a circle with the play dough ?", "words": ["the", "play", "with", "make", "play dough", "a"]}, {"sent": "and then would you like to play with the play dough now ?", "words": ["the", "to", "play", "with", "like", "you", "and", "would", "then", "play dough"]}, {"sent": "oh look at the poor play dough .", "words": ["the", "look", "play", "poor", "at", "play dough"]}, {"sent": "Oliver wants more orange play dough .", "words": ["more", "orange", "play dough", "play"]}, {"sent": "play dough has problems .", "words": ["play", "play dough"]}, {"sent": "lets see we need some play dough in there .", "words": ["need", "we", "play", "there", "in", "see", "play dough", "some"]}, {"sent": "what are you what are you doing here with play dough ?", "words": ["what", "play", "with", "you", "here", "are", "play dough"]}, {"sent": "do you have play dough ?", "words": ["do", "play", "you", "have", "play dough"]}, {"sent": "oh is there play dough on your tummy ?", "words": ["your", "is", "play", "there", "tummy", "on", "play dough"]}, {"sent": "yellow play dough .", "words": ["yellow", "play", "play dough"]}, {"sent": "wheres my blue play dough ?", "words": ["my", "play", "play dough", "blue"]}, {"sent": "do you think theyll have play dough at your new school ?", "words": ["do", "think", "your", "school", "play", "you", "have", "at", "new", "play dough"]}, {"sent": "shall we get the soft play dough ?", "words": ["the", "soft", "we", "play", "get", "play dough"]}, {"sent": "can you help me put the top on the play dough ?", "words": ["can", "the", "help", "put", "play", "you", "on", "me", "play dough"]}, {"sent": "bye play dough all done .", "words": ["all", "play", "play dough"]}, {"sent": "are you all done with the play dough ?", "words": ["the", "all", "play", "with", "you", "are", "play dough"]}, {"sent": "*CH1: play dough .", "words": ["play", "play dough"]}, {"sent": "me have play dough for snack .", "words": ["for", "play", "have", "me", "play dough"]}, {"sent": "this is play dough .", "words": ["this", "play", "play dough", "is"]}, {"sent": "oh I had forgotten there was play dough with this .", "words": ["was", "play", "with", "there", "this", "play dough"]}, {"sent": "me tummy sick of play dough now .", "words": ["of", "play", "sick", "tummy", "me", "play dough"]}, {"sent": "wheres your play dough ?", "words": ["play", "your", "play dough"]}, {"sent": "Im gonna try to make some spaghetti and meatballs because I think I might be able to do that lets see with play dough .", "words": ["do", "think", "try", "spaghetti", "see", "to", "be", "play", "with", "and", "that", "some", "try to", "make", "play dough", "because"]}, {"sent": "do you wanna mold something with the play dough ?", "words": ["the", "do", "play", "with", "you", "wanna", "play dough"]}, {"sent": "we need some more play dough .", "words": ["need", "we", "play", "more", "some", "play dough"]}, {"sent": "I just took a big chunk of play dough .", "words": ["of", "play", "big", "play dough", "a"]}, {"sent": "because we wanna make purple play dough .", "words": ["we", "play", "wanna", "make", "play dough", "because"]}, {"sent": "me like play dough .", "words": ["me", "play", "play dough", "like"]}, {"sent": "do you ever play with play dough ?", "words": ["do", "play", "with", "you", "play dough"]}, {"sent": "*CH1: play dough .", "words": ["play", "play dough"]}, {"sent": "oh it made a ring and now its covered with play dough huh ?", "words": ["play", "with", "and", "it", "play dough", "a"]}, {"sent": "you putting them in the play dough now ?", "words": ["the", "play", "them", "you", "in", "play dough"]}, {"sent": "and well go and get the soft play dough .", "words": ["the", "soft", "go", "play", "and", "get", "play dough"]}, {"sent": "but the play dough is sticking isnt it ?", "words": ["the", "is", "play", "it", "play dough", "but"]}, {"sent": "play dough ?", "words": ["play", "play dough"]}, {"sent": "did you make play dough yesterday ?", "words": ["play", "you", "did", "make", "play dough"]}, {"sent": "its like play dough but you can eat it at the end .", "words": ["can", "the", "play", "like", "you", "eat", "it", "at", "play dough", "but"]}, {"sent": "do you remember that we make things with play dough ?", "words": ["do", "we", "play", "with", "you", "that", "make", "play dough"]}, {"sent": "we use this when youre painting or playing with play dough dont we ?", "words": ["we", "play", "with", "this", "dont", "when", "play dough"]}, {"sent": "okay heres some play dough .", "words": ["some", "play", "play dough"]}, {"sent": "spoon and fork .", "words": ["spoon", "fork", "and"]}, {"sent": "find a spoon and a fork ?", "words": ["and", "spoon", "fork", "find", "a"]}, {"sent": "the spoon the knife or the fork ?", "words": ["the", "knife", "spoon", "fork"]}, {"sent": "I cant put it back on would you like a fork ?", "words": ["put", "back", "like", "you", "it", "fork", "would", "on", "a"]}, {"sent": "and you poke it with a fork .", "words": ["with", "you", "and", "it", "fork", "a"]}, {"sent": "put some fork put some food on it .", "words": ["put", "food", "it", "fork", "on", "some"]}, {"sent": "a little fork .", "words": ["fork", "little", "a"]}, {"sent": "well eat whats on your fork .", "words": ["eat", "on", "your", "fork"]}, {"sent": "just with the fork .", "words": ["the", "with", "fork"]}, {"sent": "you going to give Snoopy the fork ?", "words": ["the", "to", "you", "fork", "give"]}, {"sent": "that is a big fork because I cant find the little one .", "words": ["the", "is", "little", "that", "fork", "find", "big", "because", "a"]}, {"sent": "its like a fork lifter with sharp nails .", "words": ["with", "like", "fork", "a"]}, {"sent": "do you want this fork Mom ?", "words": ["do", "fork", "this", "you"]}, {"sent": "because you might prefer to eat the pasta with a fork .", "words": ["the", "to", "with", "you", "eat", "fork", "because", "a"]}, {"sent": "fork .", "words": ["fork"]}, {"sent": "shall I use the fork ?", "words": ["the", "fork"]}, {"sent": "a fork .", "words": ["fork", "a"]}, {"sent": "use your fork please .", "words": ["your", "fork"]}, {"sent": "I need a knife and a fork please .", "words": ["need", "knife", "and", "fork", "a"]}, {"sent": "youll hafta have youll hafta have a big fork Lara .", "words": ["have", "big", "fork", "a"]}, {"sent": "I havent got myself a knife and fork now .", "words": ["knife", "and", "myself", "fork", "a"]}, {"sent": "eating with the fork .", "words": ["the", "with", "fork"]}, {"sent": "with Purdies water bowl on it and a plate and a fork and then when Grandma comes with the pussy cat food we can feed her cant we ?", "words": ["bowl", "the", "can", "feed", "plate", "we", "with", "her", "and", "cat", "food", "it", "fork", "water", "on", "then", "when", "a"]}, {"sent": "xxx fork .", "words": ["fork"]}, {"sent": "even if you dont have booboos you might decide to eat with a fork and a spoon .", "words": ["if", "to", "with", "you", "have", "dont", "eat", "and", "spoon", "fork", "a"]}, {"sent": "oh here you gotta take the fork .", "words": ["the", "you", "here", "fork", "take"]}, {"sent": "spoon fork .", "words": ["spoon", "fork"]}, {"sent": "had I better have a fork do you think ?", "words": ["better", "do", "think", "you", "have", "fork", "a"]}, {"sent": "are you going to give us a fork ?", "words": ["to", "you", "us", "are", "fork", "give", "a"]}, {"sent": "I can use the fork .", "words": ["can", "the", "fork"]}, {"sent": "that means Mummy will hafta go get another fork .", "words": ["go", "get", "another", "fork", "will", "that"]}, {"sent": "plastic fork shes not .", "words": ["not", "fork"]}, {"sent": "or a fork ?", "words": ["fork", "a"]}, {"sent": "I can eat the pizza without a fork but I dont think I can manage the coleslaw very very politely .", "words": ["can", "the", "think", "pizza", "eat", "dont", "fork", "but", "a"]}, {"sent": "fork .", "words": ["fork"]}, {"sent": "get xxx fork or a spoon out .", "words": ["get", "spoon", "out", "fork", "a"]}, {"sent": "and Nathaniel really knows how to eat with a fork now too .", "words": ["how", "to", "with", "and", "eat", "fork", "too", "a"]}, {"sent": "a fork ?", "words": ["fork", "a"]}, {"sent": "a fork .", "words": ["fork", "a"]}, {"sent": "I put a knife and fork uhn dont I ?", "words": ["put", "knife", "and", "dont", "fork", "a"]}, {"sent": "heres your knife and fork .", "words": ["knife", "your", "fork", "and"]}, {"sent": "wheres your fork ?", "words": ["your", "fork"]}, {"sent": "Nina eats with a fork .", "words": ["with", "fork", "a"]}, {"sent": "wanna use your fork .", "words": ["fork", "your", "wanna"]}, {"sent": "heres a fork .", "words": ["fork", "a"]}, {"sent": "eat this fork .", "words": ["eat", "this", "fork"]}, {"sent": "Mommy use a fork ?", "words": ["fork", "a"]}, {"sent": "well take your fork .", "words": ["your", "take", "fork"]}, {"sent": "you can pretend somethings a fork .", "words": ["can", "you", "fork", "pretend", "a"]}, {"sent": "do you wanna try using your fork Nanette ?", "words": ["do", "your", "you", "wanna", "fork", "try"]}, {"sent": "have you got a fork ?", "words": ["have", "fork", "a", "you"]}, {"sent": "thatll just fork it out because the leaves look all right there then .", "words": ["the", "all", "look", "there", "it", "out", "fork", "then", "because"]}, {"sent": "fork .", "words": ["fork"]}, {"sent": "with a fork ?", "words": ["with", "fork", "a"]}, {"sent": "dont use your fork to eat those with .", "words": ["your", "those", "to", "with", "eat", "dont", "fork"]}, {"sent": "okay you better to put the fork and knife .", "words": ["the", "better", "put", "to", "knife", "you", "and", "fork"]}, {"sent": "hm can you get a carrot with that fork ?", "words": ["can", "with", "you", "get", "fork", "that", "a"]}, {"sent": "thats a fork .", "words": ["fork", "a"]}, {"sent": "and land on a fork ?", "words": ["on", "fork", "a", "and"]}, {"sent": "try with the fork .", "words": ["the", "fork", "try", "with"]}, {"sent": "a fork .", "words": ["fork", "a"]}, {"sent": "with whose fork ?", "words": ["with", "fork"]}, {"sent": "no fork .", "words": ["fork"]}, {"sent": "I need a knife and fork .", "words": ["need", "knife", "and", "fork", "a"]}, {"sent": "would you like a fork ?", "words": ["like", "you", "fork", "would", "a"]}, {"sent": "you can stick your fork in them .", "words": ["can", "stick", "your", "them", "you", "in", "fork"]}, {"sent": "and this is the fork we use .", "words": ["the", "is", "we", "and", "this", "fork"]}, {"sent": "this is the trowel and fork Thomas .", "words": ["the", "is", "and", "this", "fork"]}, {"sent": "lemme get your fork Naima .", "words": ["get", "your", "fork"]}, {"sent": "a fork and a spoon .", "words": ["and", "spoon", "fork", "a"]}, {"sent": "I took your fork away from you .", "words": ["away", "your", "fork", "you"]}, {"sent": "a fork or a spade ?", "words": ["fork", "a"]}, {"sent": "have you got a yellow fork ?", "words": ["you", "have", "fork", "yellow", "a"]}, {"sent": "you cant hold a fork .", "words": ["fork", "hold", "a", "you"]}, {"sent": "thats right put a piece of meat on her fork .", "words": ["put", "of", "meat", "fork", "on", "her", "a"]}, {"sent": "knife fork and spoon .", "words": ["knife", "spoon", "fork", "and"]}, {"sent": "Mummy needs a fork .", "words": ["fork", "a"]}, {"sent": "wheres your fork ?", "words": ["your", "fork"]}, {"sent": "thats a fork .", "words": ["fork", "a"]}, {"sent": "she could always hear the fridge door opening or me opening a tin and getting her plates out of the cupboard and her drawer her fork out of the drawer .", "words": ["the", "drawer", "of", "hear", "and", "out", "fork", "me", "could", "her", "she", "a", "door"]}, {"sent": "thats your err fork .", "words": ["your", "fork"]}, {"sent": "whats that on the fork ?", "words": ["on", "the", "that", "fork"]}, {"sent": "its your fork .", "words": ["your", "fork"]}, {"sent": "would you like me to go get a real fork and spoon and knife ?", "words": ["go", "to", "knife", "like", "you", "get", "and", "spoon", "fork", "would", "me", "a"]}, {"sent": "are you going to use your fork to eat your bacon with ?", "words": ["your", "to", "with", "you", "eat", "are", "fork"]}, {"sent": "he got a fork .", "words": ["he", "fork", "a"]}, {"sent": "use your spoon if you dont wanna use a fork .", "words": ["your", "if", "you", "dont", "spoon", "wanna", "fork", "a"]}, {"sent": "fork yeah .", "words": ["fork"]}, {"sent": "and youre sposta eat it with the fork .", "words": ["the", "with", "and", "eat", "it", "fork"]}, {"sent": "needta mash it down a little bit with a fork .", "words": ["with", "little", "it", "fork", "down", "a"]}, {"sent": "alright give me your fork .", "words": ["me", "give", "your", "fork"]}, {"sent": "theres a big fork .", "words": ["big", "fork", "a"]}, {"sent": "alright stir it with a fork if you want .", "words": ["if", "with", "you", "it", "fork", "a"]}, {"sent": "fork okay .", "words": ["fork"]}, {"sent": "that is a fork .", "words": ["fork", "that", "a", "is"]}, {"sent": "wanna fork ?", "words": ["fork", "wanna"]}, {"sent": "with a pretend fork and a knife ?", "words": ["knife", "with", "and", "fork", "pretend", "a"]}, {"sent": "oh youre using your fork for the pasta .", "words": ["the", "for", "your", "fork"]}, {"sent": "lets see if I have a lollipop left .", "words": ["if", "lollipop", "have", "see", "a"]}, {"sent": "yeah I dont mind if you have a lollipop tonight but one .", "words": ["if", "you", "have", "dont", "lollipop", "but", "a"]}, {"sent": "a lollipop .", "words": ["lollipop", "a"]}, {"sent": "and a lollipop .", "words": ["lollipop", "a", "and"]}, {"sent": "who was on the lollipop ?", "words": ["the", "was", "lollipop", "who", "on"]}, {"sent": "a little lollipop drip xxx ?", "words": ["lollipop", "little", "a"]}, {"sent": "lollipop .", "words": ["lollipop"]}, {"sent": "this looks like a lollipop to me .", "words": ["to", "like", "lollipop", "this", "me", "a"]}, {"sent": "a lollipop .", "words": ["lollipop", "a"]}, {"sent": "is your lollipop at home ?", "words": ["your", "is", "lollipop", "at", "home"]}, {"sent": "I think thats lollipop .", "words": ["think", "lollipop"]}, {"sent": "do you know what reminded me of the lollipop ?", "words": ["the", "do", "what", "of", "you", "lollipop", "me"]}, {"sent": "is that a lollipop ?", "words": ["that", "lollipop", "a", "is"]}, {"sent": "thats a lollipop stick isnt it ?", "words": ["stick", "it", "lollipop", "a"]}, {"sent": "lollipop .", "words": ["lollipop"]}, {"sent": "you had that lollipop .", "words": ["that", "lollipop", "you"]}, {"sent": "lollipop .", "words": ["lollipop"]}, {"sent": "its its I suppose its where youve had a lollipop .", "words": ["where", "lollipop", "a"]}, {"sent": "and Gabby would be like lollipop yeah whatever Im done .", "words": ["be", "like", "lollipop", "and", "would"]}, {"sent": "you could hold the picture but ya cant actually hold the lollipop .", "words": ["the", "hold", "picture", "you", "lollipop", "could", "but"]}, {"sent": "it looks like a lollipop .", "words": ["it", "lollipop", "like", "a"]}, {"sent": "me the lollipop .", "words": ["the", "me", "lollipop"]}, {"sent": "its melting isnt it that lollipop ?", "words": ["it", "that", "lollipop"]}, {"sent": "lollipop .", "words": ["lollipop"]}, {"sent": "theres a lollipop stick down there isnt there ?", "words": ["stick", "lollipop", "there", "down", "a"]}, {"sent": "you like to taste the lollipop ?", "words": ["the", "to", "like", "taste", "you", "lollipop"]}, {"sent": "lollipop .", "words": ["lollipop"]}, {"sent": "because it looks like a great kind of lollipop .", "words": ["of", "like", "lollipop", "it", "because", "a"]}, {"sent": "you cant play and eat your lollipop at the same time .", "words": ["the", "your", "play", "and", "eat", "you", "lollipop", "same", "at"]}, {"sent": "did Doctor Hartzell give you a lollipop ?", "words": ["lollipop", "you", "did", "give", "a"]}, {"sent": "and theres a lollipop lady .", "words": ["lollipop", "a", "and"]}, {"sent": "wheres your red lollipop ?", "words": ["red", "your", "lollipop"]}, {"sent": "no I was only asking that because in the place where I take the boys they get a lollipop .", "words": ["the", "where", "was", "lollipop", "get", "they", "in", "take", "that", "because", "a"]}, {"sent": "he wants a lollipop ?", "words": ["he", "lollipop", "a"]}, {"sent": "is that the lollipop man ?", "words": ["the", "that", "lollipop", "is"]}, {"sent": "Mommy gave you the lollipop .", "words": ["the", "lollipop", "you"]}, {"sent": "you had a chocolate lollipop didnt you ?", "words": ["chocolate", "lollipop", "a", "you"]}, {"sent": "theres lollipop in the kitchen .", "words": ["the", "in", "kitchen", "lollipop"]}, {"sent": "and who was on the lollipop ?", "words": ["the", "was", "and", "lollipop", "who", "on"]}, {"sent": "thats a lollipop lady .", "words": ["lollipop", "a"]}, {"sent": "thats your orange lollipop isnt it ?", "words": ["it", "orange", "your", "lollipop"]}, {"sent": "well you finish that lollipop now .", "words": ["lollipop", "that", "finish", "you"]}, {"sent": "the lollipop .", "words": ["the", "lollipop"]}, {"sent": "youre the lollipop man are you ?", "words": ["the", "are", "you", "lollipop"]}, {"sent": "wheres your lollipop ?", "words": ["your", "lollipop"]}, {"sent": "is this lollipop for me ?", "words": ["for", "is", "lollipop", "this", "me"]}, {"sent": "have a lollipop ?", "words": ["have", "lollipop", "a"]}, {"sent": "I think Ninas lollipop is right over here .", "words": ["think", "is", "lollipop", "here", "over"]}, {"sent": "he gave a lollipop to Nina didnt he ?", "words": ["to", "he", "lollipop", "a"]}, {"sent": "lollipop lollipop .", "words": ["lollipop"]}, {"sent": "oh lolly lollipop .", "words": ["lollipop"]}, {"sent": "got lollipop .", "words": ["lollipop"]}, {"sent": "you want a lollipop ?", "words": ["lollipop", "a", "you"]}, {"sent": "think I would have rather had a lollipop .", "words": ["think", "lollipop", "have", "would", "a"]}, {"sent": "can you pass me that lollipop please ?", "words": ["can", "lollipop", "you", "me", "that"]}, {"sent": "a lollipop .", "words": ["lollipop", "a"]}, {"sent": "thats the lollipop .", "words": ["the", "lollipop"]}, {"sent": "you got a lollipop for me ?", "words": ["for", "lollipop", "you", "me", "a"]}, {"sent": "you get a lollipop and what does mommy get at the coffee store ?", "words": ["the", "what", "does", "store", "and", "get", "lollipop", "you", "coffee", "at", "a"]}, {"sent": "thats the end of your lollipop isnt it ?", "words": ["the", "your", "of", "lollipop", "it"]}, {"sent": "heres a lollipop .", "words": ["lollipop", "a"]}, {"sent": "sit down and you can have a lollipop .", "words": ["can", "you", "and", "have", "lollipop", "sit", "down", "a"]}, {"sent": "a lollipop ?", "words": ["lollipop", "a"]}, {"sent": "you think your grapes are a lollipop .", "words": ["think", "your", "you", "lollipop", "grapes", "are", "a"]}, {"sent": "with a lollipop .", "words": ["with", "lollipop", "a"]}, {"sent": "oh you made a lollipop .", "words": ["lollipop", "a", "you"]}, {"sent": "wanna lollipop ?", "words": ["wanna", "lollipop"]}, {"sent": "and then you may have a lollipop .", "words": ["you", "have", "and", "lollipop", "then", "a"]}, {"sent": "who gave you the lollipop ?", "words": ["the", "lollipop", "you", "who"]}, {"sent": "and has a lollipop .", "words": ["lollipop", "a", "and"]}, {"sent": "lollipop .", "words": ["lollipop"]}, {"sent": "thats not lollipop Mommy thats icecream .", "words": ["not", "lollipop"]}, {"sent": "looks like a lollipop .", "words": ["lollipop", "like", "a"]}, {"sent": "oh I forgot the lollipop .", "words": ["the", "lollipop"]}, {"sent": "like a lollipop .", "words": ["lollipop", "like", "a"]}, {"sent": "oh that was fun to get off the airplane and see a great big lollipop wasnt it ?", "words": ["off", "the", "airplane", "to", "was", "and", "get", "lollipop", "big", "it", "see", "that", "a"]}, {"sent": "did he give you a lollipop ?", "words": ["you", "lollipop", "he", "did", "give", "a"]}, {"sent": "hes got a lollipop .", "words": ["lollipop", "a"]}, {"sent": "youve had a lollipop already today havent you ?", "words": ["lollipop", "you", "a"]}, {"sent": "its a lollipop again is it ?", "words": ["it", "lollipop", "is", "a"]}, {"sent": "lollipop .", "words": ["lollipop"]}, {"sent": "oh shes got a lollipop .", "words": ["lollipop", "a"]}, {"sent": "you tried a lollipop instead ?", "words": ["lollipop", "a", "you"]}, {"sent": "a lollipop .", "words": ["lollipop", "a"]}, {"sent": "did you have a lollipop ?", "words": ["lollipop", "have", "you", "did", "a"]}, {"sent": "does DoctorPeter give you a lollipop ?", "words": ["does", "lollipop", "you", "give", "a"]}, {"sent": "does it look a little bit like a lollipop that you bite and crunch ?", "words": ["does", "like", "little", "lollipop", "you", "and", "it", "look", "bite", "that", "a"]}, {"sent": "you still dont get a lollipop even if youve eaten it .", "words": ["if", "lollipop", "get", "dont", "you", "it", "a"]}, {"sent": "well give leave your lollipop here .", "words": ["here", "give", "your", "lollipop"]}, {"sent": "do you not wanna finish your lollipop ?", "words": ["do", "your", "finish", "you", "lollipop", "wanna", "not"]}, {"sent": "do you wanna give dolly a lollipop ?", "words": ["do", "you", "lollipop", "wanna", "give", "a"]}, {"sent": "draw a lollipop shall I ?", "words": ["draw", "lollipop", "a"]}, {"sent": "and mind your lollipop .", "words": ["your", "lollipop", "and"]}, {"sent": "Jwww gave you a lollipop today didnt she ?", "words": ["she", "lollipop", "a", "you"]}, {"sent": "make your tummy feel better to have a lollipop ?", "words": ["better", "your", "to", "lollipop", "have", "tummy", "make", "a"]}, {"sent": "youre a small lollipop ?", "words": ["lollipop", "a"]}, {"sent": "if this was a lollipop Id eat it up right now .", "words": ["up", "if", "was", "lollipop", "eat", "this", "it", "a"]}, {"sent": "what about a hug ?", "words": ["hug", "what", "about", "a"]}, {"sent": "give baby hug .", "words": ["hug", "give"]}, {"sent": "she needs a hug .", "words": ["hug", "she", "a"]}, {"sent": "an itll be okay hug .", "words": ["hug", "be", "an"]}, {"sent": "youre gonna hug him ?", "words": ["hug", "him"]}, {"sent": "xxx the daddys got the baby see and he can give the little girl a hug see the little girl gives a hug to the daddy there .", "words": ["the", "can", "to", "and", "little", "there", "he", "hug", "see", "give", "a"]}, {"sent": "oh are you running to give me a big hug ?", "words": ["to", "you", "are", "hug", "me", "give", "big", "a"]}, {"sent": "thank you for the hug that was very sweet .", "words": ["the", "for", "was", "you", "hug", "that"]}, {"sent": "give Daddy hug .", "words": ["hug", "give"]}, {"sent": "are we having a big hug ?", "words": ["we", "are", "hug", "big", "a"]}, {"sent": "hug .", "words": ["hug"]}, {"sent": "can ya give the baby hug ?", "words": ["can", "the", "give", "hug"]}, {"sent": "mainly just hug them .", "words": ["hug", "them"]}, {"sent": "are you giving them a big hug ?", "words": ["them", "you", "are", "hug", "big", "a"]}, {"sent": "so I think being able to hug the stuffed animals .", "words": ["the", "think", "to", "so", "hug"]}, {"sent": "I love you when you hug me .", "words": ["you", "love", "hug", "me", "when"]}, {"sent": "hug a friend .", "words": ["hug", "a"]}, {"sent": "big hug .", "words": ["hug", "big"]}, {"sent": "then hes going to hug his mommy ?", "words": ["hug", "to", "then", "his"]}, {"sent": "aw hug the baby thats so sweet .", "words": ["hug", "the", "so"]}, {"sent": "want me give you a hug ?", "words": ["you", "hug", "me", "give", "a"]}, {"sent": "give her a big hug .", "words": ["her", "hug", "give", "big", "a"]}, {"sent": "thats a nice hug .", "words": ["hug", "nice", "a"]}, {"sent": "oh then I get a big hug if you sleep on my lap .", "words": ["my", "if", "sleep", "you", "get", "on", "hug", "then", "big", "a"]}, {"sent": "did he hug you ?", "words": ["hug", "he", "did", "you"]}, {"sent": "she will take a teddy bear and hug it .", "words": ["bear", "and", "hug", "it", "will", "take", "she", "a"]}, {"sent": "give me a hug .", "words": ["hug", "me", "give", "a"]}, {"sent": "huh daddys giving you a big hug .", "words": ["hug", "big", "a", "you"]}, {"sent": "you wanna hug him ?", "words": ["hug", "him", "wanna", "you"]}, {"sent": "that is one big hug for BirthdayBear .", "words": ["for", "is", "that", "hug", "big"]}, {"sent": "I want a hug I want a hug I want a hug .", "words": ["hug", "a"]}, {"sent": "oh are you giving me a big hug ?", "words": ["you", "are", "hug", "me", "big", "a"]}, {"sent": "do baby bears or baby bees or little kittens who climb up trees kangaroos or small raccoons or bugs or goats or seals or chicks get a goodnight hug and kiss ?", "words": ["up", "do", "little", "get", "and", "who", "hug", "climb", "kiss", "a"]}, {"sent": "a hug .", "words": ["hug", "a"]}, {"sent": "a big bear hug .", "words": ["bear", "hug", "big", "a"]}, {"sent": "can I give you a hug or a kiss ?", "words": ["can", "you", "hug", "give", "kiss", "a"]}, {"sent": "come here want me to give you a hug ?", "words": ["to", "you", "here", "hug", "me", "give", "a"]}, {"sent": "nice hug .", "words": ["hug", "nice"]}, {"sent": "Mommys gonna give you a good morning hug good morning .", "words": ["you", "good", "hug", "give", "a"]}, {"sent": "oh come give me a hug .", "words": ["hug", "me", "give", "a"]}, {"sent": "can I have a hug please ?", "words": ["can", "have", "hug", "a"]}, {"sent": "aw you hug it .", "words": ["hug", "it", "you"]}, {"sent": "hug hug .", "words": ["hug"]}, {"sent": "you want a hug ?", "words": ["hug", "a", "you"]}, {"sent": "having a big hug with Mummy isnt it ?", "words": ["with", "it", "hug", "big", "a"]}, {"sent": "you and is that you giving a big hug ?", "words": ["is", "and", "you", "big", "hug", "that", "a"]}, {"sent": "so shes gonna hug onto the stem and then shes gonna slide down .", "words": ["the", "slide", "so", "and", "hug", "then", "down"]}, {"sent": "oh youre gonna hug me and make me feel better .", "words": ["better", "and", "hug", "me", "make"]}, {"sent": "big hug .", "words": ["hug", "big"]}, {"sent": "wanna hug ?", "words": ["hug", "wanna"]}, {"sent": "give her a hug .", "words": ["hug", "give", "her", "a"]}, {"sent": "can you do a hug ?", "words": ["can", "do", "you", "hug", "a"]}, {"sent": "are you giving him a hug ?", "words": ["you", "him", "are", "hug", "a"]}, {"sent": "you giving Mommy a hug ?", "words": ["hug", "a", "you"]}, {"sent": "a good morning hug .", "words": ["hug", "good", "a"]}, {"sent": "mm what a great hug .", "words": ["hug", "what", "a"]}, {"sent": "to give him a hug ?", "words": ["to", "him", "hug", "give", "a"]}, {"sent": "maybe the baby will hug the deer .", "words": ["the", "deer", "hug", "will"]}, {"sent": "oh big hug .", "words": ["hug", "big"]}, {"sent": "they all want a big big big hug .", "words": ["all", "they", "hug", "big", "a"]}, {"sent": "ah give him a hug .", "words": ["hug", "give", "him", "a"]}, {"sent": "what do ya think of the dolly wanna give the dolly a hug ?", "words": ["the", "what", "do", "think", "of", "wanna", "hug", "give", "a"]}, {"sent": "and Im going to give you the biggest hug in the whole world because you do .", "words": ["the", "do", "to", "you", "and", "in", "hug", "give", "because"]}, {"sent": "oh give him a hug .", "words": ["hug", "give", "him", "a"]}, {"sent": "can you give him a little hug ?", "words": ["can", "you", "little", "him", "hug", "give", "a"]}, {"sent": "oh can I have a hug ?", "words": ["can", "have", "hug", "a"]}, {"sent": "big hug .", "words": ["hug", "big"]}, {"sent": "you want a big hug ?", "words": ["hug", "big", "a", "you"]}, {"sent": "ah hug for you .", "words": ["hug", "for", "you"]}, {"sent": "oh give him a hug .", "words": ["hug", "give", "him", "a"]}, {"sent": "do you want a big hug ?", "words": ["do", "you", "hug", "big", "a"]}, {"sent": "hop hug .", "words": ["hug"]}, {"sent": "you wanna give her a hug ?", "words": ["you", "wanna", "hug", "give", "her", "a"]}, {"sent": "oh good hug .", "words": ["hug", "good"]}, {"sent": "does she need a hug ?", "words": ["does", "need", "hug", "she", "a"]}, {"sent": "big hug .", "words": ["hug", "big"]}, {"sent": "xxx xxx can give him a hug .", "words": ["can", "him", "hug", "give", "a"]}, {"sent": "oh hug her .", "words": ["hug", "her"]}, {"sent": "why are you giving him a hug ?", "words": ["you", "him", "are", "hug", "why", "a"]}, {"sent": "you wanna hug me ?", "words": ["hug", "me", "wanna", "you"]}, {"sent": "hug Naima oh .", "words": ["hug"]}, {"sent": "hug him .", "words": ["hug", "him"]}, {"sent": "give me hug Ces .", "words": ["hug", "me", "give"]}, {"sent": "hug \u4e00\u4e0b \u597d\u4e0d\u597d ?", "words": ["hug"]}, {"sent": "are you giving her a hug ?", "words": ["you", "are", "hug", "her", "a"]}, {"sent": "there you can hug your lion .", "words": ["can", "your", "you", "there", "hug", "lion"]}, {"sent": "can I have a hug Eleanor ?", "words": ["can", "have", "hug", "a"]}, {"sent": "hug .", "words": ["hug"]}, {"sent": "big hug time .", "words": ["hug", "big"]}, {"sent": "having a big hug now .", "words": ["hug", "big", "a"]}, {"sent": "can I have can I have a hug ?", "words": ["can", "have", "hug", "a"]}, {"sent": "here I have another kitty for you to hug .", "words": ["for", "to", "you", "here", "have", "another", "kitty", "hug"]}, {"sent": "youre giving him a hug .", "words": ["hug", "him", "a"]}, {"sent": "yeah Pebbles gave BamBam a hug .", "words": ["hug", "a"]}, {"sent": "give Daddy hug .", "words": ["hug", "give"]}, {"sent": "with a great big hug and a kiss from me to you .", "words": ["kiss", "to", "with", "and", "you", "hug", "me", "big", "a"]}, {"sent": "big hug .", "words": ["hug", "big"]}, {"sent": "so can I have a big hug ?", "words": ["can", "so", "have", "hug", "big", "a"]}, {"sent": "they both have wooly coats just like their baby lamb .", "words": ["their", "like", "have", "they", "lamb"]}, {"sent": "where is your little lamb ?", "words": ["where", "your", "is", "little", "lamb"]}, {"sent": "they both have wholly coats just like their baby lamb .", "words": ["their", "like", "have", "they", "lamb"]}, {"sent": "where is the lamb ?", "words": ["where", "the", "lamb", "is"]}, {"sent": "and a sheep or lamb .", "words": ["lamb", "sheep", "a", "and"]}, {"sent": "yeah nice lamb good lamb .", "words": ["nice", "good", "lamb"]}, {"sent": "l lamb .", "words": ["lamb"]}, {"sent": "Eric we lost the lamb .", "words": ["the", "lamb", "we"]}, {"sent": "wheres the lamb ?", "words": ["the", "lamb"]}, {"sent": "pet the lamb .", "words": ["the", "lamb"]}, {"sent": "it was a lamb .", "words": ["lamb", "was", "it", "a"]}, {"sent": "Mary had a little lamb whose fleece was white as snow .", "words": ["was", "snow", "little", "white", "a", "lamb"]}, {"sent": "Mary had a little lamb little lamb little lamb .", "words": ["lamb", "little", "a"]}, {"sent": "well put the lamb in this house .", "words": ["the", "put", "this", "in", "house", "lamb"]}, {"sent": "lamb lamb lamb lamb .", "words": ["lamb"]}, {"sent": "this one is a lamb .", "words": ["lamb", "this", "a", "is"]}, {"sent": "whereve you gone lamb ?", "words": ["lamb", "you"]}, {"sent": "or is it Mary had a little lamb do you think ?", "words": ["do", "think", "is", "little", "you", "it", "a", "lamb"]}, {"sent": "and thats just every so often because basically I dont like lamb .", "words": ["so", "like", "and", "dont", "every", "because", "lamb"]}, {"sent": "its called lamb and vegetable Mediterranean vegetable and lamb risotto .", "words": ["and", "lamb"]}, {"sent": "now why dont you ask your lamb what lamb would like to eat ?", "words": ["what", "your", "to", "like", "you", "eat", "dont", "would", "why", "lamb"]}, {"sent": "I saw a red lamb looking at me .", "words": ["red", "at", "me", "a", "lamb"]}, {"sent": "hes got a lamb in his arms .", "words": ["lamb", "his", "in", "a"]}, {"sent": "present for you lamb .", "words": ["lamb", "for", "present", "you"]}, {"sent": "mhm are you gonna buy some lamb chops ?", "words": ["you", "are", "some", "buy", "lamb"]}, {"sent": "Mary had a little lamb ?", "words": ["lamb", "little", "a"]}, {"sent": "is the lamb locked in the barn ?", "words": ["the", "in", "lamb", "is"]}, {"sent": "she does have a favorite um lamb stuffed animal kinda like this size that she sleeps with and cuddles when shes upset .", "words": ["does", "when", "with", "like", "and", "have", "this", "animal", "that", "she", "a", "lamb"]}, {"sent": "little lamb .", "words": ["lamb", "little"]}, {"sent": "little lamb .", "words": ["lamb", "little"]}, {"sent": "oh look its a lamb .", "words": ["lamb", "a", "look"]}, {"sent": "wheres lamb gone ?", "words": ["lamb"]}, {"sent": "I dont know where do you wanna put the lamb ?", "words": ["where", "do", "put", "the", "you", "dont", "wanna", "lamb"]}, {"sent": "here they want you to pat the furry little lamb .", "words": ["the", "to", "you", "here", "they", "little", "lamb"]}, {"sent": "where ya gonna put that lamb ?", "words": ["where", "put", "that", "lamb"]}, {"sent": "why dont you give your lamb a a name ?", "words": ["your", "a", "you", "dont", "give", "why", "lamb"]}, {"sent": "a wooly lamb came out to play .", "words": ["to", "play", "out", "a", "lamb"]}, {"sent": "they both have wooly coats just like their baby lamb .", "words": ["their", "like", "have", "they", "lamb"]}, {"sent": "that she says looks like a little lamb .", "words": ["lamb", "like", "little", "that", "she", "a"]}, {"sent": "well see theres the sheep or the lamb what does the lamb say ?", "words": ["the", "what", "does", "say", "sheep", "see", "lamb"]}, {"sent": "the lamb was sure to go .", "words": ["the", "go", "was", "to", "lamb"]}, {"sent": "a lamb .", "words": ["lamb", "a"]}, {"sent": "Stefan theres a little lamb bookie over there you see that ?", "words": ["little", "you", "there", "that", "see", "over", "a", "lamb"]}, {"sent": "does the lamb stand up ?", "words": ["the", "does", "up", "stand", "lamb"]}, {"sent": "they both have wooly coats just like their baby lamb .", "words": ["their", "like", "have", "they", "lamb"]}, {"sent": "little lamb .", "words": ["lamb", "little"]}, {"sent": "because theres a lamb a dog and a bear .", "words": ["a", "and", "bear", "dog", "because", "lamb"]}, {"sent": "can you say nice lamb ?", "words": ["can", "nice", "say", "you", "lamb"]}, {"sent": "pat the furry lamb .", "words": ["the", "lamb"]}, {"sent": "thats your lil lamb .", "words": ["your", "lamb"]}, {"sent": "a lamb ?", "words": ["lamb", "a"]}, {"sent": "er lamb can have some .", "words": ["can", "some", "have", "lamb"]}, {"sent": "Mary had a little lamb ?", "words": ["lamb", "little", "a"]}, {"sent": "is this chicken kebob or lamb kebob ?", "words": ["lamb", "this", "chicken", "is"]}, {"sent": "they both have wooly coats just like their baby lamb .", "words": ["their", "like", "have", "they", "lamb"]}, {"sent": "can you put your hand on the fur on the lamb ?", "words": ["can", "the", "put", "your", "hand", "you", "on", "lamb"]}, {"sent": "hes got a lamb .", "words": ["lamb", "a"]}, {"sent": "wheres the lamb ?", "words": ["the", "lamb"]}, {"sent": "can you pat the furry lamb ?", "words": ["can", "the", "lamb", "you"]}, {"sent": "oh pat the furry lamb .", "words": ["the", "lamb"]}, {"sent": "does lamb take a bath too ?", "words": ["does", "lamb", "too", "take", "a"]}, {"sent": "a lamb .", "words": ["lamb", "a"]}, {"sent": "it is like a lamb huh ?", "words": ["is", "like", "it", "a", "lamb"]}, {"sent": "remember this one where the sheep goes with the lamb .", "words": ["where", "the", "with", "this", "sheep", "lamb"]}, {"sent": "and thats a baby lamb .", "words": ["lamb", "a", "and"]}, {"sent": "are you gonna eat some lamb ?", "words": ["you", "eat", "are", "some", "lamb"]}, {"sent": "what about Larry the lamb ?", "words": ["the", "what", "about", "lamb"]}, {"sent": "just like their little baby lamb .", "words": ["lamb", "like", "little", "their"]}, {"sent": "you say lamb what would you like to eat ?", "words": ["what", "to", "say", "like", "you", "eat", "would", "lamb"]}, {"sent": "yeah theres a lamb .", "words": ["lamb", "a"]}, {"sent": "yeah lamb chops are okay .", "words": ["are", "lamb"]}, {"sent": "they both have wooly coats just like their baby lamb .", "words": ["their", "like", "have", "they", "lamb"]}, {"sent": "where can you see a picture of a lamb ?", "words": ["where", "can", "of", "lamb", "picture", "you", "see", "a"]}, {"sent": "a lamb .", "words": ["lamb", "a"]}, {"sent": "wheres lamb ?", "words": ["lamb"]}, {"sent": "and a lamb .", "words": ["lamb", "a", "and"]}, {"sent": "is that a fuzzy little lamb ?", "words": ["is", "little", "that", "a", "lamb"]}, {"sent": "thats how fuzzy tail lamb wakes up .", "words": ["up", "how", "lamb"]}, {"sent": "Larry the lamb .", "words": ["the", "lamb"]}, {"sent": "thats your lamb .", "words": ["your", "lamb"]}, {"sent": "what does the lamb say ?", "words": ["the", "what", "does", "say", "lamb"]}, {"sent": "I think that lamb is crying .", "words": ["think", "that", "is", "lamb"]}, {"sent": "or a lamb ?", "words": ["lamb", "a"]}, {"sent": "or the lamb .", "words": ["the", "lamb"]}, {"sent": "its called a lamb .", "words": ["lamb", "a"]}, {"sent": "they both have wooly coats just like their baby lamb .", "words": ["their", "like", "have", "they", "lamb"]}, {"sent": "thats what a lamb says very good .", "words": ["lamb", "what", "good", "a"]}, {"sent": "here is the lamb like you have .", "words": ["the", "is", "like", "you", "here", "have", "lamb"]}, {"sent": "one lamb in this lamb .", "words": ["this", "in", "lamb"]}, {"sent": "and heres your lamb .", "words": ["lamb", "your", "and"]}, {"sent": "is that lamb gonna ride the wagon ?", "words": ["the", "is", "ride", "that", "lamb"]}, {"sent": "they both have wooly coats just like their baby lamb .", "words": ["their", "like", "have", "they", "lamb"]}, {"sent": "and the lamb and the wise men .", "words": ["the", "lamb", "and"]}, {"sent": "you wanna give your lamb a kiss ?", "words": ["your", "you", "wanna", "give", "kiss", "a", "lamb"]}, {"sent": "just like their baby lamb .", "words": ["lamb", "like", "their"]}, {"sent": "Mary had a little lamb .", "words": ["lamb", "little", "a"]}, {"sent": "wheres lamb ?", "words": ["lamb"]}, {"sent": "a lamb .", "words": ["lamb", "a"]}, {"sent": "when little lamb went out to play .", "words": ["to", "play", "little", "out", "when", "lamb"]}, {"sent": "I know you like playing with that vacuum but you know what it ends up being too loud .", "words": ["up", "what", "with", "like", "you", "vacuum", "it", "too", "that", "loud", "but"]}, {"sent": "I dont need a vacuum cleaner any more .", "words": ["need", "any", "vacuum", "dont", "more", "a"]}, {"sent": "thats right the vacuum makes a loud noise .", "words": ["the", "vacuum", "loud", "a"]}, {"sent": "attaches into the main body of the vacuum .", "words": ["the", "of", "vacuum", "into"]}, {"sent": "can you see all the fluff and dirt that has gone inside the vacuum cleaner ?", "words": ["can", "the", "all", "inside", "you", "and", "vacuum", "see", "that"]}, {"sent": "vacuum .", "words": ["vacuum"]}, {"sent": "theyre both looking up at the vacuum cleaner on the ceiling have you ever seen a vacuum cleaner on the ceiling ?", "words": ["up", "the", "you", "vacuum", "have", "at", "on", "a"]}, {"sent": "*SI1: look at William your old vacuum cleaner .", "words": ["old", "look", "your", "vacuum", "at"]}, {"sent": "vacuum .", "words": ["vacuum"]}, {"sent": "xxx a bit pongy because it didnt have a vacuum cleaner ?", "words": ["have", "vacuum", "it", "because", "a"]}, {"sent": "he uses a vacuum cleaner all by himself .", "words": ["all", "by", "vacuum", "he", "a"]}, {"sent": "just behind the vacuum cleaner there .", "words": ["the", "there", "vacuum", "behind"]}, {"sent": "and you cant hold the vacuum cleaner thing as well .", "words": ["the", "hold", "you", "vacuum", "and"]}, {"sent": "a vacuum that poor little mouse .", "words": ["poor", "vacuum", "little", "that", "mouse", "a"]}, {"sent": "the vacuum makes a loud noise .", "words": ["the", "vacuum", "loud", "a"]}, {"sent": "he likes to vacuum .", "words": ["vacuum", "to", "he"]}, {"sent": "they must have been on the floor when Mummy used the vacuum cleaner .", "words": ["the", "have", "they", "vacuum", "on", "when"]}, {"sent": "all around it were gonna vacuum everywhere .", "words": ["all", "vacuum", "it", "around", "were"]}, {"sent": "Im gonna put the vacuum cleaner on .", "words": ["the", "put", "vacuum", "on"]}, {"sent": "vacuum .", "words": ["vacuum"]}, {"sent": "I dont think well vacuum Purdie with NooNoos .", "words": ["vacuum", "dont", "think", "with"]}, {"sent": "do you want a vacuum cleaner ?", "words": ["vacuum", "do", "a", "you"]}, {"sent": "the door is closed but the vacuum cleaner is very loud right now .", "words": ["the", "is", "vacuum", "loud", "but", "door"]}, {"sent": "because Mummy wont vacuum in there will she ?", "words": ["vacuum", "there", "in", "will", "she", "because"]}, {"sent": "the vacuum cleaner isnt working .", "words": ["the", "vacuum"]}, {"sent": "have you taken the little brush off the vacuum cleaner have you ?", "words": ["the", "off", "brush", "little", "have", "you", "vacuum"]}, {"sent": "you wanna turn the vacuum on but its too loud in here to have the vacuum on .", "words": ["the", "to", "you", "vacuum", "here", "in", "have", "wanna", "too", "on", "loud", "but"]}, {"sent": "yeah um I mean the good news about like doing it on a rug is actually you can just vacuum it right up .", "words": ["the", "can", "up", "is", "like", "you", "vacuum", "it", "about", "good", "on", "a"]}, {"sent": "she thought it was a real vacuum cleaner .", "words": ["was", "vacuum", "it", "she", "a"]}, {"sent": "Mama needta run a vacuum up in here .", "words": ["up", "vacuum", "run", "in", "here", "a"]}, {"sent": "youve picked up one of the little brushes from the vacuum .", "words": ["up", "the", "of", "little", "vacuum"]}, {"sent": "well lets vacuum these bits up shall we ?", "words": ["vacuum", "these", "up", "we"]}, {"sent": "were gonna hafta vacuum .", "words": ["vacuum", "were"]}, {"sent": "the big vacuum cleaner not even the DustBuster ?", "words": ["the", "vacuum", "big", "not"]}, {"sent": "mommys gonna hafta vacuum this whole place isnt she ?", "words": ["vacuum", "this", "she"]}, {"sent": "papa bear gave mama bear a new vacuum cleaner to keep .", "words": ["to", "bear", "vacuum", "new", "a"]}, {"sent": "wheres the vacuum cleaner ?", "words": ["the", "vacuum"]}, {"sent": "vacuum .", "words": ["vacuum"]}, {"sent": "what happened upstairs with the little vacuum ?", "words": ["the", "what", "with", "little", "vacuum"]}, {"sent": "you can either watch the vacuum alone or come with me .", "words": ["can", "the", "watch", "with", "you", "vacuum", "me"]}, {"sent": "you drink all your juice and I wont put the vacuum cleaner .", "words": ["the", "all", "put", "your", "drink", "you", "and", "vacuum", "juice"]}, {"sent": "heres your vacuum xxx .", "words": ["vacuum", "your"]}, {"sent": "vacuum cleaner .", "words": ["vacuum"]}, {"sent": "what happened with the vacuum cleaner here ?", "words": ["the", "what", "with", "vacuum", "here"]}, {"sent": "and he had the vacuum cleaner inside didnt he ?", "words": ["the", "inside", "and", "vacuum", "he"]}, {"sent": "well he fixed the vacuum cleaner except now the vacuum cleaner wouldnt vacuum the floor it would only vacuum the ceiling and that is very very silly .", "words": ["the", "is", "and", "vacuum", "it", "he", "would", "that"]}, {"sent": "do we use the vacuum cleaner ?", "words": ["the", "do", "vacuum", "we"]}, {"sent": "I dont understand it we bought the vacuum cleaner how come the vacuum cleaners not on here ?", "words": ["the", "how", "we", "vacuum", "dont", "here", "it", "on", "not"]}, {"sent": "NooNoo the vacuum cleaner .", "words": ["the", "vacuum"]}, {"sent": "yes Ive stood it behind the vacuum cleaner in the cupboard .", "words": ["the", "behind", "vacuum", "in", "it"]}, {"sent": "no the vacuum cleaner doesnt .", "words": ["the", "vacuum"]}, {"sent": "xxx xxx uses the vacuum cleaner .", "words": ["the", "vacuum"]}, {"sent": "yes the washer makes a loud noise like the vacuum does .", "words": ["the", "does", "like", "vacuum", "loud", "a"]}, {"sent": "vacuum cleaner .", "words": ["vacuum"]}, {"sent": "maybe were gonna vacuum all your hair off Zoie .", "words": ["off", "all", "your", "vacuum", "hair", "were"]}, {"sent": "oh theres the vacuum cleaner noise lets shut the door .", "words": ["the", "vacuum", "door"]}, {"sent": "and Mommy gonna hafta vacuum a little because .", "words": ["little", "vacuum", "and", "because", "a"]}, {"sent": "Floppys usin the vacuum .", "words": ["the", "vacuum"]}, {"sent": "dont you like the vacuum cleaner ?", "words": ["the", "like", "you", "vacuum", "dont"]}, {"sent": "we gotta vacuum .", "words": ["vacuum", "we"]}, {"sent": "I can hear Brian using his vacuum cleaner .", "words": ["can", "his", "vacuum", "hear"]}, {"sent": "you help mommy vacuum this later ?", "words": ["vacuum", "this", "help", "you"]}, {"sent": "thats what our vacuum cleaner does .", "words": ["vacuum", "what", "does", "our"]}, {"sent": "so Mummy wont be very pleased if she has got to walk all the way upstairs to carry the vacuum cleaner down .", "words": ["the", "all", "if", "to", "so", "be", "carry", "vacuum", "down", "walk", "she"]}, {"sent": "you were looking at my vacuum cleaner NooNoo .", "words": ["my", "you", "vacuum", "at", "were"]}, {"sent": "theres a vacuum .", "words": ["vacuum", "a"]}, {"sent": "how do you do you take it out with some kind of a vacuum ?", "words": ["do", "how", "of", "with", "you", "vacuum", "it", "out", "some", "take", "a"]}, {"sent": "I think Im gonna have to get the vacuum for your chair .", "words": ["the", "think", "for", "your", "to", "chair", "have", "get", "vacuum"]}, {"sent": "were gonna take the whole table apart were gonna vacuum underneath it .", "words": ["the", "table", "vacuum", "it", "take", "were"]}, {"sent": "thats like a vacuum Ais .", "words": ["vacuum", "like", "a"]}, {"sent": "gotta vacuum up the Cheerios so they dont hurt our feet .", "words": ["the", "up", "hurt", "so", "our", "vacuum", "they", "dont"]}, {"sent": "and after lunch Mummys going to vacuum it .", "words": ["vacuum", "to", "it", "and"]}, {"sent": "that goes to the vacuum cleaner .", "words": ["the", "to", "vacuum", "that"]}, {"sent": "oh have you just given the vacuum cleaner a kiss ?", "words": ["the", "you", "have", "vacuum", "kiss", "a"]}, {"sent": "you can vacuum .", "words": ["can", "vacuum", "you"]}, {"sent": "the vacuum is gonna suck the mouse up isnt that funny .", "words": ["the", "up", "is", "vacuum", "that", "mouse"]}, {"sent": "you saw the vacuum ?", "words": ["the", "vacuum", "you"]}, {"sent": "we hadta get the vacuum cleaner out didnt we ?", "words": ["the", "we", "get", "vacuum", "out"]}, {"sent": "I wont put the vacuum cleaner on if you drink all your juice .", "words": ["the", "all", "put", "your", "if", "drink", "you", "vacuum", "on", "juice"]}, {"sent": "tubbie toast machine and NooNoo the vacuum cleaner .", "words": ["the", "vacuum", "toast", "and"]}, {"sent": "are you talking to NooNoo the vacuum cleaner ?", "words": ["the", "to", "you", "vacuum", "are"]}, {"sent": "an you seem sort of feel like youre maybe in a vacuum or something .", "words": ["of", "like", "you", "vacuum", "in", "an", "a"]}, {"sent": "going to hafta go round with the vacuum cleaner I think and pick up all these bits .", "words": ["the", "up", "think", "all", "these", "go", "pick", "to", "with", "and", "vacuum"]}, {"sent": "we hafta make a little noise because Mommy hasta vacuum up these Cheerios .", "words": ["up", "these", "we", "little", "vacuum", "make", "because", "a"]}, {"sent": "see he gave her the dress so then she just hung it on the vacuum cleaner and gave went over to give him a kiss .", "words": ["the", "over", "give", "dress", "so", "to", "kiss", "and", "vacuum", "him", "it", "he", "on", "see", "then", "her", "she", "a"]}, {"sent": "yes I hadta move the vacuum cleaner to open the open the cupboard .", "words": ["the", "to", "vacuum", "open"]}, {"sent": "we bought some vacuum storage bags today .", "words": ["vacuum", "some", "we"]}, {"sent": "they all vacuum .", "words": ["all", "they", "vacuum"]}, {"sent": "vacuum .", "words": ["vacuum"]}, {"sent": "I think well hafta vacuum you with NooNoo .", "words": ["vacuum", "think", "with", "you"]}, {"sent": "yup thats the vacuum cleaner .", "words": ["the", "vacuum"]}, {"sent": "because like he loves to help my husband vacuum .", "words": ["help", "my", "to", "like", "vacuum", "he", "because"]}, {"sent": "vacuum how about just vacuum a little bit and maybe you can vacuum more when Erin goes home .", "words": ["can", "how", "little", "vacuum", "and", "you", "more", "about", "home", "when", "a"]}, {"sent": "clothes on the vacuum cleaner .", "words": ["on", "the", "vacuum"]}, {"sent": "it was like a big vacuum cleaner .", "words": ["was", "like", "vacuum", "it", "big", "a"]}, {"sent": "someday youll hafta clean the rug with the vacuum cleaner .", "words": ["clean", "the", "vacuum", "with"]}, {"sent": "hello lady goose .", "words": ["goose"]}, {"sent": "that that that doesnt stand up very well does it the goose .", "words": ["up", "does", "the", "stand", "it", "goose", "that"]}, {"sent": "boo to a goose .", "words": ["goose", "to", "a"]}, {"sent": "yeah it looks like a duck but its a goose .", "words": ["like", "it", "duck", "goose", "but", "a"]}, {"sent": "you silly goose .", "words": ["goose", "you"]}, {"sent": "hes a hes a drunken goose .", "words": ["goose", "a"]}, {"sent": "yes I have the goose here .", "words": ["have", "the", "here", "goose"]}, {"sent": "goose goose goose .", "words": ["goose"]}, {"sent": "and the goose goes honk .", "words": ["the", "goose", "and"]}, {"sent": "whats happened to that goose now ?", "words": ["goose", "to", "that"]}, {"sent": "its a mother goose .", "words": ["goose", "a"]}, {"sent": "its Gos the baby goose .", "words": ["the", "goose"]}, {"sent": "youknow like I dont think he knows otter and fox and eagle and fawn which are some of the and goose .", "words": ["the", "goose", "think", "of", "like", "and", "dont", "are", "he", "some", "which"]}, {"sent": "because he is a goose after all isnt he ?", "words": ["all", "is", "he", "goose", "because", "a"]}, {"sent": "theres Fern theres a spider and theres a sheep and theres the goose .", "words": ["the", "and", "sheep", "goose", "a"]}, {"sent": "thats thats a goose .", "words": ["goose", "a"]}, {"sent": "thats granny goose .", "words": ["goose"]}, {"sent": "you gave me goose bumps .", "words": ["goose", "me", "you"]}, {"sent": "what about the goose ?", "words": ["the", "what", "about", "goose"]}, {"sent": "does the goose land in trees too ?", "words": ["the", "does", "in", "too", "goose"]}, {"sent": "goose and house .", "words": ["goose", "house", "and"]}, {"sent": "go tell aunt Rhodie the old grey goose is dead .", "words": ["the", "old", "is", "go", "goose"]}, {"sent": "can you see a goose anywhere ?", "words": ["can", "goose", "you", "see", "a"]}, {"sent": "Im awake silly goose .", "words": ["goose", "awake"]}, {"sent": "what is this a Canada goose ?", "words": ["what", "is", "this", "goose", "a"]}, {"sent": "what is its a goose .", "words": ["goose", "what", "a", "is"]}, {"sent": "you silly goose .", "words": ["goose", "you"]}, {"sent": "say goose .", "words": ["goose", "say"]}, {"sent": "are you frying the goose ?", "words": ["the", "goose", "are", "you"]}, {"sent": "Lily I found a goose .", "words": ["goose", "a"]}, {"sent": "I think the goose pecked your hand didnt it ?", "words": ["the", "think", "your", "hand", "it", "goose"]}, {"sent": "and you just talk to a goose the same way .", "words": ["the", "to", "you", "and", "same", "talk", "goose", "a"]}, {"sent": "put him next to the goose ?", "words": ["the", "put", "to", "him", "goose"]}, {"sent": "oh dont be silly goose .", "words": ["goose", "dont", "be"]}, {"sent": "well what does it matter if the goose gets the market wet ?", "words": ["the", "what", "does", "wet", "if", "it", "goose"]}, {"sent": "no dont lick them silly goose .", "words": ["goose", "dont", "lick", "them"]}, {"sent": "he sees a snow goose gliding above him .", "words": ["snow", "him", "he", "above", "goose", "a"]}, {"sent": "look at the goose .", "words": ["the", "goose", "at", "look"]}, {"sent": "you a silly goose ?", "words": ["goose", "a", "you"]}, {"sent": "isnt the goose having a nightnight ?", "words": ["the", "goose", "a"]}, {"sent": "are you going to put the Mummy Mummy goose in there ?", "words": ["the", "put", "to", "you", "there", "in", "are", "goose"]}, {"sent": "where do you want this goose to go ?", "words": ["where", "do", "go", "to", "you", "this", "goose"]}, {"sent": "goose .", "words": ["goose"]}, {"sent": "theres a goose goose goose .", "words": ["goose", "a"]}, {"sent": "see the birds we see the goo the goose ?", "words": ["the", "see", "goose", "we"]}, {"sent": "boo to a goose .", "words": ["goose", "to", "a"]}, {"sent": "boo to a goose .", "words": ["goose", "to", "a"]}, {"sent": "goose goose goose .", "words": ["goose"]}, {"sent": "thats a little goose isnt it ?", "words": ["goose", "it", "little", "a"]}, {"sent": "silly goose .", "words": ["goose"]}, {"sent": "get the goose .", "words": ["get", "the", "goose"]}, {"sent": "know what the goose says ?", "words": ["the", "what", "goose"]}, {"sent": "a goose ?", "words": ["goose", "a"]}, {"sent": "hello Dada goose .", "words": ["goose"]}, {"sent": "goose .", "words": ["goose"]}, {"sent": "goose .", "words": ["goose"]}, {"sent": "a goose .", "words": ["goose", "a"]}, {"sent": "xxx a goose say ?", "words": ["goose", "say", "a"]}, {"sent": "can Mommy give the goose a hug ?", "words": ["can", "the", "hug", "goose", "give", "a"]}, {"sent": "silly goose on the loose ?", "words": ["on", "goose", "the"]}, {"sent": "the Canadian goose .", "words": ["the", "goose"]}, {"sent": "boo to a goose .", "words": ["goose", "to", "a"]}, {"sent": "you silly goose .", "words": ["goose", "you"]}, {"sent": "because youre trying to make the goose come to you you dont go to the goose .", "words": ["the", "go", "to", "you", "dont", "goose", "make", "because"]}, {"sent": "where is the goose ?", "words": ["where", "the", "goose", "is"]}, {"sent": "weve only got the the drunken goose .", "words": ["the", "goose"]}, {"sent": "do you want this goose ?", "words": ["goose", "do", "this", "you"]}, {"sent": "its probably kind of honk like a goose .", "words": ["of", "goose", "like", "a"]}, {"sent": "but I wouldnt say boo to one goose or three .", "words": ["goose", "to", "say", "but"]}, {"sent": "yeah the goose likes the pond .", "words": ["the", "goose"]}, {"sent": "get the goose .", "words": ["get", "the", "goose"]}, {"sent": "and the goose do geese do live on the water dont they ?", "words": ["the", "do", "and", "dont", "they", "water", "on", "goose"]}, {"sent": "ah the goose .", "words": ["the", "goose"]}, {"sent": "sweetcorn and a goose .", "words": ["goose", "a", "and"]}, {"sent": "um we do talk about a goose but not that much .", "words": ["do", "we", "much", "about", "talk", "goose", "that", "not", "but", "a"]}, {"sent": "you get little goose .", "words": ["get", "goose", "little", "you"]}, {"sent": "that is a goose .", "words": ["goose", "that", "a", "is"]}, {"sent": "oo heres a goose .", "words": ["goose", "a"]}, {"sent": "youre a silly goose .", "words": ["goose", "a"]}, {"sent": "honk like a goose .", "words": ["goose", "like", "a"]}, {"sent": "youre a silly goose .", "words": ["goose", "a"]}, {"sent": "look at the goose .", "words": ["the", "goose", "at", "look"]}, {"sent": "its a goose .", "words": ["goose", "a"]}, {"sent": "the giraffes neck is longer so thats how I can tell its not a goose ?", "words": ["the", "can", "how", "is", "so", "goose", "not", "a"]}, {"sent": "did you ever see a goose kissing a moose ?", "words": ["see", "you", "moose", "did", "goose", "a"]}, {"sent": "you silly goose you silly goose .", "words": ["goose", "you"]}, {"sent": "do they have a goose ?", "words": ["do", "have", "they", "goose", "a"]}, {"sent": "and the goose Im try I dunno if we actually have a goose one or if weve seen geese actual geese walking around so that might be why he knows .", "words": ["the", "if", "we", "so", "be", "and", "have", "that", "he", "around", "goose", "try", "why", "a"]}, {"sent": "it says my first mother goose book .", "words": ["my", "first", "it", "book", "goose"]}, {"sent": "I dont wanna kiss a goose .", "words": ["dont", "wanna", "goose", "kiss", "a"]}, {"sent": "yeah you saw a goose .", "words": ["goose", "a", "you"]}, {"sent": "so the goose wont get the market all wet ?", "words": ["the", "all", "wet", "so", "get", "goose"]}, {"sent": "and a goose .", "words": ["goose", "a", "and"]}, {"sent": "and we looked at the frog and the goose .", "words": ["the", "we", "frog", "and", "at", "goose"]}, {"sent": "you silly goose .", "words": ["goose", "you"]}, {"sent": "goose .", "words": ["goose"]}, {"sent": "I should be the goose ?", "words": ["the", "be", "goose"]}, {"sent": "what did that goose do to you in er Bramhall park ?", "words": ["what", "do", "to", "you", "in", "park", "did", "goose", "that"]}, {"sent": "some are fast .", "words": ["some", "fast", "are"]}, {"sent": "the man ran fast fast fast fast .", "words": ["the", "fast"]}, {"sent": "did you shake real fast shake real low ?", "words": ["fast", "shake", "did", "you"]}, {"sent": "as fast as I can .", "words": ["can", "fast"]}, {"sent": "he runs really fast .", "words": ["fast", "he"]}, {"sent": "youre a girl and Im a bear fast asleep am I ?", "words": ["fast", "bear", "and", "asleep", "am", "a"]}, {"sent": "were going fast .", "words": ["fast", "were"]}, {"sent": "they both ran very fast but Paul won because he could run .", "words": ["fast", "they", "run", "he", "but", "could", "because"]}, {"sent": "she came really fast .", "words": ["fast", "she"]}, {"sent": "the driver reduced steam but Gordon was still going too fast .", "words": ["the", "fast", "was", "too", "but"]}, {"sent": "the gingerbread boy ran down the road as fast as he could until he passed a pigsty .", "words": ["the", "fast", "he", "could", "down", "a"]}, {"sent": "RobberRed runs very fast .", "words": ["fast"]}, {"sent": "mike you wanna see how fast I go ?", "words": ["how", "fast", "go", "you", "wanna", "see"]}, {"sent": "I will sothat I can do it fast .", "words": ["can", "do", "fast", "it", "will"]}, {"sent": "you were running fast then .", "words": ["fast", "were", "then", "you"]}, {"sent": "he came up thinking xxx fast .", "words": ["up", "fast", "he"]}, {"sent": "theyre going fast asleep now .", "words": ["asleep", "fast"]}, {"sent": "big fast slope ?", "words": ["fast", "big"]}, {"sent": "you is trying to grow up to fast .", "words": ["up", "fast", "is", "to", "you"]}, {"sent": "I got fast shoes .", "words": ["fast"]}, {"sent": "Patricia dont eat too fast now okay ?", "words": ["eat", "dont", "fast", "too"]}, {"sent": "will he want his dinner if hes fast asleep ?", "words": ["fast", "if", "his", "asleep", "he", "will"]}, {"sent": "its cooked that was fast .", "words": ["was", "fast", "that"]}, {"sent": "thankyou hey kids xxx how fast that needle goes .", "words": ["that", "how", "fast"]}, {"sent": "and what happens when they go very very fast they go .", "words": ["what", "fast", "go", "and", "they", "when"]}, {"sent": "its got the thing that tells you how fast your going whether your outof of gas or not its called a speedometer .", "words": ["the", "fast", "how", "your", "of", "you", "that", "not", "a"]}, {"sent": "you turn them too fast .", "words": ["fast", "too", "them", "you"]}, {"sent": "bake me a cake as fast as you can .", "words": ["can", "fast", "cake", "you", "me", "a"]}, {"sent": "is it gonna go fast ?", "words": ["go", "fast", "it", "is"]}, {"sent": "wow the snow is melting really fast .", "words": ["the", "fast", "snow", "is"]}, {"sent": "um Alex how do you do this so so fast ?", "words": ["do", "how", "fast", "so", "you", "this"]}, {"sent": "fast because she is so .", "words": ["fast", "is", "so", "she", "because"]}, {"sent": "went very fast .", "words": ["fast"]}, {"sent": "well youre going pretty fast .", "words": ["fast", "pretty"]}, {"sent": "hes fast asleep .", "words": ["asleep", "fast"]}, {"sent": "picks it up fast .", "words": ["up", "fast", "it"]}, {"sent": "does he eat it really fast ?", "words": ["does", "fast", "eat", "it", "he"]}, {"sent": "that was fast .", "words": ["was", "fast", "that"]}, {"sent": "wow youre pretty fast .", "words": ["fast", "pretty"]}, {"sent": "oh did you drink too fast ?", "words": ["fast", "drink", "you", "did", "too"]}, {"sent": "cars are coming so theyre trying to cross the street as fast as they can .", "words": ["the", "can", "fast", "to", "so", "they", "are", "street"]}, {"sent": "so fast .", "words": ["so", "fast"]}, {"sent": "I guess hes going too fast .", "words": ["fast", "too"]}, {"sent": "oh little too fast ?", "words": ["too", "fast", "little"]}, {"sent": "this is a fast train is it ?", "words": ["fast", "train", "is", "this", "it", "a"]}, {"sent": "heh too fast .", "words": ["fast", "too"]}, {"sent": "if Daddys going very very fast .", "words": ["fast", "if"]}, {"sent": "I can do it perfectly fast enough without you .", "words": ["can", "do", "fast", "you", "it"]}, {"sent": "fast .", "words": ["fast"]}, {"sent": "that went a bit fast .", "words": ["fast", "that", "a"]}, {"sent": "you figured it out fast .", "words": ["fast", "it", "out", "you"]}, {"sent": "looks like a fast car huh ?", "words": ["car", "fast", "like", "a"]}, {"sent": "very fast ?", "words": ["fast"]}, {"sent": "fast asleep .", "words": ["asleep", "fast"]}, {"sent": "going fast .", "words": ["fast"]}, {"sent": "I can write fast .", "words": ["can", "fast", "write"]}, {"sent": "are you fast asleep ?", "words": ["asleep", "fast", "are", "you"]}, {"sent": "because you were a bit too fast for it .", "words": ["fast", "for", "you", "it", "too", "were", "because", "a"]}, {"sent": "xxx too fast .", "words": ["fast", "too"]}, {"sent": "can you make the car go fast ?", "words": ["can", "the", "fast", "go", "car", "you", "make"]}, {"sent": "lets see how fast you can go .", "words": ["can", "how", "fast", "go", "you", "see"]}, {"sent": "go very fast .", "words": ["fast", "go"]}, {"sent": "oh I dont know if I could run as fast as you were running the other day .", "words": ["the", "fast", "if", "you", "other", "dont", "run", "could", "were"]}, {"sent": "you hafta run fast .", "words": ["run", "fast", "you"]}, {"sent": "how fast can hopperoos hop ?", "words": ["can", "how", "fast"]}, {"sent": "how how can I read it you turn the pages so fast .", "words": ["can", "the", "how", "fast", "so", "you", "it", "read"]}, {"sent": "he come down those stairs fast .", "words": ["fast", "those", "he", "stairs", "down"]}, {"sent": "let us look real fast .", "words": ["us", "fast", "look"]}, {"sent": "how did you get down the stairs so fast ?", "words": ["the", "how", "fast", "so", "you", "get", "did", "stairs", "down"]}, {"sent": "he jumped up in the air and turned around and ran away as fast as he could go .", "words": ["up", "the", "fast", "go", "and", "in", "away", "he", "around", "could"]}, {"sent": "if youre a fast writer it wouldnt take you very long at all .", "words": ["all", "fast", "if", "long", "you", "it", "at", "take", "a"]}, {"sent": "they just walked in super fast before the train hit them .", "words": ["the", "fast", "train", "them", "they", "in", "hit"]}, {"sent": "do you know why hes driving very fast ?", "words": ["do", "fast", "why", "you"]}, {"sent": "read fast fast fast .", "words": ["fast", "read"]}, {"sent": "yeah he runs as fast as you .", "words": ["fast", "he", "you"]}, {"sent": "oh the red fast car .", "words": ["the", "car", "fast", "red"]}, {"sent": "Ill bet that one goes fast on the ground .", "words": ["on", "the", "fast", "that"]}, {"sent": "its very fast the water in the river .", "words": ["the", "in", "fast", "water"]}, {"sent": "you hafta twist it really fast .", "words": ["fast", "it", "you"]}, {"sent": "wow they ate that fast .", "words": ["they", "fast", "that"]}, {"sent": "that rice got stale fast .", "words": ["fast", "that"]}, {"sent": "boy that was fast .", "words": ["was", "fast", "that"]}, {"sent": "bake me a cake as fast as you can .", "words": ["can", "fast", "cake", "you", "me", "a"]}, {"sent": "it went too fast .", "words": ["fast", "it", "too"]}, {"sent": "wow youre fast .", "words": ["fast"]}, {"sent": "och it just depends on how fast Barbara can mess it up in the process .", "words": ["can", "up", "how", "fast", "the", "in", "it", "on"]}, {"sent": "well hafta fast forward it a bit dont we ?", "words": ["fast", "we", "dont", "it", "a"]}, {"sent": "Im doing it really fast .", "words": ["fast", "it"]}, {"sent": "he goes pretty fast sometimes .", "words": ["pretty", "fast", "he"]}, {"sent": "thats an ambulance it has a red light so people can see it when its coming fast fast fast .", "words": ["red", "can", "fast", "light", "so", "an", "it", "see", "when", "a"]}, {"sent": "youre such a fast fast girl .", "words": ["fast", "a"]}, {"sent": "you know we useta made a circle so fast that we useta have the most steadiest people sitting on the floor indian style .", "words": ["the", "fast", "we", "so", "you", "have", "on", "that", "a"]}, {"sent": "too fast .", "words": ["fast", "too"]}, {"sent": "the sledge was going so fast and they bumped into a big pile of snow .", "words": ["the", "fast", "into", "of", "was", "so", "snow", "and", "they", "big", "a"]}, {"sent": "he chased the wolf down the path and the wolf ran into the woods as fast as he could .", "words": ["the", "fast", "wolf", "into", "and", "he", "could", "down"]}, {"sent": "so fast .", "words": ["so", "fast"]}, {"sent": "if we put you on the radio they might start telling you the news or singing a song .", "words": ["the", "put", "if", "we", "radio", "you", "they", "on", "a"]}, {"sent": "thats my xxx radio .", "words": ["my", "radio"]}, {"sent": "how about your radio ?", "words": ["how", "your", "about", "radio"]}, {"sent": "we dont need the radio on do we ?", "words": ["the", "do", "need", "radio", "we", "dont", "on"]}, {"sent": "have you got a radio at home ?", "words": ["radio", "you", "have", "at", "home", "a"]}, {"sent": "youre gonna turn on the radio ?", "words": ["on", "the", "radio"]}, {"sent": "its not called a radio .", "words": ["radio", "not", "a"]}, {"sent": "you wanna turn on the radio now ?", "words": ["the", "radio", "you", "wanna", "on"]}, {"sent": "on the radio ?", "words": ["on", "the", "radio"]}, {"sent": "oh the radio .", "words": ["the", "radio"]}, {"sent": "no snow said the radio .", "words": ["the", "snow", "radio"]}, {"sent": "radio ?", "words": ["radio"]}, {"sent": "radio accidentally shut off .", "words": ["off", "radio"]}, {"sent": "shall we take the radio incase we wanna listen to the radio ?", "words": ["the", "listen", "radio", "we", "to", "wanna", "take"]}, {"sent": "the radio the flower vase .", "words": ["the", "flower", "radio"]}, {"sent": "radio .", "words": ["radio"]}, {"sent": "no no no no they said on the radio awuhm fifteen menin ?", "words": ["on", "they", "radio", "the"]}, {"sent": "the radio ?", "words": ["the", "radio"]}, {"sent": "see that pencil on top of the radio ?", "words": ["the", "radio", "of", "pencil", "on", "see", "that"]}, {"sent": "I found a radio .", "words": ["radio", "a"]}, {"sent": "yes theres soldiers on the radio .", "words": ["on", "the", "radio"]}, {"sent": "what do you like to hear on the radio ?", "words": ["the", "what", "do", "radio", "to", "like", "you", "hear", "on"]}, {"sent": "and the radio off too ?", "words": ["the", "off", "radio", "and", "too"]}, {"sent": "radio .", "words": ["radio"]}, {"sent": "watch the magic radio .", "words": ["the", "watch", "radio"]}, {"sent": "one for a radio one for a transistor .", "words": ["for", "radio", "a"]}, {"sent": "youve been talking about something in your foot and Ive been looking from for the aerial from the little radio and its inside your pyjamas .", "words": ["the", "for", "your", "radio", "inside", "and", "little", "foot", "in", "about"]}, {"sent": "no snow said the radio no snow said the television but snowflakes dont listen to radio and they dont watch television .", "words": ["the", "watch", "listen", "radio", "to", "snow", "and", "dont", "they", "but"]}, {"sent": "all of a sudden the radio came on .", "words": ["the", "all", "radio", "of", "on", "a"]}, {"sent": "oh theres your radio .", "words": ["your", "radio"]}, {"sent": "youre trying to turn on the radio huh ?", "words": ["on", "to", "radio", "the"]}, {"sent": "oh thats the radio magnet .", "words": ["the", "radio"]}, {"sent": "who put the radio on ?", "words": ["the", "put", "radio", "who", "on"]}, {"sent": "hes talking to the handsome pilot by the by radio .", "words": ["the", "to", "radio", "by"]}, {"sent": "its a radio .", "words": ["radio", "a"]}, {"sent": "how was Dad how did Daddy feel when the radio was taken ?", "words": ["the", "how", "radio", "was", "did", "when"]}, {"sent": "he is talking to the handsome pilot by radio .", "words": ["the", "is", "radio", "to", "by", "he"]}, {"sent": "the radio just starts working and when its time for Mummy to get up .", "words": ["the", "up", "for", "radio", "to", "and", "get", "when"]}, {"sent": "we had the radio on the other day and a band started playing and you said I like bands .", "words": ["the", "radio", "we", "like", "other", "and", "you", "on", "a"]}, {"sent": "no snow said the radio .", "words": ["the", "snow", "radio"]}, {"sent": "why did I look at my radio ?", "words": ["my", "look", "radio", "did", "at", "why"]}, {"sent": "do they read letters over the radio here ?", "words": ["the", "do", "radio", "here", "they", "read", "over"]}, {"sent": "thats a radio it doesnt look like your radio but its a radio .", "words": ["your", "radio", "like", "it", "look", "but", "a"]}, {"sent": "its radio .", "words": ["radio"]}, {"sent": "did you turn the radio on ?", "words": ["the", "radio", "you", "did", "on"]}, {"sent": "yeah I shouldnt have brought that radio today .", "words": ["have", "that", "radio"]}, {"sent": "Ill come back for the radio dear a bit later on .", "words": ["the", "for", "back", "radio", "on", "a"]}, {"sent": "on the radio .", "words": ["on", "the", "radio"]}, {"sent": "and took our radio .", "words": ["our", "radio", "and"]}, {"sent": "pretend that the radio went off and its time for the mommy and daddy to get up .", "words": ["the", "off", "up", "for", "radio", "to", "and", "get", "pretend", "that"]}, {"sent": "a radio .", "words": ["radio", "a"]}, {"sent": "radio .", "words": ["radio"]}, {"sent": "does it look like a radio ?", "words": ["does", "radio", "like", "it", "look", "a"]}, {"sent": "BC radio ?", "words": ["radio"]}, {"sent": "youll have to find some on the radio .", "words": ["the", "radio", "to", "have", "find", "on", "some"]}, {"sent": "is he listening to something on the radio in the car do you think ?", "words": ["the", "do", "think", "is", "radio", "to", "car", "you", "in", "he", "on"]}, {"sent": "thats the radio but we dont want it on at the moment do we ?", "words": ["the", "do", "we", "radio", "dont", "it", "at", "on", "but"]}, {"sent": "Pippins starting on the radio .", "words": ["on", "the", "radio"]}, {"sent": "this is a record we heard on the radio one evening isnt it coming home ?", "words": ["the", "we", "is", "radio", "this", "it", "on", "home", "a"]}, {"sent": "and what do we listen to on radio four ?", "words": ["what", "do", "listen", "we", "radio", "to", "and", "on"]}, {"sent": "and I hafta retune the radio .", "words": ["the", "radio", "and"]}, {"sent": "radio .", "words": ["radio"]}, {"sent": "Im trying to record something off the radio .", "words": ["off", "to", "radio", "the"]}, {"sent": "poor Sally in the car on the radio .", "words": ["the", "radio", "car", "poor", "in", "on"]}, {"sent": "but snowflakes dont listen to radio and they dont watch television .", "words": ["watch", "listen", "radio", "to", "and", "dont", "they", "but"]}, {"sent": "a radio .", "words": ["radio", "a"]}, {"sent": "somebody turned the radio on ?", "words": ["the", "on", "radio"]}, {"sent": "I know you like everything that comes on the radio dont you ?", "words": ["the", "radio", "like", "you", "dont", "on", "that"]}, {"sent": "surely its a police radio not a fire man Sam radio if youre a police lady .", "words": ["if", "radio", "not", "a"]}, {"sent": "do you wanna listen to your radio ?", "words": ["do", "your", "listen", "radio", "to", "you", "wanna"]}, {"sent": "the only reason is what happened is this radio didnt work on someone elses car .", "words": ["the", "what", "work", "radio", "is", "car", "this", "on"]}, {"sent": "well he was there and I spoke to him very briefly and hes to have a radio talk .", "words": ["radio", "to", "was", "and", "have", "there", "him", "he", "talk", "a"]}, {"sent": "thats a radio .", "words": ["radio", "a"]}, {"sent": "does it have a radio ?", "words": ["does", "radio", "have", "it", "a"]}, {"sent": "you have your own radio now .", "words": ["have", "your", "radio", "you"]}, {"sent": "or if we just want the radio the radiators on very low we can adjust the the temperature of the house .", "words": ["the", "can", "if", "radio", "we", "of", "house", "on"]}, {"sent": "I think you also turned on the radio in New York .", "words": ["the", "think", "radio", "you", "in", "on"]}, {"sent": "xxx with Mummy and fetch the radio ?", "words": ["the", "with", "radio", "and"]}, {"sent": "would you like to listen to the evenings news on the radio or or watch the tv ?", "words": ["the", "watch", "listen", "radio", "to", "like", "you", "tv", "would", "on"]}, {"sent": "leave the radio alone .", "words": ["the", "radio"]}, {"sent": "the man on the radio .", "words": ["the", "on", "radio"]}, {"sent": "and theres a radio .", "words": ["radio", "a", "and"]}, {"sent": "its like Melissas little radio .", "words": ["radio", "like", "little"]}, {"sent": "you watch a television but you listen to a radio .", "words": ["watch", "listen", "radio", "to", "you", "but", "a"]}, {"sent": "well it works when its its hooked up to the radio .", "words": ["up", "the", "radio", "to", "it", "when"]}, {"sent": "so I read maybe listen to the radio a little bit .", "words": ["the", "listen", "radio", "to", "so", "little", "read", "a"]}, {"sent": "no the radio doesnt work in this one its broken .", "words": ["the", "work", "radio", "broken", "this", "in"]}, {"sent": "does Gar have a radio ?", "words": ["have", "does", "radio", "a"]}, {"sent": "when we turn the radio on in the hotel the sound comes out behind the headboard of the bed doesnt it ?", "words": ["the", "radio", "we", "of", "behind", "bed", "in", "it", "out", "on", "when"]}, {"sent": "no those are my shoes see they were just shining in the dark like a radio I dont know where that radio is .", "words": ["the", "where", "my", "those", "radio", "is", "like", "they", "in", "are", "dont", "that", "dark", "see", "were", "a"]}, {"sent": "you were just telling me you got a nice a radio for your birthday didnt you ?", "words": ["for", "your", "radio", "nice", "you", "me", "were", "a"]}, {"sent": "with a good radio .", "words": ["with", "radio", "good", "a"]}, {"sent": "somethings wrong with the way the radio was made on the inside and then the fm only works when I had a regular antenna xxx .", "words": ["the", "radio", "was", "inside", "with", "and", "on", "then", "when", "a"]}, {"sent": "boy I dont even have my own tv and you are going listen to your radio too ?", "words": ["my", "your", "listen", "radio", "to", "and", "have", "dont", "tv", "you", "are", "too"]}, {"sent": "yeah you hear that radio ?", "words": ["hear", "that", "radio", "you"]}, {"sent": "Mummy went upstairs to turn the radio off .", "words": ["the", "to", "off", "radio"]}, {"sent": "radio .", "words": ["radio"]}, {"sent": "yeah this one and this one are broken the radio is broken .", "words": ["the", "radio", "is", "broken", "and", "this", "are"]}, {"sent": "did you ride your bicycle today ?", "words": ["your", "bicycle", "you", "did", "ride"]}, {"sent": "well will you bring the bicycle here please ?", "words": ["the", "bring", "bicycle", "you", "here", "will"]}, {"sent": "you shouldnt be on the sand with your bicycle .", "words": ["the", "your", "be", "with", "bicycle", "you", "on"]}, {"sent": "this is a bicycle because it has pedals on it .", "words": ["is", "bicycle", "this", "it", "on", "because", "a"]}, {"sent": "riding a bicycle and theres a car .", "words": ["car", "and", "bicycle", "a"]}, {"sent": "are you going on your bicycle or your fire engine ?", "words": ["your", "bicycle", "you", "are", "on"]}, {"sent": "there was a man on a bicycle just now did you see him ?", "words": ["was", "bicycle", "you", "there", "him", "did", "on", "see", "a"]}, {"sent": "for you to go down the street riding your bicycle with Purdie sitting in the trailer .", "words": ["the", "for", "your", "go", "to", "with", "bicycle", "you", "in", "street", "down"]}, {"sent": "have you ever seen a bear riding a bicycle ?", "words": ["bicycle", "bear", "have", "you", "a"]}, {"sent": "did you play at Spencers house on a bicycle ?", "words": ["play", "bicycle", "you", "house", "did", "at", "on", "a"]}, {"sent": "oh it wont go huh whats is this bird driving the bicycle ?", "words": ["the", "is", "go", "bird", "bicycle", "this", "it"]}, {"sent": "now we need that bicycle pump .", "words": ["that", "need", "bicycle", "we"]}, {"sent": "theres the bicycle .", "words": ["the", "bicycle"]}, {"sent": "where where do you go on your bicycle ?", "words": ["where", "do", "your", "go", "bicycle", "you", "on"]}, {"sent": "see heres a bicycle and heres you heres your feet heres your hands holding on heres your head heres your eyes heres a little horn on the bicycle okay ?", "words": ["the", "your", "head", "bicycle", "you", "and", "little", "on", "see", "a"]}, {"sent": "did Nonna give you the bicycle ?", "words": ["the", "bicycle", "you", "did", "give"]}, {"sent": "do you wanna put them in the back of the bicycle ?", "words": ["the", "do", "put", "back", "of", "them", "bicycle", "you", "in", "wanna"]}, {"sent": "the paper boy usually comes on his bicycle doesnt he ?", "words": ["the", "paper", "bicycle", "on", "he", "his"]}, {"sent": "ah the bicycle .", "words": ["the", "bicycle"]}, {"sent": "has Fraser got a bicycle ?", "words": ["bicycle", "a"]}, {"sent": "because I hadta pedal the bicycle while I was eating the ice cream .", "words": ["the", "ice cream", "was", "bicycle", "ice", "because"]}, {"sent": "I dont think weve got a bicycle on the weather chart Thomas .", "words": ["the", "think", "bicycle", "dont", "on", "a"]}, {"sent": "theres an ambulance and a bicycle and a lorry .", "words": ["an", "bicycle", "a", "and"]}, {"sent": "so we went out quick on your bicycle didnt we ?", "words": ["your", "we", "so", "bicycle", "out", "on"]}, {"sent": "you have a bicycle ?", "words": ["have", "bicycle", "a", "you"]}, {"sent": "friends bicycle ?", "words": ["bicycle"]}, {"sent": "its a story about SkateboardingSam who is a policeman and he doesnt hes a policeman who doesnt go round on a bicycle or on his foot or in a patrol car .", "words": ["is", "go", "car", "bicycle", "and", "his", "foot", "in", "he", "about", "who", "on", "story", "a"]}, {"sent": "your bicycle .", "words": ["your", "bicycle"]}, {"sent": "bicycle right .", "words": ["bicycle"]}, {"sent": "your bicycle and your truck ?", "words": ["truck", "your", "bicycle", "and"]}, {"sent": "well Nia goes home to her Mummy on her bicycle .", "words": ["to", "bicycle", "on", "her", "home"]}, {"sent": "the men from the store are gonna put your bicycle in a truck .", "words": ["the", "put", "your", "store", "truck", "bicycle", "in", "are", "a"]}, {"sent": "put it in the back of your bicycle .", "words": ["the", "put", "your", "back", "of", "bicycle", "in", "it"]}, {"sent": "a bicycle ?", "words": ["bicycle", "a"]}, {"sent": "the bicycle man .", "words": ["the", "bicycle"]}, {"sent": "well youre lying down underneath your bicycle .", "words": ["down", "your", "bicycle"]}, {"sent": "she wants a bicycle from Santa .", "words": ["she", "bicycle", "a"]}, {"sent": "somebodys riding a bicycle arent they ?", "words": ["they", "bicycle", "a"]}, {"sent": "falling off your bicycle .", "words": ["off", "your", "bicycle"]}, {"sent": "oh youre looking at the front of your bicycle .", "words": ["the", "your", "of", "bicycle", "at"]}, {"sent": "shes riding her bicycle .", "words": ["her", "bicycle"]}, {"sent": "wow hes got a bicycle doesnt he ?", "words": ["he", "bicycle", "a"]}, {"sent": "Im going to take him off the bicycle .", "words": ["off", "the", "to", "bicycle", "him", "take"]}, {"sent": "how many elephants have you seen ride a bicycle Nicole .", "words": ["how", "bicycle", "you", "have", "ride", "a"]}, {"sent": "you dont wanna go out on your bicycle ?", "words": ["your", "go", "bicycle", "you", "dont", "out", "wanna", "on"]}, {"sent": "have you got the bicycle stuck ?", "words": ["the", "bicycle", "you", "have", "stuck"]}, {"sent": "I thought he had his bicycle .", "words": ["his", "he", "bicycle"]}, {"sent": "is it MissHubbards bicycle ?", "words": ["it", "bicycle", "is"]}, {"sent": "are you going to ride the bicycle now ?", "words": ["the", "to", "bicycle", "you", "are", "ride"]}, {"sent": "look at the bicycle .", "words": ["the", "at", "bicycle", "look"]}, {"sent": "you keep falling off your bicycle .", "words": ["off", "your", "bicycle", "you"]}, {"sent": "shall I just move the bicycle so that xxx .", "words": ["the", "so", "that", "bicycle"]}, {"sent": "on the bicycle .", "words": ["on", "the", "bicycle"]}, {"sent": "do you let Leila have a turn on your bicycle ?", "words": ["do", "your", "bicycle", "you", "have", "on", "a"]}, {"sent": "and the little boy got off his bicycle an what happened here ?", "words": ["the", "off", "what", "bicycle", "little", "and", "here", "an", "his"]}, {"sent": "well you put some chocolate in the back of the bicycle earlier on and Im just wiping it out now .", "words": ["the", "put", "back", "of", "chocolate", "bicycle", "you", "and", "in", "it", "out", "on", "some"]}, {"sent": "we went to the postbox didnt we and you rode your bicycle .", "words": ["the", "your", "we", "to", "bicycle", "you", "and"]}, {"sent": "that great big bicycle about two inches from your nose .", "words": ["your", "bicycle", "that", "about", "big", "nose"]}, {"sent": "do you wanna make a bicycle for him ?", "words": ["do", "for", "bicycle", "you", "him", "wanna", "make", "a"]}, {"sent": "are you sitting back on the bicycle ?", "words": ["the", "back", "bicycle", "you", "are", "on"]}, {"sent": "but every time I every time I go to undo the strap to get you out of the bicycle you push me away .", "words": ["the", "push", "go", "of", "to", "bicycle", "you", "get", "away", "every", "out", "me", "but"]}, {"sent": "is it now a bicycle ?", "words": ["it", "bicycle", "a", "is"]}, {"sent": "in a bicycle built for three .", "words": ["in", "for", "bicycle", "a"]}, {"sent": "wheres the bicycle ?", "words": ["the", "bicycle"]}, {"sent": "theres your bicycle .", "words": ["your", "bicycle"]}, {"sent": "a bicycle .", "words": ["bicycle", "a"]}, {"sent": "were to look for a piece thats got a bicycle and a bird and part of the sail .", "words": ["the", "for", "of", "to", "bird", "bicycle", "and", "look", "were", "a"]}, {"sent": "are you going for a ride on your bicycle ?", "words": ["for", "your", "bicycle", "you", "are", "on", "ride", "a"]}, {"sent": "in the playground and you got on a tractor or bicycle or something and you cycled up towards him .", "words": ["the", "up", "bicycle", "you", "and", "in", "him", "tractor", "on", "a"]}, {"sent": "can my bicycle fit in like dis ?", "words": ["can", "my", "fit", "like", "bicycle", "in"]}, {"sent": "a sled a bicycle and a plane sometimes fast sometimes slow .", "words": ["fast", "sled", "bicycle", "and", "slow", "a"]}, {"sent": "a bicycle .", "words": ["bicycle", "a"]}, {"sent": "what are you doing in your bedroom with that bicycle ?", "words": ["what", "your", "with", "bicycle", "you", "in", "are", "that", "bedroom"]}, {"sent": "you can see the moon on the back of your bicycle .", "words": ["can", "the", "your", "moon", "back", "of", "bicycle", "you", "on", "see"]}, {"sent": "the dollys riding on little bicycle .", "words": ["the", "on", "bicycle", "little"]}, {"sent": "I can see your bicycle Thomas .", "words": ["can", "see", "your", "bicycle"]}, {"sent": "who else rides on your bicycle ?", "words": ["on", "your", "bicycle", "who"]}, {"sent": "its all of the frame of the bicycle I would imagine .", "words": ["the", "all", "of", "bicycle", "would"]}, {"sent": "can you point to the bicycle .", "words": ["can", "the", "to", "bicycle", "you"]}, {"sent": "put the bicycle in the shed .", "words": ["the", "in", "put", "bicycle"]}, {"sent": "your toy elephants sitting on your bicycle isnt it ?", "words": ["your", "bicycle", "toy", "it", "on"]}, {"sent": "a bicycle .", "words": ["bicycle", "a"]}, {"sent": "I guess I got carried away Toot said I hopped on a train and then a bus and then a plane and then a bicycle and then one thing led to another all the way to Kookooposh .", "words": ["the", "all", "train", "to", "bicycle", "and", "another", "away", "on", "then", "bus", "a"]}, {"sent": "stop laughing because I tripped over the bicycle .", "words": ["the", "stop", "bicycle", "over", "because"]}, {"sent": "youre riding very quickly on that bicycle Thomas .", "words": ["on", "that", "bicycle"]}, {"sent": "how do you do bicycle ?", "words": ["do", "how", "bicycle", "you"]}, {"sent": "you clearing a space so you can get through with the bicycle ?", "words": ["can", "the", "so", "with", "bicycle", "you", "get", "a"]}, {"sent": "are you going to start repairing the bicycle ?", "words": ["the", "to", "bicycle", "you", "are"]}, {"sent": "riding a bicycle .", "words": ["bicycle", "a"]}, {"sent": "bicycle hey .", "words": ["bicycle"]}, {"sent": "a bicycle .", "words": ["bicycle", "a"]}, {"sent": "is it like being on your bicycle or is it like being in Granddads car ?", "words": ["your", "is", "car", "like", "bicycle", "in", "it", "on"]}, {"sent": "do you have a bicycle ?", "words": ["do", "bicycle", "you", "have", "a"]}, {"sent": "oh thats something to keep slacks in place when you go on a bicycle .", "words": ["go", "to", "bicycle", "you", "in", "on", "when", "a"]}, {"sent": "he couldnt ride his bicycle without training wheels .", "words": ["his", "bicycle", "he", "ride"]}, {"sent": "and Astrid wanted to go on a bicycle and we were all saying go on the yellow one .", "words": ["the", "all", "go", "we", "to", "bicycle", "and", "on", "yellow", "were", "a"]}, {"sent": "theres a turkey .", "words": ["turkey", "a"]}, {"sent": "does the turkey have a friend ?", "words": ["the", "does", "have", "turkey", "a"]}, {"sent": "do you have any turkey sandwiches ?", "words": ["do", "any", "you", "have", "turkey"]}, {"sent": "wow look at that turkey lookit hes got feet .", "words": ["turkey", "at", "that", "look"]}, {"sent": "also on that farm he had a turkey .", "words": ["he", "turkey", "on", "that", "a"]}, {"sent": "turkey .", "words": ["turkey"]}, {"sent": "how about turkey and rice ?", "words": ["how", "and", "about", "turkey"]}, {"sent": "its a pretty impressive turkey .", "words": ["turkey", "pretty", "a"]}, {"sent": "close your mouth and eat that cereal you turkey .", "words": ["your", "mouth", "and", "eat", "you", "turkey", "close", "that", "cereal"]}, {"sent": "yes youre a turkey if you cry youre a turkey gobble gobble gobble gobble gobble gobble gobble gobble .", "words": ["if", "cry", "you", "turkey", "a"]}, {"sent": "what does a turkey say .", "words": ["what", "does", "say", "turkey", "a"]}, {"sent": "no chicken is a chicken and turkey is another kind of bird a turkey .", "words": ["is", "of", "bird", "and", "another", "turkey", "chicken", "a"]}, {"sent": "and turkey ?", "words": ["turkey", "and"]}, {"sent": "you turkey huh ?", "words": ["turkey", "you"]}, {"sent": "you turkey .", "words": ["turkey", "you"]}, {"sent": "more turkey ?", "words": ["more", "turkey"]}, {"sent": "were gonna have some turkey in a couple days .", "words": ["have", "in", "turkey", "some", "were", "a"]}, {"sent": "trot with the turkey leap with the frog .", "words": ["the", "with", "frog", "turkey"]}, {"sent": "little turkey .", "words": ["turkey", "little"]}, {"sent": "pull it way back turkey ?", "words": ["turkey", "it", "back", "pull"]}, {"sent": "a turkey .", "words": ["turkey", "a"]}, {"sent": "turkey turkey turkey .", "words": ["turkey"]}, {"sent": "what does the turkey say ?", "words": ["the", "what", "does", "say", "turkey"]}, {"sent": "I dont need this turkey leg .", "words": ["need", "dont", "this", "leg", "turkey"]}, {"sent": "thats a turkey gabil gabil gabil .", "words": ["turkey", "a"]}, {"sent": "um yeah you said the turkey she recently went .", "words": ["the", "turkey", "she", "you"]}, {"sent": "yeah he knows all about turkey right ?", "words": ["all", "he", "about", "turkey"]}, {"sent": "what does turkey say ?", "words": ["what", "does", "say", "turkey"]}, {"sent": "we gotta a cat we didnt get a turkey did we ?", "words": ["we", "get", "cat", "did", "turkey", "a"]}, {"sent": "oh yeah thats a turkey .", "words": ["turkey", "a"]}, {"sent": "turkey says coat coat coat .", "words": ["coat", "turkey"]}, {"sent": "will you have a piece of turkey turkey roll piece of chicken ?", "words": ["of", "you", "have", "turkey", "will", "chicken", "a"]}, {"sent": "did you learn any more turkey songs today ?", "words": ["any", "you", "more", "did", "turkey"]}, {"sent": "no that aint a turkey is it ?", "words": ["is", "it", "turkey", "that", "a"]}, {"sent": "we got a four legged turkey this year .", "words": ["this", "turkey", "a", "we"]}, {"sent": "what about turkey pie ?", "words": ["what", "about", "turkey"]}, {"sent": "um do you want sandwiches tonight or do you want more turkey like do you want turkey ?", "words": ["do", "like", "you", "more", "turkey"]}, {"sent": "do you want some of this turkey ?", "words": ["do", "of", "you", "this", "turkey", "some"]}, {"sent": "thats a turkey .", "words": ["turkey", "a"]}, {"sent": "do the Thanksgiving turkey .", "words": ["the", "do", "turkey"]}, {"sent": "yeah that was a paper turkey that you made that fell out of the car .", "words": ["the", "of", "was", "paper", "car", "you", "out", "turkey", "that", "a"]}, {"sent": "the turkey ?", "words": ["the", "turkey"]}, {"sent": "that looked like a a turkey actually .", "words": ["turkey", "that", "like", "a"]}, {"sent": "no you close it on my head turkey .", "words": ["my", "head", "you", "it", "turkey", "on", "close"]}, {"sent": "what about this do you think this is some turkey ?", "words": ["what", "do", "think", "is", "you", "this", "about", "turkey", "some"]}, {"sent": "look at that big turkey coming .", "words": ["look", "turkey", "that", "at", "big"]}, {"sent": "you like turkey bologna .", "words": ["turkey", "like", "you"]}, {"sent": "yes Gar is a turkey lurkey .", "words": ["turkey", "a", "is"]}, {"sent": "turkey Lurkey .", "words": ["turkey"]}, {"sent": "what does the turkey say .", "words": ["the", "what", "does", "say", "turkey"]}, {"sent": "oh a turkey who says gobble .", "words": ["who", "turkey", "a"]}, {"sent": "its roast turkey .", "words": ["turkey"]}, {"sent": "turkey .", "words": ["turkey"]}, {"sent": "you turkey .", "words": ["turkey", "you"]}, {"sent": "what else should we have with our turkey ?", "words": ["what", "we", "with", "our", "have", "turkey"]}, {"sent": "turkey Lurkey .", "words": ["turkey"]}, {"sent": "wheres the turkey ?", "words": ["the", "turkey"]}, {"sent": "gobble gobble and hello I dont I didnt hear about a turkey in the music class .", "words": ["the", "hear", "and", "dont", "in", "about", "turkey", "a"]}, {"sent": "we can make a turkey .", "words": ["can", "we", "turkey", "make", "a"]}, {"sent": "turkey .", "words": ["turkey"]}, {"sent": "that is a big cooked turkey .", "words": ["is", "that", "turkey", "big", "a"]}, {"sent": "how about your turkey ?", "words": ["how", "your", "about", "turkey"]}, {"sent": "a turkey she hasnt seen a real turkey .", "words": ["turkey", "she", "a"]}, {"sent": "you gonna cook a turkey ?", "words": ["cook", "turkey", "a", "you"]}, {"sent": "Ive heard of turkey but not tacky turkey .", "words": ["of", "not", "but", "turkey"]}, {"sent": "that turkey .", "words": ["that", "turkey"]}, {"sent": "turkey .", "words": ["turkey"]}, {"sent": "yeah thats the daddy turkey .", "words": ["the", "turkey"]}, {"sent": "well put that turkey and cheese on the sandwich and some bread on top .", "words": ["the", "put", "bread", "and", "that", "turkey", "on", "some", "cheese", "sandwich"]}, {"sent": "turkey .", "words": ["turkey"]}, {"sent": "or is it a turkey ?", "words": ["turkey", "it", "a", "is"]}, {"sent": "you want a little on your turkey ?", "words": ["your", "little", "you", "turkey", "on", "a"]}, {"sent": "probably a turkey .", "words": ["turkey", "a"]}, {"sent": "thats a turkey .", "words": ["turkey", "a"]}, {"sent": "you gonna bite some turkey ?", "words": ["bite", "some", "turkey", "you"]}, {"sent": "its turkey meat sweetheart .", "words": ["meat", "turkey"]}, {"sent": "when are you when are you gonna eat some turkey ?", "words": ["you", "eat", "are", "turkey", "some", "when"]}, {"sent": "look at how big this turkey is .", "words": ["how", "look", "is", "turkey", "this", "at", "big"]}, {"sent": "turkey .", "words": ["turkey"]}, {"sent": "do you think its a turkey ?", "words": ["do", "think", "you", "turkey", "a"]}, {"sent": "heres a turkey .", "words": ["turkey", "a"]}, {"sent": "turkey goes garble garble .", "words": ["turkey"]}, {"sent": "is that a its a turkey .", "words": ["turkey", "that", "a", "is"]}, {"sent": "the turkey is a funny bird .", "words": ["the", "is", "bird", "turkey", "a"]}, {"sent": "turkey doesnt wanna talk .", "words": ["talk", "wanna", "turkey"]}, {"sent": "its a turkey .", "words": ["turkey", "a"]}, {"sent": "tastes like turkey man with a crush on it .", "words": ["with", "like", "it", "turkey", "on", "a"]}, {"sent": "thats the noise the turkey makes .", "words": ["the", "turkey"]}, {"sent": "turkey .", "words": ["turkey"]}, {"sent": "turkey burgers ?", "words": ["turkey"]}, {"sent": "turkey .", "words": ["turkey"]}, {"sent": "turkey turkey .", "words": ["turkey"]}, {"sent": "theres turkey alright ?", "words": ["turkey"]}, {"sent": "and then they got married by a turkey .", "words": ["by", "and", "they", "turkey", "then", "a"]}, {"sent": "whats a turkey say ?", "words": ["turkey", "say", "a"]}, {"sent": "a turkey okay .", "words": ["turkey", "a"]}, {"sent": "you want turkey .", "words": ["turkey", "you"]}, {"sent": "thats not turkey .", "words": ["not", "turkey"]}, {"sent": "all right well get a turkey and well be home soon .", "words": ["all", "be", "and", "get", "turkey", "home", "a"]}, {"sent": "left over turkey .", "words": ["over", "turkey"]}, {"sent": "pop .", "words": ["pop"]}, {"sent": "will you tell me when you hear that toaster go pop ?", "words": ["pop", "go", "hear", "you", "that", "will", "me", "when"]}, {"sent": "pop pop pop .", "words": ["pop"]}, {"sent": "okay Ill pop it into a beaker for you .", "words": ["pop", "for", "into", "you", "it", "a"]}, {"sent": "pop pop pop pop .", "words": ["pop"]}, {"sent": "we hafta guess which ones going to pop up everyday .", "words": ["up", "pop", "we", "to", "which"]}, {"sent": "here pop .", "words": ["here", "pop"]}, {"sent": "did you want me to pop these four ?", "words": ["these", "pop", "to", "you", "did", "me"]}, {"sent": "oh yeah that balloon is gonna pop .", "words": ["balloon", "pop", "that", "is"]}, {"sent": "pop pop pop pop pop pop pop pop pop pop pop .", "words": ["pop"]}, {"sent": "did it pop out again ?", "words": ["out", "it", "did", "pop"]}, {"sent": "pop ?", "words": ["pop"]}, {"sent": "funnily enough when you were putting your pyjamas on and waving your arms around shouting the fire engine I actually thought you look like youre pretending to be a pop star is that what youre doing ?", "words": ["the", "what", "look", "your", "pop", "is", "to", "be", "star", "like", "you", "and", "that", "around", "on", "when", "were", "a"]}, {"sent": "pop it in here .", "words": ["here", "it", "in", "pop"]}, {"sent": "pop .", "words": ["pop"]}, {"sent": "or just pop pop ?", "words": ["pop"]}, {"sent": "it went pop bang pop .", "words": ["it", "pop"]}, {"sent": "pop .", "words": ["pop"]}, {"sent": "if we pop if we pop your finger on that one .", "words": ["pop", "your", "if", "we", "finger", "on", "that"]}, {"sent": "pop pop those bubbles .", "words": ["pop", "those", "bubbles"]}, {"sent": "pop .", "words": ["pop"]}, {"sent": "pop pop pop pop pop .", "words": ["pop"]}, {"sent": "pop ?", "words": ["pop"]}, {"sent": "pop it on the table darling .", "words": ["the", "pop", "table", "it", "on"]}, {"sent": "but youve got the pop a two first .", "words": ["the", "pop", "first", "but", "a"]}, {"sent": "Im going to pop a drop on cotton wool .", "words": ["pop", "to", "drop", "on", "a"]}, {"sent": "do you mean ice lolly pops or do you mean the lolly pop that Grandma gave you ?", "words": ["the", "do", "pop", "you", "ice", "that"]}, {"sent": "pop them in .", "words": ["in", "pop", "them"]}, {"sent": "pop pop pop pop pop pop .", "words": ["pop"]}, {"sent": "pop it up and have a look .", "words": ["up", "pop", "and", "have", "it", "look", "a"]}, {"sent": "you cant have the pop too .", "words": ["the", "pop", "you", "have", "too"]}, {"sent": "are you gonna pop him back in ?", "words": ["pop", "back", "you", "him", "are", "in"]}, {"sent": "you just pop them huh ?", "words": ["pop", "them", "you"]}, {"sent": "doesnt always pop up .", "words": ["up", "pop"]}, {"sent": "it didnt pop though did it ?", "words": ["it", "did", "pop"]}, {"sent": "pop pop pop pop .", "words": ["pop"]}, {"sent": "pop pop pop pop pop .", "words": ["pop"]}, {"sent": "then pop pop pop pop we saw cats e f and g .", "words": ["then", "pop", "and", "we"]}, {"sent": "does it not pop every time ?", "words": ["does", "pop", "it", "every", "not"]}, {"sent": "rabbit go pop .", "words": ["pop", "go"]}, {"sent": "does that make it pop up ?", "words": ["up", "does", "pop", "that", "it", "make"]}, {"sent": "pop .", "words": ["pop"]}, {"sent": "it went pop .", "words": ["it", "pop"]}, {"sent": "pop your legs under the table .", "words": ["the", "pop", "your", "table", "under"]}, {"sent": "pop .", "words": ["pop"]}, {"sent": "hey pop .", "words": ["pop"]}, {"sent": "if it goes on the light it might pop .", "words": ["the", "pop", "if", "light", "it", "on"]}, {"sent": "right pop the beaker xxx on the train track now .", "words": ["the", "on", "pop", "train"]}, {"sent": "well just pop them on the floor shall we ?", "words": ["the", "pop", "we", "them", "on"]}, {"sent": "she says I wanna eat frenchfries and a cheeseburger and drink pop .", "words": ["pop", "drink", "and", "eat", "wanna", "she", "a"]}, {"sent": "right pop those Smarties on some of these biscuits please .", "words": ["these", "pop", "those", "of", "on", "some"]}, {"sent": "pop .", "words": ["pop"]}, {"sent": "pop ?", "words": ["pop"]}, {"sent": "pop it in here .", "words": ["here", "it", "in", "pop"]}, {"sent": "thats a wooden block with a nice picture of Purdie on it and you pop it on the ink pad like that and then we press it onto the paper .", "words": ["the", "pop", "block", "we", "of", "paper", "picture", "nice", "with", "like", "and", "you", "it", "on", "then", "that", "a"]}, {"sent": "pop em pop pop pop pop .", "words": ["pop"]}, {"sent": "we hafta pop it in first .", "words": ["pop", "we", "first", "in", "it"]}, {"sent": "we put you in a big towel we pop some of it over your head when youve had your hair washed .", "words": ["put", "pop", "your", "when", "we", "of", "head", "you", "in", "towel", "it", "over", "some", "hair", "big", "a"]}, {"sent": "a pop ?", "words": ["pop", "a"]}, {"sent": "pop it in the bin .", "words": ["the", "it", "in", "pop"]}, {"sent": "pop him on your paper .", "words": ["pop", "your", "paper", "him", "on"]}, {"sent": "pop no good .", "words": ["pop", "good"]}, {"sent": "pop pop pop .", "words": ["pop"]}, {"sent": "pop .", "words": ["pop"]}, {"sent": "the spark is so hot it makes the air around it pop and thats the sound of thunder .", "words": ["the", "pop", "is", "of", "so", "and", "hot", "it", "around"]}, {"sent": "pop goes the weasel ?", "words": ["the", "pop"]}, {"sent": "lets pop a drop more water in .", "words": ["pop", "more", "in", "drop", "water", "a"]}, {"sent": "and well pop xxx into this xxx .", "words": ["this", "pop", "into", "and"]}, {"sent": "now do you wanna get a little spoon of flour and pop it into that container please .", "words": ["do", "pop", "into", "of", "you", "get", "little", "and", "it", "spoon", "wanna", "that", "a"]}, {"sent": "more pop .", "words": ["more", "pop"]}, {"sent": "the toast has just gone pop .", "words": ["the", "pop", "toast"]}, {"sent": "corn times snack time pop pop pop pop popcorn .", "words": ["pop", "popcorn", "corn"]}, {"sent": "pop it in there .", "words": ["it", "in", "pop", "there"]}, {"sent": "pop pop pop pop .", "words": ["pop"]}, {"sent": "pop it with your nose .", "words": ["pop", "your", "with", "it", "nose"]}, {"sent": "I am very hungry pop when is this spot picking going to stop .", "words": ["pop", "is", "to", "stop", "hungry", "this", "am", "when"]}, {"sent": "if you turn it upside down and pop some icecream in there mm we could eat it couldnt we ?", "words": ["pop", "if", "we", "and", "you", "there", "in", "it", "eat", "some", "could", "down"]}, {"sent": "you can pop all these cards out look .", "words": ["can", "all", "these", "pop", "you", "out", "look"]}, {"sent": "pop your strawberry xxx .", "words": ["pop", "your", "strawberry"]}, {"sent": "pop .", "words": ["pop"]}, {"sent": "go and pop mine .", "words": ["mine", "pop", "and", "go"]}, {"sent": "pop pop pop pop pop pop pop pop pop pop pop pop .", "words": ["pop"]}, {"sent": "pop .", "words": ["pop"]}, {"sent": "pop .", "words": ["pop"]}, {"sent": "pop pop pop pop pop pop .", "words": ["pop"]}, {"sent": "he goes pop pop pop pop pop pop pop .", "words": ["pop", "he"]}, {"sent": "pop pop pop pop pop mine ?", "words": ["mine", "pop"]}, {"sent": "you pop them out .", "words": ["pop", "them", "out", "you"]}, {"sent": "pop pop pop pop pop pop .", "words": ["pop"]}, {"sent": "pop them back .", "words": ["pop", "them", "back"]}, {"sent": "and go pop .", "words": ["go", "pop", "and"]}, {"sent": "pop it back on there please .", "words": ["pop", "back", "there", "it", "on"]}, {"sent": "you want me to pop you in your mouth ?", "words": ["pop", "your", "to", "mouth", "you", "in", "me"]}, {"sent": "ah pop .", "words": ["pop"]}, {"sent": "is it pop ?", "words": ["it", "pop", "is"]}, {"sent": "now close the door and pop .", "words": ["the", "pop", "and", "close", "door"]}, {"sent": "cant rip the book .", "words": ["the", "book", "rip"]}, {"sent": "rip .", "words": ["rip"]}, {"sent": "not sposta rip puzzles are you ?", "words": ["rip", "are", "not", "you"]}, {"sent": "dont rip the book .", "words": ["the", "dont", "rip", "book"]}, {"sent": "dont rip too much up Thomas because we need it xxx dont we ?", "words": ["up", "need", "we", "rip", "dont", "it", "too", "much", "because"]}, {"sent": "dont rip it because Mummy wants to keep it .", "words": ["to", "rip", "dont", "it", "because"]}, {"sent": "dont rip it .", "words": ["dont", "rip", "it"]}, {"sent": "no do rip it because .", "words": ["do", "rip", "it", "because"]}, {"sent": "rip it off this package .", "words": ["off", "this", "rip", "it"]}, {"sent": "oh just leave it like that because if you rip your books then you wont have them .", "words": ["your", "if", "rip", "them", "like", "you", "have", "it", "then", "that", "because"]}, {"sent": "why you wanna rip it ?", "words": ["rip", "you", "it", "wanna", "why"]}, {"sent": "now we hafta rip open this bag .", "words": ["open", "this", "rip", "we"]}, {"sent": "please dont rip that out .", "words": ["dont", "rip", "that", "out"]}, {"sent": "dont rip em .", "words": ["dont", "rip"]}, {"sent": "let me turn pull the page out because you might rip it .", "words": ["the", "rip", "pull", "you", "it", "out", "me", "because"]}, {"sent": "dont rip it .", "words": ["dont", "rip", "it"]}, {"sent": "rip it rip it .", "words": ["rip", "it"]}, {"sent": "please dont rip it .", "words": ["dont", "rip", "it"]}, {"sent": "thats enough thats enough tape let me rip it here .", "words": ["rip", "tape", "here", "it", "me"]}, {"sent": "I dont want you to rip the box .", "words": ["the", "box", "to", "rip", "you", "dont"]}, {"sent": "now if you rip it itll all go wont it ?", "words": ["all", "if", "go", "rip", "you", "it"]}, {"sent": "I dont think it is sposta rip off like that .", "words": ["off", "think", "is", "rip", "like", "dont", "it", "that"]}, {"sent": "did you rip it ?", "words": ["rip", "it", "did", "you"]}, {"sent": "you did rip it .", "words": ["rip", "it", "did", "you"]}, {"sent": "you can youll rip your pants .", "words": ["can", "your", "rip", "you", "pants"]}, {"sent": "dont rip it .", "words": ["dont", "rip", "it"]}, {"sent": "uh oh we dont wanna rip that here .", "words": ["we", "rip", "here", "dont", "wanna", "that"]}, {"sent": "because if there its a real book he will rip the pages out so .", "words": ["the", "if", "so", "rip", "will", "there", "he", "out", "book", "because", "a"]}, {"sent": "she was gonna rip that one .", "words": ["was", "rip", "that", "she"]}, {"sent": "yeah well dont rip it .", "words": ["dont", "rip", "it"]}, {"sent": "whoops thats a rip .", "words": ["rip", "a"]}, {"sent": "dont rip pages .", "words": ["dont", "rip"]}, {"sent": "dont rip it .", "words": ["dont", "rip", "it"]}, {"sent": "dont rip the box .", "words": ["the", "dont", "rip", "box"]}, {"sent": "please dont rip that off .", "words": ["off", "dont", "rip", "that"]}, {"sent": "dont rip that Thomas .", "words": ["dont", "rip", "that"]}, {"sent": "dont rip that anymore than it is already ripped .", "words": ["is", "rip", "dont", "it", "that"]}, {"sent": "dont dont rip it .", "words": ["dont", "rip", "it"]}, {"sent": "dont rip it Thomas .", "words": ["dont", "rip", "it"]}, {"sent": "no Im not gonna rip a magazine .", "words": ["rip", "not", "a"]}, {"sent": "youre going to rip it and then it willnt be nice to play with because itll be broken if its ripped .", "words": ["if", "to", "rip", "be", "nice", "play", "with", "and", "broken", "it", "then", "because"]}, {"sent": "you gonna rip it open ?", "words": ["open", "rip", "it", "you"]}, {"sent": "oh Thomas dont rip them up .", "words": ["up", "dont", "rip", "them"]}, {"sent": "yeah you do you got a little rip on Gil youre right .", "words": ["do", "rip", "little", "you", "on", "a"]}, {"sent": "just rip it out ?", "words": ["rip", "it", "out"]}, {"sent": "wee rip .", "words": ["rip"]}, {"sent": "xxx dont rip it .", "words": ["dont", "rip", "it"]}, {"sent": "now take it off because youre gonna rip it .", "words": ["off", "rip", "it", "take", "because"]}, {"sent": "you dont dont rip the toys do you ?", "words": ["the", "do", "rip", "you", "dont"]}, {"sent": "youre a wee rip taking bites outof bickies .", "words": ["rip", "a"]}, {"sent": "dont rip that .", "words": ["dont", "rip", "that"]}, {"sent": "dont rip it .", "words": ["dont", "rip", "it"]}, {"sent": "because I dont want you to rip it Nicole .", "words": ["to", "rip", "you", "dont", "it", "because"]}, {"sent": "what are you saying you wee rip ?", "words": ["what", "rip", "are", "you"]}, {"sent": "Megan lets not rip that honey .", "words": ["rip", "that", "not"]}, {"sent": "you were gonna rip the paper from there .", "words": ["the", "rip", "paper", "you", "there", "were"]}, {"sent": "give me one good reason I shouldnt rip you apart Scar .", "words": ["rip", "you", "good", "me", "give"]}, {"sent": "did you rip it open ?", "words": ["open", "rip", "you", "it", "did"]}, {"sent": "I know you didnt rip it .", "words": ["rip", "it", "you"]}, {"sent": "so you dont rip it .", "words": ["so", "rip", "you", "dont", "it"]}, {"sent": "can I rip this off ?", "words": ["can", "this", "rip", "off"]}, {"sent": "maybe if you rip this this bit here because thats just the flap .", "words": ["the", "if", "rip", "you", "here", "this", "because"]}, {"sent": "please dont rip it .", "words": ["dont", "rip", "it"]}, {"sent": "let me rip it for you .", "words": ["for", "rip", "you", "it", "me"]}, {"sent": "be careful not to rip these pockets .", "words": ["these", "to", "be", "rip", "careful", "not"]}, {"sent": "dont rip it .", "words": ["dont", "rip", "it"]}, {"sent": "youre not allowed to rip them are you ?", "words": ["to", "rip", "them", "you", "are", "not"]}, {"sent": "I dont want you to rip its tail .", "words": ["to", "dont", "rip", "you"]}, {"sent": "dont pull it because youll rip just rip it .", "words": ["rip", "pull", "dont", "it", "because"]}, {"sent": "big rip .", "words": ["rip", "big"]}, {"sent": "no that one youre going to rip with your teeth .", "words": ["your", "to", "rip", "with", "that"]}, {"sent": "easy dont rip the page easy .", "words": ["the", "dont", "rip"]}, {"sent": "here you can rip up this bill .", "words": ["can", "up", "rip", "you", "here", "this"]}, {"sent": "because thats all she just wants to like rip at them and chew them right now .", "words": ["all", "to", "rip", "them", "like", "and", "at", "she", "because"]}, {"sent": "xxx pull it rip it xxx and stop that xxx that off a your Tyrones book its something xxx gimme those I meant to tell yall we xxx our locker while out there .", "words": ["off", "your", "those", "we", "to", "rip", "stop", "and", "pull", "our", "there", "it", "out", "book", "that", "a"]}, {"sent": "no dont rip them off .", "words": ["off", "dont", "rip", "them"]}, {"sent": "dont rip .", "words": ["dont", "rip"]}, {"sent": "dont rip him .", "words": ["dont", "rip", "him"]}, {"sent": "there is a good place to rip it .", "words": ["is", "to", "rip", "there", "it", "good", "a"]}, {"sent": "where do you think we should put the book so we dont lose it or break rip it ?", "words": ["where", "do", "think", "put", "the", "we", "so", "rip", "you", "dont", "it", "book", "break"]}, {"sent": "lookee Edw dont rip the pages .", "words": ["the", "dont", "rip"]}, {"sent": "its ripped but you didnt rip it darling .", "words": ["rip", "it", "but", "you"]}, {"sent": "dont rip .", "words": ["dont", "rip"]}, {"sent": "rip it up .", "words": ["up", "rip", "it"]}, {"sent": "if you rip the book well hafta throw the book away .", "words": ["the", "if", "throw", "rip", "you", "away", "book"]}, {"sent": "dont rip it any more .", "words": ["rip", "any", "dont", "more", "it"]}, {"sent": "oh please dont rip that paper sweetie pie .", "words": ["dont", "rip", "that", "paper"]}, {"sent": "because youll itll catch and probably rip further down .", "words": ["rip", "catch", "and", "down", "because"]}, {"sent": "okay well youre gonna rip this off and thats all .", "words": ["off", "all", "rip", "and", "this"]}, {"sent": "no do not rip it .", "words": ["do", "rip", "it", "not"]}, {"sent": "you want me to rip the tape Vaivy ?", "words": ["the", "to", "rip", "tape", "you", "me"]}, {"sent": "yeah you did rip it .", "words": ["rip", "it", "did", "you"]}, {"sent": "dont rip em up .", "words": ["up", "dont", "rip"]}, {"sent": "theres a rip in it .", "words": ["in", "rip", "it", "a"]}, {"sent": "no rip no more .", "words": ["rip", "more"]}, {"sent": "dont rip it .", "words": ["dont", "rip", "it"]}, {"sent": "can I rip this ?", "words": ["can", "this", "rip"]}, {"sent": "youre going to rip your book this way .", "words": ["your", "to", "rip", "this", "book"]}, {"sent": "gonna sit down next to your broom and your mop ?", "words": ["your", "to", "and", "broom", "mop", "sit", "down"]}, {"sent": "you know what a mop is ?", "words": ["what", "is", "you", "mop", "a"]}, {"sent": "a top and a mop .", "words": ["mop", "and", "a"]}, {"sent": "as soon as I um finish sweeping we can do the mop mop .", "words": ["can", "do", "the", "we", "finish", "mop"]}, {"sent": "I was just ready to mop up .", "words": ["to", "was", "up", "mop"]}, {"sent": "gotta sweep and mop .", "words": ["sweep", "mop", "and"]}, {"sent": "just one mop .", "words": ["mop"]}, {"sent": "lets put it by the mop .", "words": ["the", "put", "by", "it", "mop"]}, {"sent": "mop it up .", "words": ["up", "it", "mop"]}, {"sent": "is that a mop ?", "words": ["mop", "that", "a", "is"]}, {"sent": "a mop right .", "words": ["mop", "a"]}, {"sent": "luckily he has got a mop in the other hand so he can mop it up .", "words": ["the", "can", "up", "hand", "so", "other", "in", "it", "he", "mop", "a"]}, {"sent": "mop .", "words": ["mop"]}, {"sent": "this is a mop .", "words": ["mop", "this", "a", "is"]}, {"sent": "hafta mop it all up wont I ?", "words": ["up", "all", "it", "mop"]}, {"sent": "can you mop it up ?", "words": ["can", "up", "you", "it", "mop"]}, {"sent": "mm okay lemme mop that .", "words": ["that", "mop"]}, {"sent": "sweep and mop ?", "words": ["sweep", "mop", "and"]}, {"sent": "mop .", "words": ["mop"]}, {"sent": "Im gonna needta mop this floor .", "words": ["this", "mop"]}, {"sent": "I gotta sweep the floor before I mop it .", "words": ["the", "it", "sweep", "mop"]}, {"sent": "just spray that down use put the mop it lifts every wee stain off the floor .", "words": ["the", "off", "put", "it", "every", "mop", "down", "that"]}, {"sent": "if you dont dump it on the floor and make Daddy mop till noon .", "words": ["the", "if", "you", "and", "dont", "it", "mop", "on", "dump", "make"]}, {"sent": "are you gonna mop up the orange juice ?", "words": ["up", "the", "you", "are", "orange", "mop", "juice"]}, {"sent": "were going to mop the cookie monster .", "words": ["the", "cookie", "to", "mop", "were"]}, {"sent": "to mop it up .", "words": ["up", "to", "it", "mop"]}, {"sent": "can mommy have the mop ?", "words": ["can", "the", "have", "mop"]}, {"sent": "mop mop .", "words": ["mop"]}, {"sent": "why are you saying map and mop and pap and pat ?", "words": ["and", "you", "are", "mop", "why"]}, {"sent": "if you mop it up I cant come in .", "words": ["up", "if", "you", "in", "it", "mop"]}, {"sent": "I think when when work periods over though youre gonna hafta mop it up .", "words": ["up", "think", "work", "when", "it", "mop", "over"]}, {"sent": "my friends wee boy his hair was uh a mop of curls .", "words": ["my", "of", "was", "mop", "his", "hair", "a"]}, {"sent": "fortunately Mummy didnt hafta mop it .", "words": ["it", "mop"]}, {"sent": "mop it up .", "words": ["up", "it", "mop"]}, {"sent": "lets just mop some of this up .", "words": ["up", "of", "this", "mop", "some"]}, {"sent": "JJ can you say mop ?", "words": ["can", "say", "mop", "you"]}, {"sent": "Patrick keep the mop on the floor .", "words": ["the", "on", "mop"]}, {"sent": "ah now the fox with the mop and bucket I think is going to mop up where the tap has leaked .", "words": ["the", "up", "think", "where", "bucket", "is", "to", "with", "and", "mop"]}, {"sent": "Im gonna mop the floor .", "words": ["the", "mop"]}, {"sent": "someones bringing a mop but its too late .", "words": ["but", "too", "mop", "a"]}, {"sent": "I needta um just mop up with you an interview and permission slip .", "words": ["up", "with", "you", "and", "an", "mop"]}, {"sent": "you still gotta mop this floor .", "words": ["this", "mop", "you"]}, {"sent": "let Mommy mop the floor before you get down .", "words": ["the", "you", "get", "mop", "down"]}, {"sent": "you put the mop in the bucket .", "words": ["the", "put", "bucket", "you", "in", "mop"]}, {"sent": "and we dont hafta mop .", "words": ["dont", "we", "mop", "and"]}, {"sent": "lookit there he made the mop and a tree his row his paddle and the puppy dog in the bucket he was having fun wasnt he ?", "words": ["the", "tree", "bucket", "was", "and", "there", "in", "he", "puppy", "mop", "his", "dog", "a"]}, {"sent": "you can be the mop .", "words": ["can", "the", "be", "you", "mop"]}, {"sent": "Daddy does not wanna hafta mop the floor .", "words": ["the", "does", "mop", "wanna", "not"]}, {"sent": "no mop .", "words": ["mop"]}, {"sent": "oh hes got a mop on his head .", "words": ["head", "his", "mop", "on", "a"]}, {"sent": "what does the mop go in though ?", "words": ["the", "what", "does", "go", "in", "mop"]}, {"sent": "when we went shopping to buy that mop and the bucket ?", "words": ["the", "when", "we", "bucket", "to", "and", "mop", "that", "buy"]}, {"sent": "Ill just get a cloth to mop up that little bit .", "words": ["up", "to", "little", "get", "mop", "that", "a"]}, {"sent": "mop .", "words": ["mop"]}, {"sent": "okay here you can have one mop .", "words": ["can", "you", "here", "have", "mop"]}, {"sent": "to mop up the milk .", "words": ["up", "the", "to", "mop", "milk"]}, {"sent": "mummie is shaking dust outof her mop .", "words": ["her", "mop", "is"]}, {"sent": "its a mop .", "words": ["mop", "a"]}, {"sent": "mop tub .", "words": ["mop"]}, {"sent": "can you wait one minute so Mommy can mop the floor ?", "words": ["can", "the", "wait", "so", "you", "mop"]}, {"sent": "go get a diaper to mop up the ginger ale .", "words": ["up", "the", "go", "diaper", "to", "get", "mop", "a"]}, {"sent": "Eleanors a mop .", "words": ["mop", "a"]}, {"sent": "ohdear hafta mop it up now .", "words": ["up", "it", "mop"]}, {"sent": "its a mop .", "words": ["mop", "a"]}, {"sent": "eat the dust mop ?", "words": ["eat", "the", "mop"]}, {"sent": "Jwww held the mop .", "words": ["the", "mop"]}, {"sent": "thats a mop .", "words": ["mop", "a"]}, {"sent": "its a little dust mop .", "words": ["mop", "little", "a"]}, {"sent": "and mop up that bit of water on the floor .", "words": ["up", "the", "of", "and", "mop", "water", "on", "that"]}, {"sent": "I guess you gotta sweep and mop daily when you have little kids .", "words": ["sweep", "and", "you", "have", "little", "mop", "when"]}, {"sent": "xxx mop these things up .", "words": ["up", "these", "mop"]}, {"sent": "we can mop it up with a little water and paper towel .", "words": ["can", "up", "we", "paper", "with", "little", "and", "towel", "it", "mop", "water", "a"]}, {"sent": "yep mop mop xxx xxx xxx here because its not as wet .", "words": ["wet", "here", "mop", "not", "because"]}, {"sent": "there you go there come here you look like you have a mop on your head .", "words": ["your", "go", "head", "like", "you", "here", "there", "have", "mop", "look", "on", "a"]}, {"sent": "and Daddyll hafta mop the floor .", "words": ["the", "mop", "and"]}, {"sent": "you wanna do the mop mop ?", "words": ["the", "do", "you", "mop", "wanna"]}, {"sent": "you gonna let Mommy mop the floor while you eat your breakfast ?", "words": ["the", "your", "you", "eat", "mop"]}, {"sent": "Im gonna get the mop .", "words": ["get", "the", "mop"]}, {"sent": "mop .", "words": ["mop"]}, {"sent": "what does the mop come in ?", "words": ["the", "what", "does", "in", "mop"]}, {"sent": "look like a rag mop .", "words": ["mop", "like", "a", "look"]}, {"sent": "mop for cleaning .", "words": ["for", "mop"]}, {"sent": "you gonna put the mop in the kitchen ?", "words": ["the", "put", "you", "in", "mop", "kitchen"]}, {"sent": "you dont need a mop do you ?", "words": ["do", "need", "you", "dont", "mop", "a"]}, {"sent": "mop .", "words": ["mop"]}, {"sent": "mop .", "words": ["mop"]}, {"sent": "well you use something like that to mop up all the juice .", "words": ["up", "all", "the", "to", "like", "you", "mop", "juice", "that"]}, {"sent": "you you can be the mop .", "words": ["can", "the", "be", "you", "mop"]}, {"sent": "we can mop it up .", "words": ["can", "up", "we", "it", "mop"]}, {"sent": "a mug and a mop .", "words": ["mop", "and", "a"]}, {"sent": "mop .", "words": ["mop"]}, {"sent": "thats a dust mop .", "words": ["mop", "a"]}, {"sent": "well why dont we put them back in the bag and then back in the cracker and then we can pull the cracker again later and then Daddy will think its the first time weve pulled the cracker and he wont realize weve already opened it will he ?", "words": ["the", "can", "put", "think", "back", "we", "cracker", "them", "first", "and", "pull", "dont", "in", "it", "he", "will", "then", "why"]}, {"sent": "would you like a cracker ?", "words": ["cracker", "like", "you", "would", "a"]}, {"sent": "thats not a cracker .", "words": ["cracker", "not", "a"]}, {"sent": "uh please dont talk with a mouth full of cracker .", "words": ["full", "of", "mouth", "with", "cracker", "dont", "talk", "a"]}, {"sent": "oh cracker and cheese is in nere .", "words": ["is", "cracker", "and", "in", "cheese"]}, {"sent": "you want this kind of cracker ?", "words": ["of", "this", "cracker", "you"]}, {"sent": "you want another cracker ?", "words": ["another", "cracker", "you"]}, {"sent": "would you like a cracker ?", "words": ["cracker", "like", "you", "would", "a"]}, {"sent": "xxx this cheese because dis cheese cracker .", "words": ["this", "cracker", "cheese", "because"]}, {"sent": "well the cookie maybe she doesnt really eat cookies but her brother does once in a while so she may see that or or she might know it as a cracker cause she eats crackers .", "words": ["the", "cookie", "does", "so", "cracker", "eat", "in", "that", "it", "see", "her", "she", "but", "a"]}, {"sent": "you had your cracker before you had your grape juice .", "words": ["juice", "cracker", "your", "you"]}, {"sent": "oh Mommy will have a cracker too .", "words": ["cracker", "have", "too", "will", "a"]}, {"sent": "whatd I just do to the cracker ?", "words": ["to", "do", "cracker", "the"]}, {"sent": "more cracker ?", "words": ["more", "cracker"]}, {"sent": "apple butter on a cracker ?", "words": ["cracker", "butter", "on", "apple", "a"]}, {"sent": "a graham cracker is a yummy treat .", "words": ["cracker", "is", "a"]}, {"sent": "yeah its a cracker huh ?", "words": ["cracker", "a"]}, {"sent": "do you want a cracker or a cookie ?", "words": ["cookie", "do", "cracker", "you", "a"]}, {"sent": "would you like a cracker ?", "words": ["cracker", "like", "you", "would", "a"]}, {"sent": "would you like a cracker ?", "words": ["cracker", "like", "you", "would", "a"]}, {"sent": "eat your cracker .", "words": ["eat", "cracker", "your"]}, {"sent": "then you can have a cracker .", "words": ["can", "cracker", "you", "have", "then", "a"]}, {"sent": "use a little bit on the top but first we hafta finish your cracker or otherwise theyll get paint all over it you cant eat a painty cracker right ?", "words": ["the", "all", "your", "paint", "we", "first", "cracker", "finish", "little", "get", "you", "eat", "it", "on", "over", "but", "a"]}, {"sent": "a cracker ?", "words": ["cracker", "a"]}, {"sent": "a cracker .", "words": ["cracker", "a"]}, {"sent": "I want a bite of your cracker .", "words": ["your", "of", "cracker", "bite", "a"]}, {"sent": "these wheat cracker ?", "words": ["these", "cracker"]}, {"sent": "what about that cracker ?", "words": ["what", "cracker", "that", "about"]}, {"sent": "you just want to stand up and eat your cracker huh ?", "words": ["up", "your", "to", "stand", "cracker", "and", "you", "eat", "want to"]}, {"sent": "I had a cracker its .", "words": ["cracker", "a"]}, {"sent": "ask the person next to you please pass it you had a cracker right ?", "words": ["the", "to", "cracker", "you", "it", "a"]}, {"sent": "what did I do to the cracker ?", "words": ["the", "what", "do", "to", "cracker", "did"]}, {"sent": "Im a little cracker .", "words": ["cracker", "little", "a"]}, {"sent": "want a cracker ?", "words": ["cracker", "a"]}, {"sent": "there is no cracker Laura .", "words": ["there", "cracker", "is"]}, {"sent": "would you like a cracker ?", "words": ["cracker", "like", "you", "would", "a"]}, {"sent": "can you eat your cracker ?", "words": ["can", "your", "cracker", "you", "eat"]}, {"sent": "you want your cracker ?", "words": ["cracker", "your", "you"]}, {"sent": "do you wanna have some almond butter on your cracker ?", "words": ["do", "your", "cracker", "you", "have", "butter", "wanna", "on", "some"]}, {"sent": "Polly want a cracker ?", "words": ["cracker", "a"]}, {"sent": "do you want a cracker ?", "words": ["do", "cracker", "a", "you"]}, {"sent": "did you have a cracker ?", "words": ["cracker", "you", "have", "did", "a"]}, {"sent": "would you like to have a cracker ?", "words": ["to", "cracker", "like", "you", "have", "would", "a"]}, {"sent": "would you like your cracker ?", "words": ["your", "cracker", "like", "you", "would"]}, {"sent": "eat your cracker Baby .", "words": ["eat", "cracker", "your"]}, {"sent": "how about a cracker ?", "words": ["how", "cracker", "about", "a"]}, {"sent": "want a cracker ?", "words": ["cracker", "a"]}, {"sent": "what is that cracker called ?", "words": ["what", "cracker", "that", "is"]}, {"sent": "do you want another cracker ?", "words": ["another", "do", "cracker", "you"]}, {"sent": "nobody wants to see your mushed up cracker .", "words": ["up", "your", "to", "cracker", "see"]}, {"sent": "would you like a cracker ?", "words": ["cracker", "like", "you", "would", "a"]}, {"sent": "cracker .", "words": ["cracker"]}, {"sent": "you can have an animal cracker .", "words": ["can", "cracker", "you", "have", "an", "animal"]}, {"sent": "say cracker .", "words": ["cracker", "say"]}, {"sent": "eat your cracker and drink your juice .", "words": ["your", "drink", "cracker", "and", "eat", "juice"]}, {"sent": "a cracker .", "words": ["cracker", "a"]}, {"sent": "you dropped your cracker .", "words": ["cracker", "your", "you"]}, {"sent": "lets go graham cracker .", "words": ["cracker", "go"]}, {"sent": "honey if you wash the cracker off you wont be able to eat it .", "words": ["the", "off", "if", "be", "to", "cracker", "you", "eat", "it", "wash"]}, {"sent": "oh lovely because then the cracker that you made .", "words": ["the", "cracker", "you", "then", "that", "because"]}, {"sent": "I think your cracker fell apart .", "words": ["think", "cracker", "your"]}, {"sent": "cracker yeah .", "words": ["cracker"]}, {"sent": "no more cracker ?", "words": ["more", "cracker"]}, {"sent": "cracker please .", "words": ["cracker"]}, {"sent": "all done with your cracker ?", "words": ["your", "all", "cracker", "with"]}, {"sent": "eat your cracker .", "words": ["eat", "cracker", "your"]}, {"sent": "peeping outof a cracker isnt he ?", "words": ["cracker", "he", "a"]}, {"sent": "right can I have a bite of this cracker ?", "words": ["can", "of", "cracker", "have", "this", "bite", "a"]}, {"sent": "here Violet Ill give you a graham cracker .", "words": ["cracker", "you", "here", "give", "a"]}, {"sent": "hurry up and eat your cracker there .", "words": ["up", "hurry", "your", "cracker", "and", "eat", "there"]}, {"sent": "now you get the cracker .", "words": ["get", "the", "cracker", "you"]}, {"sent": "on some cracker good idea .", "words": ["on", "some", "cracker", "good"]}, {"sent": "can you say cracker ?", "words": ["can", "cracker", "say", "you"]}, {"sent": "say cracker .", "words": ["cracker", "say"]}, {"sent": "and I got you a um little cheese and cracker and grape selection .", "words": ["cracker", "little", "you", "and", "cheese", "a"]}, {"sent": "okay eat your cracker .", "words": ["eat", "cracker", "your"]}, {"sent": "remember that kind of cracker ?", "words": ["of", "cracker", "that"]}, {"sent": "you want another animal cracker ?", "words": ["another", "animal", "cracker", "you"]}, {"sent": "watch the cracker .", "words": ["the", "watch", "cracker"]}, {"sent": "xxx xxx xxx cracker xxx .", "words": ["cracker"]}, {"sent": "what is your cracker doing ?", "words": ["what", "cracker", "your", "is"]}, {"sent": "yes you may have a cracker .", "words": ["have", "cracker", "a", "you"]}, {"sent": "you dont want a graham cracker ?", "words": ["dont", "cracker", "a", "you"]}, {"sent": "you could have almond butter on your cracker .", "words": ["your", "cracker", "you", "have", "butter", "on", "could"]}, {"sent": "my cracker .", "words": ["my", "cracker"]}, {"sent": "is the cracker good ?", "words": ["the", "cracker", "good", "is"]}, {"sent": "do you want a cracker or a cookie ?", "words": ["cookie", "do", "cracker", "you", "a"]}, {"sent": "heres your cracker .", "words": ["cracker", "your"]}, {"sent": "theres another Christmas cracker here look .", "words": ["another", "cracker", "here", "look"]}, {"sent": "cracker .", "words": ["cracker"]}, {"sent": "you could have bear food on a cracker .", "words": ["cracker", "bear", "have", "you", "food", "on", "could", "a"]}, {"sent": "from meat to cracker to .", "words": ["to", "cracker", "meat"]}, {"sent": "a Christmas cracker box ?", "words": ["cracker", "box", "a"]}, {"sent": "and we got a graham cracker .", "words": ["cracker", "we", "a", "and"]}, {"sent": "and you hold one end of the Christmas cracker and Mummy holds the other and we pull pull pull pull pull till it snaps and go ow .", "words": ["the", "hold", "we", "of", "go", "cracker", "you", "and", "other", "pull", "it"]}, {"sent": "say cracker .", "words": ["cracker", "say"]}, {"sent": "say cracker .", "words": ["cracker", "say"]}, {"sent": "cracker ?", "words": ["cracker"]}, {"sent": "oh cracker and summer vegetables that sounds so delicious .", "words": ["so", "cracker", "that", "and"]}, {"sent": "uhoh there goes turtle .", "words": ["there", "turtle"]}, {"sent": "you dont like the turtle either ?", "words": ["the", "like", "you", "dont", "turtle"]}, {"sent": "did you see a turtle at the zoo ?", "words": ["the", "zoo", "you", "did", "at", "see", "turtle", "a"]}, {"sent": "when you stroke her under her turtle chin she blinks her eyes and makes small gentle sounds of pleasure .", "words": ["when", "of", "chin", "you", "and", "under", "turtle", "gentle", "her", "she"]}, {"sent": "where is your turtle ?", "words": ["where", "turtle", "your", "is"]}, {"sent": "the bird market turtle okay then lets take two books okay ?", "words": ["the", "bird", "turtle", "then", "take"]}, {"sent": "you should buy another turtle .", "words": ["another", "turtle", "buy", "you"]}, {"sent": "the turtle and this alright ?", "words": ["the", "turtle", "this", "and"]}, {"sent": "do you know which the turtle is ?", "words": ["the", "do", "is", "you", "turtle", "which"]}, {"sent": "the turtle will go home .", "words": ["the", "go", "will", "turtle", "home"]}, {"sent": "why you want the turtle ?", "words": ["the", "turtle", "why", "you"]}, {"sent": "turtle ?", "words": ["turtle"]}, {"sent": "thats what the turtle eats .", "words": ["the", "what", "turtle"]}, {"sent": "the turtle is hers the budgie is his and the dog is .", "words": ["the", "is", "hers", "and", "his", "turtle", "dog"]}, {"sent": "so that turtle has a friend .", "words": ["so", "that", "turtle", "a"]}, {"sent": "the turtle is hers the budgie is his and the dog is .", "words": ["the", "is", "hers", "and", "his", "turtle", "dog"]}, {"sent": "wanna do turtle with this ?", "words": ["do", "with", "this", "wanna", "turtle"]}, {"sent": "you pushed them all off the turtle youre hitting the turtle ?", "words": ["off", "all", "the", "them", "you", "turtle"]}, {"sent": "heres turtle .", "words": ["turtle"]}, {"sent": "thats a turtle a turtle where should I make him right here ?", "words": ["where", "here", "him", "turtle", "make", "a"]}, {"sent": "a turtle crawls like that ?", "words": ["turtle", "that", "like", "a"]}, {"sent": "a turtle ?", "words": ["turtle", "a"]}, {"sent": "the turtle is hers the budgie is his and the dog is .", "words": ["the", "is", "hers", "and", "his", "turtle", "dog"]}, {"sent": "was that turtle hungry ?", "words": ["was", "hungry", "that", "turtle"]}, {"sent": "do you know a little turtle who goes to school ?", "words": ["do", "to", "school", "little", "you", "who", "turtle", "a"]}, {"sent": "I dont think thats a turtle .", "words": ["dont", "think", "turtle", "a"]}, {"sent": "turtle .", "words": ["turtle"]}, {"sent": "the turtle is hers the budgie is his and the dog is .", "words": ["the", "is", "hers", "and", "his", "turtle", "dog"]}, {"sent": "oh no they looked out to see the turtle where did the turtle go ?", "words": ["the", "where", "go", "to", "they", "did", "out", "see", "turtle"]}, {"sent": "got your turtle ?", "words": ["turtle", "your"]}, {"sent": "animal turtle .", "words": ["turtle", "animal"]}, {"sent": "hey did you see a turtle at the zoo ?", "words": ["the", "zoo", "you", "did", "at", "see", "turtle", "a"]}, {"sent": "turtle .", "words": ["turtle"]}, {"sent": "now the blue turtle wants a turn .", "words": ["the", "turtle", "a", "blue"]}, {"sent": "is it a swimming pool with a turtle shell on its back ?", "words": ["back", "is", "with", "pool", "it", "on", "turtle", "a"]}, {"sent": "no that is a turtle .", "words": ["turtle", "that", "a", "is"]}, {"sent": "along came a turtle .", "words": ["turtle", "a"]}, {"sent": "a turtle sundae ?", "words": ["turtle", "a"]}, {"sent": "er I dont want dis I want the turtle and this .", "words": ["the", "and", "dont", "this", "turtle"]}, {"sent": "you gonna finish your turtle ?", "words": ["turtle", "your", "finish", "you"]}, {"sent": "yes a turtle .", "words": ["turtle", "a"]}, {"sent": "its a turtle .", "words": ["turtle", "a"]}, {"sent": "the turtle is hers the budgie is his and the dog is .", "words": ["the", "is", "hers", "and", "his", "turtle", "dog"]}, {"sent": "hes a spitting turtle .", "words": ["turtle", "a"]}, {"sent": "this is the basket put over the frog by the kids who picked up the net that dropped over the turtle that slid into the pond that ate the snake that dropped from the branch who swallowed .", "words": ["the", "up", "put", "into", "is", "frog", "by", "this", "that", "basket", "who", "turtle", "over"]}, {"sent": "we saw a turtle thats right .", "words": ["turtle", "a", "we"]}, {"sent": "what do you hafta do to take care of your turtle ?", "words": ["what", "do", "your", "of", "to", "you", "turtle", "take"]}, {"sent": "is that a turtle Bax ?", "words": ["turtle", "that", "a", "is"]}, {"sent": "yeah so he had those those were some of his earliest words so turtle hes seen live too .", "words": ["those", "of", "so", "he", "turtle", "too", "his", "some", "were"]}, {"sent": "turtle .", "words": ["turtle"]}, {"sent": "okay turtle .", "words": ["turtle"]}, {"sent": "a ninja turtle ?", "words": ["turtle", "a"]}, {"sent": "Im gonna go cook up some more pizza with my turtle .", "words": ["up", "my", "go", "with", "pizza", "cook", "more", "some", "turtle"]}, {"sent": "its a turtle and hes saying try the basket .", "words": ["the", "and", "basket", "turtle", "try", "a"]}, {"sent": "you have a gray turtle oh .", "words": ["have", "turtle", "a", "you"]}, {"sent": "thats a turtle .", "words": ["turtle", "a"]}, {"sent": "thats a turtle .", "words": ["turtle", "a"]}, {"sent": "a cat and a turtle makes a catle .", "words": ["cat", "turtle", "and", "a"]}, {"sent": "a turtle .", "words": ["turtle", "a"]}, {"sent": "the turtle is hers the budgie is his and the dog is .", "words": ["the", "is", "hers", "and", "his", "turtle", "dog"]}, {"sent": "you want me to be a turtle ?", "words": ["to", "be", "you", "me", "turtle", "a"]}, {"sent": "thats a turtle .", "words": ["turtle", "a"]}, {"sent": "turtle .", "words": ["turtle"]}, {"sent": "lets see theres the turtle .", "words": ["the", "see", "turtle"]}, {"sent": "turtle doesnt work .", "words": ["turtle", "work"]}, {"sent": "its a turtle .", "words": ["turtle", "a"]}, {"sent": "they cant find her oh now the birds looking for the turtle they looked everywhere they looked in parrot cages .", "words": ["the", "for", "they", "in", "find", "turtle", "her"]}, {"sent": "thats a turtle .", "words": ["turtle", "a"]}, {"sent": "come to think of it we havent fed the turtle xxx .", "words": ["the", "think", "we", "of", "to", "it", "turtle"]}, {"sent": "turtle .", "words": ["turtle"]}, {"sent": "you find a turtle ?", "words": ["turtle", "a", "find", "you"]}, {"sent": "see the turtle ?", "words": ["the", "see", "turtle"]}, {"sent": "turtle .", "words": ["turtle"]}, {"sent": "turtle .", "words": ["turtle"]}, {"sent": "turtle .", "words": ["turtle"]}, {"sent": "a turtle ?", "words": ["turtle", "a"]}, {"sent": "he takes Rosalee thats the turtle outof her basket and puts her gently down on top of the cages this is what Gestan has been waiting for he flies straight onto Rosalees back flaps his wings and in his great croaking voice he begins to call out Rosalee Rosalee .", "words": ["the", "what", "for", "back", "is", "of", "to", "and", "his", "this", "in", "he", "basket", "out", "on", "turtle", "down", "her"]}, {"sent": "see the turtle ?", "words": ["the", "see", "turtle"]}, {"sent": "I said turtle .", "words": ["turtle"]}, {"sent": "well that turtle goes on it .", "words": ["on", "turtle", "it", "that"]}, {"sent": "well its a turtle .", "words": ["turtle", "a"]}, {"sent": "turtle ?", "words": ["turtle"]}, {"sent": "this is a nice turtle huh ?", "words": ["is", "nice", "this", "turtle", "a"]}, {"sent": "a turtle whats he doing inside ?", "words": ["turtle", "inside", "he", "a"]}, {"sent": "and you got turtle ?", "words": ["turtle", "you", "and"]}, {"sent": "I have my green turtle .", "words": ["have", "turtle", "my", "green"]}, {"sent": "he just wanted a turn to play with the turtle right ?", "words": ["the", "to", "play", "with", "he", "turtle", "a"]}, {"sent": "the turtle ?", "words": ["the", "turtle"]}, {"sent": "go get the turtle .", "words": ["get", "the", "turtle", "go"]}, {"sent": "see now ya even got a turtle on your bracelet .", "words": ["your", "on", "see", "turtle", "a"]}, {"sent": "turtle .", "words": ["turtle"]}, {"sent": "a turtle .", "words": ["turtle", "a"]}, {"sent": "can you put the penny in the piggybank ?", "words": ["can", "the", "put", "you", "penny", "in"]}, {"sent": "wait what happened to your penny ?", "words": ["what", "wait", "your", "to", "penny"]}, {"sent": "Im gonna put it in a penny on his head .", "words": ["put", "head", "his", "penny", "in", "it", "on", "a"]}, {"sent": "now put the penny back in the piggybank .", "words": ["the", "put", "back", "penny", "in"]}, {"sent": "lookit the penny penny .", "words": ["the", "penny"]}, {"sent": "a purse with a penny in it .", "words": ["purse", "with", "penny", "in", "it", "a"]}, {"sent": "the penny is stuck the penny is stuck .", "words": ["the", "penny", "stuck", "is"]}, {"sent": "you give her a penny xxx a penny .", "words": ["you", "penny", "give", "her", "a"]}, {"sent": "then Ill give you a penny okay ?", "words": ["then", "you", "penny", "give", "a"]}, {"sent": "a penny in the truck ?", "words": ["the", "truck", "penny", "in", "a"]}, {"sent": "and you gave me back the penny .", "words": ["the", "back", "you", "and", "penny", "me"]}, {"sent": "he seems to be um generally pretty observant like if you show him how to do things a few times the thing with the penny was pretty interesting .", "words": ["the", "do", "how", "if", "show", "to", "be", "was", "with", "like", "you", "penny", "him", "pretty", "he", "a"]}, {"sent": "one a penny two a penny .", "words": ["penny", "a"]}, {"sent": "wait you got the penny and youre gonna dig for him ?", "words": ["the", "wait", "for", "and", "you", "penny", "him"]}, {"sent": "a penny huh ?", "words": ["penny", "a"]}, {"sent": "is that a hole in the penny ?", "words": ["the", "is", "penny", "in", "that", "a"]}, {"sent": "did that penny drink up her coffee ?", "words": ["up", "drink", "her", "penny", "coffee", "did", "that"]}, {"sent": "where did the penny go ?", "words": ["where", "the", "go", "penny", "did"]}, {"sent": "oh you see the penny .", "words": ["the", "see", "penny", "you"]}, {"sent": "put a penny in the slot .", "words": ["the", "put", "penny", "in", "a"]}, {"sent": "where is your penny .", "words": ["where", "penny", "your", "is"]}, {"sent": "did your penny fall in my coffee ?", "words": ["my", "your", "fall", "penny", "in", "coffee", "did"]}, {"sent": "penny .", "words": ["penny"]}, {"sent": "this is a penny to put in the piggybank .", "words": ["the", "put", "is", "to", "this", "penny", "in", "a"]}, {"sent": "you have a penny ?", "words": ["have", "penny", "a", "you"]}, {"sent": "that is a penny .", "words": ["penny", "that", "a", "is"]}, {"sent": "Ill get a penny out of my piggybank because I have sixteen cents by accident .", "words": ["my", "of", "by", "get", "penny", "have", "out", "because", "a"]}, {"sent": "xxx penny .", "words": ["penny"]}, {"sent": "that was no penny .", "words": ["was", "penny", "that"]}, {"sent": "a penny .", "words": ["penny", "a"]}, {"sent": "put the penny in the piggybank .", "words": ["the", "penny", "put", "in"]}, {"sent": "Ill give you a penny .", "words": ["penny", "give", "a", "you"]}, {"sent": "it is a penny .", "words": ["penny", "it", "a", "is"]}, {"sent": "baby penny .", "words": ["penny"]}, {"sent": "who gave you the penny ?", "words": ["the", "penny", "you", "who"]}, {"sent": "I gave you the last penny .", "words": ["the", "penny", "last", "you"]}, {"sent": "xxx penny .", "words": ["penny"]}, {"sent": "can you see the penny ?", "words": ["can", "the", "you", "penny", "see"]}, {"sent": "found your penny on the floor .", "words": ["on", "penny", "your", "the"]}, {"sent": "thats not a penny .", "words": ["penny", "not", "a"]}, {"sent": "how many pennies in your penny bank ?", "words": ["in", "how", "penny", "your"]}, {"sent": "thatll save you a penny on nappies .", "words": ["on", "penny", "a", "you"]}, {"sent": "is that a penny ?", "words": ["penny", "that", "a", "is"]}, {"sent": "put penny in the bank .", "words": ["the", "penny", "put", "in"]}, {"sent": "keep the penny change .", "words": ["the", "penny"]}, {"sent": "do you have a penny ?", "words": ["do", "you", "have", "penny", "a"]}, {"sent": "where did the penny go ?", "words": ["where", "the", "go", "penny", "did"]}, {"sent": "you have you put your penny in the slot .", "words": ["the", "put", "your", "you", "have", "penny", "in"]}, {"sent": "its a quarter a nickel thats thirty cents and a penny thats thirty one cents .", "words": ["penny", "and", "a"]}, {"sent": "wheres a penny ?", "words": ["penny", "a"]}, {"sent": "thanks penny .", "words": ["penny"]}, {"sent": "right one penny cant buy nothing xxx ?", "words": ["penny", "buy"]}, {"sent": "a penny for your bank .", "words": ["penny", "for", "your", "a"]}, {"sent": "says the pieman to Simple Simon show me first your penny .", "words": ["the", "your", "show", "to", "first", "penny", "me"]}, {"sent": "the penny xxx .", "words": ["the", "penny"]}, {"sent": "why dont you get a penny then if you wanna do these little games on here then then we need a coin okay ?", "words": ["do", "these", "need", "if", "we", "you", "get", "dont", "penny", "little", "here", "wanna", "on", "then", "why", "a"]}, {"sent": "there is a penny .", "words": ["there", "penny", "a", "is"]}, {"sent": "there we are another penny through the slot .", "words": ["the", "we", "another", "there", "penny", "are"]}, {"sent": "I just have one penny .", "words": ["have", "penny"]}, {"sent": "here is a penny .", "words": ["here", "penny", "a", "is"]}, {"sent": "a theres your penny .", "words": ["penny", "your", "a"]}, {"sent": "penny come here .", "words": ["here", "penny"]}, {"sent": "see mom is holding the penny .", "words": ["the", "see", "penny", "is"]}, {"sent": "youve got a penny in the piggybank .", "words": ["the", "penny", "in", "a"]}, {"sent": "a penny ?", "words": ["penny", "a"]}, {"sent": "put the penny in the piggybank .", "words": ["the", "penny", "put", "in"]}, {"sent": "can I have a penny ?", "words": ["can", "penny", "have", "a"]}, {"sent": "a nickel and a penny and a dime .", "words": ["penny", "and", "a"]}, {"sent": "wheres the penny gone ?", "words": ["the", "penny"]}, {"sent": "I think its a sort of penny .", "words": ["of", "penny", "think", "a"]}, {"sent": "not a penny .", "words": ["penny", "not", "a"]}, {"sent": "what penny ?", "words": ["what", "penny"]}, {"sent": "its a penny in there Mommy you can put a penny in there .", "words": ["can", "put", "you", "penny", "in", "there", "a"]}, {"sent": "and he has got what we call a penny tray .", "words": ["what", "we", "and", "penny", "he", "tray", "a"]}, {"sent": "find a penny ?", "words": ["penny", "a", "find"]}, {"sent": "two a penny .", "words": ["penny", "a"]}, {"sent": "what do you need a penny for ?", "words": ["what", "do", "for", "need", "you", "penny", "a"]}, {"sent": "Adam put the penny in the cup .", "words": ["the", "cup", "put", "penny", "in"]}, {"sent": "do have a penny in there ?", "words": ["do", "have", "penny", "in", "there", "a"]}, {"sent": "who put that big hole in the penny ?", "words": ["the", "put", "penny", "that", "in", "who", "big"]}, {"sent": "two a penny .", "words": ["penny", "a"]}, {"sent": "Abe found your penny on the floor .", "words": ["on", "penny", "your", "the"]}, {"sent": "one pence one penny .", "words": ["penny"]}, {"sent": "a penny .", "words": ["penny", "a"]}, {"sent": "it is a penny .", "words": ["penny", "it", "a", "is"]}, {"sent": "you were posting a penny through something .", "words": ["penny", "were", "a", "you"]}, {"sent": "well there is a penny in there .", "words": ["is", "there", "penny", "in", "a"]}, {"sent": "oh you know what here is a penny .", "words": ["what", "is", "you", "here", "penny", "a"]}, {"sent": "one penny two pennys three pennys .", "words": ["penny"]}, {"sent": "there is a pretty penny .", "words": ["is", "there", "penny", "pretty", "a"]}, {"sent": "oh this is where the penny goes .", "words": ["where", "the", "is", "this", "penny"]}, {"sent": "get each and every penny ?", "words": ["each", "and", "get", "penny", "every"]}, {"sent": "you want a penny ?", "words": ["penny", "a", "you"]}, {"sent": "Abe watch this penny .", "words": ["this", "penny", "watch"]}, {"sent": "and finally the penny clicked .", "words": ["the", "penny", "and"]}, {"sent": "and hes waiting for penny to come back and give him and feed him .", "words": ["for", "back", "to", "and", "penny", "him", "feed", "give"]}, {"sent": "yeah you can have that penny .", "words": ["can", "you", "have", "penny", "that"]}, {"sent": "well like if the paint gets if the bug catch it cant breathe .", "words": ["the", "bug", "paint", "if", "catch", "like", "it"]}, {"sent": "aw poor bug .", "words": ["bug", "poor"]}, {"sent": "you wanna bug that ?", "words": ["bug", "that", "wanna", "you"]}, {"sent": "is it does it say bug up there ?", "words": ["bug", "does", "up", "is", "say", "there", "it"]}, {"sent": "bug ouch ?", "words": ["bug"]}, {"sent": "yeah but is it the bug or the bunny that sleeps in the little bed ?", "words": ["the", "bug", "bunny", "is", "bed", "little", "in", "it", "that", "but"]}, {"sent": "bug yeah .", "words": ["bug"]}, {"sent": "probably a bug .", "words": ["bug", "a"]}, {"sent": "do I think that little bug guy is dizzy ?", "words": ["bug", "do", "think", "is", "little", "that"]}, {"sent": "the bug is walking on the apple isnt he ?", "words": ["the", "bug", "is", "he", "on", "apple"]}, {"sent": "what did he just do to that bug ?", "words": ["bug", "what", "do", "to", "he", "did", "that"]}, {"sent": "aye the stomach bug .", "words": ["the", "bug"]}, {"sent": "doodle bug doodle bug doodle doodle .", "words": ["bug"]}, {"sent": "some poison so the bug cant come back ?", "words": ["the", "bug", "back", "so", "some"]}, {"sent": "oh you almost stepped on the bug .", "words": ["on", "the", "bug", "you"]}, {"sent": "where are you going Jilly bug ?", "words": ["where", "are", "bug", "you"]}, {"sent": "footed bug .", "words": ["bug"]}, {"sent": "except that the big blue bug had a red one on or an orange .", "words": ["the", "bug", "red", "blue", "that", "an", "orange", "on", "big", "a"]}, {"sent": "a bug .", "words": ["bug", "a"]}, {"sent": "the little bug went away ?", "words": ["the", "away", "bug", "little"]}, {"sent": "do you think hed like to meet the bug ?", "words": ["the", "do", "think", "bug", "to", "like", "you"]}, {"sent": "ah that is a bug .", "words": ["bug", "that", "a", "is"]}, {"sent": "what kind of bug I wonder .", "words": ["of", "what", "bug"]}, {"sent": "ah a bug bugs .", "words": ["bug", "a"]}, {"sent": "I heard bug but maybe hes saying burgers are coming .", "words": ["bug", "are", "but"]}, {"sent": "its down in my bug ?", "words": ["bug", "in", "down", "my"]}, {"sent": "maybe it is a tiny little bug .", "words": ["tiny", "bug", "is", "little", "it", "a"]}, {"sent": "bug .", "words": ["bug"]}, {"sent": "theres a bug .", "words": ["bug", "a"]}, {"sent": "dont drop bug .", "words": ["bug", "dont", "drop"]}, {"sent": "the big bug and the little bug in beddie .", "words": ["the", "bug", "little", "and", "in", "big"]}, {"sent": "thats a spittle bug .", "words": ["bug", "a"]}, {"sent": "we didnt bring the monkeys this time Jilly bug .", "words": ["the", "bug", "we", "bring", "this"]}, {"sent": "theres the bug .", "words": ["the", "bug"]}, {"sent": "yes thats a bug .", "words": ["bug", "a"]}, {"sent": "uh bug watch .", "words": ["bug", "watch"]}, {"sent": "what kind of bug ?", "words": ["of", "what", "bug"]}, {"sent": "a little black bug doing what he oughta .", "words": ["bug", "what", "little", "black", "he", "a"]}, {"sent": "thats a bug .", "words": ["bug", "a"]}, {"sent": "large stink bug .", "words": ["bug"]}, {"sent": "some kind of bug .", "words": ["of", "some", "bug"]}, {"sent": "where is the bug ?", "words": ["where", "the", "bug", "is"]}, {"sent": "you got hurt by that bug ?", "words": ["bug", "hurt", "by", "you", "that"]}, {"sent": "show Brittany the bug .", "words": ["show", "the", "bug"]}, {"sent": "a boat and a bug a ladybug .", "words": ["bug", "boat", "and", "a"]}, {"sent": "hes dead but hes still a bug .", "words": ["bug", "but", "a"]}, {"sent": "look at that bug Alexander .", "words": ["bug", "at", "that", "look"]}, {"sent": "Im squash you like a bug Weener ?", "words": ["bug", "like", "a", "you"]}, {"sent": "a bug .", "words": ["bug", "a"]}, {"sent": "oh you like the bug one ?", "words": ["the", "bug", "like", "you"]}, {"sent": "can you say bug ?", "words": ["can", "say", "bug", "you"]}, {"sent": "lady bug .", "words": ["bug"]}, {"sent": "yeah hes gonna catch a bug aint he ?", "words": ["bug", "catch", "he", "a"]}, {"sent": "oh were gonna talk about the bug some more okay .", "words": ["the", "bug", "more", "about", "talk", "some", "were"]}, {"sent": "and I thought it was maybe just like a bug or something .", "words": ["bug", "was", "like", "and", "it", "a"]}, {"sent": "these are the bug taxis .", "words": ["the", "these", "are", "bug"]}, {"sent": "one bug is all alone .", "words": ["bug", "all", "is"]}, {"sent": "okay thats bug .", "words": ["bug"]}, {"sent": "June bug .", "words": ["bug"]}, {"sent": "you want a bug ?", "words": ["bug", "a", "you"]}, {"sent": "what kind of bug is that ?", "words": ["bug", "what", "is", "of", "that"]}, {"sent": "theres a bug .", "words": ["bug", "a"]}, {"sent": "a bug .", "words": ["bug", "a"]}, {"sent": "whats the spider going to do to the bug ?", "words": ["the", "to", "do", "bug"]}, {"sent": "bug .", "words": ["bug"]}, {"sent": "Max rescued his very favorite piece of gum on a string from the bottom of his bug box .", "words": ["the", "bug", "bottom", "box", "of", "on", "gum", "his", "a"]}, {"sent": "and this a bug .", "words": ["bug", "this", "a", "and"]}, {"sent": "mommy bug bug dey ?", "words": ["bug"]}, {"sent": "ew a bug .", "words": ["bug", "a"]}, {"sent": "there should be another bug .", "words": ["another", "there", "be", "bug"]}, {"sent": "hi bug .", "words": ["bug"]}, {"sent": "no thats the bug .", "words": ["the", "bug"]}, {"sent": "bug .", "words": ["bug"]}, {"sent": "is that bug whats he doing ?", "words": ["bug", "he", "that", "is"]}, {"sent": "what bug ?", "words": ["bug", "what"]}, {"sent": "its a bug .", "words": ["bug", "a"]}, {"sent": "bug .", "words": ["bug"]}, {"sent": "that spiders trying to eat that bug .", "words": ["eat", "to", "that", "bug"]}, {"sent": "bug bug bug those frogs are after the bugs .", "words": ["bug", "the", "are", "those"]}, {"sent": "the little bug ?", "words": ["the", "bug", "little"]}, {"sent": "aw what a little cuddle bug .", "words": ["bug", "what", "little", "a"]}, {"sent": "here sweetheart lets show Daddy the bug .", "words": ["here", "the", "bug", "show"]}, {"sent": "June bug xxx .", "words": ["bug"]}, {"sent": "and look the bug has d a little dolly bug .", "words": ["the", "bug", "and", "little", "look", "a"]}, {"sent": "tummy bug .", "words": ["bug", "tummy"]}, {"sent": "a bug in a rug .", "words": ["bug", "in", "a"]}, {"sent": "hi bug .", "words": ["bug"]}, {"sent": "the zugly bug .", "words": ["the", "bug"]}, {"sent": "no not bug .", "words": ["bug", "not"]}, {"sent": "thats a giant water bug .", "words": ["water", "bug", "a"]}, {"sent": "hes called bunny bug ?", "words": ["bug", "bunny"]}, {"sent": "Spuds the potato bug .", "words": ["the", "bug", "potato"]}, {"sent": "a black oh a little black bug .", "words": ["black", "bug", "little", "a"]}, {"sent": "oh my drink looks like it has a bug in it .", "words": ["bug", "my", "drink", "like", "in", "it", "a"]}, {"sent": "just spell bug for me .", "words": ["bug", "me", "for"]}, {"sent": "can you look for it where we had where we found the main part of the baby bug ?", "words": ["can", "where", "the", "for", "bug", "we", "of", "you", "it", "look"]}, {"sent": "oh I know thats that bug Thomas .", "words": ["bug", "that"]}, {"sent": "is kitty inside the house ?", "words": ["the", "is", "inside", "kitty", "house"]}, {"sent": "do you think that theyd really wanna get that kitty cat back now ?", "words": ["do", "think", "back", "you", "get", "cat", "kitty", "wanna", "that"]}, {"sent": "yep and what does the kitty say ?", "words": ["the", "what", "does", "say", "and", "kitty"]}, {"sent": "kitty .", "words": ["kitty"]}, {"sent": "there is no kitty .", "words": ["there", "kitty", "is"]}, {"sent": "theres a big kitty in there .", "words": ["there", "in", "kitty", "big", "a"]}, {"sent": "somebody catch that kitty ?", "words": ["kitty", "catch", "that"]}, {"sent": "wheres the kitty ?", "words": ["the", "kitty"]}, {"sent": "is kitty in the pigpen ?", "words": ["the", "in", "kitty", "is"]}, {"sent": "theres a kitty yes .", "words": ["kitty", "a"]}, {"sent": "oh have you seen kitty put it in her mouth ?", "words": ["put", "mouth", "you", "have", "in", "it", "kitty", "her"]}, {"sent": "he didnt run after the kitty .", "words": ["the", "run", "kitty", "he"]}, {"sent": "no its not a kitty .", "words": ["kitty", "not", "a"]}, {"sent": "poor kitty .", "words": ["kitty", "poor"]}, {"sent": "which kitty ?", "words": ["kitty", "which"]}, {"sent": "is there a kitty cat who likes to play with the mouse ?", "words": ["the", "is", "to", "play", "with", "there", "cat", "kitty", "who", "mouse", "a"]}, {"sent": "hi kitty .", "words": ["kitty"]}, {"sent": "come here kitty kitty kitty .", "words": ["here", "kitty"]}, {"sent": "kitty .", "words": ["kitty"]}, {"sent": "you cannot go get that kitty under there because theres dangerous things under there .", "words": ["go", "you", "get", "under", "there", "kitty", "that", "because"]}, {"sent": "I got the kitty cat sounds again .", "words": ["the", "cat", "kitty"]}, {"sent": "yeh where did kitty go ?", "words": ["where", "kitty", "did", "go"]}, {"sent": "you know what the kitty is doing ?", "words": ["the", "what", "is", "you", "kitty"]}, {"sent": "you see the kitty ?", "words": ["the", "see", "kitty", "you"]}, {"sent": "in the farm there was a kitty .", "words": ["the", "was", "there", "in", "kitty", "a"]}, {"sent": "theres a kitty .", "words": ["kitty", "a"]}, {"sent": "thats the eye of the kitty .", "words": ["the", "of", "kitty", "eye"]}, {"sent": "you want to do the kitty ?", "words": ["the", "do", "to", "you", "kitty", "want to"]}, {"sent": "what a nice kitty .", "words": ["what", "nice", "kitty", "a"]}, {"sent": "and the policeman leans forward to quickly grab the kitty and the kitty runs away .", "words": ["the", "to", "and", "kitty", "away"]}, {"sent": "whats the kitty say ?", "words": ["the", "kitty", "say"]}, {"sent": "its a kitty cat with a fiddle .", "words": ["cat", "kitty", "with", "a"]}, {"sent": "do you wanna give some fish to the kitty ?", "words": ["the", "do", "to", "you", "kitty", "wanna", "fish", "some", "give"]}, {"sent": "and the kitty jumped away .", "words": ["the", "kitty", "away", "and"]}, {"sent": "kitty .", "words": ["kitty"]}, {"sent": "kitty .", "words": ["kitty"]}, {"sent": "still blowing away and this little kitty is watching all the leaves blow away .", "words": ["the", "all", "is", "little", "and", "this", "away", "kitty", "blow"]}, {"sent": "say byebye kitty .", "words": ["kitty", "say"]}, {"sent": "all the birds are yelling at the kitty huh ?", "words": ["the", "all", "are", "kitty", "at"]}, {"sent": "you wanna lay your head down on the kitty ?", "words": ["the", "your", "head", "you", "kitty", "wanna", "on", "down"]}, {"sent": "and kitty kitty .", "words": ["kitty", "and"]}, {"sent": "its a fake kitty .", "words": ["kitty", "a"]}, {"sent": "kitty .", "words": ["kitty"]}, {"sent": "here kitty kitty kitty kitty here kitty kitty kitty .", "words": ["here", "kitty"]}, {"sent": "come here and get the kitty .", "words": ["the", "and", "here", "get", "kitty"]}, {"sent": "whats the kitty doing ?", "words": ["the", "kitty"]}, {"sent": "why dont you go get that kitty ?", "words": ["go", "you", "get", "dont", "kitty", "that", "why"]}, {"sent": "wheres the kitty ?", "words": ["the", "kitty"]}, {"sent": "and then you helped them to get better from their little bites from the kitty ?", "words": ["the", "better", "their", "to", "them", "you", "and", "get", "little", "kitty", "then"]}, {"sent": "yeah real kitty .", "words": ["kitty"]}, {"sent": "is kitty eating ?", "words": ["kitty", "is"]}, {"sent": "kitty .", "words": ["kitty"]}, {"sent": "this kitty ?", "words": ["this", "kitty"]}, {"sent": "wheres the kitty Theo ?", "words": ["the", "kitty"]}, {"sent": "kitty kitty kitty kitty kit wid .", "words": ["kitty"]}, {"sent": "is that the kitty ?", "words": ["the", "kitty", "that", "is"]}, {"sent": "you can see the kitty from here .", "words": ["can", "the", "you", "here", "kitty", "see"]}, {"sent": "Brittany youre being ugly to the kitty now .", "words": ["the", "to", "kitty"]}, {"sent": "kitty cat Dannys got it .", "words": ["cat", "kitty", "it"]}, {"sent": "kitty kitty kitty .", "words": ["kitty"]}, {"sent": "kitty kitty widdy widdy wid .", "words": ["kitty"]}, {"sent": "is that your kitty ?", "words": ["your", "kitty", "that", "is"]}, {"sent": "kitty kitty kitty .", "words": ["kitty"]}, {"sent": "that one looks like a kitty .", "words": ["kitty", "that", "like", "a"]}, {"sent": "say here kitty kitty kitty .", "words": ["here", "kitty", "say"]}, {"sent": "how about the kitty ?", "words": ["the", "how", "kitty", "about"]}, {"sent": "kitty .", "words": ["kitty"]}, {"sent": "Kim say kitty .", "words": ["kitty", "say"]}, {"sent": "off the kitty .", "words": ["off", "the", "kitty"]}, {"sent": "no kitty doesnt want that .", "words": ["kitty", "that"]}, {"sent": "thats your favorite kitty Cheeto .", "words": ["kitty", "your"]}, {"sent": "you think theres a kitty inside the tent ?", "words": ["the", "think", "inside", "you", "kitty", "a"]}, {"sent": "kitty cat .", "words": ["cat", "kitty"]}, {"sent": "I hear the kitty .", "words": ["the", "kitty", "hear"]}, {"sent": "wheres your kitty go ?", "words": ["kitty", "your", "go"]}, {"sent": "the kitty he didnt fall in yet .", "words": ["the", "fall", "in", "kitty", "he"]}, {"sent": "I think youre showing off for that kitty .", "words": ["off", "think", "for", "kitty", "that"]}, {"sent": "pet the kitty .", "words": ["the", "kitty"]}, {"sent": "kitty cat .", "words": ["cat", "kitty"]}, {"sent": "theres the kitty kitty .", "words": ["the", "kitty"]}, {"sent": "kitty is in the bag .", "words": ["the", "in", "kitty", "is"]}, {"sent": "kitty .", "words": ["kitty"]}, {"sent": "well the kitty cats up the tree .", "words": ["the", "tree", "kitty", "up"]}, {"sent": "kitty kitty .", "words": ["kitty"]}, {"sent": "you like that kitty .", "words": ["kitty", "that", "like", "you"]}, {"sent": "the other little kitty cat said Im running .", "words": ["the", "little", "other", "cat", "kitty"]}, {"sent": "and the kitty says meow .", "words": ["the", "kitty", "and"]}, {"sent": "yes kitty yes kitty three sleepy kids .", "words": ["sleepy", "kitty"]}, {"sent": "nice kitty .", "words": ["kitty", "nice"]}, {"sent": "those kitty cats ?", "words": ["kitty", "those"]}, {"sent": "oh you a little kitty cat ?", "words": ["little", "you", "cat", "kitty", "a"]}, {"sent": "do you like kitty cat ?", "words": ["do", "like", "you", "cat", "kitty"]}, {"sent": "is there a kitty there ?", "words": ["there", "kitty", "a", "is"]}, {"sent": "can you show me the kitty ?", "words": ["can", "the", "show", "you", "kitty", "me"]}, {"sent": "see the kitty ?", "words": ["the", "see", "kitty"]}, {"sent": "here comes the kitty .", "words": ["here", "the", "kitty"]}, {"sent": "theres a kitty in the basket .", "words": ["the", "in", "kitty", "basket", "a"]}, {"sent": "no its not a kitty .", "words": ["kitty", "not", "a"]}, {"sent": "and you wrote on the pen and you gave it to AuntRoberta .", "words": ["the", "to", "you", "and", "it", "pen", "on"]}, {"sent": "has he eaten Jeannines pen ?", "words": ["he", "pen"]}, {"sent": "no thats not a pen .", "words": ["pen", "not", "a"]}, {"sent": "Mummys pen .", "words": ["pen"]}, {"sent": "pen .", "words": ["pen"]}, {"sent": "Mummy has lost her pen now .", "words": ["her", "pen"]}, {"sent": "would you like a pen ?", "words": ["like", "would", "you", "pen", "a"]}, {"sent": "let me just put the lid on my fountain pen and let me draw xxx .", "words": ["the", "put", "my", "draw", "and", "pen", "on", "me"]}, {"sent": "now you hold on this pen and well write Thomas .", "words": ["hold", "write", "you", "and", "this", "pen", "on"]}, {"sent": "wheres my pen ?", "words": ["my", "pen"]}, {"sent": "can I just get this pen out Fraser ?", "words": ["can", "get", "this", "out", "pen"]}, {"sent": "you want the pen to do it ?", "words": ["the", "do", "to", "you", "it", "pen"]}, {"sent": "with that pen ?", "words": ["that", "with", "pen"]}, {"sent": "you broke my last pen .", "words": ["pen", "my", "last", "you"]}, {"sent": "lets leave the pen up here .", "words": ["the", "up", "here", "pen"]}, {"sent": "we could spell some things with a pen .", "words": ["we", "with", "pen", "some", "could", "a"]}, {"sent": "get a big pen .", "words": ["get", "pen", "big", "a"]}, {"sent": "get another pen .", "words": ["get", "another", "pen"]}, {"sent": "because then youll break the pen .", "words": ["the", "pen", "then", "break", "because"]}, {"sent": "yeah just while were playing with the pens because you get pen everywhere .", "words": ["the", "with", "you", "get", "pen", "were", "because"]}, {"sent": "theres a pen cap but theres no pen .", "words": ["pen", "but", "a"]}, {"sent": "its my pen .", "words": ["my", "pen"]}, {"sent": "you want a pen ?", "words": ["pen", "a", "you"]}, {"sent": "I want ta go get another pen .", "words": ["get", "pen", "another", "go"]}, {"sent": "Eleanor this is not a pen is it ?", "words": ["is", "this", "it", "pen", "not", "a"]}, {"sent": "Ill hafta take the pen away if you do that .", "words": ["the", "do", "if", "you", "that", "away", "pen", "take"]}, {"sent": "I got a pen ?", "words": ["pen", "a"]}, {"sent": "its a play pen .", "words": ["pen", "play", "play pen", "a"]}, {"sent": "a pen a pen .", "words": ["pen", "a"]}, {"sent": "your pen rather .", "words": ["your", "pen"]}, {"sent": "have a pen .", "words": ["have", "pen", "a"]}, {"sent": "are you getting in that pen again ?", "words": ["you", "in", "are", "pen", "that"]}, {"sent": "yeah its your own special pen for writing your name .", "words": ["for", "your", "pen"]}, {"sent": "theres your pen .", "words": ["your", "pen"]}, {"sent": "put the pen down when youre doing that .", "words": ["the", "put", "that", "pen", "down", "when"]}, {"sent": "wheres my pen ?", "words": ["my", "pen"]}, {"sent": "written in gold pen .", "words": ["in", "pen"]}, {"sent": "youd like a pen to draw on some envelopes .", "words": ["to", "draw", "like", "pen", "on", "some", "a"]}, {"sent": "pen .", "words": ["pen"]}, {"sent": "I done closed my pen up .", "words": ["up", "my", "pen"]}, {"sent": "when you get your pen built .", "words": ["your", "you", "get", "pen", "when"]}, {"sent": "is that your pen ?", "words": ["your", "pen", "that", "is"]}, {"sent": "she just Lois took the pencil away so Lois so Jenny wouldnt play with the pen .", "words": ["the", "so", "play", "with", "pencil", "away", "pen", "she"]}, {"sent": "you wanna hold the pen ?", "words": ["the", "hold", "you", "wanna", "pen"]}, {"sent": "what do you want a pen for ?", "words": ["what", "do", "for", "you", "pen", "a"]}, {"sent": "and Ill get a pen .", "words": ["get", "pen", "a", "and"]}, {"sent": "you mustnt put pen in your mouth .", "words": ["put", "your", "mouth", "you", "in", "pen"]}, {"sent": "wheres another pen ?", "words": ["another", "pen"]}, {"sent": "can you get me the pen ?", "words": ["can", "the", "you", "get", "pen", "me"]}, {"sent": "you gonna put it in his pen ?", "words": ["put", "you", "in", "it", "pen", "his"]}, {"sent": "I told you not to get pen on your clothes Thomas .", "words": ["your", "to", "you", "get", "pen", "on", "not"]}, {"sent": "youve got pen on your chin now .", "words": ["on", "your", "chin", "pen"]}, {"sent": "its a pen mark .", "words": ["pen", "a"]}, {"sent": "how did she manage to get pen on her I wonder .", "words": ["how", "to", "get", "did", "pen", "on", "her", "she"]}, {"sent": "make a pattern with a pen .", "words": ["pen", "make", "with", "a"]}, {"sent": "you want ta give me my pen while you look for another one ?", "words": ["my", "look", "for", "you", "another", "pen", "me", "give"]}, {"sent": "gimme a pen .", "words": ["pen", "a"]}, {"sent": "put the pen in your hand .", "words": ["the", "put", "your", "hand", "in", "pen"]}, {"sent": "is that a purple pen ?", "words": ["pen", "that", "a", "is"]}, {"sent": "shall we give zep a pen ?", "words": ["pen", "give", "a", "we"]}, {"sent": "pen cap can you put it on ?", "words": ["can", "put", "you", "it", "pen", "on"]}, {"sent": "pen .", "words": ["pen"]}, {"sent": "okay let me get a pen for myself .", "words": ["for", "get", "myself", "pen", "me", "a"]}, {"sent": "alright you put the top back on that pen .", "words": ["the", "put", "back", "you", "pen", "on", "that"]}, {"sent": "I accidenty took your pen .", "words": ["your", "pen"]}, {"sent": "theres a pen .", "words": ["pen", "a"]}, {"sent": "why dont you bring a piece of paper and pen ?", "words": ["of", "paper", "bring", "you", "and", "dont", "pen", "why", "a"]}, {"sent": "are you giving the pen to BirthdayBear ?", "words": ["the", "to", "you", "are", "pen"]}, {"sent": "the pen has fallen down has it ?", "words": ["the", "it", "down", "pen"]}, {"sent": "youve never used a pen before .", "words": ["pen", "a"]}, {"sent": "why dont you get a pen and you can draw her eyes and her ears on .", "words": ["can", "draw", "you", "get", "dont", "and", "pen", "on", "her", "why", "a"]}, {"sent": "now youve broken Daddys pen .", "words": ["pen", "broken"]}, {"sent": "no Kalie what I meant was heres a pen .", "words": ["was", "what", "pen", "a"]}, {"sent": "wheres the pen gone Lara ?", "words": ["the", "pen"]}, {"sent": "a pen .", "words": ["pen", "a"]}, {"sent": "this is your pen .", "words": ["this", "pen", "your", "is"]}, {"sent": "let me have a look at this pen .", "words": ["pen", "have", "this", "at", "look", "me", "a"]}, {"sent": "can you write to Mama from Eleanor on there with a pen ?", "words": ["can", "to", "with", "write", "you", "there", "pen", "on", "a"]}, {"sent": "is the pen off or on ?", "words": ["the", "off", "is", "pen", "on"]}, {"sent": "she got a pen ?", "words": ["pen", "she", "a"]}, {"sent": "figure to try with the other pen ?", "words": ["the", "to", "with", "other", "pen", "try"]}, {"sent": "you asked for the blue pen especially .", "words": ["the", "for", "blue", "you", "pen"]}, {"sent": "you found my pen .", "words": ["pen", "my", "you"]}, {"sent": "no pen .", "words": ["pen"]}, {"sent": "would you like to use the red pen ?", "words": ["the", "red", "to", "pen", "like", "you", "would"]}, {"sent": "another pen in the box .", "words": ["the", "box", "another", "in", "pen"]}, {"sent": "what are you doing with that pen ?", "words": ["what", "with", "you", "are", "pen", "that"]}, {"sent": "let me see if I can find a pen for you to do it .", "words": ["can", "do", "for", "if", "to", "me", "pen", "you", "it", "find", "see", "a"]}, {"sent": "did you see a black pen ?", "words": ["you", "black", "did", "pen", "see", "a"]}, {"sent": "Mummys pen .", "words": ["pen"]}, {"sent": "oh who gave you this pen ?", "words": ["this", "pen", "you", "who"]}, {"sent": "right hold the pen properly like Dada says .", "words": ["the", "hold", "like", "pen"]}, {"sent": "whos pen ?", "words": ["pen"]}, {"sent": "would you like Carolines pen ?", "words": ["pen", "like", "you", "would"]}, {"sent": "youve dropped your pen .", "words": ["your", "pen"]}, {"sent": "Ill give you a pen in a minute .", "words": ["you", "in", "pen", "give", "a"]}, {"sent": "pass the pig pen .", "words": ["the", "pen", "pig"]}, {"sent": "Is it ice cream ?", "words": ["ice", "ice cream", "it"]}, {"sent": "ice cream ?", "words": ["ice", "ice cream"]}, {"sent": "she wont have um ice cream with wa um with the ice .", "words": ["the", "ice cream", "with", "have", "ice", "she"]}, {"sent": "the ice cream cone .", "words": ["the", "ice cream", "ice"]}, {"sent": "whos going to get the ice cream cone ?", "words": ["the", "ice cream", "to", "get", "ice"]}, {"sent": "they going on the beach or are they having an ice cream ?", "words": ["the", "ice cream", "ice", "they", "beach", "are", "an", "on"]}, {"sent": "ice cream ?", "words": ["ice", "ice cream"]}, {"sent": "yeah ice cream cone ?", "words": ["ice", "ice cream"]}, {"sent": "recognize the ice cream cone ?", "words": ["the", "ice cream", "ice"]}, {"sent": "mm ice cream .", "words": ["ice", "ice cream"]}, {"sent": "and let see what other kind of ice cream .", "words": ["what", "ice cream", "of", "other", "and", "ice", "see"]}, {"sent": "you could work as an ice cream man couldnt you ?", "words": ["ice cream", "work", "you", "an", "ice", "could"]}, {"sent": "my ice cream van has broken down .", "words": ["ice cream", "my", "broken", "ice", "down"]}, {"sent": "are you cooking ice cream ?", "words": ["ice", "ice cream", "are", "you"]}, {"sent": "in the ice cream ?", "words": ["the", "ice cream", "in", "ice"]}, {"sent": "what kind of ice cream are we eating ?", "words": ["what", "ice cream", "we", "of", "are", "ice"]}, {"sent": "oh ice cream cone .", "words": ["ice", "ice cream"]}, {"sent": "do you like ice cream ?", "words": ["do", "ice cream", "like", "you", "ice"]}, {"sent": "ice cream well well have that for dessert with our cookie right ?", "words": ["cookie", "ice cream", "for", "with", "our", "have", "ice", "that"]}, {"sent": "okay wheres the ice cream .", "words": ["the", "ice cream", "ice"]}, {"sent": "that is ice cream .", "words": ["ice", "ice cream", "that", "is"]}, {"sent": "thats the ice cream and the biscuit is called the cone .", "words": ["the", "ice cream", "is", "and", "ice"]}, {"sent": "ice cream ?", "words": ["ice", "ice cream"]}, {"sent": "heres the lady ice cream .", "words": ["the", "ice cream", "ice"]}, {"sent": "is that ice cream ?", "words": ["ice", "ice cream", "that", "is"]}, {"sent": "ice cream .", "words": ["ice", "ice cream"]}, {"sent": "what kind of ice cream ?", "words": ["of", "what", "ice", "ice cream"]}, {"sent": "ice cream .", "words": ["ice", "ice cream"]}, {"sent": "the ice cream bus van has come ?", "words": ["the", "ice cream", "ice", "bus"]}, {"sent": "oh heres the ice cream cone .", "words": ["the", "ice cream", "ice"]}, {"sent": "how do you make the ice cream ?", "words": ["the", "do", "how", "ice cream", "you", "ice", "make"]}, {"sent": "xxx hes looking for little children to run up to want ice cream .", "words": ["up", "ice cream", "for", "to", "little", "run", "ice"]}, {"sent": "on melted ice cream .", "words": ["on", "ice cream", "ice"]}, {"sent": "Id like an ice cream for the little boy .", "words": ["the", "ice cream", "for", "like", "little", "an", "ice"]}, {"sent": "you want some ice cream too .", "words": ["ice cream", "you", "too", "ice", "some"]}, {"sent": "you got an ice cream sandwich .", "words": ["ice cream", "you", "an", "ice", "sandwich"]}, {"sent": "put the ice cream on the cone ?", "words": ["the", "ice cream", "put", "ice", "on"]}, {"sent": "ice cream sandwich huh ?", "words": ["ice", "ice cream", "sandwich"]}, {"sent": "um like youknow I think that hell like recognize the pizza and ice cream .", "words": ["the", "ice cream", "think", "like", "pizza", "and", "ice", "that"]}, {"sent": "oh youd like some ice cream to go with your cookie ?", "words": ["cookie", "ice cream", "your", "go", "to", "with", "like", "ice", "some"]}, {"sent": "thats an ice cream cone .", "words": ["ice", "ice cream", "an"]}, {"sent": "is this my ice cream ?", "words": ["ice cream", "my", "is", "this", "ice"]}, {"sent": "um but we dont like eat ice cream on a cone .", "words": ["ice cream", "we", "like", "eat", "dont", "on", "ice", "but", "a"]}, {"sent": "if I eat sardines and ice cream I feel grouchy all day .", "words": ["all", "ice cream", "if", "and", "eat", "ice"]}, {"sent": "wheres the ice cream one ?", "words": ["the", "ice cream", "ice"]}, {"sent": "I want ice cream with cookies in it .", "words": ["ice cream", "with", "in", "it", "ice"]}, {"sent": "why are you putting the ice cream in the oven ?", "words": ["the", "ice cream", "you", "in", "are", "ice", "oven", "why"]}, {"sent": "look at all this ice cream .", "words": ["all", "ice cream", "look", "this", "at", "ice"]}, {"sent": "is that ice cream cone better than a slice of pizza ?", "words": ["better", "ice cream", "is", "of", "pizza", "ice", "that", "a"]}, {"sent": "not the ice cream though .", "words": ["the", "ice cream", "ice", "not"]}, {"sent": "it does sort of look like ice cream .", "words": ["does", "ice cream", "of", "like", "it", "look", "ice"]}, {"sent": "oh mommy has ice cream too .", "words": ["ice", "ice cream", "too"]}, {"sent": "yeah you want the ice cream .", "words": ["the", "ice cream", "ice", "you"]}, {"sent": "the ice cream factory ?", "words": ["the", "ice cream", "ice"]}, {"sent": "they can sit on there and eat their dinner their ice cream .", "words": ["can", "ice cream", "their", "ice", "and", "eat", "they", "there", "on", "sit"]}, {"sent": "you sure like ice cream dont you ?", "words": ["ice cream", "like", "you", "dont", "ice"]}, {"sent": "theres no ice cream .", "words": ["ice", "ice cream"]}, {"sent": "it was an amazing ice cream cone .", "words": ["ice cream", "was", "an", "it", "ice"]}, {"sent": "just checked the ice cream havent we ?", "words": ["the", "ice cream", "ice", "we"]}, {"sent": "ice cream .", "words": ["ice", "ice cream"]}, {"sent": "recognize ice cream ?", "words": ["ice", "ice cream"]}, {"sent": "and you spotted the ice cream with the chocolate flake sitting in the top .", "words": ["the", "ice cream", "chocolate", "with", "you", "and", "in", "ice"]}, {"sent": "otherwise all the ice cream runs out doesnt it ?", "words": ["the", "all", "ice cream", "it", "out", "ice"]}, {"sent": "ice cream .", "words": ["ice", "ice cream"]}, {"sent": "Ernie gets Rubber Duckie and an ice cream .", "words": ["ice", "ice cream", "an", "and"]}, {"sent": "I think I can hear the ice cream man .", "words": ["can", "the", "think", "ice cream", "hear", "ice"]}, {"sent": "going to have some ice cream .", "words": ["ice cream", "to", "have", "ice", "some"]}, {"sent": "For strawberry ice cream ?", "words": ["ice", "ice cream", "strawberry"]}, {"sent": "there we are theres an ice cream cone .", "words": ["ice cream", "we", "there", "an", "are", "ice"]}, {"sent": "baby likes the ice cream in her hair .", "words": ["the", "ice cream", "in", "ice", "hair", "her"]}, {"sent": "ice cream .", "words": ["ice", "ice cream"]}, {"sent": "eating an ice cream and drinking lemonade .", "words": ["ice", "ice cream", "an", "and"]}, {"sent": "dont ask me for ice cream or anything like that later .", "words": ["ice cream", "for", "like", "dont", "ice", "me", "that"]}, {"sent": "um ice cream .", "words": ["ice", "ice cream"]}, {"sent": "wheres the ice cream ?", "words": ["the", "ice cream", "ice"]}, {"sent": "are you done with that ice cream already ?", "words": ["ice cream", "with", "you", "are", "ice", "that"]}, {"sent": "ice cream .", "words": ["ice", "ice cream"]}, {"sent": "grape soda spam beets and ice cream .", "words": ["ice", "ice cream", "soda", "and"]}, {"sent": "Nathaniel put his spoon in the ice cream .", "words": ["the", "ice cream", "put", "ice", "in", "spoon", "his"]}, {"sent": "theres no ice cream .", "words": ["ice", "ice cream"]}, {"sent": "it was nice ice cream that wasnt it ?", "words": ["ice cream", "was", "nice", "it", "ice", "that"]}, {"sent": "I can hear the ice cream van .", "words": ["can", "the", "ice cream", "hear", "ice"]}, {"sent": "you had ice cream .", "words": ["ice", "ice cream", "you"]}, {"sent": "can you make it look like ice cream ?", "words": ["can", "ice cream", "like", "you", "it", "look", "ice", "make"]}, {"sent": "ice cream .", "words": ["ice", "ice cream"]}, {"sent": "the ice cream cone goes with it .", "words": ["the", "ice cream", "with", "it", "ice"]}, {"sent": "he ate through one piece of chocolate cake one ice cream cone one pickle one slice of Swiss cheese one slice of salami one lollipop one piece of cherry pie one sausage one cupcake and one slice of watermelon .", "words": ["ice cream", "cake", "of", "chocolate", "pickle", "lollipop", "and", "he", "ice", "cheese"]}, {"sent": "youre not gonna get to have a picture of you eating your ice cream .", "words": ["ice cream", "your", "of", "to", "picture", "you", "get", "have", "ice", "not", "a"]}, {"sent": "you gonna eat the hamburger and the ice cream ?", "words": ["the", "ice cream", "hamburger", "eat", "and", "you", "ice"]}, {"sent": "theyre buying ice cream from an ice cream van .", "words": ["ice", "ice cream", "an"]}, {"sent": "you gonna put ice cream on it ?", "words": ["ice cream", "put", "you", "on", "it", "ice"]}, {"sent": "before it melts like an ice cream .", "words": ["ice cream", "like", "an", "it", "ice"]}, {"sent": "that aint chairs thats for ice cream .", "words": ["ice", "ice cream", "for", "that"]}, {"sent": "we go to ice cream how do we get to the ice cream shop ?", "words": ["the", "do", "how", "ice cream", "go", "we", "to", "get", "ice"]}, {"sent": "behind the rocking chair .", "words": ["the", "rocking chair", "behind", "chair"]}, {"sent": "thats a rocking chair .", "words": ["rocking chair", "chair", "a"]}, {"sent": "come lets sit on this nice rocking chair and have a cuddle .", "words": ["nice", "rocking chair", "chair", "and", "have", "this", "on", "sit", "a"]}, {"sent": "can you put the baby in the rocking chair ?", "words": ["can", "the", "put", "rocking chair", "chair", "you", "in"]}, {"sent": "rocking chair ?", "words": ["rocking chair", "chair"]}, {"sent": "wait a minute this rocking chair is too what ?", "words": ["what", "wait", "is", "rocking chair", "chair", "this", "too", "a"]}, {"sent": "here is your own little rocking chair .", "words": ["your", "is", "rocking chair", "chair", "little", "here"]}, {"sent": "are you behind the rocking chair ?", "words": ["the", "rocking chair", "behind", "chair", "you", "are"]}, {"sent": "mm shes in a rocking chair .", "words": ["in", "rocking chair", "chair", "a"]}, {"sent": "on her rocking chair .", "words": ["on", "rocking chair", "her", "chair"]}, {"sent": "rocking chair ?", "words": ["rocking chair", "chair"]}, {"sent": "rocking chair .", "words": ["rocking chair", "chair"]}, {"sent": "BirthdayBear back in the rocking chair .", "words": ["the", "back", "rocking chair", "chair", "in"]}, {"sent": "are you gonna sit in your rocking chair ?", "words": ["your", "rocking chair", "chair", "you", "in", "are", "sit"]}, {"sent": "I spy a rocking chair .", "words": ["rocking chair", "chair", "a"]}, {"sent": "and then you went and rock the rocking chair .", "words": ["the", "rocking chair", "chair", "you", "and", "then", "rock"]}, {"sent": "but it isnt a rocking chair .", "words": ["rocking chair", "chair", "it", "but", "a"]}, {"sent": "rocking rocking a rocking chair .", "words": ["rocking chair", "chair", "a"]}, {"sent": "its behind the rocking chair .", "words": ["the", "rocking chair", "behind", "chair"]}, {"sent": "thank you for the presents that you put in the rocking chair .", "words": ["the", "put", "for", "rocking chair", "chair", "you", "in", "that"]}, {"sent": "and the rocking chair xxx .", "words": ["the", "rocking chair", "chair", "and"]}, {"sent": "its over there by your rocking chair .", "words": ["your", "rocking chair", "chair", "by", "there", "over"]}, {"sent": "its there by your rocking chair .", "words": ["your", "rocking chair", "chair", "by", "there"]}, {"sent": "it is fun to put the pillow in the rocking chair is not it .", "words": ["the", "put", "is", "to", "rocking chair", "chair", "in", "it", "not", "pillow"]}, {"sent": "its a wood rocking chair .", "words": ["rocking chair", "chair", "a"]}, {"sent": "you have a funny way of playing with that pillow in that rocking chair .", "words": ["pillow", "of", "rocking chair", "with", "chair", "you", "have", "in", "that", "a"]}, {"sent": "we hafta get out of this rocking chair dont we .", "words": ["we", "of", "rocking chair", "chair", "get", "this", "dont", "out"]}, {"sent": "KingFreddies sitting in the rocking chair .", "words": ["the", "in", "rocking chair", "chair"]}, {"sent": "Stefan rocking rocking in the rocking chair .", "words": ["the", "in", "rocking chair", "chair"]}, {"sent": "does Birthday Bear like the rocking chair ?", "words": ["the", "does", "rocking chair", "like", "chair"]}, {"sent": "I can see Thomass face behind the rocking chair .", "words": ["can", "the", "face", "rocking chair", "behind", "chair", "see"]}, {"sent": "nice to rock in rocking chair .", "words": ["to", "nice", "rocking chair", "chair", "in", "rock"]}, {"sent": "rocking chair .", "words": ["rocking chair", "chair"]}, {"sent": "you wanna go in the rocking chair ?", "words": ["the", "go", "rocking chair", "chair", "you", "in", "wanna"]}, {"sent": "rocking chair ?", "words": ["rocking chair", "chair"]}, {"sent": "oh whos gonna sit in the rocking chair ?", "words": ["the", "rocking chair", "chair", "in", "sit"]}, {"sent": "she climbed into the little rocking chair .", "words": ["the", "into", "rocking chair", "chair", "little", "she"]}, {"sent": "thats what do you do with the rocking chair ?", "words": ["the", "what", "do", "rocking chair", "with", "chair", "you"]}, {"sent": "because you asked FatherChristmas to leave your presents in the rocking chair didnt you ?", "words": ["the", "your", "to", "rocking chair", "chair", "you", "in", "because"]}, {"sent": "I will rock you on the rocking chair just one second Honey .", "words": ["the", "rocking chair", "chair", "you", "on", "will", "rock"]}, {"sent": "and she said this rocking chair is too .", "words": ["is", "rocking chair", "chair", "and", "this", "too", "she"]}, {"sent": "you wanna rock on the rocking chair honey ?", "words": ["the", "rocking chair", "chair", "you", "wanna", "on", "rock"]}, {"sent": "Im on the rocking chair .", "words": ["on", "the", "rocking chair", "chair"]}, {"sent": "I like this rocking chair .", "words": ["this", "rocking chair", "like", "chair"]}, {"sent": "rocking in the rocking chair .", "words": ["the", "in", "rocking chair", "chair"]}, {"sent": "but it went around the back of the rocking chair .", "words": ["the", "back", "of", "rocking chair", "chair", "it", "around", "but"]}, {"sent": "oh are we having a rock on the rocking chair ?", "words": ["the", "we", "rocking chair", "chair", "are", "on", "rock", "a"]}, {"sent": "I push the rocking chair .", "words": ["the", "rocking chair", "push", "chair"]}, {"sent": "want Mama to put you on the rocking chair ?", "words": ["the", "put", "to", "rocking chair", "chair", "you", "on"]}, {"sent": "rocking chair .", "words": ["rocking chair", "chair"]}, {"sent": "that rocking chair ?", "words": ["rocking chair", "that", "chair"]}, {"sent": "whose rocking chair is this ?", "words": ["this", "rocking chair", "chair", "is"]}, {"sent": "are you gonna sit in that rocking chair ?", "words": ["rocking chair", "chair", "you", "in", "are", "sit", "that"]}, {"sent": "oh be careful now because if youre sitting in the rocking chair and the cushion isnt on it you might fall off .", "words": ["the", "off", "if", "be", "rocking chair", "fall", "chair", "and", "you", "in", "it", "careful", "on", "because"]}, {"sent": "he hasnt figured out yet that his tray and rocking chair dont quite fit together .", "words": ["fit", "rocking chair", "chair", "and", "dont", "he", "out", "tray", "his", "that"]}, {"sent": "shes rocking in that little rocking chair for a long time right ?", "words": ["for", "rocking chair", "chair", "little", "long", "in", "that", "a"]}, {"sent": "okay heres a rocking chair .", "words": ["rocking chair", "chair", "a"]}, {"sent": "that are on the rocking chair .", "words": ["the", "rocking chair", "chair", "are", "on", "that"]}, {"sent": "diyou hit it with the rocking chair ?", "words": ["the", "rocking chair", "with", "chair", "it", "hit"]}, {"sent": "do you wanna go sit in the rocking chair ?", "words": ["the", "do", "go", "rocking chair", "chair", "you", "in", "wanna", "sit"]}, {"sent": "thats your rocking chair .", "words": ["rocking chair", "your", "chair"]}, {"sent": "you want me I dont really fit in that rocking chair my bottom is too big .", "words": ["my", "bottom", "is", "fit", "rocking chair", "chair", "you", "dont", "in", "big", "too", "me", "that"]}, {"sent": "the rocking chair .", "words": ["the", "rocking chair", "chair"]}, {"sent": "rocking chair ?", "words": ["rocking chair", "chair"]}, {"sent": "oh its over there behind the rocking chair .", "words": ["the", "rocking chair", "behind", "chair", "there", "over"]}, {"sent": "and when we came downstairs ah werent there a lot of things under the tree and in the rocking chair ?", "words": ["the", "tree", "we", "of", "a lot", "rocking chair", "chair", "and", "there", "under", "in", "when", "a"]}, {"sent": "rocking the rocking chair .", "words": ["the", "rocking chair", "chair"]}, {"sent": "rocking chair .", "words": ["rocking chair", "chair"]}, {"sent": "you wanna put Floppy on the rocking chair honey ?", "words": ["the", "put", "rocking chair", "chair", "you", "wanna", "on"]}, {"sent": "youre rocking the rocking chair ?", "words": ["the", "rocking chair", "chair"]}, {"sent": "the other one is a rocking chair .", "words": ["the", "is", "rocking chair", "chair", "other", "a"]}, {"sent": "oh that was good rocking chair .", "words": ["was", "rocking chair", "chair", "good", "that"]}, {"sent": "rocking chair .", "words": ["rocking chair", "chair"]}, {"sent": "in the rocking chair woo woo woo woo .", "words": ["the", "in", "rocking chair", "chair"]}, {"sent": "we dont have your rocking chair back .", "words": ["your", "back", "we", "rocking chair", "chair", "have", "dont"]}, {"sent": "she sat in the biggest rocking chair .", "words": ["the", "rocking chair", "chair", "in", "she"]}, {"sent": "you like to rock in the rocking chair .", "words": ["the", "to", "rocking chair", "like", "chair", "you", "in", "rock"]}, {"sent": "sit in your rocking chair and read your book .", "words": ["your", "rocking chair", "chair", "and", "in", "read", "book", "sit"]}, {"sent": "can you clean the rocking chair ?", "words": ["can", "the", "clean", "rocking chair", "chair", "you"]}, {"sent": "and uh theres uh uh like uh d uh like uh a dolls house and a rocking chair .", "words": ["rocking chair", "like", "chair", "and", "house", "a"]}, {"sent": "and into the rocking chair .", "words": ["the", "into", "rocking chair", "chair", "and"]}, {"sent": "you stay here you wanna sit on the rocking chair ?", "words": ["the", "rocking chair", "chair", "you", "here", "wanna", "on", "stay", "sit"]}, {"sent": "and he left TinkyWinky in your bedroom and a little note to say Ive left the others in the rocking chair .", "words": ["the", "your", "to", "rocking chair", "say", "chair", "and", "little", "in", "he", "a", "bedroom"]}, {"sent": "the rocking chair .", "words": ["the", "rocking chair", "chair"]}, {"sent": "rocking chair .", "words": ["rocking chair", "chair"]}, {"sent": "rocking chair .", "words": ["rocking chair", "chair"]}, {"sent": "Birthday Bears in the rocking chair why dont you go wipe his nose ?", "words": ["the", "go", "rocking chair", "chair", "you", "dont", "in", "his", "wipe", "why", "nose"]}, {"sent": "thank you said bear and he sat down in the rocking chair .", "words": ["the", "rocking chair", "chair", "bear", "and", "you", "in", "he", "down"]}, {"sent": "it was just you were pointing to the rocking chair when you said FatherChristmas sit there .", "words": ["the", "when", "to", "was", "rocking chair", "chair", "you", "there", "it", "sit", "were"]}, {"sent": "back and forth they rocked theres the rocking chair .", "words": ["the", "back", "rocking chair", "chair", "and", "they"]}, {"sent": "she got into the next rocking chair and that was the medium sized rocking chair .", "words": ["the", "into", "was", "rocking chair", "chair", "and", "that", "she"]}, {"sent": "you rocking the big rocking chair .", "words": ["the", "rocking chair", "chair", "you", "big"]}, {"sent": "by chair rocking chair turkeys by .", "words": ["rocking chair", "chair", "by"]}, {"sent": "here is your rocking chair young miss .", "words": ["your", "is", "rocking chair", "chair", "here"]}, {"sent": "rocking chair ?", "words": ["rocking chair", "chair"]}, {"sent": "okay just pour it in .", "words": ["it", "in", "pour"]}, {"sent": "would you pour me a ah an aperitif ?", "words": ["pour", "you", "an", "would", "me", "a"]}, {"sent": "and then you go pour pour pour pour pour .", "words": ["go", "pour", "you", "and", "then"]}, {"sent": "can you pour some more ?", "words": ["can", "pour", "you", "more", "some"]}, {"sent": "okay hang on let me pour the sugar out .", "words": ["the", "pour", "out", "on", "me"]}, {"sent": "pour juice in .", "words": ["in", "pour", "juice"]}, {"sent": "and you pour the tea in the cup .", "words": ["the", "cup", "pour", "you", "and", "in"]}, {"sent": "youre gonna pour some milk ?", "words": ["milk", "some", "pour"]}, {"sent": "okay Ill pour some in your cup .", "words": ["cup", "your", "pour", "in", "some"]}, {"sent": "lets pour some for Violet .", "words": ["some", "pour", "for"]}, {"sent": "or do you wanna pour it in ?", "words": ["do", "pour", "you", "in", "it", "wanna"]}, {"sent": "pour it .", "words": ["it", "pour"]}, {"sent": "well pour some milk on there .", "words": ["pour", "there", "on", "milk", "some"]}, {"sent": "pour xxx .", "words": ["pour"]}, {"sent": "pour it in .", "words": ["it", "in", "pour"]}, {"sent": "you melt chocolate and then you pour the melted chocolate into the little moulds and then you make chocolate and they came with little cellophane wrappers because they would be nice to do for easter time wouldnt they ?", "words": ["the", "do", "for", "into", "be", "to", "chocolate", "pour", "with", "nice", "and", "you", "little", "they", "would", "then", "make", "because"]}, {"sent": "let me get up a minute and make pour the tea out .", "words": ["up", "the", "pour", "and", "get", "out", "me", "make", "a"]}, {"sent": "pour me some tea .", "words": ["me", "pour", "some"]}, {"sent": "dont just pour them on .", "words": ["on", "dont", "pour", "them"]}, {"sent": "see he pour ei um he poured s some here and den he drink it and den he put it back and den you took some from dere you got a bad cold .", "words": ["put", "drink", "back", "pour", "bad", "and", "here", "you", "it", "he", "see", "cold", "a", "some"]}, {"sent": "you pour me some tea .", "words": ["me", "pour", "some", "you"]}, {"sent": "you pour the milk in .", "words": ["the", "pour", "you", "in", "milk"]}, {"sent": "okay where shall I pour it ?", "words": ["where", "it", "pour"]}, {"sent": "Im going to pour it .", "words": ["to", "pour", "it"]}, {"sent": "pour water .", "words": ["pour", "water"]}, {"sent": "I always forget to pour my tea and then it gets stewed .", "words": ["my", "to", "pour", "and", "it", "then"]}, {"sent": "Im just going to pour myself a cup of tea .", "words": ["cup", "of", "to", "pour", "myself", "a"]}, {"sent": "or do you wanna pour it ?", "words": ["do", "pour", "you", "it", "wanna"]}, {"sent": "I will pour poah you some tea .", "words": ["will", "some", "pour", "you"]}, {"sent": "can you pour some juice ?", "words": ["can", "pour", "you", "some", "juice"]}, {"sent": "you wanna help me pour it ?", "words": ["help", "pour", "you", "it", "wanna", "me"]}, {"sent": "wanna pour it in a cup ?", "words": ["cup", "pour", "in", "it", "wanna", "a"]}, {"sent": "would you please pour the water in the yellow cup please .", "words": ["the", "cup", "pour", "would", "you", "in", "water", "yellow"]}, {"sent": "pour us some water .", "words": ["water", "us", "pour", "some"]}, {"sent": "wanna pour me something to drink ?", "words": ["drink", "to", "pour", "wanna", "me"]}, {"sent": "are you going to pour that drink in to a cup ?", "words": ["cup", "drink", "to", "pour", "you", "in", "are", "that", "a"]}, {"sent": "is there a cup we can pour it ?", "words": ["cup", "can", "we", "is", "pour", "there", "it", "a"]}, {"sent": "it doesnt pour out .", "words": ["it", "pour", "out"]}, {"sent": "pour me something to drink outof the pitcher .", "words": ["the", "drink", "to", "pour", "me"]}, {"sent": "pour coffee into the cup .", "words": ["the", "cup", "into", "pour", "coffee"]}, {"sent": "theres coffee and then I pour it in the cup and its allgone .", "words": ["the", "cup", "pour", "and", "in", "it", "coffee", "then"]}, {"sent": "Ill pour it .", "words": ["it", "pour"]}, {"sent": "I better pour that cup of tea hadnt I ?", "words": ["cup", "better", "of", "pour", "that"]}, {"sent": "pour it in there shall we ?", "words": ["we", "pour", "there", "in", "it"]}, {"sent": "I dont think were gonna pour them out I think well let you pick them outof the box okay ?", "words": ["the", "think", "box", "pick", "pour", "them", "you", "dont", "out", "were"]}, {"sent": "lets not pour it in the sink right now .", "words": ["the", "pour", "in", "it", "sink", "not"]}, {"sent": "pour ?", "words": ["pour"]}, {"sent": "before you before you pour Dimitra hers ?", "words": ["hers", "pour", "you"]}, {"sent": "trying to pour something out ?", "words": ["to", "pour", "out"]}, {"sent": "I wanna pour .", "words": ["pour", "wanna"]}, {"sent": "would you pour me some ?", "words": ["pour", "you", "would", "me", "some"]}, {"sent": "will you pour me out a cup of tea ?", "words": ["cup", "of", "pour", "you", "out", "will", "me", "a"]}, {"sent": "pour it in .", "words": ["it", "in", "pour"]}, {"sent": "pour some of that on her leg .", "words": ["of", "pour", "her", "leg", "on", "some", "that"]}, {"sent": "lets pour out the milk and then we can check out the butter .", "words": ["the", "can", "we", "pour", "and", "butter", "out", "milk", "then"]}, {"sent": "tip me over and pour me out .", "words": ["pour", "and", "out", "me", "over"]}, {"sent": "now you pour but you dont put your hand in the bowl .", "words": ["the", "bowl", "put", "your", "hand", "pour", "you", "dont", "in", "but"]}, {"sent": "pour it out ?", "words": ["it", "pour", "out"]}, {"sent": "you wanna pour daddy some more ?", "words": ["pour", "you", "more", "wanna", "some"]}, {"sent": "okay just a second let me get pour one more glass .", "words": ["pour", "get", "more", "glass", "me", "a"]}, {"sent": "who should pour ?", "words": ["pour", "who"]}, {"sent": "but lets pour one or two sweeties on .", "words": ["on", "pour", "but"]}, {"sent": "you gonna pour your juice ?", "words": ["juice", "pour", "your", "you"]}, {"sent": "can I go ahead and pour it in here ?", "words": ["can", "go", "pour", "and", "here", "in", "it"]}, {"sent": "Mummys going to just pour herself a cup of tea .", "words": ["cup", "of", "to", "pour", "a"]}, {"sent": "you pour your milk when youre ready for it .", "words": ["for", "your", "pour", "you", "it", "milk", "when"]}, {"sent": "would you like me to pour some of your apple juice ?", "words": ["your", "of", "to", "pour", "like", "you", "would", "apple", "me", "juice", "some"]}, {"sent": "pour it in .", "words": ["it", "in", "pour"]}, {"sent": "dont pour all the flowers out because they were so pretty .", "words": ["the", "all", "so", "pour", "dont", "they", "pretty", "out", "were", "because"]}, {"sent": "you think you could pour us some lemonade ?", "words": ["think", "pour", "you", "us", "some", "could"]}, {"sent": "well well pour some milk onto the green saucer for Jess .", "words": ["the", "for", "pour", "green", "milk", "some"]}, {"sent": "you pour it .", "words": ["it", "pour", "you"]}, {"sent": "den pour pour the fire on people .", "words": ["the", "on", "pour"]}, {"sent": "I am going to pour myself a cup of tea .", "words": ["cup", "of", "to", "pour", "am", "myself", "a"]}, {"sent": "snip the top of that sachet and well pour it in .", "words": ["the", "of", "pour", "and", "in", "it", "that"]}, {"sent": "Mommy is gonna pour this milk into her cereal .", "words": ["into", "is", "pour", "this", "milk", "her", "cereal"]}, {"sent": "well you pour yourself a cup of tea then .", "words": ["cup", "yourself", "of", "pour", "you", "then", "a"]}, {"sent": "oh please dont pour it out out out outside the tub Naima its gonna make the floor all wet .", "words": ["the", "all", "wet", "outside", "pour", "dont", "it", "out", "make"]}, {"sent": "now you can pour some in .", "words": ["can", "pour", "you", "in", "some"]}, {"sent": "want me to pour my milk and pass it to you then you pour your milk .", "words": ["my", "your", "to", "pour", "and", "you", "it", "milk", "me", "then"]}, {"sent": "are you going to pour some more ?", "words": ["to", "pour", "you", "more", "are", "some"]}, {"sent": "go on pick it up Ill tell you what hes going to do hes going to pour the milk into that churn there .", "words": ["up", "what", "do", "the", "milk", "into", "go", "pick", "to", "pour", "you", "there", "it", "on", "that"]}, {"sent": "pour .", "words": ["pour"]}, {"sent": "pour a little drink pour mommy drink .", "words": ["pour", "drink", "little", "a"]}, {"sent": "okay now pour the block in to your cup .", "words": ["the", "cup", "block", "your", "to", "pour", "in"]}, {"sent": "can you pour me some coffee ?", "words": ["can", "pour", "you", "coffee", "me", "some"]}, {"sent": "Mummys going to go and pour herself a cup of tea .", "words": ["cup", "go", "of", "to", "pour", "and", "a"]}, {"sent": "okay ready Im ready to pour the water in get ready .", "words": ["the", "to", "pour", "get", "in", "water"]}, {"sent": "pour the tea then pour the coffee .", "words": ["the", "then", "pour", "coffee"]}, {"sent": "are you gonna try to pour it on my head ?", "words": ["my", "to", "head", "pour", "you", "are", "it", "on", "try", "try to"]}, {"sent": "get the cup please and Ill pour it in .", "words": ["the", "cup", "pour", "and", "get", "in", "it"]}, {"sent": "pour something to drink .", "words": ["to", "pour", "drink"]}, {"sent": "so I pour the pasta into the colander and it catches all the pasta but all the water drains through ?", "words": ["the", "all", "into", "so", "pour", "and", "it", "water", "but"]}, {"sent": "pour them in .", "words": ["in", "pour", "them"]}, {"sent": "Im gonna pour some okay ?", "words": ["some", "pour"]}, {"sent": "tip me over and pour me out .", "words": ["pour", "and", "out", "me", "over"]}, {"sent": "pour some more .", "words": ["some", "more", "pour"]}, {"sent": "pour ?", "words": ["pour"]}, {"sent": "gimme duh butterfly .", "words": ["butterfly"]}, {"sent": "shoo butterfly said Sergeant Major loudly .", "words": ["butterfly"]}, {"sent": "on butterfly .", "words": ["on", "butterfly"]}, {"sent": "theres the butterfly .", "words": ["the", "butterfly"]}, {"sent": "it looks like weve got a bee attached to that butterfly .", "words": ["butterfly", "to", "bee", "like", "it", "that", "a"]}, {"sent": "oh and another butterfly .", "words": ["another", "butterfly", "and"]}, {"sent": "xxx have a butterfly .", "words": ["have", "butterfly", "a"]}, {"sent": "and butterfly .", "words": ["butterfly", "and"]}, {"sent": "thats a butterfly .", "words": ["butterfly", "a"]}, {"sent": "be a dead butterfly if you do that to it .", "words": ["butterfly", "do", "if", "be", "to", "you", "it", "that", "a"]}, {"sent": "a butterfly .", "words": ["butterfly", "a"]}, {"sent": "on the wall am I the most beautiful butterfly of them all ?", "words": ["the", "butterfly", "all", "of", "them", "am", "on"]}, {"sent": "has the frog got the butterfly ?", "words": ["the", "butterfly", "frog"]}, {"sent": "has the butterfly got hair ?", "words": ["the", "hair", "butterfly"]}, {"sent": "theres a butterfly and a caterpillar .", "words": ["butterfly", "and", "a"]}, {"sent": "and the next day he was a beautiful butterfly .", "words": ["the", "butterfly", "was", "and", "he", "a"]}, {"sent": "a butterfly doesnt it ?", "words": ["butterfly", "it", "a"]}, {"sent": "now can we find some butterfly ?", "words": ["can", "butterfly", "we", "find", "some"]}, {"sent": "yeah I see a butterfly .", "words": ["butterfly", "see", "a"]}, {"sent": "yes its a butterfly .", "words": ["butterfly", "a"]}, {"sent": "a butterfly ?", "words": ["butterfly", "a"]}, {"sent": "a butterfly has wings doesnt it ?", "words": ["butterfly", "it", "a"]}, {"sent": "purple said the butterfly .", "words": ["the", "butterfly"]}, {"sent": "wheres the butterfly ?", "words": ["the", "butterfly"]}, {"sent": "what turns into a butterfly when it grows ?", "words": ["butterfly", "what", "into", "it", "when", "a"]}, {"sent": "I see butterfly .", "words": ["butterfly", "see"]}, {"sent": "the butterfly Lily did you tell Amanda what your teachers name is ?", "words": ["the", "what", "butterfly", "your", "is", "you", "did"]}, {"sent": "butterfly biscuit ?", "words": ["butterfly"]}, {"sent": "a butterfly .", "words": ["butterfly", "a"]}, {"sent": "the butterfly is flying ?", "words": ["the", "butterfly", "is"]}, {"sent": "like the soft soft whisper of a butterfly .", "words": ["the", "butterfly", "soft", "of", "like", "a"]}, {"sent": "this is a butterfly ?", "words": ["butterfly", "this", "a", "is"]}, {"sent": "theres another one running from a butterfly .", "words": ["another", "butterfly", "a"]}, {"sent": "a butterfly .", "words": ["butterfly", "a"]}, {"sent": "a butterfly ?", "words": ["butterfly", "a"]}, {"sent": "a butterfly or just a fly ?", "words": ["butterfly", "a"]}, {"sent": "butterfly biscuit ?", "words": ["butterfly"]}, {"sent": "where is the butterfly by the way ?", "words": ["where", "the", "butterfly", "is", "by"]}, {"sent": "have you ever seen a butterfly ?", "words": ["have", "butterfly", "a", "you"]}, {"sent": "and b is for butterfly .", "words": ["butterfly", "for", "is", "and"]}, {"sent": "along came a butterfly and then there were nine little ladybugs sitting on a gate .", "words": ["butterfly", "and", "little", "there", "on", "then", "were", "a"]}, {"sent": "wheres the butterfly ?", "words": ["the", "butterfly"]}, {"sent": "a pretty blue butterfly .", "words": ["butterfly", "blue", "pretty", "a"]}, {"sent": "a butterfly .", "words": ["butterfly", "a"]}, {"sent": "look at that butterfly .", "words": ["butterfly", "at", "that", "look"]}, {"sent": "its a butterfly is it ?", "words": ["butterfly", "it", "is", "a"]}, {"sent": "its a butterfly .", "words": ["butterfly", "a"]}, {"sent": "I found a butterfly .", "words": ["butterfly", "a"]}, {"sent": "and then theres the butterfly .", "words": ["the", "then", "butterfly", "and"]}, {"sent": "what a pretty butterfly huh ?", "words": ["butterfly", "what", "pretty", "a"]}, {"sent": "whats our favorite story about a butterfly ?", "words": ["butterfly", "our", "about", "story", "a"]}, {"sent": "wheres the butterfly ?", "words": ["the", "butterfly"]}, {"sent": "heres a butterfly hat .", "words": ["butterfly", "hat", "a"]}, {"sent": "a butterfly .", "words": ["butterfly", "a"]}, {"sent": "can you put the butterfly in the box ?", "words": ["can", "the", "put", "butterfly", "box", "you", "in"]}, {"sent": "you find the butterfly .", "words": ["the", "butterfly", "find", "you"]}, {"sent": "hm where was that other butterfly ?", "words": ["where", "butterfly", "was", "other", "that"]}, {"sent": "its a pink butterfly .", "words": ["butterfly", "a"]}, {"sent": "an apple or a butterfly ?", "words": ["butterfly", "apple", "an", "a"]}, {"sent": "thats a butterfly .", "words": ["butterfly", "a"]}, {"sent": "this is the largest butterfly there is in the whole pack .", "words": ["the", "butterfly", "is", "this", "there", "in"]}, {"sent": "can you show me the butterfly ?", "words": ["can", "the", "butterfly", "show", "you", "me"]}, {"sent": "a butterfly .", "words": ["butterfly", "a"]}, {"sent": "it has got a butterfly on it and a picture of the sun .", "words": ["butterfly", "the", "of", "sun", "picture", "and", "it", "on", "a"]}, {"sent": "ten little ladybugs sitting on a vine along came a butterfly and then there were .", "words": ["butterfly", "and", "little", "there", "on", "then", "were", "a"]}, {"sent": "hes a butterfly Joseph see ?", "words": ["butterfly", "see", "a"]}, {"sent": "oh I guess that looks like a butterfly .", "words": ["butterfly", "that", "like", "a"]}, {"sent": "the butterfly flutters in the breeze .", "words": ["the", "in", "butterfly"]}, {"sent": "where did you make your butterfly ?", "words": ["where", "butterfly", "your", "you", "did", "make"]}, {"sent": "butterfly pillow .", "words": ["butterfly", "pillow"]}, {"sent": "you would like to make a butterfly face ?", "words": ["butterfly", "face", "to", "like", "you", "would", "make", "a"]}, {"sent": "huh a zebra butterfly and theres miss spider .", "words": ["butterfly", "zebra", "and", "a"]}, {"sent": "are you gonna make a butterfly ?", "words": ["butterfly", "you", "are", "make", "a"]}, {"sent": "they look like frogs eyes oh a butterfly .", "words": ["butterfly", "like", "they", "look", "a"]}, {"sent": "dis butterfly is hers .", "words": ["butterfly", "hers", "is"]}, {"sent": "do you think the frog might have the butterfly ?", "words": ["the", "do", "think", "butterfly", "frog", "you", "have"]}, {"sent": "keep your backpack on Ill put your butterfly wings over your backpack .", "words": ["butterfly", "put", "your", "on", "over"]}, {"sent": "oh we could put the butterfly .", "words": ["the", "butterfly", "put", "we", "could"]}, {"sent": "and I think you said the butterfly has whiskers .", "words": ["the", "butterfly", "think", "you", "and"]}, {"sent": "youre a butterfly .", "words": ["butterfly", "a"]}, {"sent": "theres no butterfly .", "words": ["butterfly"]}, {"sent": "miss xxx butterfly okay .", "words": ["butterfly"]}, {"sent": "wheres the butterfly Fraser ?", "words": ["the", "butterfly"]}, {"sent": "a butterfly .", "words": ["butterfly", "a"]}, {"sent": "and thats a butterfly isnt it ?", "words": ["butterfly", "it", "a", "and"]}, {"sent": "he was a pretty butterfly .", "words": ["butterfly", "was", "pretty", "he", "a"]}, {"sent": "it has got oh its the butterfly bib .", "words": ["the", "bib", "it", "butterfly"]}, {"sent": "b for butterfly .", "words": ["butterfly", "for"]}, {"sent": "thats a butterfly that one isnt it ?", "words": ["butterfly", "it", "that", "a"]}, {"sent": "a butterfly .", "words": ["butterfly", "a"]}, {"sent": "then once as I bent above a brook to try to catch a fish on bent tin hook a butterfly fluttered by .", "words": ["butterfly", "to", "catch", "by", "on", "fish", "above", "then", "try", "try to", "a"]}, {"sent": "and saw a lovely beautiful butterfly .", "words": ["butterfly", "a", "and"]}, {"sent": "butterfly in the tummy ?", "words": ["butterfly", "the", "in", "tummy"]}, {"sent": "a butterfly driving a train ?", "words": ["butterfly", "train", "a"]}, {"sent": "wheres the butterfly gone ?", "words": ["the", "butterfly"]}, {"sent": "can you find the butterfly ?", "words": ["can", "the", "butterfly", "you", "find"]}, {"sent": "our pup is smart our pup can get a butterfly without a net .", "words": ["can", "butterfly", "is", "our", "get", "a"]}, {"sent": "theres the butterfly .", "words": ["the", "butterfly"]}, {"sent": "another butterfly .", "words": ["another", "butterfly"]}, {"sent": "something and pancake bake xxx .", "words": ["pancake", "and"]}, {"sent": "thats your pancake and your sausage .", "words": ["pancake", "your", "and"]}, {"sent": "you want a pancake ?", "words": ["pancake", "a", "you"]}, {"sent": "is it a pancake ?", "words": ["it", "pancake", "a", "is"]}, {"sent": "um how about that pancake xxx ?", "words": ["pancake", "how", "that", "about"]}, {"sent": "its like a pancake .", "words": ["pancake", "like", "a"]}, {"sent": "shall we take them to the pancake house ?", "words": ["the", "pancake", "we", "to", "them", "house", "take"]}, {"sent": "you want my pancake ?", "words": ["my", "pancake", "you"]}, {"sent": "I have pancake batter .", "words": ["have", "pancake"]}, {"sent": "a pancake freak .", "words": ["pancake", "a"]}, {"sent": "youve got a pancake over here if you want it .", "words": ["pancake", "if", "you", "here", "it", "over", "a"]}, {"sent": "except your mommy probably uses a pancake turner right ?", "words": ["pancake", "your", "a"]}, {"sent": "did you wake up this morning and think I want a large pancake ?", "words": ["up", "think", "pancake", "you", "and", "this", "did", "wake", "a"]}, {"sent": "well I think it must have been ShroveTuesday pancake day .", "words": ["have", "think", "it", "pancake"]}, {"sent": "she answered if you wanna have a pancake you will hafta help me what should I do ?", "words": ["what", "help", "do", "pancake", "if", "you", "have", "wanna", "will", "me", "she", "a"]}, {"sent": "pancake .", "words": ["pancake"]}, {"sent": "thats Dauaus pancake alright .", "words": ["pancake"]}, {"sent": "this is called a pancake grill .", "words": ["this", "pancake", "a", "is"]}, {"sent": "who wants a pancake ?", "words": ["pancake", "a", "who"]}, {"sent": "and a pancake landed on that little boys face .", "words": ["pancake", "face", "and", "little", "on", "that", "a"]}, {"sent": "but it seems like when you tell him its like a pancake it was like oh okay .", "words": ["pancake", "was", "like", "you", "him", "it", "when", "but", "a"]}, {"sent": "a pancake ?", "words": ["pancake", "a"]}, {"sent": "Nathaniel hasnt finished the pancake thats right .", "words": ["the", "pancake"]}, {"sent": "if you give a pig a pancake .", "words": ["pancake", "if", "pig", "you", "give", "a"]}, {"sent": "roll the pancake .", "words": ["the", "pancake"]}, {"sent": "when did you have a pancake last ?", "words": ["pancake", "last", "you", "have", "did", "when", "a"]}, {"sent": "do you think its a pancake or something ?", "words": ["do", "think", "pancake", "you", "a"]}, {"sent": "are you making me a pancake ?", "words": ["pancake", "you", "are", "me", "a"]}, {"sent": "if you give a pig a pancake xxx .", "words": ["pancake", "if", "pig", "you", "give", "a"]}, {"sent": "thats the pancake syrup .", "words": ["the", "pancake"]}, {"sent": "we could write a book about you called chair chair instead of pancake pancake Shem said I want a big chair for breakfast .", "words": ["for", "pancake", "we", "of", "write", "you", "chair", "about", "book", "could", "big", "a"]}, {"sent": "hey I may sit on him and flatten him out like a pancake .", "words": ["pancake", "like", "and", "him", "out", "on", "sit", "a"]}, {"sent": "or its all like uh fruit and vegetables I think and a pancake .", "words": ["all", "think", "pancake", "like", "and", "a"]}, {"sent": "the whole building got covered up with pancake .", "words": ["the", "pancake", "up", "with"]}, {"sent": "for the pancake kid .", "words": ["the", "for", "pancake"]}, {"sent": "bake a pancake .", "words": ["pancake", "a"]}, {"sent": "did I eat some of your pancake ?", "words": ["your", "pancake", "of", "eat", "did", "some"]}, {"sent": "this pancake has a face on it .", "words": ["pancake", "face", "this", "it", "on", "a"]}, {"sent": "whats pancake ?", "words": ["pancake"]}, {"sent": "help her make the pancake .", "words": ["the", "help", "pancake", "her", "make"]}, {"sent": "I bet youre gonna make a pancake .", "words": ["pancake", "make", "a"]}, {"sent": "bring me your plate said his mother she slipped the pancake from the frying pan on to jacks plate shall I spread it with strawberry jam for you ?", "words": ["the", "for", "plate", "your", "pancake", "strawberry", "to", "with", "bring", "you", "on", "it", "his", "me", "she"]}, {"sent": "its like a pancake .", "words": ["pancake", "like", "a"]}, {"sent": "large pancake for breakfast today I have alotof work to do she answered if you want a pancake .", "words": ["do", "for", "work", "pancake", "if", "to", "you", "have", "she", "a"]}, {"sent": "thats Nathaniels pancake pan thats right .", "words": ["pancake"]}, {"sent": "so when jack woke up he thought how hungry I am I could eat a large pancake for breakfast .", "words": ["up", "how", "for", "pancake", "so", "hungry", "eat", "he", "am", "could", "when", "a"]}, {"sent": "ya go to pancake house .", "words": ["house", "to", "pancake", "go"]}, {"sent": "a bag of flour an egg bottle of milk and a round brown thing that says pancake .", "words": ["pancake", "egg", "brown", "of", "and", "an", "milk", "bottle", "that", "a"]}, {"sent": "pancake .", "words": ["pancake"]}, {"sent": "mhm little pancake .", "words": ["pancake", "little"]}, {"sent": "lets see if I heat up a pancake three times will it actually turn into rubber ?", "words": ["up", "pancake", "if", "into", "it", "will", "see", "a"]}, {"sent": "would you like a pancake ?", "words": ["pancake", "like", "you", "would", "a"]}, {"sent": "whoa theres the pancake book .", "words": ["the", "book", "pancake"]}, {"sent": "Im so hungry said jack lets make the pancake but his mother said go to the woodshed and bring me a few logs and some kindling wood .", "words": ["the", "pancake", "go", "so", "to", "hungry", "bring", "and", "some", "his", "me", "make", "but", "a"]}, {"sent": "what shape is a pancake ?", "words": ["what", "pancake", "a", "is"]}, {"sent": "Daddys gonna have his pancake now .", "words": ["have", "his", "pancake"]}, {"sent": "we dont have anything but we had pancakes at the pancake house .", "words": ["the", "pancake", "we", "have", "dont", "house", "at", "but"]}, {"sent": "look theres a little kitty is he waiting for a pancake ?", "words": ["for", "pancake", "is", "little", "kitty", "he", "look", "a"]}, {"sent": "or is that a pancake ?", "words": ["pancake", "that", "a", "is"]}, {"sent": "xxx the pancake book xxx .", "words": ["the", "book", "pancake"]}, {"sent": "we got that at the pancake house .", "words": ["the", "pancake", "we", "house", "at", "that"]}, {"sent": "was that a pancake ?", "words": ["was", "pancake", "that", "a"]}, {"sent": "Id like to have a big pancake now wouldnt you ?", "words": ["pancake", "to", "like", "you", "have", "big", "a"]}, {"sent": "jack waited and waited just as he was ready to give up the hen said tsk tsk again and there was an egg jack carefully took the egg from the nest and ran back to the house heres an egg mother lets make a pancake I am so hungry .", "words": ["the", "an", "am", "there", "hen", "up", "to", "was", "so", "hungry", "and", "make", "a", "pancake", "egg", "back", "he", "house", "give"]}, {"sent": "the pancake will give you energy .", "words": ["the", "pancake", "you", "will", "give"]}, {"sent": "a pancake fell on his head ?", "words": ["pancake", "head", "his", "on", "a"]}, {"sent": "here have your pancake .", "words": ["here", "pancake", "have", "your"]}, {"sent": "is this the pancake book ?", "words": ["the", "pancake", "is", "this", "book"]}, {"sent": "shall we see if this pancake is done ?", "words": ["pancake", "if", "we", "is", "this", "see"]}, {"sent": "is that a pancake ?", "words": ["pancake", "that", "a", "is"]}, {"sent": "in the bread box now weve got hot cross buns lemon and sultana pancake .", "words": ["the", "bread", "box", "pancake", "and", "hot", "in"]}, {"sent": "Naima do you want some pancake too ?", "words": ["do", "pancake", "you", "too", "some"]}, {"sent": "make it flat like a pancake .", "words": ["pancake", "like", "it", "make", "a"]}, {"sent": "shall we put do you wanna save this mat from the pancake house ?", "words": ["the", "do", "put", "pancake", "we", "you", "this", "house", "wanna"]}, {"sent": "Ill turn the pancake over ready ?", "words": ["the", "over", "pancake"]}, {"sent": "you just eat your pancake .", "words": ["eat", "pancake", "your", "you"]}, {"sent": "a large pancake ?", "words": ["pancake", "a"]}, {"sent": "no Im gonna make a pancake out of .", "words": ["pancake", "of", "out", "make", "a"]}, {"sent": "theres another pancake .", "words": ["another", "pancake"]}, {"sent": "pat pat pat pancake .", "words": ["pancake"]}, {"sent": "you want more pancake ?", "words": ["more", "pancake", "you"]}, {"sent": "okay what is all this stuff that you needta make a pancake huh ?", "words": ["all", "what", "pancake", "is", "you", "this", "make", "that", "a"]}, {"sent": "now she says you hafta help me make this pancake if you want it .", "words": ["help", "pancake", "if", "you", "this", "it", "me", "make", "she"]}, {"sent": "plate and looks like a piece of toast and a pancake huh ?", "words": ["plate", "pancake", "toast", "of", "like", "and", "a"]}, {"sent": "lets go find a book you know mommy said you had a new book called the pancake book ?", "words": ["the", "pancake", "go", "you", "find", "book", "new", "a"]}, {"sent": "youll be swimming with pancake energy .", "words": ["be", "with", "pancake"]}, {"sent": "he has a pancake on his face .", "words": ["pancake", "face", "his", "he", "on", "a"]}, {"sent": "thats the strangest looking pancake Ive ever seen .", "words": ["the", "pancake"]}, {"sent": "okay weve gotta finish our pancake book .", "words": ["book", "pancake", "finish", "our"]}, {"sent": "heres the flour jack said now lets make the pancake Im hungry but we need an egg his mother answered .", "words": ["the", "pancake", "need", "egg", "we", "hungry", "an", "his", "make", "but"]}, {"sent": "when was the last time you had a pancake ?", "words": ["the", "pancake", "was", "last", "you", "when", "a"]}, {"sent": "no it is not a pancake .", "words": ["pancake", "is", "it", "not", "a"]}, {"sent": "and chances are if she ask you for some syrup shell want a pancake to go with it xxx .", "words": ["for", "pancake", "if", "go", "to", "with", "you", "and", "are", "it", "some", "she", "a"]}, {"sent": "after a minute or two she looked at the underside of the pancake .", "words": ["the", "pancake", "of", "at", "she", "a"]}, {"sent": "okay well tell you what how about if we get dressed and then we can read the pancake book okay ?", "words": ["can", "what", "how", "the", "pancake", "if", "we", "you", "get", "and", "read", "about", "book", "then"]}, {"sent": "the pancake book ?", "words": ["the", "book", "pancake"]}, {"sent": "xxx dad you useta make buttermilk pancake .", "words": ["pancake", "make", "you"]}, {"sent": "a pancake .", "words": ["pancake", "a"]}, {"sent": "television in the refrigerator ?", "words": ["the", "in", "refrigerator"]}, {"sent": "where is the refrigerator ?", "words": ["where", "the", "refrigerator", "is"]}, {"sent": "show me in the refrigerator .", "words": ["the", "refrigerator", "show", "in", "me"]}, {"sent": "I was gonna say that I would have a lot of Chewies in my refrigerator .", "words": ["my", "refrigerator", "of", "was", "a lot", "say", "have", "in", "would", "that", "a"]}, {"sent": "refrigerator is too small .", "words": ["too", "refrigerator", "is"]}, {"sent": "his feet are in the refrigerator .", "words": ["the", "refrigerator", "in", "are", "his"]}, {"sent": "its hiding behind the refrigerator .", "words": ["the", "behind", "refrigerator"]}, {"sent": "for the refrigerator .", "words": ["the", "for", "refrigerator"]}, {"sent": "you want it open the refrigerator ?", "words": ["the", "refrigerator", "open", "you", "it"]}, {"sent": "did we just take them out of the refrigerator and eat them like that ?", "words": ["the", "refrigerator", "we", "of", "them", "like", "and", "eat", "that", "did", "out", "take"]}, {"sent": "in the refrigerator .", "words": ["the", "in", "refrigerator"]}, {"sent": "Mommy forgot to put them in the refrigerator so theyre gonna be warm .", "words": ["the", "put", "refrigerator", "to", "so", "be", "them", "in"]}, {"sent": "back in the refrigerator .", "words": ["the", "in", "refrigerator", "back"]}, {"sent": "its in the bottom of the refrigerator .", "words": ["the", "bottom", "refrigerator", "of", "in"]}, {"sent": "help me close that refrigerator .", "words": ["help", "refrigerator", "me", "close", "that"]}, {"sent": "I wouldnt put something in the refrigerator unless the refrigerator temperature was xxx .", "words": ["the", "put", "refrigerator", "was", "in"]}, {"sent": "where is the refrigerator .", "words": ["where", "the", "refrigerator", "is"]}, {"sent": "you have a sheep on your refrigerator door honey .", "words": ["your", "refrigerator", "you", "have", "sheep", "on", "door", "a"]}, {"sent": "you were in front of the refrigerator saying tikatika .", "words": ["the", "refrigerator", "of", "you", "in", "were"]}, {"sent": "ya well ya got a stove and a sink and a refrigerator and a house and a bathroom and a kitchen and a couch .", "words": ["refrigerator", "couch", "and", "sink", "stove", "house", "kitchen", "bathroom", "a"]}, {"sent": "well youre just standing looking at all the things in the refrigerator .", "words": ["the", "all", "refrigerator", "in", "at"]}, {"sent": "Im gonna put this egg back in the refrigerator .", "words": ["the", "put", "egg", "back", "refrigerator", "this", "in"]}, {"sent": "you have some in the refrigerator .", "words": ["the", "refrigerator", "you", "have", "in", "some"]}, {"sent": "its on the refrigerator hon .", "words": ["on", "the", "refrigerator"]}, {"sent": "in the refrigerator right ?", "words": ["the", "in", "refrigerator"]}, {"sent": "we have not gone and opened the refrigerator and looked in it in a long time ?", "words": ["the", "refrigerator", "we", "long", "and", "have", "in", "it", "not", "a"]}, {"sent": "Mommy hasta open the refrigerator .", "words": ["open", "the", "refrigerator"]}, {"sent": "would get the ice cubes outof the um refrigerator there ?", "words": ["the", "refrigerator", "get", "there", "would", "ice"]}, {"sent": "then we can put it on the refrigerator .", "words": ["can", "the", "put", "refrigerator", "we", "it", "on", "then"]}, {"sent": "oh what was inside the refrigerator ?", "words": ["the", "what", "refrigerator", "was", "inside"]}, {"sent": "peaches are back in the refrigerator .", "words": ["the", "refrigerator", "back", "in", "are"]}, {"sent": "thats why I opened up the refrigerator .", "words": ["up", "the", "refrigerator", "why"]}, {"sent": "come outof the refrigerator please .", "words": ["the", "refrigerator"]}, {"sent": "on top a duh refrigerator .", "words": ["on", "refrigerator", "a"]}, {"sent": "he might walk into the refrigerator .", "words": ["the", "into", "refrigerator", "he", "walk"]}, {"sent": "let us play in the refrigerator .", "words": ["the", "refrigerator", "play", "us", "in"]}, {"sent": "you playing in the refrigerator again ?", "words": ["the", "in", "refrigerator", "you"]}, {"sent": "Im gonna close this refrigerator now okay ?", "words": ["this", "close", "refrigerator"]}, {"sent": "and its like a refrigerator that opens .", "words": ["refrigerator", "like", "and", "that", "a"]}, {"sent": "refrigerator .", "words": ["refrigerator"]}, {"sent": "two cats in the refrigerator ?", "words": ["the", "in", "refrigerator"]}, {"sent": "in the refrigerator .", "words": ["the", "in", "refrigerator"]}, {"sent": "thats the refrigerator .", "words": ["the", "refrigerator"]}, {"sent": "Mommy go in the refrigerator and get you some .", "words": ["the", "refrigerator", "go", "and", "get", "you", "in", "some"]}, {"sent": "thats a refrigerator .", "words": ["refrigerator", "a"]}, {"sent": "let us get into the refrigerator .", "words": ["the", "into", "refrigerator", "get", "us"]}, {"sent": "wanna put them in the refrigerator with me ?", "words": ["the", "put", "refrigerator", "them", "with", "in", "wanna", "me"]}, {"sent": "the cookies must be in the refrigerator .", "words": ["the", "be", "in", "refrigerator"]}, {"sent": "theyre over there in the in front of the refrigerator .", "words": ["the", "refrigerator", "of", "there", "in", "over"]}, {"sent": "this is a refrigerator .", "words": ["this", "refrigerator", "a", "is"]}, {"sent": "shut that refrigerator and go .", "words": ["go", "that", "refrigerator", "and"]}, {"sent": "give it to Mommy well put it in the refrigerator .", "words": ["the", "put", "refrigerator", "to", "in", "it", "give"]}, {"sent": "let us go to the refrigerator .", "words": ["the", "refrigerator", "go", "to", "us"]}, {"sent": "lets put your juice in the refrigerator .", "words": ["the", "put", "your", "refrigerator", "in", "juice"]}, {"sent": "refrigerator door .", "words": ["refrigerator", "door"]}, {"sent": "you like to bang your head on that refrigerator ?", "words": ["your", "refrigerator", "to", "head", "like", "you", "on", "that"]}, {"sent": "not your tummy no not your tummy your refrigerator .", "words": ["your", "not", "refrigerator", "tummy"]}, {"sent": "oh is that your refrigerator ?", "words": ["your", "that", "refrigerator", "is"]}, {"sent": "what else are you putting in the refrigerator ?", "words": ["the", "what", "refrigerator", "you", "in", "are"]}, {"sent": "how about put it in the refrigerator .", "words": ["the", "how", "put", "refrigerator", "in", "it", "about"]}, {"sent": "where is the refrigerator .", "words": ["where", "the", "refrigerator", "is"]}, {"sent": "xxx at your house xxx say do you have chairs on top of your refrigerator at home ?", "words": ["do", "your", "refrigerator", "of", "say", "you", "have", "house", "at", "on", "home"]}, {"sent": "Nomi dont put those things in the refrigerator .", "words": ["the", "put", "those", "refrigerator", "dont", "in"]}, {"sent": "but I dont know where a refrigerator would go on this .", "words": ["where", "refrigerator", "go", "dont", "this", "would", "on", "but", "a"]}, {"sent": "refrigerator microwave ?", "words": ["refrigerator"]}, {"sent": "so you know Ill tell h her where are the eggs we xxx put in the refrigerator before they go bad .", "words": ["where", "the", "put", "refrigerator", "we", "go", "so", "bad", "you", "they", "in", "are", "her"]}, {"sent": "whats that in the refrigerator you mean ?", "words": ["the", "refrigerator", "you", "in", "that"]}, {"sent": "you gonna play in the refrigerator ?", "words": ["the", "refrigerator", "play", "you", "in"]}, {"sent": "yes would you close the refrigerator please .", "words": ["the", "refrigerator", "you", "would", "close"]}, {"sent": "whats inside the refrigerator can you see ?", "words": ["the", "can", "refrigerator", "inside", "you", "see"]}, {"sent": "come close this refrigerator .", "words": ["this", "close", "refrigerator"]}, {"sent": "it is called a refrigerator .", "words": ["it", "refrigerator", "a", "is"]}, {"sent": "are they in the refrigerator ?", "words": ["the", "refrigerator", "they", "in", "are"]}, {"sent": "its not a refrigerator .", "words": ["not", "refrigerator", "a"]}, {"sent": "well wait a minute then it says for ready to use phyllo store overnight in the refrigerator .", "words": ["the", "wait", "for", "refrigerator", "to", "store", "in", "it", "then", "a"]}, {"sent": "are you talking about the refrigerator ?", "words": ["the", "refrigerator", "you", "are", "about"]}, {"sent": "the refrigerator .", "words": ["the", "refrigerator"]}, {"sent": "we gotta close the refrigerator .", "words": ["the", "close", "refrigerator", "we"]}, {"sent": "be a good big girl and put it right in the refrigerator .", "words": ["the", "put", "refrigerator", "be", "and", "in", "it", "good", "big", "a"]}, {"sent": "milk in the refrigerator .", "words": ["milk", "the", "in", "refrigerator"]}, {"sent": "that peach was in the refrigerator all night .", "words": ["the", "all", "refrigerator", "was", "in", "that"]}, {"sent": "you want to open the refrigerator ?", "words": ["the", "refrigerator", "open", "to", "you", "want to"]}, {"sent": "do you know how to close that refrigerator ?", "words": ["do", "how", "refrigerator", "to", "you", "close", "that"]}, {"sent": "like that refrigerator ?", "words": ["that", "like", "refrigerator"]}, {"sent": "youre gonna put Bert in in the refrigerator ?", "words": ["the", "in", "put", "refrigerator"]}, {"sent": "because you can close the refrigerator all the time .", "words": ["can", "the", "all", "refrigerator", "you", "close", "because"]}, {"sent": "in the refrigerator xxx .", "words": ["the", "in", "refrigerator"]}, {"sent": "okay what do you wanna put this refrigerator ?", "words": ["what", "do", "put", "refrigerator", "you", "this", "wanna"]}, {"sent": "well it might be on the uh in the refrigerator or something ?", "words": ["the", "refrigerator", "be", "in", "it", "on"]}, {"sent": "refrigerator .", "words": ["refrigerator"]}, {"sent": "close the refrigerator .", "words": ["the", "close", "refrigerator"]}, {"sent": "what color is the refrigerator ?", "words": ["the", "what", "refrigerator", "is"]}, {"sent": "put this in the refrigerator for me to get xxx theres cherry cola u .", "words": ["the", "put", "for", "refrigerator", "to", "get", "this", "in", "me"]}, {"sent": "in the refrigerator ?", "words": ["the", "in", "refrigerator"]}, {"sent": "but theyre tasty when theyre coming out of the refrigerator .", "words": ["the", "refrigerator", "of", "out", "when", "but"]}, {"sent": "play in the refrigerator .", "words": ["the", "in", "play", "refrigerator"]}, {"sent": "its still gonna take anything thats heart shaped takes a while though to make because we hafta put it in the refrigerator to chill .", "words": ["the", "put", "refrigerator", "we", "to", "in", "it", "take", "make", "because", "a"]}, {"sent": "Nomi lets put Jenkos bottles in the refrigerator .", "words": ["the", "in", "put", "refrigerator"]}, {"sent": "xxx bring out the sprinkler .", "words": ["the", "bring", "out", "sprinkler"]}, {"sent": "its like a sprinkler ?", "words": ["sprinkler", "like", "a"]}, {"sent": "oh youre thinking of the other day when Daddy pressed the button to turn the sprinkler on at the park next to Dadajis house .", "words": ["the", "button", "of", "to", "other", "sprinkler", "park", "house", "at", "on", "when"]}, {"sent": "this is the sprinkler .", "words": ["the", "this", "sprinkler", "is"]}, {"sent": "might just be from last night lets see how the sprinkler is doing .", "words": ["the", "how", "is", "be", "last", "sprinkler", "see"]}, {"sent": "sprinkler is on now .", "words": ["on", "is", "sprinkler"]}, {"sent": "the sprinkler is coming on .", "words": ["the", "on", "is", "sprinkler"]}, {"sent": "its a sprinkler sprinkling water .", "words": ["water", "sprinkler", "a"]}, {"sent": "and their Daddy useta have a water sprinkler in the middle of the lawn .", "words": ["the", "their", "of", "and", "have", "sprinkler", "in", "water", "a"]}, {"sent": "in the do you remember in the what in the sprinkler ?", "words": ["the", "do", "what", "you", "sprinkler", "in"]}, {"sent": "you ran outside and played in the sprinkler with who ?", "words": ["the", "outside", "with", "and", "you", "sprinkler", "in", "who"]}, {"sent": "a sprinkler ?", "words": ["sprinkler", "a"]}, {"sent": "a sprinkler and a hoe in the car ?", "words": ["the", "car", "sprinkler", "and", "in", "a"]}, {"sent": "wearing glasses daddys driving the car and then well put a hoe and a sprinkler theres part of the hoe oh you want a hoe and a sprinkler down here ?", "words": ["the", "put", "of", "car", "and", "glasses", "sprinkler", "you", "here", "then", "down", "a"]}, {"sent": "what do you do with the sprinkler ?", "words": ["the", "what", "do", "with", "you", "sprinkler"]}, {"sent": "he cant get in the bed with the sprinkler though right ?", "words": ["the", "with", "bed", "sprinkler", "get", "in", "he"]}, {"sent": "xxx xxx sprinkler .", "words": ["sprinkler"]}, {"sent": "they did they get to play in the sprinkler ?", "words": ["the", "to", "play", "sprinkler", "get", "they", "in", "did"]}, {"sent": "thats because it was so hot out you got to play in the cool sprinkler .", "words": ["the", "was", "so", "to", "play", "you", "sprinkler", "hot", "it", "in", "out", "because"]}, {"sent": "do you remember when you went to to school and you played that game with the sprinkler ?", "words": ["the", "do", "game", "to", "school", "with", "you", "and", "sprinkler", "that", "when"]}, {"sent": "that lil Elmo sprinkler whats he holding in his hand ?", "words": ["hand", "sprinkler", "in", "he", "his", "that"]}, {"sent": "your bottle sounds like a sprinkler ?", "words": ["your", "like", "sprinkler", "bottle", "a"]}, {"sent": "Daddy pressed a button to make the sprinkler go on didnt he ?", "words": ["the", "button", "go", "to", "sprinkler", "he", "on", "make", "a"]}, {"sent": "I think maybe we should go get the sprinkler .", "words": ["the", "think", "go", "we", "sprinkler", "get"]}, {"sent": "sprinkler .", "words": ["sprinkler"]}, {"sent": "yep thats the sprinkler okay .", "words": ["the", "sprinkler"]}, {"sent": "sprinkler .", "words": ["sprinkler"]}, {"sent": "sit sprinkler thats right its not sitting .", "words": ["sit", "not", "sprinkler"]}, {"sent": "just checking out the sprinkler .", "words": ["the", "out", "sprinkler"]}, {"sent": "sprinkler does make things wet were giving the plants water .", "words": ["the", "wet", "does", "sprinkler", "water", "make", "were"]}, {"sent": "yeah the sprinkler is on .", "words": ["the", "on", "is", "sprinkler"]}, {"sent": "the sprinkler well the watering can is sprinkling water .", "words": ["the", "can", "is", "sprinkler", "water"]}, {"sent": "you put your hands in the water that the sprinkler was sprinkling ?", "words": ["the", "put", "your", "was", "you", "sprinkler", "in", "water", "that"]}, {"sent": "wanna go with me xxx well put on the sprinkler ?", "words": ["the", "put", "go", "with", "sprinkler", "wanna", "on", "me"]}, {"sent": "see the sprinkler ?", "words": ["the", "see", "sprinkler"]}, {"sent": "the sprinkler Alexander .", "words": ["the", "sprinkler"]}, {"sent": "thats the sprinkler Alexander .", "words": ["the", "sprinkler"]}, {"sent": "heres the sprinkler .", "words": ["the", "sprinkler"]}, {"sent": "thats the sprinkler .", "words": ["the", "sprinkler"]}, {"sent": "I dont think youve ever seen a sprinkler have you ?", "words": ["think", "sprinkler", "have", "dont", "you", "a"]}, {"sent": "when Mummy was a little girl we had a sprinkler at home .", "words": ["we", "was", "little", "sprinkler", "at", "home", "when", "a"]}, {"sent": "the sprinkler is coming on .", "words": ["the", "on", "is", "sprinkler"]}, {"sent": "and there was a shortage of water and he couldnt use his water sprinkler .", "words": ["of", "was", "and", "sprinkler", "there", "he", "water", "his", "a"]}, {"sent": "so this is a sprinkler .", "words": ["is", "so", "sprinkler", "this", "a"]}, {"sent": "there isnt room if you put the sprinkler in as well .", "words": ["the", "put", "if", "you", "sprinkler", "there", "in", "room"]}, {"sent": "maybe it is a sprinkler huh ?", "words": ["sprinkler", "it", "a", "is"]}, {"sent": "sprinkler .", "words": ["sprinkler"]}, {"sent": "shall we call it sprinkler ?", "words": ["sprinkler", "it", "we"]}, {"sent": "but you liked being in the sprinker sprinkler right ?", "words": ["the", "sprinkler", "you", "in", "but"]}, {"sent": "wheres her shovel ?", "words": ["her", "shovel"]}, {"sent": "you want the digger to come and help shovel some snow .", "words": ["the", "help", "to", "snow", "and", "you", "shovel", "some"]}, {"sent": "the power shovel pick the dirt up ?", "words": ["the", "pick", "up", "shovel"]}, {"sent": "theres a steam shovel and theres a horse trailer .", "words": ["horse", "shovel", "and", "a"]}, {"sent": "shovel .", "words": ["shovel"]}, {"sent": "you can see his brush and shovel on the side .", "words": ["can", "the", "brush", "you", "and", "on", "shovel", "his", "see"]}, {"sent": "I have a red shovel .", "words": ["have", "shovel", "red", "a"]}, {"sent": "see the shovel and the bucket ?", "words": ["the", "bucket", "and", "shovel", "see"]}, {"sent": "like the shovel from that new bag .", "words": ["the", "like", "shovel", "new", "that"]}, {"sent": "I was just thinking that the next time they make a mess Ill shout use your brush and shovel .", "words": ["the", "your", "brush", "was", "and", "they", "that", "shovel", "make", "a"]}, {"sent": "whatre you supposed to do with that shovel ?", "words": ["do", "to", "with", "you", "shovel", "that"]}, {"sent": "pail and shovel you found shovel wheres pail ?", "words": ["shovel", "you", "and"]}, {"sent": "yeah a boy and hes got a shovel .", "words": ["shovel", "and", "a"]}, {"sent": "I think she has her own shovel .", "words": ["think", "her", "she", "shovel"]}, {"sent": "you want another pail and shovel ?", "words": ["another", "shovel", "and", "you"]}, {"sent": "a shovel .", "words": ["shovel", "a"]}, {"sent": "good job there in that picture theres shovel and wheres pail hes red ?", "words": ["red", "picture", "and", "there", "in", "good", "shovel", "that"]}, {"sent": "thats a shovel .", "words": ["shovel", "a"]}, {"sent": "where did you shovel ?", "words": ["where", "shovel", "did", "you"]}, {"sent": "oh grannys got a shovel .", "words": ["shovel", "a"]}, {"sent": "will this young man over here do me a favor and bring the shovel from the sand corner .", "words": ["the", "do", "bring", "and", "here", "this", "shovel", "will", "me", "over", "a"]}, {"sent": "a shovel .", "words": ["shovel", "a"]}, {"sent": "didjou wanna give the shovel to xxx ?", "words": ["the", "to", "wanna", "shovel", "give"]}, {"sent": "what you were doing with the shovel .", "words": ["the", "what", "with", "you", "shovel", "were"]}, {"sent": "that is a shovel .", "words": ["shovel", "that", "a", "is"]}, {"sent": "what are you going to do with the pail and shovel ?", "words": ["the", "what", "do", "to", "with", "you", "and", "are", "shovel"]}, {"sent": "thats a steam shovel ?", "words": ["shovel", "a"]}, {"sent": "shes got a shovel and a bucket too .", "words": ["bucket", "and", "shovel", "too", "a"]}, {"sent": "are you having trouble with that shovel ?", "words": ["with", "you", "are", "shovel", "that"]}, {"sent": "can shovel up .", "words": ["can", "up", "shovel"]}, {"sent": "I didnt know they had a brush and shovel .", "words": ["brush", "and", "they", "shovel", "a"]}, {"sent": "she has a shovel .", "words": ["shovel", "she", "a"]}, {"sent": "what did you do with your pail and shovel today ?", "words": ["what", "do", "your", "with", "you", "and", "did", "shovel"]}, {"sent": "and with the shovel the shovel .", "words": ["the", "shovel", "with", "and"]}, {"sent": "the rake helps me put things into the shovel .", "words": ["the", "put", "into", "shovel", "me"]}, {"sent": "youre reminding me that there was a shovel in the laundry .", "words": ["the", "was", "there", "in", "shovel", "me", "that", "a"]}, {"sent": "you mean a shovel ?", "words": ["shovel", "a", "you"]}, {"sent": "you shovel oh .", "words": ["shovel", "you"]}, {"sent": "arm thats a shovel I think .", "words": ["shovel", "think", "arm", "a"]}, {"sent": "*AD2: the shovel I need the shovel .", "words": ["the", "need", "shovel"]}, {"sent": "and a shovel for the bucket .", "words": ["the", "for", "bucket", "and", "shovel", "a"]}, {"sent": "you have a shovel ?", "words": ["have", "shovel", "a", "you"]}, {"sent": "dont you hafta have your shovel in a special way ?", "words": ["your", "you", "have", "dont", "in", "shovel", "a"]}, {"sent": "right you see he could use his brush and shovel now because hes dropped something .", "words": ["brush", "you", "and", "he", "shovel", "his", "see", "could", "because"]}, {"sent": "with the big shovel ?", "words": ["the", "big", "with", "shovel"]}, {"sent": "xxx a shovel and a pick axe .", "words": ["pick", "shovel", "and", "a"]}, {"sent": "but thats named the shovel .", "words": ["the", "but", "shovel"]}, {"sent": "a shovel ?", "words": ["shovel", "a"]}, {"sent": "I mean they they cant go in the house if they dont and they cant go outof the house if they dont shovel the walk .", "words": ["the", "if", "go", "and", "they", "in", "dont", "house", "shovel", "walk"]}, {"sent": "did you shovel a lot of rocks ?", "words": ["of", "a lot", "you", "did", "shovel", "a"]}, {"sent": "theres a bucket and a shovel .", "words": ["bucket", "shovel", "and", "a"]}, {"sent": "with that someone who took his power shovel .", "words": ["with", "shovel", "his", "who", "that"]}, {"sent": "why dont you shovel them up with your digger ?", "words": ["up", "your", "them", "with", "you", "dont", "shovel", "why"]}, {"sent": "what do you do with a shovel ?", "words": ["what", "do", "with", "you", "shovel", "a"]}, {"sent": "I dont have any other pail and shovel .", "words": ["any", "and", "have", "dont", "other", "shovel"]}, {"sent": "yeah bring the shovel inside oh I hear the baby crying .", "words": ["the", "inside", "bring", "hear", "shovel"]}, {"sent": "Ill get you Ill get you the spade and shovel .", "words": ["the", "and", "get", "you", "shovel"]}, {"sent": "that is a shovel you know what that is .", "words": ["what", "is", "you", "shovel", "that", "a"]}, {"sent": "well no its got a more like a shovel on the front hasnt it ?", "words": ["the", "like", "more", "it", "shovel", "on", "a"]}, {"sent": "what shovel ?", "words": ["what", "shovel"]}, {"sent": "leave that shovel there .", "words": ["there", "that", "shovel"]}, {"sent": "oh that hes got a shovel .", "words": ["shovel", "that", "a"]}, {"sent": "do we have a shovel ?", "words": ["do", "we", "have", "shovel", "a"]}, {"sent": "Mey look at the shovel .", "words": ["the", "shovel", "at", "look"]}, {"sent": "youre pulling a shovel .", "words": ["shovel", "a"]}, {"sent": "*AD1: big bird is that a big bird shovel ?", "words": ["is", "bird", "big", "shovel", "that", "a"]}, {"sent": "with the shovel and the pick .", "words": ["the", "pick", "with", "and", "shovel"]}, {"sent": "a shovel and a pail and what else ?", "words": ["shovel", "what", "and", "a"]}, {"sent": "I think the idea Warren is to pick them up in that shovel really .", "words": ["the", "up", "think", "is", "pick", "to", "them", "in", "shovel", "that"]}, {"sent": "thats the shovel .", "words": ["the", "shovel"]}, {"sent": "they also gave you pail and shovel to play at the beach .", "words": ["the", "to", "play", "and", "you", "they", "beach", "at", "shovel"]}, {"sent": "where did you shovel ?", "words": ["where", "shovel", "did", "you"]}, {"sent": "okay and heres a shovel for me .", "words": ["for", "and", "shovel", "me", "a"]}, {"sent": "where did you shovel yesterday ?", "words": ["where", "shovel", "did", "you"]}, {"sent": "pretty shovel big shovel .", "words": ["big", "pretty", "shovel"]}, {"sent": "wheres your shovel ?", "words": ["your", "shovel"]}, {"sent": "theres a little shovel in there I was trying to get out .", "words": ["was", "to", "little", "get", "there", "in", "out", "shovel", "a"]}, {"sent": "how about a shovel ?", "words": ["shovel", "how", "about", "a"]}, {"sent": "a shovel ?", "words": ["shovel", "a"]}, {"sent": "okay lets look for a pail and shovel .", "words": ["for", "look", "and", "shovel", "a"]}, {"sent": "you need the shovel ?", "words": ["the", "shovel", "need", "you"]}, {"sent": "oh no he went into Roger Rhinos power shovel .", "words": ["he", "into", "shovel"]}, {"sent": "you had a new shovel .", "words": ["shovel", "new", "a", "you"]}, {"sent": "oh its a steam shovel now .", "words": ["shovel", "a"]}, {"sent": "grannys got a shovel .", "words": ["shovel", "a"]}, {"sent": "on a shovel ?", "words": ["on", "shovel", "a"]}, {"sent": "shovel you mean .", "words": ["you", "shovel"]}, {"sent": "and whos that with a shovel trying to get it out ?", "words": ["to", "with", "and", "get", "it", "out", "shovel", "that", "a"]}, {"sent": "go get the shovel wheres the shovel ?", "words": ["get", "the", "shovel", "go"]}, {"sent": "do you need a shovel do you think ?", "words": ["do", "think", "need", "you", "shovel", "a"]}, {"sent": "oh this big shovel ?", "words": ["this", "big", "shovel"]}, {"sent": "thats not a shovel .", "words": ["shovel", "not", "a"]}, {"sent": "this shovel opens and closes but this one doesnt do that .", "words": ["do", "and", "this", "shovel", "that", "but"]}, {"sent": "why does he shovel the snow ?", "words": ["the", "does", "snow", "he", "shovel", "why"]}, {"sent": "well it looks more like a shovel .", "words": ["like", "more", "it", "shovel", "a"]}, {"sent": "I guess John had a um tractor shovel or something .", "words": ["shovel", "tractor", "a"]}, {"sent": "heres your shovel .", "words": ["your", "shovel"]}, {"sent": "youve got crayon on it as well .", "words": ["crayon", "on", "it"]}, {"sent": "here put the crayon .", "words": ["here", "the", "put", "crayon"]}, {"sent": "that still has crayon .", "words": ["crayon", "that"]}, {"sent": "in here is your green crayon .", "words": ["your", "is", "crayon", "here", "in", "green"]}, {"sent": "what do we do with a crayon ?", "words": ["what", "do", "we", "crayon", "with", "a"]}, {"sent": "give the crayon to the lady .", "words": ["the", "to", "give", "crayon"]}, {"sent": "crayon .", "words": ["crayon"]}, {"sent": "one green crayon .", "words": ["crayon", "green"]}, {"sent": "I need another crayon to make believe with .", "words": ["need", "crayon", "to", "with", "another", "make"]}, {"sent": "put the crayon down on the paper .", "words": ["the", "put", "crayon", "paper", "on", "down"]}, {"sent": "use your crayon .", "words": ["crayon", "your"]}, {"sent": "thats the green crayon dyou wanna go get the green crayon for me ?", "words": ["the", "for", "go", "crayon", "get", "wanna", "green", "me"]}, {"sent": "hold the crayon .", "words": ["the", "hold", "crayon"]}, {"sent": "next to the orange crayon .", "words": ["the", "to", "orange", "crayon"]}, {"sent": "get the crayon .", "words": ["get", "the", "crayon"]}, {"sent": "so you put your put your crayon on the bunny .", "words": ["the", "put", "bunny", "your", "crayon", "so", "you", "on"]}, {"sent": "here Ill give you another crayon .", "words": ["crayon", "you", "here", "another", "give"]}, {"sent": "have you got crayon in it ?", "words": ["crayon", "you", "have", "in", "it"]}, {"sent": "put that crayon back in that box and lets do something else .", "words": ["do", "put", "box", "back", "crayon", "and", "in", "that"]}, {"sent": "look like that with your crayon .", "words": ["your", "crayon", "with", "like", "look", "that"]}, {"sent": "you have the red crayon can you find the red crayon ?", "words": ["the", "red", "can", "crayon", "you", "have", "find"]}, {"sent": "like the crayon ?", "words": ["the", "like", "crayon"]}, {"sent": "right look watch the crayon .", "words": ["the", "watch", "crayon", "look"]}, {"sent": "you wanna use my crayon ?", "words": ["crayon", "my", "wanna", "you"]}, {"sent": "yellow crayon I see .", "words": ["crayon", "yellow", "see"]}, {"sent": "here you take you get the orange crayon .", "words": ["the", "crayon", "you", "here", "get", "orange", "take"]}, {"sent": "put the crayon in the box .", "words": ["the", "put", "box", "crayon", "in"]}, {"sent": "wanna have the red crayon ?", "words": ["the", "red", "crayon", "have", "wanna"]}, {"sent": "can you put this crayon away for Mommy ?", "words": ["can", "put", "for", "crayon", "you", "this", "away"]}, {"sent": "your purple crayon ?", "words": ["crayon", "your"]}, {"sent": "are they crayon ?", "words": ["crayon", "they", "are"]}, {"sent": "you can have a crayon .", "words": ["can", "crayon", "you", "have", "a"]}, {"sent": "may I have that crayon ?", "words": ["have", "that", "crayon"]}, {"sent": "what happened to your crayon ?", "words": ["to", "what", "your", "crayon"]}, {"sent": "thats a blue crayon .", "words": ["crayon", "blue", "a"]}, {"sent": "thats a crayon .", "words": ["crayon", "a"]}, {"sent": "let us put the crayon away for that .", "words": ["the", "put", "for", "crayon", "us", "away", "that"]}, {"sent": "you draw a crayon you can draw a crayon and Ill draw a crayon .", "words": ["can", "crayon", "draw", "you", "and", "a"]}, {"sent": "what color is this crayon ?", "words": ["crayon", "what", "this", "is"]}, {"sent": "its a crayon .", "words": ["crayon", "a"]}, {"sent": "heres another crayon .", "words": ["another", "crayon"]}, {"sent": "but you see I hadnt got a white crayon had I so I hadta make it a black cat instead of a black of white cat .", "words": ["crayon", "so", "of", "you", "black", "cat", "it", "see", "white", "make", "but", "a"]}, {"sent": "you need to go and pick up that crayon .", "words": ["up", "need", "go", "pick", "to", "crayon", "need to", "and", "you", "that"]}, {"sent": "its a crayon sharpener really isnt it ?", "words": ["crayon", "it", "a"]}, {"sent": "no hold the crayon .", "words": ["the", "hold", "crayon"]}, {"sent": "sleep with the crayon ?", "words": ["sleep", "the", "with", "crayon"]}, {"sent": "howcome you got the crayon all over her ?", "words": ["the", "all", "crayon", "her", "you", "over"]}, {"sent": "whoa that is a fat crayon .", "words": ["crayon", "that", "a", "is"]}, {"sent": "crayon it all in .", "words": ["crayon", "all", "in", "it"]}, {"sent": "no this a regular crayon .", "words": ["crayon", "this", "a"]}, {"sent": "crayon .", "words": ["crayon"]}, {"sent": "if you put your crayon down .", "words": ["put", "your", "if", "crayon", "you", "down"]}, {"sent": "I think the crayon will work without you peeling off all of the paper .", "words": ["the", "off", "think", "all", "work", "crayon", "of", "paper", "you", "will"]}, {"sent": "uhhh wash crayon marks from walls as soon as possible if crayon marks remain after washing use a .", "words": ["crayon", "wash", "if", "a"]}, {"sent": "I dont know where your black crayon is Fraser .", "words": ["where", "your", "is", "crayon", "black", "dont"]}, {"sent": "now you gonna color with the red crayon ?", "words": ["the", "red", "crayon", "with", "you"]}, {"sent": "crayon .", "words": ["crayon"]}, {"sent": "thats where the crayon broke on the floor .", "words": ["where", "the", "on", "crayon"]}, {"sent": "you got it your crayon upside down .", "words": ["your", "crayon", "you", "it", "down"]}, {"sent": "did you lose a crayon honey ?", "words": ["crayon", "did", "a", "you"]}, {"sent": "put the crayon back and well read okay ?", "words": ["the", "put", "back", "crayon", "and", "read"]}, {"sent": "did you see that pink flower and think about the pink crayon ?", "words": ["the", "think", "crayon", "flower", "you", "and", "did", "about", "see", "that"]}, {"sent": "Thomas is making a crayon out of a piece of tinsel .", "words": ["is", "crayon", "of", "out", "a"]}, {"sent": "with a crayon ?", "words": ["crayon", "with", "a"]}, {"sent": "we need to get those crayon marks off .", "words": ["off", "need", "those", "we", "crayon", "to", "need to", "get"]}, {"sent": "crayon .", "words": ["crayon"]}, {"sent": "can you lean over and just color the donkeys with your crayon ?", "words": ["can", "the", "your", "crayon", "with", "and", "you", "over"]}, {"sent": "ooh theres a red crayon .", "words": ["red", "crayon", "a"]}, {"sent": "let us get a crayon .", "words": ["get", "us", "crayon", "a"]}, {"sent": "wanna do it with the crayon ?", "words": ["the", "do", "crayon", "with", "it", "wanna"]}, {"sent": "can you why dont you try and get a red crayon ?", "words": ["can", "red", "crayon", "you", "and", "dont", "get", "try", "why", "a"]}, {"sent": "this is an orange crayon .", "words": ["is", "crayon", "this", "an", "orange"]}, {"sent": "can you say crayon ?", "words": ["can", "say", "crayon", "you"]}, {"sent": "thats not thats not even a real thats not a real crayon .", "words": ["crayon", "not", "a"]}, {"sent": "that is not a red crayon Nathaniel .", "words": ["red", "is", "crayon", "that", "not", "a"]}, {"sent": "its a crayon ring .", "words": ["crayon", "a"]}, {"sent": "and the other ones a yellow crayon .", "words": ["the", "crayon", "other", "and", "yellow", "a"]}, {"sent": "what black crayon ?", "words": ["black", "what", "crayon"]}, {"sent": "see a crayon ?", "words": ["crayon", "see", "a"]}, {"sent": "its my crayon .", "words": ["crayon", "my"]}, {"sent": "get get a crayon out of here .", "words": ["of", "crayon", "get", "here", "out", "a"]}, {"sent": "that is a crayon .", "words": ["crayon", "that", "a", "is"]}, {"sent": "oh this little crayon got broken .", "words": ["crayon", "this", "broken", "little"]}, {"sent": "and a crayon and I will show you how to write it .", "words": ["how", "crayon", "show", "to", "write", "and", "you", "it", "will", "a"]}, {"sent": "theres a crayon here .", "words": ["crayon", "here", "a"]}, {"sent": "you get the crayon and well put it in the box .", "words": ["the", "put", "box", "crayon", "and", "get", "you", "in", "it"]}, {"sent": "some crayon ?", "words": ["crayon", "some"]}, {"sent": "get the blue crayon .", "words": ["get", "the", "crayon", "blue"]}, {"sent": "but luckily he kept his wits and his purple crayon .", "words": ["crayon", "and", "he", "his", "but"]}, {"sent": "I found another crayon Eve .", "words": ["another", "crayon"]}, {"sent": "do not eat crayon .", "words": ["eat", "do", "not", "crayon"]}, {"sent": "crayon .", "words": ["crayon"]}, {"sent": "oh youve got a crayon in each hand .", "words": ["each", "hand", "crayon", "in", "a"]}, {"sent": "should we get another crayon Laura ?", "words": ["get", "crayon", "another", "we"]}, {"sent": "hes back with his purple crayon .", "words": ["his", "with", "crayon", "back"]}, {"sent": "its a broken green crayon .", "words": ["crayon", "green", "broken", "a"]}, {"sent": "dont eat that little crayon bit .", "words": ["crayon", "little", "eat", "dont", "that"]}, {"sent": "uhoh its in your pants .", "words": ["in", "your", "pants"]}, {"sent": "and also I recognized those are not your sweat pants .", "words": ["your", "those", "and", "are", "pants", "not"]}, {"sent": "that is his pants .", "words": ["his", "that", "pants", "is"]}, {"sent": "what does she wear with her pants ?", "words": ["what", "does", "with", "pants", "her", "she"]}, {"sent": "the pants are like your pants with the buttons right there ?", "words": ["the", "your", "with", "like", "there", "are", "pants"]}, {"sent": "lets take your pants down .", "words": ["your", "down", "take", "pants"]}, {"sent": "Ill get you some clean pants in a minute .", "words": ["clean", "you", "get", "in", "pants", "some", "a"]}, {"sent": "right well pull your pants up .", "words": ["up", "your", "pants", "pull"]}, {"sent": "you wear those pants Im gonna xxx .", "words": ["those", "pants", "you"]}, {"sent": "and how about lets see these arent pants is this your pants drawer ?", "words": ["these", "how", "your", "drawer", "is", "and", "this", "pants", "about", "see"]}, {"sent": "lets see these pants what size are these ?", "words": ["what", "these", "are", "pants", "see"]}, {"sent": "let me pull your pants up .", "words": ["up", "your", "pull", "pants", "me"]}, {"sent": "be careful youre walking around with your pants down by your ankles .", "words": ["your", "be", "careful", "with", "by", "pants", "around", "down"]}, {"sent": "you havent just dirtied your pants again ?", "words": ["your", "pants", "you"]}, {"sent": "you dont do a weewee in your pants .", "words": ["do", "your", "you", "dont", "in", "pants", "a"]}, {"sent": "here is his pants .", "words": ["here", "his", "pants", "is"]}, {"sent": "mommys got it on her pants see ?", "words": ["it", "pants", "on", "see", "her"]}, {"sent": "pants .", "words": ["pants"]}, {"sent": "is he wearing pants ?", "words": ["he", "pants", "is"]}, {"sent": "shall we put some pants on ?", "words": ["put", "we", "pants", "on", "some"]}, {"sent": "you cant take his pants off .", "words": ["off", "you", "pants", "his", "take"]}, {"sent": "can I fix your pants ?", "words": ["can", "fix", "your", "pants"]}, {"sent": "go and get some pants out of your drawer .", "words": ["your", "drawer", "go", "of", "and", "get", "pants", "out", "some"]}, {"sent": "and then I guess maybe this weekend we can assess how we are with shorts and pants and everything .", "words": ["can", "how", "shorts", "we", "with", "and", "this", "are", "pants", "then"]}, {"sent": "shes got no pants on .", "words": ["on", "pants"]}, {"sent": "training pants ?", "words": ["pants"]}, {"sent": "yes you have your blue pants on .", "words": ["your", "blue", "have", "you", "pants", "on"]}, {"sent": "what is this on your pants by the way ?", "words": ["the", "what", "your", "is", "by", "this", "pants", "on"]}, {"sent": "now put his pants on .", "words": ["his", "put", "pants", "on"]}, {"sent": "put the ladys pants on put the ladys pants on .", "words": ["the", "put", "pants", "on"]}, {"sent": "comere please I know youre trying to pull your pants up but you know what ?", "words": ["up", "what", "your", "to", "pull", "you", "pants", "but"]}, {"sent": "come here and lets put your pants on .", "words": ["put", "your", "and", "here", "pants", "on"]}, {"sent": "poopie pants .", "words": ["pants"]}, {"sent": "a zipper on his pants .", "words": ["his", "zipper", "pants", "on", "a"]}, {"sent": "leave your pants on please .", "words": ["on", "your", "pants"]}, {"sent": "here take your pants off sweetie .", "words": ["off", "your", "here", "pants", "take"]}, {"sent": "pj pants .", "words": ["pants"]}, {"sent": "come on get outof them damn pants .", "words": ["on", "get", "them", "pants"]}, {"sent": "bite your pants ?", "words": ["bite", "your", "pants"]}, {"sent": "see if he has a pair of pants or a shirt .", "words": ["if", "of", "shirt", "he", "pants", "see", "a"]}, {"sent": "can you find pants ?", "words": ["can", "pants", "find", "you"]}, {"sent": "can you put the ladys pants on ?", "words": ["can", "the", "put", "you", "pants", "on"]}, {"sent": "where these white pants ?", "words": ["where", "these", "white", "pants"]}, {"sent": "does Nina wear pants ?", "words": ["does", "pants"]}, {"sent": "let me pull up your pants .", "words": ["up", "your", "pull", "pants", "me"]}, {"sent": "she shes wearing pants .", "words": ["pants", "she"]}, {"sent": "oh and theres pants .", "words": ["pants", "and"]}, {"sent": "wet your pants ?", "words": ["wet", "your", "pants"]}, {"sent": "you could clip it on your pants if you wanted .", "words": ["your", "if", "you", "it", "pants", "on", "could"]}, {"sent": "here did you split your pants too along with everything else today ?", "words": ["your", "with", "you", "here", "pants", "did", "too"]}, {"sent": "say pants .", "words": ["say", "pants"]}, {"sent": "take the pants off .", "words": ["the", "take", "off", "pants"]}, {"sent": "pants on .", "words": ["on", "pants"]}, {"sent": "Ewwws got pants on .", "words": ["on", "pants"]}, {"sent": "yellow hat green shirt blue pants purple socks red shoes oops .", "words": ["red", "shirt", "blue", "pants", "green", "hat", "yellow"]}, {"sent": "theyre Awwws pants .", "words": ["pants"]}, {"sent": "can you put the ladys pants on ?", "words": ["can", "the", "put", "you", "pants", "on"]}, {"sent": "lemme check your pants .", "words": ["your", "pants"]}, {"sent": "and its got a little clip you put on the back of your pants .", "words": ["the", "put", "your", "back", "of", "you", "little", "and", "pants", "on", "a"]}, {"sent": "shes saying that sounds too loud for me Im gonna go hide in your pants .", "words": ["for", "your", "hide", "go", "in", "pants", "too", "me", "that", "loud"]}, {"sent": "and your pants and your coat .", "words": ["your", "coat", "pants", "and"]}, {"sent": "how bout his pants ?", "words": ["his", "how", "pants"]}, {"sent": "shall we put this top and the pants on ?", "words": ["the", "put", "we", "and", "this", "pants", "on"]}, {"sent": "the pants ?", "words": ["the", "pants"]}, {"sent": "can you pick up those pants and bring them into your room so we can put them on ?", "words": ["can", "up", "put", "those", "into", "your", "we", "pick", "so", "them", "bring", "you", "and", "room", "pants", "on"]}, {"sent": "um I was asking her to help me pick out some clothes for the dolls but she lost interest after pants and shoes .", "words": ["the", "help", "for", "pick", "to", "was", "and", "pants", "out", "me", "her", "she", "but", "some"]}, {"sent": "Mama bought you those pants darling .", "words": ["those", "pants", "you"]}, {"sent": "okay can you put you can put her pants on cant you ?", "words": ["can", "put", "you", "pants", "on", "her"]}, {"sent": "his pants .", "words": ["his", "pants"]}, {"sent": "because youve got your pants round your knees havent you ?", "words": ["pants", "your", "because", "you"]}, {"sent": "those are your pants .", "words": ["your", "are", "those", "pants"]}, {"sent": "there you help the pants .", "words": ["the", "help", "you", "there", "pants"]}, {"sent": "take the pants off .", "words": ["the", "take", "off", "pants"]}, {"sent": "socks and pants .", "words": ["pants", "and"]}, {"sent": "I wanna pull your pants up .", "words": ["up", "your", "pull", "pants", "wanna"]}, {"sent": "heres some big kid pants .", "words": ["some", "big", "pants"]}, {"sent": "shes got her big girl pants .", "words": ["big", "her", "pants"]}, {"sent": "do you want to take your pants off and sit down ?", "words": ["off", "do", "your", "to", "you", "and", "pants", "want to", "sit", "down", "take"]}, {"sent": "there are is the gun in your pants ?", "words": ["the", "your", "is", "there", "in", "are", "pants"]}, {"sent": "oh my gosh he laid her egg in her pants .", "words": ["my", "egg", "in", "he", "pants", "her"]}, {"sent": "then put de udder pants on .", "words": ["on", "then", "pants", "put"]}, {"sent": "just put your trousers back on with no pants .", "words": ["put", "your", "back", "with", "pants", "on"]}, {"sent": "youve got pants on already .", "words": ["on", "pants"]}, {"sent": "you dont want her pants off do you ?", "words": ["off", "do", "you", "dont", "pants", "her"]}, {"sent": "fix your pants .", "words": ["fix", "your", "pants"]}, {"sent": "where are her pants ?", "words": ["where", "are", "her", "pants"]}, {"sent": "well change your diaper and Ill change my pants too .", "words": ["my", "your", "diaper", "and", "pants", "too"]}, {"sent": "no put the pants down here .", "words": ["the", "put", "here", "pants", "down"]}, {"sent": "do you have messy pants ?", "words": ["have", "do", "pants", "you"]}, {"sent": "can you put his pants on Elbert ?", "words": ["can", "put", "you", "on", "pants", "his"]}, {"sent": "then you can wear pants .", "words": ["can", "then", "pants", "you"]}, {"sent": "do you wanna put some pants on Fraser ?", "words": ["do", "put", "you", "pants", "wanna", "on", "some"]}, {"sent": "do you wanna go and get some pants for me ?", "words": ["do", "for", "go", "me", "and", "you", "get", "pants", "wanna", "some"]}, {"sent": "I like these pants do you like these pants ?", "words": ["these", "do", "like", "you", "pants"]}, {"sent": "the pants ?", "words": ["the", "pants"]}, {"sent": "arent they nice all these pants we got from the Liebermans .", "words": ["the", "all", "these", "we", "nice", "they", "pants"]}, {"sent": "maybe Ill put on oh these pants look better .", "words": ["these", "put", "better", "pants", "look", "on"]}, {"sent": "okay lets put your pants on .", "words": ["on", "put", "your", "pants"]}, {"sent": "the blue the blue and your shirt and your pants .", "words": ["the", "your", "shirt", "and", "blue", "pants"]}, {"sent": "hes traveling the house by basket .", "words": ["the", "basket", "house", "by"]}, {"sent": "and this big old basket that he keeps his bats in .", "words": ["old", "and", "this", "that", "in", "basket", "he", "his", "big"]}, {"sent": "do you ever get it into the basket ?", "words": ["the", "do", "into", "you", "get", "it", "basket"]}, {"sent": "I go and get shopping basket and xxx .", "words": ["get", "basket", "and", "go"]}, {"sent": "what should go in the basket Lily ?", "words": ["the", "what", "go", "in", "basket"]}, {"sent": "and then youll be able to pull the basket out even more easily .", "words": ["the", "to", "be", "pull", "and", "more", "basket", "out", "then"]}, {"sent": "what do you roll in the basket ?", "words": ["the", "what", "do", "you", "in", "basket"]}, {"sent": "shopping basket .", "words": ["basket"]}, {"sent": "can you see some bananas in the fruit basket ?", "words": ["can", "the", "you", "in", "basket", "see", "some"]}, {"sent": "xxx I was trying to find you a basket no we dont have a basket .", "words": ["we", "to", "was", "you", "have", "dont", "basket", "find", "a"]}, {"sent": "and well put the clothes back in the laundry basket .", "words": ["the", "put", "back", "and", "in", "basket"]}, {"sent": "lets put some of these foods back in the shopping basket shall we ?", "words": ["the", "these", "put", "back", "we", "of", "in", "basket", "some"]}, {"sent": "do you remember when you got an Easter basket ?", "words": ["do", "you", "an", "basket", "when"]}, {"sent": "I wonder if hes in Purdies basket .", "words": ["in", "basket", "if"]}, {"sent": "the soccer balls in your basket .", "words": ["the", "in", "your", "basket"]}, {"sent": "got the ball outof the basket and threw it .", "words": ["the", "and", "ball", "it", "basket"]}, {"sent": "oh give me those apples you have in your basket said the kangaroo .", "words": ["the", "those", "your", "you", "have", "in", "basket", "me", "give"]}, {"sent": "Im gonna put everything in the basket okay ?", "words": ["the", "in", "put", "basket"]}, {"sent": "I dont they dont have any bagels right there on this on this in this basket .", "words": ["any", "have", "dont", "they", "there", "this", "in", "basket", "on"]}, {"sent": "still in the basket .", "words": ["the", "in", "basket"]}, {"sent": "and on this side of the basket what has it got pictures of ?", "words": ["the", "what", "of", "and", "this", "it", "basket", "on"]}, {"sent": "thats a wire basket like we use at the supermarket isnt it ?", "words": ["the", "we", "like", "it", "basket", "at", "a"]}, {"sent": "whats in the basket ?", "words": ["the", "in", "basket"]}, {"sent": "you wanna put all this stuff in your basket ?", "words": ["all", "put", "your", "you", "this", "in", "basket", "wanna"]}, {"sent": "is spot in the basket ?", "words": ["the", "in", "basket", "is"]}, {"sent": "wheres the picnic basket ?", "words": ["the", "basket"]}, {"sent": "yave a lil basket .", "words": ["basket", "a"]}, {"sent": "in here in the basket ?", "words": ["here", "the", "in", "basket"]}, {"sent": "a basket .", "words": ["basket", "a"]}, {"sent": "and they looked in the lunch basket they looked everywhere but the turtle wasnt anywhere no one had seen oop .", "words": ["the", "and", "they", "in", "basket", "turtle", "but"]}, {"sent": "this is the basket .", "words": ["the", "this", "basket", "is"]}, {"sent": "you put the things in the basket .", "words": ["the", "put", "you", "in", "basket"]}, {"sent": "the basket .", "words": ["the", "basket"]}, {"sent": "play basket ball .", "words": ["ball", "play", "basket"]}, {"sent": "but you know the sort of basket I mean dont you ?", "words": ["the", "of", "you", "dont", "basket", "but"]}, {"sent": "the cutlery goes in that basket you see .", "words": ["the", "you", "in", "basket", "see", "that"]}, {"sent": "are you gonna keep showing me whats in your basket ?", "words": ["your", "you", "in", "are", "basket", "me"]}, {"sent": "thats the wastepaper basket .", "words": ["the", "basket"]}, {"sent": "with a flower basket .", "words": ["basket", "flower", "with", "a"]}, {"sent": "shall we put shall we put these things in the basket ?", "words": ["the", "these", "put", "we", "in", "basket"]}, {"sent": "his paws were soon full and he was just beginning to wish he had left his suitcase at home when he saw the manager coming toward him pushing a large basket on wheels .", "words": ["the", "when", "full", "was", "to", "and", "wish", "him", "on", "he", "basket", "at", "his", "were", "a", "home"]}, {"sent": "yeah all the food back in your yellow basket .", "words": ["the", "all", "your", "back", "food", "in", "basket", "yellow"]}, {"sent": "are you going to put some food in the basket then aswell ?", "words": ["the", "put", "to", "you", "food", "are", "in", "basket", "some", "then"]}, {"sent": "a basket .", "words": ["basket", "a"]}, {"sent": "put all the tins in your basket and then you can see what youve got .", "words": ["the", "all", "put", "can", "your", "what", "and", "you", "in", "basket", "see", "then"]}, {"sent": "in the basket please .", "words": ["the", "in", "basket"]}, {"sent": "what else could we eat outof the basket ?", "words": ["the", "what", "we", "eat", "basket", "could"]}, {"sent": "Eleanor can you put those clothes in the laundry basket please .", "words": ["can", "the", "put", "those", "you", "in", "basket"]}, {"sent": "and thats what it looks like when we were inside the basket .", "words": ["the", "what", "we", "inside", "like", "and", "it", "basket", "when", "were"]}, {"sent": "and that went right into the basket as well .", "words": ["the", "into", "and", "basket", "that"]}, {"sent": "whatre you taking with you in your basket ?", "words": ["your", "with", "you", "in", "basket"]}, {"sent": "and while youre about it could you take this basket of eggs to the ReverendTims ?", "words": ["the", "of", "to", "you", "and", "this", "it", "basket", "about", "could", "take"]}, {"sent": "should we yeah lets put all the food in the basket .", "words": ["the", "all", "put", "we", "food", "in", "basket"]}, {"sent": "WinniethePooh fell down in the basket .", "words": ["the", "in", "down", "basket"]}, {"sent": "the basket fell down ?", "words": ["the", "down", "basket"]}, {"sent": "or put it in the basket and then put it in the bag .", "words": ["the", "put", "and", "in", "it", "basket", "then"]}, {"sent": "what else have you got in the shopping basket Thomas ?", "words": ["the", "what", "you", "have", "in", "basket"]}, {"sent": "you throw the ball into the basket dont you ?", "words": ["the", "throw", "into", "you", "ball", "dont", "basket"]}, {"sent": "you have a shopping basket like that dont you ?", "words": ["like", "you", "have", "dont", "basket", "that", "a"]}, {"sent": "no the basket .", "words": ["the", "basket"]}, {"sent": "hes spinning around in the laundry basket ?", "words": ["the", "in", "basket", "around"]}, {"sent": "in the basket .", "words": ["the", "in", "basket"]}, {"sent": "what happened to the pot basket ball ?", "words": ["the", "what", "to", "ball", "basket"]}, {"sent": "it is a basket .", "words": ["it", "basket", "a", "is"]}, {"sent": "put it in the basket .", "words": ["the", "put", "in", "it", "basket"]}, {"sent": "no you put it in the basket like Mama does .", "words": ["the", "does", "put", "like", "you", "in", "it", "basket"]}, {"sent": "do you wanna put some things in your basket ?", "words": ["do", "put", "your", "you", "in", "basket", "wanna", "some"]}, {"sent": "perhaps we can put it in the basket first .", "words": ["can", "the", "put", "we", "first", "in", "it", "basket"]}, {"sent": "I thought it was a basket .", "words": ["was", "it", "basket", "a"]}, {"sent": "Kangaa is carrying a basket on her back it looks like .", "words": ["back", "is", "like", "it", "basket", "on", "her", "a"]}, {"sent": "and shopping from your shopping basket .", "words": ["basket", "your", "and"]}, {"sent": "no no you didnt get this one in your Easter basket .", "words": ["your", "you", "get", "this", "in", "basket"]}, {"sent": "how many candles have we had in this basket ?", "words": ["how", "we", "have", "this", "in", "basket"]}, {"sent": "would you like to take the fruit from Dimitra and put it in the basket ?", "words": ["the", "put", "to", "like", "you", "and", "in", "it", "basket", "would", "take"]}, {"sent": "can you see underneath the hanging basket ?", "words": ["can", "the", "you", "basket", "see"]}, {"sent": "its a basket .", "words": ["basket", "a"]}, {"sent": "I need to find the laundry basket empty one now .", "words": ["the", "need", "need to", "to", "empty", "basket", "find"]}, {"sent": "put it back in the basket .", "words": ["the", "put", "back", "in", "it", "basket"]}, {"sent": "put her in her basket and then when we saw them the next time they were actually driving .", "words": ["the", "put", "when", "we", "them", "and", "they", "in", "basket", "then", "her", "were"]}, {"sent": "youknow a stand for a Moses basket ?", "words": ["stand", "for", "basket", "a"]}, {"sent": "there are definitely some grapes in the basket .", "words": ["the", "there", "in", "are", "grapes", "basket", "some"]}, {"sent": "because we need its in the washing basket .", "words": ["the", "need", "we", "in", "basket", "because"]}, {"sent": "whatve you got in your shopping basket Becky ?", "words": ["basket", "in", "your", "you"]}, {"sent": "dont take the basket off her .", "words": ["the", "off", "her", "dont", "basket", "take"]}, {"sent": "xxx put that in a basket ?", "words": ["put", "in", "basket", "that", "a"]}, {"sent": "put it in the basket .", "words": ["the", "put", "in", "it", "basket"]}, {"sent": "a shopping basket ?", "words": ["basket", "a"]}, {"sent": "you were hiding in the basket ?", "words": ["the", "you", "in", "basket", "were"]}, {"sent": "he was in the dogs basket .", "words": ["the", "was", "in", "he", "basket"]}, {"sent": "is that your shopping basket ?", "words": ["your", "basket", "that", "is"]}, {"sent": "you wanna put the shirt in the clothes basket like mommy has ?", "words": ["the", "put", "shirt", "like", "you", "in", "basket", "wanna"]}, {"sent": "thats a basket .", "words": ["basket", "a"]}, {"sent": "you may go around with the basket for me today please Ingrid .", "words": ["the", "for", "go", "with", "you", "basket", "around", "me"]}, {"sent": "all the things that normally go inside the basket .", "words": ["the", "all", "go", "inside", "basket", "that"]}, {"sent": "and um yeah and then she brought them over and put them in the little shopping basket .", "words": ["the", "put", "them", "and", "little", "in", "basket", "then", "over", "she"]}, {"sent": "I may have the basket where the marbles belong ?", "words": ["have", "the", "basket", "where"]}, {"sent": "the big whales in the basket aswell .", "words": ["the", "in", "big", "basket"]}, {"sent": "Id better put that in the basket in a minute .", "words": ["the", "better", "put", "in", "basket", "that", "a"]}, {"sent": "wow thats a garbage truck .", "words": ["truck", "garbage", "a"]}, {"sent": "its garbage day .", "words": ["garbage"]}, {"sent": "Mommy has to open the garage door for the garbage guys .", "words": ["the", "for", "open", "to", "garage", "garbage", "door"]}, {"sent": "put the garbage in these .", "words": ["the", "these", "put", "in", "garbage"]}, {"sent": "we could save all our garbage .", "words": ["all", "we", "our", "could", "garbage"]}, {"sent": "maybe well go to Target later because Mommy needs to get a garbage can .", "words": ["can", "go", "to", "get", "garbage", "because", "a"]}, {"sent": "where do I put the garbage ?", "words": ["where", "do", "put", "the", "garbage"]}, {"sent": "I put dat cloth in the garbage ?", "words": ["the", "in", "put", "garbage"]}, {"sent": "empty garbage ?", "words": ["garbage", "empty"]}, {"sent": "took what and threw it in the garbage ?", "words": ["the", "what", "and", "in", "it", "garbage"]}, {"sent": "here goes the garbage .", "words": ["here", "the", "garbage"]}, {"sent": "and then it was putting it through like a sieve almost and it was taking all the garbage out of the sand and also flattening the sand .", "words": ["the", "all", "of", "was", "like", "and", "it", "out", "then", "garbage", "a"]}, {"sent": "and I sometimes come in your garbage cans at night when youre not looking .", "words": ["your", "not", "and", "in", "at", "when", "garbage"]}, {"sent": "and this is a drinker theres his seat and theres a garbage can .", "words": ["can", "is", "and", "this", "his", "garbage", "a"]}, {"sent": "get off that garbage .", "words": ["get", "garbage", "that", "off"]}, {"sent": "thats a garbage can .", "words": ["can", "garbage", "a"]}, {"sent": "*SI1: no we cant see it cant see the garbage truck .", "words": ["the", "we", "truck", "it", "see", "garbage"]}, {"sent": "it can fit in this garbage can .", "words": ["can", "fit", "this", "in", "it", "garbage"]}, {"sent": "in front of a garbage truck ?", "words": ["of", "truck", "in", "garbage", "a"]}, {"sent": "well Ill clean up some of this garbage .", "words": ["clean", "up", "of", "this", "some", "garbage"]}, {"sent": "you wanna put that in the garbage ?", "words": ["the", "put", "you", "in", "wanna", "that", "garbage"]}, {"sent": "hes getting our garbage bye garbage .", "words": ["garbage", "our"]}, {"sent": "like a garbage can .", "words": ["can", "garbage", "like", "a"]}, {"sent": "what are garbage cans ?", "words": ["what", "are", "garbage"]}, {"sent": "the flys are flying around the garbage truck .", "words": ["the", "truck", "are", "around", "garbage"]}, {"sent": "and this is a garbage can ?", "words": ["can", "is", "and", "this", "garbage", "a"]}, {"sent": "here is that green garbage truck .", "words": ["is", "truck", "here", "green", "that", "garbage"]}, {"sent": "there is a red bus and there is a green garbage truck .", "words": ["red", "is", "truck", "and", "there", "green", "bus", "garbage", "a"]}, {"sent": "*SI1: yeah garbage truck garbage truck .", "words": ["truck", "garbage"]}, {"sent": "now lets find out whats in the garbage disposal .", "words": ["the", "in", "out", "find", "garbage"]}, {"sent": "may be its the garbage truck to pick up the garbage .", "words": ["the", "up", "pick", "be", "to", "truck", "garbage"]}, {"sent": "its your garbage you put it away .", "words": ["put", "your", "you", "it", "away", "garbage"]}, {"sent": "garbage is what its what ?", "words": ["what", "garbage", "is"]}, {"sent": "your daddy threw more marbles in the garbage after that ?", "words": ["the", "your", "in", "more", "that", "garbage"]}, {"sent": "one type of skimmer I guess this is the skimmer machine lifts the sheet of oil and oily debris that means garbage out of the water .", "words": ["the", "is", "of", "and", "this", "out", "water", "that", "garbage"]}, {"sent": "and what happens to the garbage when its gets in the garbage can it gets what ?", "words": ["the", "what", "can", "to", "and", "in", "it", "when", "garbage"]}, {"sent": "and in the other one there was the person who puts the tissue very carefully when theyre finished with it put it very carefully in the garbage .", "words": ["the", "put", "tissue", "was", "with", "other", "and", "there", "in", "it", "who", "when", "garbage"]}, {"sent": "then the garbage gets all chewed up right ?", "words": ["the", "all", "up", "then", "garbage"]}, {"sent": "yeah thats garbage .", "words": ["garbage"]}, {"sent": "those thats garbage there .", "words": ["there", "those", "garbage"]}, {"sent": "Im gonna throw this in the garbage can .", "words": ["the", "can", "throw", "this", "in", "garbage"]}, {"sent": "oo theyre taking the garbage out .", "words": ["the", "garbage", "out"]}, {"sent": "xxx to go dispose of this garbage with the .", "words": ["the", "go", "of", "to", "with", "this", "garbage"]}, {"sent": "all kinds of metal garbage .", "words": ["of", "all", "garbage"]}, {"sent": "and cherry pits in the garbage disposal .", "words": ["the", "in", "garbage", "and"]}, {"sent": "this is garbage .", "words": ["this", "garbage", "is"]}, {"sent": "leave the garbage can alone alright .", "words": ["the", "can", "garbage"]}, {"sent": "whos that in that garbage can trash can ?", "words": ["can", "trash", "in", "that", "garbage"]}, {"sent": "why dont you go put it in the garbage ?", "words": ["the", "put", "go", "you", "dont", "in", "it", "garbage", "why"]}, {"sent": "the garbage truck right .", "words": ["the", "truck", "garbage"]}, {"sent": "oh you do have a tv so you get to watch Sesame Street look at all this garbage .", "words": ["do", "all", "watch", "so", "to", "you", "have", "get", "tv", "at", "this", "look", "garbage", "a"]}, {"sent": "thats the garbage truck again .", "words": ["the", "truck", "garbage"]}, {"sent": "dont you help Daddy take the garbage out ?", "words": ["the", "help", "you", "dont", "out", "take", "garbage"]}, {"sent": "not in the garbage can .", "words": ["the", "can", "not", "in", "garbage"]}, {"sent": "scrape that plate in the garbage not on that floor .", "words": ["the", "plate", "not", "in", "on", "that", "garbage"]}, {"sent": "I put it in the garbage it was just one little piece .", "words": ["the", "put", "was", "little", "in", "it", "garbage"]}, {"sent": "garbage ?", "words": ["garbage"]}, {"sent": "*SI1: a garbage truck .", "words": ["truck", "garbage", "a"]}, {"sent": "thats alright put it in the garbage can .", "words": ["the", "can", "put", "in", "it", "garbage"]}, {"sent": "that garbage that basket garbage .", "words": ["basket", "that", "garbage"]}, {"sent": "*SI1: cant see the garbage truck .", "words": ["the", "see", "truck", "garbage"]}, {"sent": "Mommys throwing away some things in the garbage okay ?", "words": ["the", "in", "away", "some", "garbage"]}, {"sent": "that is a garbage truck outside .", "words": ["is", "outside", "truck", "that", "garbage", "a"]}, {"sent": "are they putting the garbage in the truck ?", "words": ["the", "truck", "they", "in", "are", "garbage"]}, {"sent": "thats a garbage truck .", "words": ["truck", "garbage", "a"]}, {"sent": "thats the garbage truck .", "words": ["the", "truck", "garbage"]}, {"sent": "oh did you put it in the garbage ?", "words": ["the", "put", "you", "in", "it", "did", "garbage"]}, {"sent": "garbage .", "words": ["garbage"]}, {"sent": "yeah what else is the garbage can ?", "words": ["the", "what", "can", "is", "garbage"]}, {"sent": "okay is that all the garbage we have lets see .", "words": ["the", "all", "is", "we", "have", "see", "that", "garbage"]}, {"sent": "yes Mommy is emptying the garbage .", "words": ["the", "garbage", "is"]}, {"sent": "bad doggy knocked over the garbage .", "words": ["the", "over", "garbage", "bad"]}, {"sent": "we have still have garbage in our car .", "words": ["we", "car", "our", "have", "in", "garbage"]}, {"sent": "I think everythings thrown in the garbage .", "words": ["the", "in", "think", "garbage"]}, {"sent": "this is a garbage truck this is a street sweeper .", "words": ["is", "truck", "this", "street", "garbage", "a"]}, {"sent": "xxx the garbage truck .", "words": ["the", "truck", "garbage"]}, {"sent": "no thats the garbage .", "words": ["the", "garbage"]}, {"sent": "put our garbage in plastic bags .", "words": ["in", "put", "garbage", "our"]}, {"sent": "garbage truck .", "words": ["truck", "garbage"]}, {"sent": "we could throw them in the garbage .", "words": ["the", "throw", "we", "them", "in", "could", "garbage"]}, {"sent": "you playing soccer with a garbage can ?", "words": ["can", "with", "you", "garbage", "a"]}, {"sent": "is thats the garbage truck ?", "words": ["the", "truck", "garbage", "is"]}, {"sent": "you gonna open the garbage ?", "words": ["open", "the", "garbage", "you"]}, {"sent": "a garbage man ?", "words": ["garbage", "a"]}, {"sent": "puppy is on the garbage can .", "words": ["the", "can", "is", "puppy", "on", "garbage"]}, {"sent": "dont put that cookie in the garbage ?", "words": ["the", "cookie", "put", "dont", "in", "that", "garbage"]}, {"sent": "here throw this paper away in the garbage .", "words": ["the", "throw", "paper", "here", "this", "in", "away", "garbage"]}, {"sent": "*SI1: wanna see the garbage truck ?", "words": ["the", "truck", "wanna", "see", "garbage"]}, {"sent": "this wheel goes on the garbage truck .", "words": ["the", "truck", "this", "on", "garbage"]}, {"sent": "you want the garbage truck ?", "words": ["the", "truck", "garbage", "you"]}, {"sent": "green green buckets of garbage .", "words": ["of", "garbage", "green"]}, {"sent": "he saved my garbage .", "words": ["my", "he", "garbage"]}, {"sent": "hurry up or I put them in the garbage .", "words": ["up", "the", "put", "hurry", "them", "in", "garbage"]}, {"sent": "Oscar sleeps in the garbage .", "words": ["the", "in", "garbage"]}, {"sent": "you gonna put the garbage out Jack ?", "words": ["the", "put", "you", "out", "garbage"]}, {"sent": "this is our little garbage machine .", "words": ["is", "our", "little", "this", "garbage"]}, {"sent": "in the garbage .", "words": ["the", "in", "garbage"]}, {"sent": "you can help them make a star .", "words": ["can", "help", "star", "them", "you", "make", "a"]}, {"sent": "Mama listen to twinkle twinkle little star .", "words": ["to", "star", "listen", "little"]}, {"sent": "look at the star .", "words": ["the", "star", "at", "look"]}, {"sent": "star yes .", "words": ["star"]}, {"sent": "you are such a star .", "words": ["star", "are", "a", "you"]}, {"sent": "star .", "words": ["star"]}, {"sent": "theres a star .", "words": ["star", "a"]}, {"sent": "find the star .", "words": ["the", "star", "find"]}, {"sent": "thats the star .", "words": ["the", "star"]}, {"sent": "who do you know thats a tv star and has a rubber ducky ?", "words": ["do", "star", "you", "and", "tv", "who", "a"]}, {"sent": "can you see the tree with the star on top ?", "words": ["can", "the", "tree", "star", "with", "you", "on", "see"]}, {"sent": "hooray you found a circle now see if you can find a red star .", "words": ["can", "red", "if", "star", "you", "find", "see", "a"]}, {"sent": "good job Joseph can you give me the star ?", "words": ["can", "the", "star", "you", "good", "me", "give"]}, {"sent": "star .", "words": ["star"]}, {"sent": "yes Im not quite sure why theres a star out but .", "words": ["star", "out", "but", "not", "why", "a"]}, {"sent": "star .", "words": ["star"]}, {"sent": "lets use the star see .", "words": ["the", "star", "see"]}, {"sent": "and looking through their window you can see they have a Christmas tree with a star on top .", "words": ["can", "tree", "their", "star", "with", "you", "and", "they", "have", "on", "see", "window", "a"]}, {"sent": "a yellow star .", "words": ["yellow", "star", "a"]}, {"sent": "has that got a star ?", "words": ["star", "that", "a"]}, {"sent": "thats a beautiful star .", "words": ["star", "a"]}, {"sent": "star .", "words": ["star"]}, {"sent": "a two pointed star .", "words": ["star", "a"]}, {"sent": "twinkle twinkle little star .", "words": ["star", "little"]}, {"sent": "where does the star go ?", "words": ["where", "does", "the", "go", "star"]}, {"sent": "Et the star press a picture .", "words": ["the", "star", "picture", "a"]}, {"sent": "youre a shining star when you smile youre a shining star you can shine for me .", "words": ["can", "for", "star", "you", "me", "when", "smile", "a"]}, {"sent": "because if you look at a star through a telescope .", "words": ["if", "star", "you", "at", "look", "because", "a"]}, {"sent": "do you remember the wise men looked up and they saw a star in the east and they followed it didnt they ?", "words": ["the", "do", "up", "star", "and", "you", "they", "in", "it", "a"]}, {"sent": "and you have a star .", "words": ["star", "you", "have", "and", "a"]}, {"sent": "you slipped on the cards all the rattles and the shaky egg and this star fish is kind of a rattle too then Im gonna put all the animals that move around in another spot .", "words": ["the", "put", "of", "star", "another", "on", "then", "that", "all", "is", "and", "you", "too", "a", "egg", "this", "in", "around", "fish"]}, {"sent": "what color is the star ?", "words": ["the", "what", "star", "is"]}, {"sent": "thats a star its a star .", "words": ["star", "a"]}, {"sent": "this is a star .", "words": ["this", "star", "a", "is"]}, {"sent": "cut out in the shape of a star .", "words": ["the", "cut", "of", "star", "in", "out", "a"]}, {"sent": "yeah thats a star .", "words": ["star", "a"]}, {"sent": "and a star .", "words": ["star", "a", "and"]}, {"sent": "star darling .", "words": ["star"]}, {"sent": "hey you got a little star there .", "words": ["star", "little", "you", "there", "a"]}, {"sent": "I bet you think youre a film star Faye .", "words": ["star", "think", "a", "you"]}, {"sent": "star whats wrong with you this morning ?", "words": ["star", "with", "this", "you"]}, {"sent": "stick a star there today .", "words": ["star", "stick", "there", "a"]}, {"sent": "because we needta stick a star on there for you .", "words": ["stick", "for", "we", "star", "you", "there", "on", "because", "a"]}, {"sent": "oh this is a star yeah .", "words": ["this", "star", "a", "is"]}, {"sent": "thats a star .", "words": ["star", "a"]}, {"sent": "yeah but look at the star now .", "words": ["the", "look", "star", "at", "but"]}, {"sent": "star .", "words": ["star"]}, {"sent": "a star isnt it ?", "words": ["star", "it", "a"]}, {"sent": "but we did the star already .", "words": ["the", "we", "star", "did", "but"]}, {"sent": "the crabs hiding the twinkle star .", "words": ["the", "star"]}, {"sent": "put the star in .", "words": ["the", "star", "put", "in"]}, {"sent": "a star ?", "words": ["star", "a"]}, {"sent": "how about twinkle twinkle little star ?", "words": ["star", "how", "about", "little"]}, {"sent": "that star goes upstairs doesnt it ?", "words": ["star", "it", "that"]}, {"sent": "thats a star what colors that star ?", "words": ["star", "what", "that", "a"]}, {"sent": "his ears move like twinkle twinkle little star ?", "words": ["his", "star", "like", "little"]}, {"sent": "a star .", "words": ["star", "a"]}, {"sent": "you gonna be a star like your brother ?", "words": ["your", "be", "star", "like", "you", "a"]}, {"sent": "thats a lil star .", "words": ["star", "a"]}, {"sent": "you see it falling thorough the sky with a little trail of light behind it and its called a shooting star .", "words": ["the", "sky", "of", "light", "star", "with", "behind", "you", "little", "and", "it", "see", "a"]}, {"sent": "yeah your star has problems honey .", "words": ["star", "your"]}, {"sent": "did you have a star ?", "words": ["star", "you", "have", "did", "a"]}, {"sent": "that makes a star right ?", "words": ["star", "that", "a"]}, {"sent": "that yellow book has got a star on it hasnt it ?", "words": ["star", "on", "it", "book", "yellow", "that", "a"]}, {"sent": "see the star ?", "words": ["the", "see", "star"]}, {"sent": "see the star ?", "words": ["the", "see", "star"]}, {"sent": "twinkle twinkle little star .", "words": ["star", "little"]}, {"sent": "its a star .", "words": ["star", "a"]}, {"sent": "a yellow star .", "words": ["yellow", "star", "a"]}, {"sent": "then make a bird and a star .", "words": ["star", "bird", "and", "then", "make", "a"]}, {"sent": "and star .", "words": ["star", "and"]}, {"sent": "can you tell me when we see a star .", "words": ["can", "we", "see", "star", "you", "me", "when", "a"]}, {"sent": "lets see if I can get you a star for that .", "words": ["can", "for", "if", "star", "you", "get", "see", "that", "a"]}, {"sent": "star .", "words": ["star"]}, {"sent": "theres your mister star guy .", "words": ["star", "your"]}, {"sent": "she looks like a movie star .", "words": ["star", "she", "like", "a"]}, {"sent": "star .", "words": ["star"]}, {"sent": "s for star .", "words": ["star", "for"]}, {"sent": "star ones ?", "words": ["star"]}, {"sent": "oh youve got a little star .", "words": ["star", "little", "a"]}, {"sent": "tiger look hes got a star fish .", "words": ["look", "star", "tiger", "fish", "a"]}, {"sent": "little star .", "words": ["star", "little"]}, {"sent": "that star is smiling .", "words": ["star", "that", "is"]}, {"sent": "can you say star ?", "words": ["can", "star", "say", "you"]}, {"sent": "so we were talking about tomorrow youre gonna be a new new twinkle star .", "words": ["we", "so", "be", "star", "about", "new", "were", "a"]}, {"sent": "or is it the kings who follow the star .", "words": ["the", "is", "star", "it", "who"]}, {"sent": "you color that star ?", "words": ["star", "that", "you"]}, {"sent": "do a star .", "words": ["do", "star", "a"]}, {"sent": "I got kungfu star .", "words": ["star"]}, {"sent": "a chocolate star ?", "words": ["star", "chocolate", "a"]}, {"sent": "theres a star .", "words": ["star", "a"]}, {"sent": "is that a star ?", "words": ["star", "that", "a", "is"]}, {"sent": "its star shaped .", "words": ["star"]}, {"sent": "thats a star a starfish .", "words": ["star", "a"]}, {"sent": "a star .", "words": ["star", "a"]}, {"sent": "a star on top of what ?", "words": ["what", "of", "star", "on", "a"]}, {"sent": "well sing twinkle twinkle little star .", "words": ["sing", "star", "little"]}, {"sent": "that is like your star .", "words": ["your", "is", "star", "like", "that"]}, {"sent": "like twinkle twinkle little star .", "words": ["star", "like", "little"]}, {"sent": "could I have that jar before it breaks please .", "words": ["jar", "have", "it", "could", "that"]}, {"sent": "theyve been in a jar for a long time in the fridge .", "words": ["the", "for", "long", "in", "jar", "a"]}, {"sent": "and then they rocked the jar the other way .", "words": ["the", "and", "other", "they", "then", "jar"]}, {"sent": "while they slept a hungry snake went looking for dinner on his way he found a nice big jar .", "words": ["for", "jar", "nice", "hungry", "his", "they", "he", "on", "big", "a"]}, {"sent": "peanutbutter jar .", "words": ["jar"]}, {"sent": "yeah when its inside the jar it looks really large doesnt it ?", "words": ["the", "jar", "inside", "it", "when"]}, {"sent": "carry moon beams home in a jar ?", "words": ["moon", "carry", "in", "home", "jar", "a"]}, {"sent": "Im putting whats left in this bag into this jar sothat the ants cant get it .", "words": ["the", "into", "get", "this", "in", "it", "jar"]}, {"sent": "just the end of the jar okay ?", "words": ["the", "of", "jar"]}, {"sent": "in a jar or what ?", "words": ["what", "in", "jar", "a"]}, {"sent": "flower jar .", "words": ["flower", "jar"]}, {"sent": "the jar is all broken see ?", "words": ["the", "all", "is", "broken", "see", "jar"]}, {"sent": "flower jar .", "words": ["flower", "jar"]}, {"sent": "thats a jar of peanutbutter .", "words": ["of", "jar", "a"]}, {"sent": "or would you put in a jar ?", "words": ["put", "you", "in", "would", "jar", "a"]}, {"sent": "this dumb dog is looking in the jar theyre looking everywhere but they cant find him see ?", "words": ["the", "is", "this", "in", "they", "him", "find", "see", "jar", "dog", "but"]}, {"sent": "Ill give you your drinks in a jam jar I think .", "words": ["think", "your", "you", "in", "give", "jar", "a"]}, {"sent": "she puts them down and I pick it up and put it in the jar and she gets the same one .", "words": ["up", "the", "put", "pick", "them", "and", "in", "it", "same", "down", "jar", "she"]}, {"sent": "you could always pour some of those from the jar .", "words": ["the", "those", "of", "pour", "you", "some", "could", "jar"]}, {"sent": "oh yes its in a jar .", "words": ["in", "jar", "a"]}, {"sent": "as much like the jar of peanutbutter .", "words": ["the", "jar", "of", "like", "much"]}, {"sent": "down the beach she had a grasshopper in a jar .", "words": ["the", "beach", "in", "down", "jar", "she", "a"]}, {"sent": "knocked the jar over and it went on the floor .", "words": ["the", "jar", "and", "it", "on", "over"]}, {"sent": "are you holding the jar of jam ?", "words": ["the", "of", "you", "are", "jar"]}, {"sent": "if I hand you those would you like to put them into the jar ?", "words": ["the", "put", "those", "if", "into", "hand", "to", "them", "like", "you", "would", "jar"]}, {"sent": "come and look at the jar the onion .", "words": ["the", "and", "at", "look", "jar"]}, {"sent": "in the biscuit jar .", "words": ["the", "in", "jar"]}, {"sent": "oh yeah well you finished that jar .", "words": ["jar", "that", "you"]}, {"sent": "put put them in the jar .", "words": ["the", "put", "them", "in", "jar"]}, {"sent": "jar .", "words": ["jar"]}, {"sent": "this is the bottom of the jar of marmalade .", "words": ["the", "bottom", "is", "of", "this", "jar"]}, {"sent": "just pick them up and put them in the jar .", "words": ["up", "the", "put", "pick", "them", "and", "in", "jar"]}, {"sent": "jar .", "words": ["jar"]}, {"sent": "you wanna put all those crayons back in he jar ?", "words": ["all", "put", "those", "back", "you", "in", "he", "wanna", "jar"]}, {"sent": "well I think its a pot in front of the Marmite jar isnt it ?", "words": ["the", "think", "of", "in", "it", "jar", "a"]}, {"sent": "well we needta put them back in the sweetie jar Thomas .", "words": ["the", "put", "back", "we", "them", "in", "jar"]}, {"sent": "hes still in jar food so .", "words": ["so", "in", "jar", "food"]}, {"sent": "uhoh there goes the jar .", "words": ["the", "there", "jar"]}, {"sent": "no more jar thats why hes mad at him see ?", "words": ["mad", "more", "him", "at", "see", "jar", "why"]}, {"sent": "those crayons all go in the jar .", "words": ["the", "all", "those", "go", "in", "jar"]}, {"sent": "thats a jar the empty jar .", "words": ["the", "empty", "jar", "a"]}, {"sent": "I think we know by the shape of the jar .", "words": ["the", "think", "we", "of", "by", "jar"]}, {"sent": "the snake reached the big mouse but it was only a cold hard rock and when he got back the jar was empty .", "words": ["the", "when", "back", "jar", "was", "and", "empty", "it", "mouse", "he", "hard", "rock", "big", "cold", "but", "a"]}, {"sent": "but there was no sign of Paddingtons jar .", "words": ["of", "was", "there", "jar", "but"]}, {"sent": "are you going to put them in the sweetie jar ?", "words": ["the", "put", "to", "them", "you", "in", "are", "jar"]}, {"sent": "its marmalade thats a jar of marmalade .", "words": ["of", "jar", "a"]}, {"sent": "you like to play with the peanutbutter jar .", "words": ["the", "to", "play", "with", "like", "you", "jar"]}, {"sent": "jar .", "words": ["jar"]}, {"sent": "theres the jar .", "words": ["the", "jar"]}, {"sent": "thats the little bear in the jar of .", "words": ["the", "of", "bear", "little", "in", "jar"]}, {"sent": "Daddy made you a whole jar of it .", "words": ["of", "you", "it", "jar", "a"]}, {"sent": "thats the lid from a jam jar .", "words": ["the", "jar", "a"]}, {"sent": "well you just hold the bread with the jam rather than the jar of jam because if you drop it oh its a mess .", "words": ["the", "bread", "hold", "if", "of", "with", "you", "it", "drop", "jar", "because", "a"]}, {"sent": "and were moving a few tea bags from the tea caddy into the storage jar arent we ?", "words": ["the", "into", "jar", "we", "and", "were", "a"]}, {"sent": "it is a jar .", "words": ["it", "jar", "a", "is"]}, {"sent": "choose a different jar .", "words": ["jar", "a"]}, {"sent": "will you bring up some seeds tomorrow so we can plant it in the jar ?", "words": ["up", "can", "the", "we", "so", "bring", "you", "in", "it", "will", "some", "plant", "jar"]}, {"sent": "hes looking at him like this grrrr because see the dog broke the jar and the little boy wanted the jar so hes mad now what are they gonna do ?", "words": ["the", "what", "do", "so", "mad", "like", "and", "little", "this", "him", "are", "they", "at", "see", "jar", "dog", "because"]}, {"sent": "carry moon beams home in a jar ?", "words": ["moon", "carry", "in", "home", "jar", "a"]}, {"sent": "whats funny about an empty jar ?", "words": ["an", "jar", "about", "empty"]}, {"sent": "Ive got to rinse the jar now .", "words": ["the", "to", "jar"]}, {"sent": "and this is a jar of peanut butter .", "words": ["is", "of", "and", "this", "butter", "peanut butter", "jar", "a"]}, {"sent": "now well get some red jam out of the jar and put .", "words": ["red", "the", "put", "of", "and", "get", "out", "some", "jar"]}, {"sent": "and he put it into the specimen jar .", "words": ["the", "put", "into", "and", "it", "he", "jar"]}, {"sent": "Jerry Jordans jelly jar and jam begin that way .", "words": ["jelly", "that", "jar", "and"]}, {"sent": "one two three he said and dropped them into the jar .", "words": ["the", "into", "them", "and", "he", "jar"]}, {"sent": "he put the mouse in the jar .", "words": ["the", "put", "in", "he", "jar", "mouse"]}, {"sent": "sweeties in the jar .", "words": ["the", "in", "jar"]}, {"sent": "theres the jar .", "words": ["the", "jar"]}, {"sent": "oh you had a glass jar didnt you ?", "words": ["jar", "glass", "a", "you"]}, {"sent": "he hurried off to get the big mouse and while he was gone the mice rocked the jar one way .", "words": ["off", "the", "jar", "to", "was", "and", "get", "he", "big", "mouse"]}, {"sent": "but all good things come to an end and when it was time to leave he stood for a moment holding up an empty marmalade jar .", "words": ["up", "all", "for", "jar", "to", "was", "and", "empty", "an", "it", "he", "good", "when", "but", "a"]}, {"sent": "you like that jar .", "words": ["jar", "that", "like", "you"]}, {"sent": "in this specimen jar youknow xxx .", "words": ["this", "in", "jar"]}, {"sent": "can you see the jar ?", "words": ["can", "the", "you", "see", "jar"]}, {"sent": "but you mustnt drop the jar because its made of glass .", "words": ["the", "of", "because", "you", "drop", "glass", "jar", "but"]}, {"sent": "because the doggie broke the jar see ?", "words": ["the", "see", "jar", "because"]}, {"sent": "a jar .", "words": ["jar", "a"]}, {"sent": "oh back in the jar .", "words": ["the", "in", "jar", "back"]}, {"sent": "and we put it in this jar .", "words": ["put", "we", "and", "this", "in", "it", "jar"]}, {"sent": "the jar rolled away .", "words": ["the", "away", "jar"]}, {"sent": "you can hold the jam jar for a couple of minutes .", "words": ["can", "the", "for", "hold", "of", "you", "jar", "a"]}, {"sent": "mm no you cant have the jar .", "words": ["have", "the", "jar", "you"]}, {"sent": "carry moon beams home in a jar ?", "words": ["moon", "carry", "in", "home", "jar", "a"]}, {"sent": "say byebye to the jar .", "words": ["the", "to", "say", "jar"]}, {"sent": "assemble a plastic jar and lid .", "words": ["jar", "and", "a"]}, {"sent": "oh might come off of the jar .", "words": ["off", "of", "jar", "the"]}, {"sent": "thats also a jar of some kind of jam but what kind of jam is it ?", "words": ["what", "is", "of", "it", "some", "jar", "but", "a"]}, {"sent": "she xxx mustve had the jelly jar open .", "words": ["the", "jelly", "open", "jar", "she"]}, {"sent": "and were putting it in a jar and were shaking .", "words": ["jar", "and", "in", "it", "were", "a"]}, {"sent": "Jerry Jordans jelly jar and jam begin that way .", "words": ["jelly", "that", "jar", "and"]}, {"sent": "the jar doesnt taste that good does it ?", "words": ["the", "does", "jar", "taste", "it", "good", "that"]}, {"sent": "I think this is called summer vegetable dinner that jar .", "words": ["think", "jar", "is", "this", "that"]}, {"sent": "youre putting the jar round your back ?", "words": ["the", "jar", "back", "your"]}, {"sent": "I think Ill pour this jar of coffee into a storage jar Thomas .", "words": ["think", "into", "of", "pour", "this", "coffee", "jar", "a"]}, {"sent": "carry moon beams home in a jar ?", "words": ["moon", "carry", "in", "home", "jar", "a"]}, {"sent": "yeah the coffee jar .", "words": ["the", "coffee", "jar"]}, {"sent": "I hafta get some kleenex so I can blow my nose .", "words": ["can", "my", "kleenex", "so", "get", "some", "blow", "nose"]}, {"sent": "hes getting you a piece of kleenex .", "words": ["of", "kleenex", "a", "you"]}, {"sent": "here lets get ya kleenex xxx .", "words": ["here", "get", "kleenex"]}, {"sent": "Ill get you a kleenex okay ?", "words": ["get", "kleenex", "a", "you"]}, {"sent": "can I get a kleenex for your nose ?", "words": ["can", "for", "your", "kleenex", "nose", "get", "a"]}, {"sent": "want me to get your little packet of kleenex that you take to Shirleys ?", "words": ["your", "kleenex", "of", "to", "little", "get", "you", "me", "take", "that"]}, {"sent": "how about a piece of kleenex and some water okay ?", "words": ["how", "kleenex", "of", "and", "about", "water", "some", "a"]}, {"sent": "you cant have the kleenex either .", "words": ["have", "the", "kleenex", "you"]}, {"sent": "oh you got kleenex of your own there ?", "words": ["your", "kleenex", "of", "you", "there"]}, {"sent": "what are ya doing with the Kleenex all the kleenex out on your bed and put it back in the box .", "words": ["the", "what", "all", "put", "your", "box", "kleenex", "back", "with", "bed", "and", "in", "are", "it", "out", "on"]}, {"sent": "the other day you needed kleenex too .", "words": ["the", "kleenex", "you", "other", "too"]}, {"sent": "no Abe here use this kleenex .", "words": ["here", "this", "kleenex"]}, {"sent": "Mommy get you a kleenex for your nose .", "words": ["for", "your", "kleenex", "nose", "you", "get", "a"]}, {"sent": "lets get you a kleenex .", "words": ["get", "kleenex", "a", "you"]}, {"sent": "you want a kleenex ?", "words": ["kleenex", "a", "you"]}, {"sent": "that is just a kleenex .", "words": ["kleenex", "that", "a", "is"]}, {"sent": "Im gonna throw away this kleenex .", "words": ["this", "away", "throw", "kleenex"]}, {"sent": "would you bring me a piece of kleenex so I can wipe off Annas nose ?", "words": ["can", "off", "kleenex", "of", "so", "nose", "bring", "you", "would", "me", "wipe", "a"]}, {"sent": "kleenex .", "words": ["kleenex"]}, {"sent": "yeah mommy was carrying five boxes of kleenex .", "words": ["of", "was", "kleenex"]}, {"sent": "you need a kleenex ?", "words": ["kleenex", "need", "a", "you"]}, {"sent": "we better get some more kleenex .", "words": ["better", "kleenex", "we", "get", "more", "some"]}, {"sent": "Im going to get a kleenex .", "words": ["get", "to", "kleenex", "a"]}, {"sent": "okay can I get you a kleenex ?", "words": ["can", "kleenex", "you", "get", "a"]}, {"sent": "yes Ill get you a kleenex .", "words": ["get", "kleenex", "a", "you"]}, {"sent": "xxx we need a kleenex .", "words": ["kleenex", "need", "a", "we"]}, {"sent": "yes we hafta use kleenex .", "words": ["kleenex", "we"]}, {"sent": "well like her socks were gonna glue the kleenex next to xxx .", "words": ["the", "kleenex", "to", "like", "glue", "her", "were"]}, {"sent": "get da kleenex .", "words": ["get", "kleenex"]}, {"sent": "give me the kleenex .", "words": ["the", "me", "give", "kleenex"]}, {"sent": "let me get you a kleenex .", "words": ["kleenex", "you", "get", "me", "a"]}, {"sent": "kleenex ?", "words": ["kleenex"]}, {"sent": "a kleenex .", "words": ["kleenex", "a"]}, {"sent": "give me the kleenex .", "words": ["the", "me", "give", "kleenex"]}, {"sent": "heres a kleenex .", "words": ["kleenex", "a"]}, {"sent": "need another kleenex ?", "words": ["another", "need", "kleenex"]}, {"sent": "would you like to use a kleenex for a blanket ?", "words": ["for", "kleenex", "to", "like", "you", "would", "blanket", "a"]}, {"sent": "get me another kleenex .", "words": ["get", "me", "kleenex", "another"]}, {"sent": "do you want some kleenex too ?", "words": ["do", "kleenex", "you", "too", "some"]}, {"sent": "Im going to get a kleenex for your nose .", "words": ["for", "your", "kleenex", "nose", "to", "get", "a"]}, {"sent": "lets get a little kleenex for you .", "words": ["for", "kleenex", "little", "get", "you", "a"]}, {"sent": "more kleenex to play with .", "words": ["kleenex", "to", "play", "with", "more"]}, {"sent": "he doesnt need a kleenex .", "words": ["need", "he", "kleenex", "a"]}, {"sent": "because you just dont have a pillow with the kleenex .", "words": ["the", "pillow", "kleenex", "with", "you", "have", "dont", "because", "a"]}, {"sent": "I put all the kleenex right there .", "words": ["the", "all", "put", "kleenex", "there"]}, {"sent": "here lets put some kleenex .", "words": ["here", "some", "put", "kleenex"]}, {"sent": "she put the kleenex in the bucket and the buckets full .", "words": ["the", "put", "kleenex", "bucket", "full", "and", "in", "she"]}, {"sent": "no you dont need the kleenex .", "words": ["the", "need", "kleenex", "you", "dont"]}, {"sent": "going to eat the kleenex ?", "words": ["eat", "to", "kleenex", "the"]}, {"sent": "give me a kleenex please .", "words": ["me", "give", "kleenex", "a"]}, {"sent": "get me a kleenex JoAnne in the bathroom .", "words": ["the", "kleenex", "get", "in", "bathroom", "me", "a"]}, {"sent": "why dont you get a kleenex .", "words": ["kleenex", "you", "get", "dont", "why", "a"]}, {"sent": "here come get a kleenex and wipe your mouth .", "words": ["your", "kleenex", "mouth", "and", "here", "get", "wipe", "a"]}, {"sent": "the part that you open sothat you can pull the kleenex up through there .", "words": ["the", "can", "up", "kleenex", "open", "you", "pull", "there", "that"]}, {"sent": "its a kleenex yes .", "words": ["kleenex", "a"]}, {"sent": "thats my kleenex .", "words": ["my", "kleenex"]}, {"sent": "why dont you get a kleenex ?", "words": ["kleenex", "you", "get", "dont", "why", "a"]}, {"sent": "okay come on Ill get you a kleenex .", "words": ["kleenex", "you", "get", "on", "a"]}, {"sent": "let me get you a kleenex .", "words": ["kleenex", "you", "get", "me", "a"]}, {"sent": "Desie would you happen to have any kleenex or kleenex or tissue ?", "words": ["kleenex", "tissue", "to", "any", "you", "have", "would"]}, {"sent": "or some uh kleenex tissue .", "words": ["some", "kleenex", "tissue"]}, {"sent": "kleenex .", "words": ["kleenex"]}, {"sent": "you need a kleenex ?", "words": ["kleenex", "need", "a", "you"]}, {"sent": "first we better get a kleenex for your nose .", "words": ["better", "for", "your", "kleenex", "we", "nose", "first", "get", "a"]}, {"sent": "first we better get a kleenex for your nose .", "words": ["better", "for", "your", "kleenex", "we", "nose", "first", "get", "a"]}, {"sent": "Mommy hasta get her kleenex .", "words": ["get", "her", "kleenex"]}, {"sent": "Mommy get kleenex for your nose .", "words": ["for", "your", "kleenex", "get", "nose"]}, {"sent": "let me get a kleenex .", "words": ["get", "me", "kleenex", "a"]}, {"sent": "heres a kleenex .", "words": ["kleenex", "a"]}, {"sent": "stay there Ill get you a kleenex .", "words": ["kleenex", "you", "get", "there", "stay", "a"]}, {"sent": "Ill get you a kleenex .", "words": ["get", "kleenex", "a", "you"]}, {"sent": "shouldnt be dirty I can give you a kleenex or something .", "words": ["can", "kleenex", "be", "you", "dirty", "give", "a"]}, {"sent": "kleenex ?", "words": ["kleenex"]}, {"sent": "I needta get a kleenex .", "words": ["get", "kleenex", "a"]}, {"sent": "oh Nathaniel wait until I get this kleenex over there you dope .", "words": ["wait", "kleenex", "you", "get", "this", "there", "over"]}, {"sent": "need a kleenex ?", "words": ["need", "kleenex", "a"]}, {"sent": "kleenex .", "words": ["kleenex"]}, {"sent": "were gonna need alotof kleenex .", "words": ["need", "were", "kleenex"]}, {"sent": "would you like a kleenex ?", "words": ["kleenex", "like", "you", "would", "a"]}, {"sent": "xxx oops buscar kleenex .", "words": ["kleenex"]}, {"sent": "let us use the little kleenex and wipe your chin here .", "words": ["the", "your", "kleenex", "chin", "and", "little", "us", "here", "wipe"]}, {"sent": "here do you want a kleenex ?", "words": ["do", "kleenex", "you", "here", "a"]}, {"sent": "here have a kleenex .", "words": ["here", "have", "kleenex", "a"]}, {"sent": "do you need a kleenex Maisie ?", "words": ["do", "need", "kleenex", "you", "a"]}, {"sent": "here is the kleenex just incase you need it .", "words": ["the", "need", "kleenex", "is", "you", "here", "it"]}, {"sent": "who can get me a kleenex ?", "words": ["can", "kleenex", "get", "who", "me", "a"]}, {"sent": "xxx kleenex napkins right ?", "words": ["kleenex"]}, {"sent": "heres a kleenex a hankie .", "words": ["kleenex", "a"]}, {"sent": "see the sink .", "words": ["the", "see", "sink"]}, {"sent": "you smell sink .", "words": ["sink", "you"]}, {"sent": "at the big at the big sink okay Zoie ?", "words": ["the", "big", "sink", "at"]}, {"sent": "thats a sink .", "words": ["sink", "a"]}, {"sent": "sink your song .", "words": ["your", "sink"]}, {"sent": "do you see the sink ?", "words": ["the", "do", "you", "sink", "see"]}, {"sent": "right in the sink ?", "words": ["the", "in", "sink"]}, {"sent": "thats the sink down there .", "words": ["the", "there", "down", "sink"]}, {"sent": "put the plug in the sink and fill the sink up .", "words": ["the", "up", "put", "and", "in", "sink"]}, {"sent": "in the sink .", "words": ["the", "in", "sink"]}, {"sent": "whats that ball doing in the sink .", "words": ["the", "ball", "in", "sink", "that"]}, {"sent": "the sink .", "words": ["the", "sink"]}, {"sent": "Mommy put it in the sink because its allgone .", "words": ["the", "put", "in", "it", "sink", "because"]}, {"sent": "dont you think youd sink with all those on ?", "words": ["all", "think", "those", "with", "you", "dont", "sink", "on"]}, {"sent": "sink .", "words": ["sink"]}, {"sent": "yes what what do you put in the sink ?", "words": ["the", "what", "do", "put", "you", "in", "sink"]}, {"sent": "can we pretend this is our sink ?", "words": ["can", "is", "we", "our", "this", "sink", "pretend"]}, {"sent": "it goes to the sink ?", "words": ["the", "to", "it", "sink"]}, {"sent": "yeah the kitchen sink might be the best .", "words": ["the", "be", "sink", "kitchen"]}, {"sent": "climb in the sink and have a bath in the sink ?", "words": ["the", "and", "have", "in", "sink", "climb", "a"]}, {"sent": "can I have the sink please ?", "words": ["can", "the", "have", "sink"]}, {"sent": "put your hands in the sink and wash them .", "words": ["the", "put", "your", "them", "and", "in", "sink", "wash"]}, {"sent": "the sink in the utility .", "words": ["the", "in", "sink"]}, {"sent": "youre sposta put that in the sink .", "words": ["the", "put", "in", "sink", "that"]}, {"sent": "you wanna put it in the sink .", "words": ["the", "put", "you", "in", "it", "sink", "wanna"]}, {"sent": "here weve got a sink .", "words": ["here", "sink", "a"]}, {"sent": "it would sink .", "words": ["it", "sink", "would"]}, {"sent": "thats a sink is this a sink ?", "words": ["this", "sink", "is", "a"]}, {"sent": "willnt sink with those on will you ?", "words": ["those", "will", "with", "you", "sink", "on"]}, {"sent": "sink .", "words": ["sink"]}, {"sent": "a sink .", "words": ["sink", "a"]}, {"sent": "and a dentist chair does have a sink on the chair doesnt it .", "words": ["the", "does", "chair", "and", "have", "it", "sink", "on", "a"]}, {"sent": "on the sink .", "words": ["on", "the", "sink"]}, {"sent": "they just sink .", "words": ["they", "sink"]}, {"sent": "after supper after your diaper you can have bath in the sink .", "words": ["can", "the", "your", "diaper", "you", "have", "in", "sink"]}, {"sent": "what does it do to the sink Shem ?", "words": ["the", "what", "do", "does", "to", "it", "sink"]}, {"sent": "you like the sink ?", "words": ["the", "sink", "like", "you"]}, {"sent": "xxx put it in the sink ?", "words": ["the", "put", "in", "it", "sink"]}, {"sent": "oh its on in the little cabinet on the side of the sink .", "words": ["the", "of", "little", "in", "sink", "on"]}, {"sent": "put it back in the sink .", "words": ["the", "put", "back", "in", "it", "sink"]}, {"sent": "the sink is right here .", "words": ["the", "here", "sink", "is"]}, {"sent": "um right Im gonna put this in the um in the sink .", "words": ["the", "put", "this", "in", "sink"]}, {"sent": "does she let you stand on the stool and help her at the kitchen sink ?", "words": ["the", "does", "help", "stand", "you", "and", "sink", "kitchen", "at", "on", "her", "she"]}, {"sent": "you brush your teeth over the sink .", "words": ["the", "your", "brush", "you", "sink", "over"]}, {"sent": "can we use the kitchen sink ?", "words": ["can", "the", "we", "sink", "kitchen"]}, {"sent": "a big sink ?", "words": ["big", "sink", "a"]}, {"sent": "oh did you have a bath in the sink last weekend ?", "words": ["the", "last", "you", "have", "in", "sink", "did", "a"]}, {"sent": "were gonna get out of the sink .", "words": ["the", "of", "get", "sink", "out", "were"]}, {"sent": "you got a sink ?", "words": ["sink", "a", "you"]}, {"sent": "Roman heres the sink heres the couch where do they go ?", "words": ["the", "do", "where", "go", "couch", "they", "sink"]}, {"sent": "pour it um into the sink and then xxx .", "words": ["the", "into", "pour", "and", "it", "sink", "then"]}, {"sent": "tray off with the sink ?", "words": ["off", "the", "with", "sink", "tray"]}, {"sent": "put the boat in the sink so it can dry off in the sink .", "words": ["the", "can", "put", "off", "so", "dry", "in", "it", "sink", "boat"]}, {"sent": "it fell off of the sink into the bath tub ?", "words": ["off", "the", "into", "of", "it", "sink"]}, {"sent": "the sink .", "words": ["the", "sink"]}, {"sent": "please get outof the sink .", "words": ["get", "the", "sink"]}, {"sent": "pour it in the sink thats it .", "words": ["the", "pour", "in", "it", "sink"]}, {"sent": "want me to take the sink away from ya ?", "words": ["the", "to", "away", "sink", "me", "take"]}, {"sent": "theres the sink refrigerator .", "words": ["the", "sink", "refrigerator"]}, {"sent": "in the sink .", "words": ["the", "in", "sink"]}, {"sent": "now where you show me where the sink goes .", "words": ["where", "the", "show", "you", "sink", "me"]}, {"sent": "you know I made macaroni and cheese for her last night in the microwave but then I didnt have a sink to wash the bowl in so then Im like in the bathroom trying to wash the sink and .", "words": ["the", "bowl", "like", "sink", "have", "then", "wash", "for", "to", "so", "and", "you", "cheese", "a", "last", "in", "bathroom", "her", "but"]}, {"sent": "would you like to sit stand on your chair by the sink and wash your hands .", "words": ["the", "your", "to", "stand", "like", "chair", "you", "by", "and", "sink", "would", "on", "sit", "wash"]}, {"sent": "where shall we put the sink ?", "words": ["where", "the", "put", "we", "sink"]}, {"sent": "but shes got a stove and a sink and she cooks .", "words": ["and", "sink", "stove", "she", "but", "a"]}, {"sent": "Sarah its gonna land right in the sink .", "words": ["the", "in", "sink"]}, {"sent": "dont put your feet in the sink .", "words": ["the", "put", "your", "dont", "in", "sink"]}, {"sent": "the water is running all out of the sink .", "words": ["the", "all", "is", "of", "sink", "out", "water"]}, {"sent": "under the sink .", "words": ["the", "under", "sink"]}, {"sent": "shes near the sink right now .", "words": ["the", "sink"]}, {"sent": "look under the sink .", "words": ["the", "under", "sink", "look"]}, {"sent": "must wash that one properly in the sink .", "words": ["the", "wash", "in", "sink", "that"]}, {"sent": "okay you cleanup house at the sink .", "words": ["the", "you", "house", "sink", "at"]}, {"sent": "heres your sink .", "words": ["your", "sink"]}, {"sent": "in the sink ?", "words": ["the", "in", "sink"]}, {"sent": "Daddy is gonna get right against the sink .", "words": ["get", "the", "sink", "is"]}, {"sent": "bring the bottle in here darling and put it in the sink .", "words": ["the", "put", "bring", "and", "here", "in", "it", "sink", "bottle"]}, {"sent": "hes at the sink .", "words": ["the", "sink", "at"]}, {"sent": "and the sink .", "words": ["the", "sink", "and"]}, {"sent": "by the sink ?", "words": ["the", "sink", "by"]}, {"sent": "spill it in the sink .", "words": ["the", "in", "it", "sink", "spill"]}, {"sent": "gotta look under the sink to see if theres any soap hold on .", "words": ["the", "if", "hold", "to", "any", "soap", "under", "sink", "look", "on", "see"]}, {"sent": "wheres the sink ?", "words": ["the", "sink"]}, {"sent": "sink .", "words": ["sink"]}, {"sent": "the sink and the toilet .", "words": ["the", "sink", "and"]}, {"sent": "can you put Amys bottle in the sink please .", "words": ["can", "the", "put", "you", "in", "sink", "bottle"]}, {"sent": "what are you show us what are you going to do about the sink ?", "words": ["the", "what", "do", "show", "to", "you", "us", "are", "sink", "about"]}, {"sent": "youre not allowed into a house not unless theres a sink in it .", "words": ["into", "in", "it", "sink", "house", "not", "a"]}, {"sent": "I dont think so I think youre right but we have a little sink .", "words": ["think", "we", "so", "little", "have", "dont", "sink", "but", "a"]}, {"sent": "it overflowed the sink didnt it ?", "words": ["the", "it", "sink"]}, {"sent": "no no no no just the brush in the sink I dont want that in the sink thatll make a big mess .", "words": ["the", "brush", "dont", "in", "make", "sink", "big", "that", "a"]}, {"sent": "I just put my cup in the sink .", "words": ["cup", "the", "put", "my", "in", "sink"]}, {"sent": "over by the sink .", "words": ["the", "over", "sink", "by"]}, {"sent": "put it in the sink .", "words": ["the", "put", "in", "it", "sink"]}, {"sent": "but the wee sink and all is lovely with it isnt it ?", "words": ["the", "all", "is", "with", "and", "it", "sink", "but"]}, {"sent": "no you cant dump my purse .", "words": ["my", "dump", "purse", "you"]}, {"sent": "oh I forgot the purse ?", "words": ["the", "purse"]}, {"sent": "you cant youre not playing with the purse .", "words": ["the", "purse", "with", "you", "not"]}, {"sent": "do you have a purple plastic purse its a star .", "words": ["do", "purse", "star", "you", "have", "a"]}, {"sent": "mermaid purse right .", "words": ["purse"]}, {"sent": "it was in my Santa stocking and its a little yellow purse with kittens on it .", "words": ["my", "purse", "was", "with", "and", "little", "in", "it", "on", "yellow", "a"]}, {"sent": "are you going to put all that money back into Mummys purse ?", "words": ["money", "all", "put", "into", "back", "purse", "to", "you", "are", "that"]}, {"sent": "you wanna use the baby purse ?", "words": ["the", "purse", "wanna", "you"]}, {"sent": "theres her purse .", "words": ["her", "purse"]}, {"sent": "that goes in mommas purse .", "words": ["in", "that", "purse"]}, {"sent": "can you put everything in the purse ?", "words": ["can", "the", "put", "purse", "you", "in"]}, {"sent": "wheres the purse ?", "words": ["the", "purse"]}, {"sent": "purse .", "words": ["purse"]}, {"sent": "is that your purse ?", "words": ["your", "that", "purse", "is"]}, {"sent": "carry it carry a purse ?", "words": ["carry", "it", "purse", "a"]}, {"sent": "but my purse is here .", "words": ["my", "purse", "is", "here", "but"]}, {"sent": "Lilys purple plastic purse .", "words": ["purse"]}, {"sent": "xxx then you got a purse .", "words": ["then", "purse", "a", "you"]}, {"sent": "purse .", "words": ["purse"]}, {"sent": "my purse .", "words": ["my", "purse"]}, {"sent": "the purse was broken right your coin purse was broken .", "words": ["the", "your", "purse", "was", "broken"]}, {"sent": "would you like to start putting it back into my purse please ?", "words": ["my", "into", "back", "purse", "to", "like", "you", "it", "would"]}, {"sent": "he has had Mummys purse .", "words": ["he", "purse"]}, {"sent": "fold all of this back in it like that and hook it back up and then itll look just like a little purse see here the picture ?", "words": ["the", "of", "like", "it", "then", "that", "up", "all", "and", "little", "here", "see", "a", "back", "purse", "picture", "this", "in", "look"]}, {"sent": "but Ive found my purse .", "words": ["my", "purse", "but"]}, {"sent": "there it is right there thats where were going alright here we go up the stairs again look and see if anybodys here no still not here well go here errr boom and Ill get my purse and then well go to see your mommy okay ?", "words": ["the", "go", "stairs", "not", "where", "there", "it", "then", "were", "up", "if", "is", "to", "and", "here", "see", "my", "your", "purse", "we", "get", "look"]}, {"sent": "Mommy you should put your purse in the bedroom .", "words": ["the", "put", "your", "purse", "you", "in", "bedroom"]}, {"sent": "she need something to hold the purse with .", "words": ["the", "need", "hold", "purse", "to", "with", "she"]}, {"sent": "Nomi put the truck on the purse .", "words": ["the", "put", "purse", "truck", "on"]}, {"sent": "another purse ?", "words": ["another", "purse"]}, {"sent": "well you can make it your purse too .", "words": ["can", "your", "purse", "you", "it", "too", "make"]}, {"sent": "heres your purse sweetheart .", "words": ["your", "purse"]}, {"sent": "have you got my purse ?", "words": ["have", "my", "purse", "you"]}, {"sent": "you know the little toy purse that we brought from Spain ?", "words": ["the", "purse", "we", "little", "you", "toy", "that"]}, {"sent": "you dont want the little purse on ?", "words": ["the", "purse", "little", "you", "dont", "on"]}, {"sent": "so Mummys little red and black purse had a black face and it had a red body with black spots .", "words": ["red", "face", "purse", "so", "with", "and", "black", "little", "it", "a"]}, {"sent": "and all the ladys like NanaBealby would have a little basket ready and a purse ready and theyd go trotting out and Id go with her .", "words": ["the", "all", "purse", "go", "with", "like", "and", "have", "little", "basket", "out", "would", "her", "a"]}, {"sent": "a purse ?", "words": ["purse", "a"]}, {"sent": "shes gonna say she wants her purse huh okay well well turn and find uhoh .", "words": ["purse", "say", "and", "find", "her", "she"]}, {"sent": "is that your purse ?", "words": ["your", "that", "purse", "is"]}, {"sent": "oh you want me to get some real money from my purse pay you .", "words": ["money", "my", "purse", "to", "you", "get", "me", "some"]}, {"sent": "Mommys purse .", "words": ["purse"]}, {"sent": "oh you can sit on mommas lap because when you were the purse its hard to sit .", "words": ["can", "the", "were", "purse", "to", "you", "hard", "on", "sit", "when", "because"]}, {"sent": "where the purse go ?", "words": ["where", "the", "purse", "go"]}, {"sent": "Mamas purse .", "words": ["purse"]}, {"sent": "its a purse .", "words": ["purse", "a"]}, {"sent": "in your purse ?", "words": ["in", "your", "purse"]}, {"sent": "purse ?", "words": ["purse"]}, {"sent": "that is Nanas purse .", "words": ["that", "purse", "is"]}, {"sent": "well Ive got my purse now .", "words": ["my", "purse"]}, {"sent": "my purse .", "words": ["my", "purse"]}, {"sent": "oh that purse is looped around your foot .", "words": ["your", "purse", "is", "foot", "around", "that"]}, {"sent": "its your purse huh ?", "words": ["your", "purse"]}, {"sent": "Im just getting some money for my purse .", "words": ["money", "my", "for", "purse", "some"]}, {"sent": "or back in my purse .", "words": ["my", "in", "purse", "back"]}, {"sent": "youre breaking the purse David .", "words": ["the", "purse"]}, {"sent": "okay well heres a little girl and shes all dressed up with a hat and a purse .", "words": ["up", "all", "purse", "with", "and", "little", "hat", "a"]}, {"sent": "Ill get all your stuff out of your purse .", "words": ["all", "your", "purse", "of", "get", "out"]}, {"sent": "open the purse .", "words": ["open", "the", "purse"]}, {"sent": "take your purse out .", "words": ["your", "out", "take", "purse"]}, {"sent": "she have xxx in her purse .", "words": ["purse", "have", "in", "her", "she"]}, {"sent": "if that makes sense like uh cooking toys or cell phones her purse .", "words": ["if", "purse", "like", "that", "her"]}, {"sent": "why dont you get your purse your purse with your pennys in and go to the shop and buy some food for Mummy ?", "words": ["the", "for", "your", "purse", "go", "to", "with", "you", "get", "dont", "in", "and", "food", "some", "buy", "why"]}, {"sent": "got a new purse have you ?", "words": ["purse", "you", "have", "new", "a"]}, {"sent": "that purse is in there and .", "words": ["purse", "is", "and", "there", "in", "that"]}, {"sent": "he really liked this purse .", "words": ["this", "he", "purse"]}, {"sent": "the purse .", "words": ["the", "purse"]}, {"sent": "dont xxx my purse is already in the room ready .", "words": ["the", "my", "purse", "is", "dont", "in", "room"]}, {"sent": "is that your purse ?", "words": ["your", "that", "purse", "is"]}, {"sent": "hey theyre not in my purse .", "words": ["in", "purse", "not", "my"]}, {"sent": "a pink purse ?", "words": ["purse", "a"]}, {"sent": "and if you havent got enough pennies in your purse or your pocket or your wallet to buy things then you cant have them because taking things without paying .", "words": ["your", "if", "purse", "to", "them", "you", "and", "have", "in", "then", "buy", "because"]}, {"sent": "the thing for the picture is in my purse .", "words": ["the", "my", "for", "purse", "is", "picture", "in"]}, {"sent": "in this purse .", "words": ["this", "in", "purse"]}, {"sent": "its my purse .", "words": ["my", "purse"]}, {"sent": "lets see if weve got a book of stamps in the back of Mummys purse shall we ?", "words": ["the", "if", "back", "purse", "of", "we", "in", "book", "see", "a"]}, {"sent": "would you like it if the dog jumped out of your purse Laura ?", "words": ["the", "your", "if", "purse", "of", "like", "you", "it", "out", "would", "dog"]}, {"sent": "this is a purse .", "words": ["this", "purse", "a", "is"]}, {"sent": "thats Mummys purse .", "words": ["purse"]}, {"sent": "see if that fits in your tiny purse .", "words": ["tiny", "your", "if", "purse", "in", "see", "that"]}, {"sent": "yep theyre taking him away off to jail cause he did something bad he stole a purse .", "words": ["off", "purse", "to", "bad", "him", "away", "he", "did", "a"]}, {"sent": "Ill put it in your purse .", "words": ["put", "your", "purse", "in", "it"]}, {"sent": "wheres Gigis purse I dont know .", "words": ["dont", "purse"]}, {"sent": "dont put it in my purse .", "words": ["put", "my", "purse", "dont", "in", "it"]}, {"sent": "I think I left my purse in the den .", "words": ["the", "think", "my", "purse", "in"]}, {"sent": "Kim has a shopping cart and a purse with pretend money in it .", "words": ["money", "purse", "with", "and", "in", "it", "pretend", "a"]}, {"sent": "do you wanna put it in your purse ?", "words": ["do", "put", "your", "purse", "you", "in", "it", "wanna"]}, {"sent": "a purse ?", "words": ["purse", "a"]}, {"sent": "it was a purse .", "words": ["was", "it", "purse", "a"]}, {"sent": "I meant what is this letter on the purse ?", "words": ["the", "what", "purse", "is", "this", "on"]}, {"sent": "thats Mommys purse .", "words": ["purse"]}, {"sent": "thats a Minnie Mouse purse ?", "words": ["purse", "a"]}, {"sent": "I dont have this purse .", "words": ["have", "dont", "purse", "this"]}, {"sent": "you didnt want the baby purse today .", "words": ["the", "purse", "you"]}, {"sent": "would you like her to put it in the baby purse ?", "words": ["the", "put", "purse", "to", "like", "you", "in", "it", "would", "her"]}, {"sent": "do you mind just grabbing my purse please ?", "words": ["do", "my", "purse", "you"]}, {"sent": "in my purse .", "words": ["in", "purse", "my"]}, {"sent": "why dont you go get Sammys dish ?", "words": ["go", "you", "get", "dont", "dish", "why"]}, {"sent": "bring the whole dish out wont you do that ?", "words": ["the", "do", "bring", "you", "dish", "out", "that"]}, {"sent": "so wheres his dish .", "words": ["his", "so", "dish"]}, {"sent": "right shall we dish the dinner up ?", "words": ["the", "dish", "up", "we"]}, {"sent": "shes trying to use spoon to dish this .", "words": ["to", "dish", "spoon", "this"]}, {"sent": "whats going to go in the dish ?", "words": ["the", "go", "to", "dish", "in"]}, {"sent": "that is a little dish .", "words": ["is", "little", "dish", "that", "a"]}, {"sent": "we put it in the dish .", "words": ["the", "put", "we", "dish", "in", "it"]}, {"sent": "do you have a dish ?", "words": ["do", "you", "have", "dish", "a"]}, {"sent": "well dont have mine in a dish .", "words": ["have", "dont", "in", "mine", "dish", "a"]}, {"sent": "a dish would a dish help ?", "words": ["dish", "help", "would", "a"]}, {"sent": "is that dish ?", "words": ["dish", "that", "is"]}, {"sent": "dish ran away with the spoon xxx .", "words": ["the", "with", "dish", "away", "spoon"]}, {"sent": "a dish and a .", "words": ["dish", "and", "a"]}, {"sent": "yeah this is a dish cloth .", "words": ["this", "dish", "a", "is"]}, {"sent": "in a dish ?", "words": ["dish", "in", "a"]}, {"sent": "I put it in a butter dish .", "words": ["put", "dish", "in", "it", "butter", "a"]}, {"sent": "with a dish on the top .", "words": ["the", "with", "dish", "on", "a"]}, {"sent": "lets put a little bit in this dish .", "words": ["put", "little", "this", "in", "dish", "a"]}, {"sent": "yes the dish ran away with theres the moon .", "words": ["the", "moon", "with", "dish", "away"]}, {"sent": "you put the cats dish down the laundry shoot ?", "words": ["the", "put", "you", "dish", "down"]}, {"sent": "whichever way you like okay but now I need that dish back in the kitchen so I can put the gravy in it because I cant .", "words": ["the", "can", "put", "need", "back", "so", "like", "because", "you", "dish", "in", "it", "kitchen", "that", "but"]}, {"sent": "okay Ill get you a dish of your own .", "words": ["your", "of", "you", "get", "dish", "a"]}, {"sent": "remember when you fed the pigeons you didnt hafta put it in a dish .", "words": ["the", "put", "you", "dish", "in", "it", "when", "a"]}, {"sent": "and whats this in the dish ?", "words": ["the", "and", "this", "in", "dish"]}, {"sent": "because this is mummies best dish thing .", "words": ["this", "dish", "because", "is"]}, {"sent": "let me see you wash the dish .", "words": ["the", "me", "you", "dish", "see", "wash"]}, {"sent": "on a dish .", "words": ["on", "dish", "a"]}, {"sent": "I think Id rather put it in a dish .", "words": ["think", "put", "dish", "in", "it", "a"]}, {"sent": "dont you need to pull the dish a little closer to yourself ?", "words": ["the", "yourself", "need", "need to", "to", "pull", "you", "dont", "dish", "little", "a"]}, {"sent": "a dish yes it is a dish .", "words": ["dish", "it", "is", "a"]}, {"sent": "and your soap dish .", "words": ["dish", "your", "soap", "and"]}, {"sent": "thats the dogs dish .", "words": ["the", "dish"]}, {"sent": "put the dish up there .", "words": ["the", "up", "put", "dish", "there"]}, {"sent": "whats this soap dish doing in here ?", "words": ["soap", "here", "this", "dish", "in"]}, {"sent": "there is a dish in there .", "words": ["is", "there", "dish", "in", "a"]}, {"sent": "you youre dish xxx you xxx .", "words": ["dish", "you"]}, {"sent": "she likes to go and like steal the dish towels out of the kitchen and wrap her dolls up in them um .", "words": ["the", "up", "go", "of", "to", "them", "like", "and", "dish", "in", "out", "kitchen", "her", "she"]}, {"sent": "put put some on the dish .", "words": ["the", "put", "dish", "on", "some"]}, {"sent": "thats your dish and thats mine .", "words": ["dish", "your", "mine", "and"]}, {"sent": "can you turn out the whole dish of beads ?", "words": ["can", "the", "of", "you", "dish", "out", "beads"]}, {"sent": "have you got a dish with MickeyMouse on and a plate ?", "words": ["plate", "with", "you", "have", "dish", "and", "on", "a"]}, {"sent": "is that the sugar dish ?", "words": ["the", "dish", "that", "is"]}, {"sent": "a blue dish cloth .", "words": ["dish", "blue", "a"]}, {"sent": "did the dish fall off the table ?", "words": ["the", "off", "table", "fall", "dish", "did"]}, {"sent": "and he played with that little dish .", "words": ["with", "little", "and", "dish", "he", "that"]}, {"sent": "heres a milk dish .", "words": ["milk", "dish", "a"]}, {"sent": "yes youre going to have your Kabooms in a dish .", "words": ["your", "to", "have", "dish", "in", "a"]}, {"sent": "do you know what she eats in that dish ?", "words": ["do", "what", "you", "dish", "in", "that", "she"]}, {"sent": "digger mans swimming in the dish bowl ?", "words": ["the", "dish", "in", "bowl"]}, {"sent": "dish ?", "words": ["dish"]}, {"sent": "and the dish ran away with the spoon .", "words": ["the", "with", "and", "dish", "away", "spoon"]}, {"sent": "Im making another dish for the other cat .", "words": ["the", "for", "other", "another", "dish", "cat"]}, {"sent": "its not a pail anymore I might as well make a dish out of it .", "words": ["of", "dish", "it", "out", "make", "not", "a"]}, {"sent": "I can hold up the fish and a little toy ship and some milk on a dish and look I can hop up and down on the ball .", "words": ["can", "the", "up", "look", "hold", "and", "little", "dish", "toy", "on", "ball", "fish", "milk", "some", "down", "a"]}, {"sent": "if you want the little dish you can use it if not you dont hafta .", "words": ["the", "can", "if", "little", "you", "dish", "dont", "it", "not"]}, {"sent": "a dish mop .", "words": ["dish", "mop", "a"]}, {"sent": "a dish ?", "words": ["dish", "a"]}, {"sent": "he he the dish ran away with the spoon .", "words": ["the", "with", "dish", "away", "he", "spoon"]}, {"sent": "dish is like a plate .", "words": ["plate", "is", "like", "dish", "a"]}, {"sent": "in other words you would like to put a hot dish on it ?", "words": ["put", "to", "like", "you", "other", "dish", "in", "hot", "it", "would", "on", "a"]}, {"sent": "or the food dish ?", "words": ["the", "dish", "food"]}, {"sent": "thats Grandmas blancmange dish .", "words": ["dish"]}, {"sent": "what are the dish and the spoon having to do with that ?", "words": ["the", "what", "do", "to", "with", "and", "dish", "are", "spoon", "that"]}, {"sent": "bah noo dish no .", "words": ["dish"]}, {"sent": "your your dish isnt empty but if youd like to eat it straight from the box I dont mind .", "words": ["the", "your", "if", "box", "to", "like", "empty", "eat", "dish", "dont", "it", "but"]}, {"sent": "with the spoon and the dish ?", "words": ["the", "with", "and", "dish", "spoon"]}, {"sent": "I dont like give him the dish .", "words": ["the", "like", "dont", "him", "dish", "give"]}, {"sent": "dish .", "words": ["dish"]}, {"sent": "Ill just scrape the last of butter out of the dish .", "words": ["the", "of", "last", "dish", "butter", "out"]}, {"sent": "and the dish ran away .", "words": ["the", "dish", "away", "and"]}, {"sent": "the little dog laughed to see such a sight and the dish ran away with the spoon .", "words": ["the", "to", "with", "little", "and", "dish", "away", "spoon", "see", "dog", "a"]}, {"sent": "no milk just a dish with blueberries .", "words": ["milk", "dish", "with", "a"]}, {"sent": "he picked up the cake and the rake and the gown the milk and the strings and the books and the dish and the fan and the cup and the ship and the fish .", "words": ["up", "the", "cup", "cake", "and", "dish", "he", "fish", "milk"]}, {"sent": "dish .", "words": ["dish"]}, {"sent": "not in a dish ?", "words": ["dish", "in", "not", "a"]}, {"sent": "wheres the dish ?", "words": ["the", "dish"]}, {"sent": "its a little dish .", "words": ["dish", "little", "a"]}, {"sent": "dont bang the dish .", "words": ["the", "dont", "dish"]}, {"sent": "thats a dish rag .", "words": ["dish", "a"]}, {"sent": "and on your soap dish .", "words": ["your", "soap", "and", "dish", "on"]}, {"sent": "I think the other ice cream is sposta be in a dish .", "words": ["the", "ice cream", "think", "is", "be", "other", "dish", "in", "ice", "a"]}, {"sent": "and the dish ran away with the spoon .", "words": ["the", "with", "and", "dish", "away", "spoon"]}, {"sent": "this is a meal in a dish .", "words": ["is", "this", "in", "dish", "a"]}, {"sent": "Hunka Munka tried every tin spoon in turn the fish was glued to the dish .", "words": ["the", "was", "to", "dish", "in", "every", "spoon", "fish"]}, {"sent": "heres his cat dish down here .", "words": ["here", "cat", "dish", "his", "down"]}, {"sent": "careful keep your soap dish up straight .", "words": ["up", "your", "soap", "dish", "careful"]}, {"sent": "you wanna get a dish ?", "words": ["you", "get", "dish", "wanna", "a"]}, {"sent": "can you wash the dish ?", "words": ["can", "the", "you", "dish", "wash"]}, {"sent": "the dish ran away with the spoon boo boo .", "words": ["the", "with", "dish", "away", "spoon"]}, {"sent": "the little dish .", "words": ["the", "dish", "little"]}, {"sent": "or the dish .", "words": ["the", "dish"]}, {"sent": "okay hes gonna dish out the water to everybody .", "words": ["the", "to", "dish", "out", "water"]}, {"sent": "hes eating the food in the dish .", "words": ["the", "in", "food", "dish"]}, {"sent": "I see a dish .", "words": ["see", "dish", "a"]}, {"sent": "youre putting his dish on his head ?", "words": ["his", "dish", "on", "head"]}, {"sent": "she got up in her crib the other day and just sang the whole hey diddle diddle .", "words": ["up", "the", "crib", "other", "and", "in", "her", "she"]}, {"sent": "and there was you upstairs in a beautiful little crib with all soft covers .", "words": ["all", "soft", "was", "crib", "with", "you", "and", "there", "in", "little", "a"]}, {"sent": "a crib for Nathan okay .", "words": ["crib", "for", "a"]}, {"sent": "Im gonna throw this crib out you know .", "words": ["throw", "crib", "you", "this", "out"]}, {"sent": "itll be bigger than your crib .", "words": ["be", "your", "crib"]}, {"sent": "oh we hafta find a tiny baby for that crib .", "words": ["tiny", "for", "we", "crib", "find", "that", "a"]}, {"sent": "Ana has one its a crib right ?", "words": ["crib", "a"]}, {"sent": "hi Ethan are you in your crib ?", "words": ["your", "crib", "you", "in", "are"]}, {"sent": "do you wanna put the crib in the doll house ?", "words": ["the", "do", "put", "crib", "doll", "you", "in", "house", "wanna"]}, {"sent": "xxx underneath the crib and see if there are any balls oh theres a big ball but not the blue ball .", "words": ["the", "if", "crib", "any", "and", "ball", "there", "blue", "are", "see", "big", "not", "but", "a"]}, {"sent": "crib .", "words": ["crib"]}, {"sent": "in the crib .", "words": ["the", "crib", "in"]}, {"sent": "do you take a carriage or a crib to the park ?", "words": ["the", "do", "crib", "to", "you", "park", "take", "a"]}, {"sent": "Lily took her things back she pinched his tail and she yelled insulting comments into his crib .", "words": ["into", "back", "crib", "and", "his", "her", "she"]}, {"sent": "think theyre in your crib .", "words": ["in", "think", "your", "crib"]}, {"sent": "but more mostly around the crib .", "words": ["the", "crib", "more", "around", "but"]}, {"sent": "get your pillow outof your crib .", "words": ["get", "crib", "your", "pillow"]}, {"sent": "lets put the crib in Ninas room .", "words": ["the", "put", "crib", "in", "room"]}, {"sent": "for the night in a special old crib she has there .", "words": ["the", "old", "for", "crib", "there", "in", "she", "a"]}, {"sent": "um so shell sit in her crib and shell have that and fall asleep with that book .", "words": ["so", "crib", "with", "fall", "and", "have", "asleep", "in", "that", "book", "sit", "her"]}, {"sent": "its a crib .", "words": ["crib", "a"]}, {"sent": "the crib ?", "words": ["the", "crib"]}, {"sent": "in the crib xxx no the babys crib .", "words": ["the", "crib", "in"]}, {"sent": "pack me off to college in my crib .", "words": ["off", "my", "to", "crib", "in", "me"]}, {"sent": "whats the car doing in the crib ?", "words": ["the", "car", "in", "crib"]}, {"sent": "you can take it with you in the crib .", "words": ["can", "the", "crib", "with", "you", "in", "it", "take"]}, {"sent": "hes under your crib .", "words": ["under", "your", "crib"]}, {"sent": "in the crib okay .", "words": ["the", "crib", "in"]}, {"sent": "I think the girl might actually be in your crib .", "words": ["the", "think", "your", "be", "crib", "in"]}, {"sent": "thats not the one that you had in your crib last night sweetie .", "words": ["the", "your", "crib", "last", "you", "in", "that", "not"]}, {"sent": "hey the kittycat is in your crib .", "words": ["the", "your", "is", "crib", "in"]}, {"sent": "oh that kittys going in the crib .", "words": ["the", "crib", "in", "that"]}, {"sent": "is that your crib ?", "words": ["your", "crib", "that", "is"]}, {"sent": "next to your crib right ?", "words": ["to", "your", "crib"]}, {"sent": "youre gonna land right inside your crib .", "words": ["crib", "inside", "your"]}, {"sent": "here the xxx can be his crib .", "words": ["the", "can", "be", "crib", "here", "his"]}, {"sent": "do you get worried when shes not in her crib ?", "words": ["do", "crib", "her", "you", "get", "in", "when", "not"]}, {"sent": "after he is done farming he brings the corn and stores it in the corn crib .", "words": ["the", "is", "crib", "and", "in", "it", "he", "corn"]}, {"sent": "oh hes outgrown the crib .", "words": ["the", "crib"]}, {"sent": "theres the crib .", "words": ["the", "crib"]}, {"sent": "itll be funny if she falls outof the crib ?", "words": ["the", "if", "be", "crib", "she"]}, {"sent": "heres her crib .", "words": ["crib", "her"]}, {"sent": "here you sit in your crib for just a minute .", "words": ["for", "your", "crib", "you", "here", "in", "sit", "a"]}, {"sent": "do you get outof your crib sometime ?", "words": ["do", "your", "crib", "you", "get"]}, {"sent": "Jwww doesnt sleep in a crib .", "words": ["sleep", "crib", "in", "a"]}, {"sent": "do you think we should put the baby crib in the bedroom ?", "words": ["the", "do", "think", "put", "we", "crib", "you", "in", "bedroom"]}, {"sent": "threw it in the crib at the baby Jesus didnt he ?", "words": ["the", "crib", "in", "it", "he", "at"]}, {"sent": "I didnt mess your crib up .", "words": ["up", "crib", "your"]}, {"sent": "no he has uh one book we often let him have in his crib .", "words": ["we", "crib", "have", "his", "him", "in", "he", "book"]}, {"sent": "yeah but he loves to si Imean he has hes go bo youknow like board books in his crib that he sits and reads and reads .", "words": ["go", "to", "crib", "like", "and", "in", "he", "his", "that", "but"]}, {"sent": "theyre all sleeping in their nice little cozy crib .", "words": ["all", "their", "crib", "nice", "little", "in"]}, {"sent": "you mean therere two babies in that crib now .", "words": ["crib", "in", "that", "you"]}, {"sent": "who should go in the crib ?", "words": ["the", "go", "crib", "in", "who"]}, {"sent": "I have my crib .", "words": ["have", "crib", "my"]}, {"sent": "yeah if you if you get outof your crib I do because at night ones sposta go to sleep right ?", "words": ["do", "your", "if", "go", "sleep", "crib", "to", "you", "get", "at", "because"]}, {"sent": "theyre in your crib .", "words": ["crib", "in", "your"]}, {"sent": "no crib for a bed .", "words": ["crib", "for", "bed", "a"]}, {"sent": "do you wanna come outof your little crib ?", "words": ["do", "your", "crib", "little", "you", "wanna"]}, {"sent": "it goes under the crib .", "words": ["the", "under", "it", "crib"]}, {"sent": "thats a little crib er a playpen .", "words": ["crib", "little", "a"]}, {"sent": "you wanna go into the crib ?", "words": ["the", "into", "go", "crib", "you", "wanna"]}, {"sent": "whats that in the crib ?", "words": ["the", "crib", "in", "that"]}, {"sent": "put the baby in the crib ?", "words": ["the", "in", "put", "crib"]}, {"sent": "is it under your crib nope .", "words": ["your", "is", "crib", "under", "it"]}, {"sent": "yes Im gonna put that one in your crib .", "words": ["put", "your", "crib", "in", "that"]}, {"sent": "whos whos that crib belong to ?", "words": ["to", "crib", "that"]}, {"sent": "there is his crib .", "words": ["his", "there", "crib", "is"]}, {"sent": "oh it goes on the crib .", "words": ["on", "the", "it", "crib"]}, {"sent": "hes wearing a hat and hes in a crib ?", "words": ["crib", "and", "in", "hat", "a"]}, {"sent": "is that a crib ?", "words": ["crib", "that", "a", "is"]}, {"sent": "thats a crib made outof plastic .", "words": ["crib", "a"]}, {"sent": "in her crib ?", "words": ["crib", "in", "her"]}, {"sent": "do you wanna sleep in the sleeping bag or in the crib tonight ?", "words": ["the", "do", "sleep", "crib", "you", "in", "wanna"]}, {"sent": "what is your crib made outof ?", "words": ["what", "your", "crib", "is"]}, {"sent": "put that back in the crib .", "words": ["the", "put", "back", "crib", "in", "that"]}, {"sent": "a babys crib ?", "words": ["crib", "a"]}, {"sent": "is that the crib thats the babys crib ?", "words": ["the", "crib", "that", "is"]}, {"sent": "should we put the baby bunny in the crib ?", "words": ["the", "put", "bunny", "we", "crib", "in"]}, {"sent": "yeah that baby goes in that crib like this .", "words": ["crib", "like", "this", "in", "that"]}, {"sent": "yeah you should stay in your crib at night right ?", "words": ["your", "crib", "you", "in", "at", "stay"]}, {"sent": "are you adjusting the crib ?", "words": ["the", "crib", "are", "you"]}, {"sent": "she has a crib and a feeding chair and a bassinet .", "words": ["crib", "chair", "and", "she", "a"]}, {"sent": "ya know we only get to play with Manuela for a small period of time so I dont think we should stay in our crib the whole time .", "words": ["the", "think", "for", "we", "of", "to", "so", "play", "with", "crib", "our", "get", "dont", "in", "stay", "a"]}, {"sent": "and your crib is made outof what ?", "words": ["what", "your", "is", "crib", "and"]}, {"sent": "theres the baby in the crib .", "words": ["the", "crib", "in"]}, {"sent": "well that will be close to your crib xxx .", "words": ["your", "be", "to", "crib", "will", "close", "that"]}, {"sent": "but she has a few in her crib just like two or three of them .", "words": ["of", "crib", "them", "like", "in", "her", "she", "but", "a"]}, {"sent": "thats a crib .", "words": ["crib", "a"]}, {"sent": "so your crib is made outof wood .", "words": ["so", "your", "is", "crib"]}, {"sent": "I think thats a I think thats a crib for a baby .", "words": ["crib", "think", "for", "a"]}, {"sent": "I think were gonna put this blankie in the crib for you to cuddle .", "words": ["the", "think", "put", "for", "crib", "to", "you", "this", "in", "were"]}, {"sent": "I got a good idea and he play in his crib in the morning .", "words": ["the", "crib", "play", "and", "in", "he", "good", "his", "a"]}, {"sent": "and you talk for the baby while hes playing in his crib .", "words": ["the", "for", "crib", "you", "and", "in", "talk", "his"]}, {"sent": "she has um not the same but she has a bunch of like the small um picture books and like the hard books Idontknow she has like a bin of books that she can dig into and sometimes Ill put her in her crib with just a bunch of books and shell just flip through them for a little while .", "words": ["the", "put", "into", "of", "like", "same", "hard", "not", "can", "crib", "with", "that", "she", "for", "and", "little", "a", "picture", "them", "in", "her", "but"]}, {"sent": "no she doesnt put you in a crib .", "words": ["put", "crib", "you", "in", "she", "a"]}, {"sent": "put the baby in the crib ?", "words": ["the", "in", "put", "crib"]}, {"sent": "why are you behind the crib sweetiepie ?", "words": ["the", "crib", "behind", "you", "are", "why"]}, {"sent": "not so much of like a she is not so much of like a shell shell I guess shell like if she is in her crib when she is like youknow kind of going to sleep she will kind of like talk her self to sleep then .", "words": ["if", "when", "is", "of", "so", "crib", "to", "her", "like", "sleep", "in", "talk", "she", "will", "then", "much", "not", "a"]}, {"sent": "yeah this thats right its a sled .", "words": ["this", "sled", "a"]}, {"sent": "and heres a little boy little girl riding on a sled up here oh .", "words": ["up", "sled", "little", "and", "here", "on", "a"]}, {"sent": "looks like a sled to me .", "words": ["sled", "to", "like", "me", "a"]}, {"sent": "its called a sled .", "words": ["sled", "a"]}, {"sent": "thats a sled .", "words": ["sled", "a"]}, {"sent": "whats the name of the sled ?", "words": ["the", "of", "sled"]}, {"sent": "its a yellow sled .", "words": ["yellow", "sled", "a"]}, {"sent": "next year well get you a sled so youll know what a sledding is okay ?", "words": ["what", "sled", "is", "so", "you", "get", "a"]}, {"sent": "shes on her yellow sled .", "words": ["on", "yellow", "her", "sled"]}, {"sent": "is that your new sled ?", "words": ["your", "sled", "is", "new", "that"]}, {"sent": "Alex do you like your sled ?", "words": ["do", "sled", "your", "like", "you"]}, {"sent": "yeah hes on the sled hes going on the snow .", "words": ["on", "the", "snow", "sled"]}, {"sent": "its an awesome sled I gotta get that kind .", "words": ["get", "that", "an", "sled"]}, {"sent": "youve been on your sled havent you ?", "words": ["on", "your", "sled", "you"]}, {"sent": "a sled ?", "words": ["sled", "a"]}, {"sent": "sled ?", "words": ["sled"]}, {"sent": "you wanna go down the hill on a sled ?", "words": ["the", "sled", "go", "you", "wanna", "on", "down", "a"]}, {"sent": "shes got a big sled .", "words": ["sled", "big", "a"]}, {"sent": "you went on the sled ?", "words": ["on", "the", "sled", "you"]}, {"sent": "its like a sled that was a stretcher right .", "words": ["sled", "was", "like", "that", "a"]}, {"sent": "you went on the sled and then what who caught you ?", "words": ["the", "what", "sled", "and", "you", "who", "on", "then"]}, {"sent": "a sled ?", "words": ["sled", "a"]}, {"sent": "its called a sled .", "words": ["sled", "a"]}, {"sent": "oh it looks like a sled .", "words": ["it", "sled", "like", "a"]}, {"sent": "these guys are riding in the sled .", "words": ["the", "these", "sled", "in", "are"]}, {"sent": "yup theyre all riding in the sled same with thing with these guys this one this one and this one hm ?", "words": ["the", "all", "these", "sled", "with", "and", "this", "in", "same"]}, {"sent": "when its snowing you ride on one of those like a sled .", "words": ["those", "sled", "of", "like", "ride", "you", "on", "when", "a"]}, {"sent": "like a sled .", "words": ["sled", "like", "a"]}, {"sent": "sled .", "words": ["sled"]}, {"sent": "sled .", "words": ["sled"]}, {"sent": "can you say sled ?", "words": ["can", "say", "sled", "you"]}, {"sent": "thats a sled honey .", "words": ["sled", "a"]}, {"sent": "is that a sled ?", "words": ["sled", "that", "a", "is"]}, {"sent": "and how does a sled go ?", "words": ["does", "how", "sled", "go", "and", "a"]}, {"sent": "how does a sled go ?", "words": ["does", "how", "sled", "go", "a"]}, {"sent": "a sled .", "words": ["sled", "a"]}, {"sent": "so shes riding through the snow on a sled and then oh .", "words": ["the", "sled", "so", "snow", "and", "on", "then", "a"]}, {"sent": "a sled .", "words": ["sled", "a"]}, {"sent": "see now Miffys going down the hill in the yellow sled in the snow .", "words": ["the", "sled", "yellow", "snow", "in", "see", "down"]}, {"sent": "huh a sled .", "words": ["sled", "a"]}, {"sent": "does he take a sled ?", "words": ["does", "sled", "he", "take", "a"]}, {"sent": "they even used the magic carpet as a sled to slide down a nearby hill .", "words": ["the", "slide", "sled", "to", "they", "down", "a"]}, {"sent": "they were clumsy and they didnt know how to ride their sled and they crashed head first into the snowbank .", "words": ["the", "how", "sled", "their", "into", "to", "head", "first", "ride", "and", "they", "were"]}, {"sent": "whats that sled ?", "words": ["sled", "that"]}, {"sent": "Sigs didnt have the the a sled .", "words": ["have", "the", "sled", "a"]}, {"sent": "that sled is being pulled by a mommy .", "words": ["sled", "is", "by", "that", "a"]}, {"sent": "I think youre remembering when it was winter and Mommy pulled you and the sled up the driveway and you saw Tahoe .", "words": ["the", "up", "think", "sled", "was", "and", "you", "it", "when"]}, {"sent": "snow sled ?", "words": ["snow", "sled"]}, {"sent": "yeah we did buy a snow sled at the toy store .", "words": ["the", "sled", "we", "store", "snow", "toy", "did", "at", "buy", "a"]}, {"sent": "wee they dont even need a sled they just go on their belly .", "words": ["need", "sled", "their", "go", "dont", "they", "on", "a"]}, {"sent": "he went on a sled didnt he ?", "words": ["on", "he", "sled", "a"]}, {"sent": "have you been on a sled ?", "words": ["sled", "you", "have", "on", "a"]}, {"sent": "thats a sled .", "words": ["sled", "a"]}, {"sent": "Spots riding on the snow on the sled .", "words": ["on", "the", "snow", "sled"]}, {"sent": "and hes got a sled .", "words": ["sled", "a", "and"]}, {"sent": "and we never took you and Jwww on your sled did we ?", "words": ["your", "sled", "we", "you", "and", "did", "on"]}, {"sent": "have you been on a sled ?", "words": ["sled", "you", "have", "on", "a"]}, {"sent": "youre on the sled .", "words": ["on", "the", "sled"]}, {"sent": "Daddys pulling you on the sled isnt he ?", "words": ["the", "sled", "you", "he", "on"]}, {"sent": "is there a baby on the sled ?", "words": ["the", "sled", "is", "there", "on", "a"]}, {"sent": "or on a sled ?", "words": ["on", "sled", "a"]}, {"sent": "theyre going down a little mountain on a sled arent they ?", "words": ["sled", "little", "they", "on", "down", "a"]}, {"sent": "who rode the sled with you ?", "words": ["the", "sled", "with", "you", "who"]}, {"sent": "heres a sled .", "words": ["sled", "a"]}, {"sent": "you got a sled too yeah .", "words": ["too", "sled", "a", "you"]}, {"sent": "Donna did you see um did Betty have the sled out ?", "words": ["the", "sled", "you", "have", "did", "out", "see"]}, {"sent": "who had the sled out ?", "words": ["the", "sled", "out", "who"]}, {"sent": "sled ?", "words": ["sled"]}, {"sent": "uhhuh is there a sled in there ?", "words": ["sled", "is", "there", "in", "a"]}, {"sent": "heres a sled .", "words": ["sled", "a"]}, {"sent": "and they put her on the sled and took her to the doctor .", "words": ["the", "put", "sled", "to", "and", "they", "on", "her"]}, {"sent": "what sled ?", "words": ["what", "sled"]}, {"sent": "that was a long sled ride .", "words": ["sled", "was", "long", "ride", "that", "a"]}, {"sent": "you took one long sled ride the whole time you were outside ?", "words": ["the", "sled", "outside", "long", "you", "ride", "were"]}, {"sent": "oh you took one sled ride huh ?", "words": ["sled", "ride", "you"]}, {"sent": "are you gonna get our sled and go sliding down the hill ?", "words": ["the", "sled", "go", "our", "get", "you", "and", "are", "down"]}, {"sent": "take take the sled out there and pull him on the sled .", "words": ["the", "sled", "and", "pull", "there", "him", "out", "on", "take"]}, {"sent": "when it snows sometimes and you can go out and play in your sled thats winter .", "words": ["can", "your", "sled", "go", "play", "you", "and", "in", "it", "out", "when"]}, {"sent": "did you go on the sled ?", "words": ["the", "sled", "go", "you", "did", "on"]}, {"sent": "a stretcher sled ?", "words": ["sled", "a"]}, {"sent": "on the sled what did you do on the sled ?", "words": ["the", "what", "do", "sled", "you", "did", "on"]}, {"sent": "is it a sled ?", "words": ["it", "sled", "a", "is"]}, {"sent": "oh and they sled in the snow ?", "words": ["the", "sled", "snow", "and", "they", "in"]}, {"sent": "either your sled or .", "words": ["sled", "your"]}, {"sent": "is it called a sled ?", "words": ["it", "sled", "a", "is"]}, {"sent": "okay scan this sled I mean slide .", "words": ["this", "slide", "sled"]}, {"sent": "whose sled may bring toys to children .", "words": ["to", "sled", "bring"]}, {"sent": "a moose xxx xxx .", "words": ["moose", "a"]}, {"sent": "right there next to the moose ?", "words": ["to", "there", "moose", "the"]}, {"sent": "see the moose ?", "words": ["the", "see", "moose"]}, {"sent": "theres the moose nose .", "words": ["the", "nose", "moose"]}, {"sent": "moose and Foose .", "words": ["and", "moose"]}, {"sent": "its a moose .", "words": ["moose", "a"]}, {"sent": "that is a big moose huh .", "words": ["is", "moose", "that", "big", "a"]}, {"sent": "the moose .", "words": ["the", "moose"]}, {"sent": "so pj said I dont wanna be a bear of a bird or a beaver or a pig or a moose .", "words": ["of", "so", "be", "bird", "bear", "pig", "dont", "moose", "wanna", "a"]}, {"sent": "moose .", "words": ["moose"]}, {"sent": "and pj went to live with the moose but pj .", "words": ["the", "to", "with", "moose", "and", "but"]}, {"sent": "that is a moose .", "words": ["that", "moose", "a", "is"]}, {"sent": "a moose right and what are these ?", "words": ["what", "these", "moose", "and", "are", "a"]}, {"sent": "so its cow dog moose .", "words": ["so", "dog", "cow", "moose"]}, {"sent": "thats a moose .", "words": ["moose", "a"]}, {"sent": "the moose is going to eat it huh .", "words": ["the", "is", "to", "moose", "eat", "it"]}, {"sent": "where are you going with that moose ?", "words": ["where", "with", "you", "moose", "are", "that"]}, {"sent": "so pj said I dont wanna be a bear or a bird or a beaver or a pig or a moose or a opossum I wanna be a skunk .", "words": ["so", "be", "bird", "bear", "pig", "dont", "moose", "wanna", "a"]}, {"sent": "an and I couldnt come up with noise a moose .", "words": ["up", "with", "and", "moose", "an", "a"]}, {"sent": "you want that moose ?", "words": ["that", "moose", "you"]}, {"sent": "that is a moose .", "words": ["that", "moose", "a", "is"]}, {"sent": "oh no youre sitting on the moose .", "words": ["on", "the", "moose"]}, {"sent": "can you say moose ?", "words": ["can", "say", "moose", "you"]}, {"sent": "moose .", "words": ["moose"]}, {"sent": "what do you think a moose says ?", "words": ["what", "do", "think", "you", "moose", "a"]}, {"sent": "yeah they have moose .", "words": ["have", "they", "moose"]}, {"sent": "moose .", "words": ["moose"]}, {"sent": "yes a moose .", "words": ["moose", "a"]}, {"sent": "wheres the moose eye ?", "words": ["the", "eye", "moose"]}, {"sent": "thats a moose .", "words": ["moose", "a"]}, {"sent": "moose .", "words": ["moose"]}, {"sent": "moose .", "words": ["moose"]}, {"sent": "can you make a moose call ?", "words": ["can", "moose", "you", "make", "a"]}, {"sent": "that is a moose .", "words": ["that", "moose", "a", "is"]}, {"sent": "xxx wow a big moose .", "words": ["big", "moose", "a"]}, {"sent": "mhm a moose and a porcupine theyre gonna finish up all the pie .", "words": ["up", "all", "the", "finish", "moose", "and", "a"]}, {"sent": "moose .", "words": ["moose"]}, {"sent": "there is a moose and a bunny .", "words": ["bunny", "is", "and", "moose", "there", "a"]}, {"sent": "oh look there is a moose .", "words": ["is", "moose", "there", "look", "a"]}, {"sent": "moose .", "words": ["moose"]}, {"sent": "that is a big moose .", "words": ["is", "moose", "that", "big", "a"]}, {"sent": "a moose .", "words": ["moose", "a"]}, {"sent": "oh that is a moose .", "words": ["that", "moose", "a", "is"]}, {"sent": "not that is a moose .", "words": ["is", "moose", "that", "not", "a"]}, {"sent": "a moose and a goose together have juice .", "words": ["moose", "have", "and", "goose", "juice", "a"]}, {"sent": "three moose .", "words": ["moose"]}, {"sent": "and a moose .", "words": ["moose", "a", "and"]}, {"sent": "moose .", "words": ["moose"]}, {"sent": "this is a moose .", "words": ["this", "moose", "a", "is"]}, {"sent": "moose .", "words": ["moose"]}, {"sent": "say moose .", "words": ["say", "moose"]}, {"sent": "or a moose see these horns he has ?", "words": ["these", "moose", "he", "see", "a"]}, {"sent": "and the moose has big brown antlers .", "words": ["the", "brown", "moose", "and", "big"]}, {"sent": "I dont know what sound a moose makes so well turn some more in this book .", "words": ["what", "so", "moose", "dont", "more", "in", "this", "book", "some", "a"]}, {"sent": "that is a moose .", "words": ["that", "moose", "a", "is"]}, {"sent": "I like the moose .", "words": ["the", "like", "moose"]}, {"sent": "candles look loose worried the moose .", "words": ["the", "moose", "look"]}, {"sent": "um she didnt know I dont think she knows what a um she doesnt know what a uh a moose .", "words": ["what", "think", "moose", "dont", "she", "a"]}, {"sent": "moose .", "words": ["moose"]}, {"sent": "maybe the moose wants the icecream .", "words": ["the", "moose"]}, {"sent": "a moose .", "words": ["moose", "a"]}, {"sent": "but um but like the moose like we were just at my husbands cousins house and his house is decorated with moose .", "words": ["the", "my", "we", "is", "with", "like", "moose", "and", "house", "at", "his", "were", "but"]}, {"sent": "say moose ?", "words": ["say", "moose"]}, {"sent": "moose moose .", "words": ["moose"]}, {"sent": "yep the moose is gonna eat the pie up and what is the other animal he left ?", "words": ["the", "up", "what", "is", "moose", "eat", "and", "other", "animal", "he"]}, {"sent": "look at that moose .", "words": ["at", "that", "moose", "look"]}, {"sent": "a moose hes got a beard .", "words": ["moose", "a"]}, {"sent": "moose .", "words": ["moose"]}, {"sent": "moose .", "words": ["moose"]}, {"sent": "moose .", "words": ["moose"]}, {"sent": "the moose right we like to play with the moose youre right .", "words": ["the", "we", "to", "play", "with", "like", "moose"]}, {"sent": "or is that a moose ?", "words": ["that", "moose", "a", "is"]}, {"sent": "um and I pulled out the moose which she recognized she recognized it .", "words": ["the", "moose", "and", "it", "out", "which", "she"]}, {"sent": "she saw a moose head .", "words": ["head", "moose", "she", "a"]}, {"sent": "you like the moose ?", "words": ["the", "moose", "like", "you"]}, {"sent": "Mommy does the moose .", "words": ["the", "does", "moose"]}, {"sent": "that is a moose .", "words": ["that", "moose", "a", "is"]}, {"sent": "because a moose has big antlers .", "words": ["big", "moose", "because", "a"]}, {"sent": "we saw the moose ?", "words": ["the", "moose", "we"]}, {"sent": "do moose .", "words": ["do", "moose"]}, {"sent": "yum the moose is eating the pizza .", "words": ["the", "pizza", "is", "moose"]}, {"sent": "thats that thats a moose .", "words": ["that", "moose", "a"]}, {"sent": "thats a moose .", "words": ["moose", "a"]}, {"sent": "hello moose .", "words": ["moose"]}, {"sent": "is that a moose ?", "words": ["that", "moose", "a", "is"]}, {"sent": "a moose .", "words": ["moose", "a"]}, {"sent": "big hug for the moose .", "words": ["the", "for", "moose", "hug", "big"]}, {"sent": "I dont know what the moose says the moose doesnt make that much noise .", "words": ["the", "what", "much", "moose", "dont", "that", "make"]}, {"sent": "didjou tell Manuela that we went to a the restaurant with the moose ?", "words": ["the", "we", "to", "with", "moose", "that", "a"]}, {"sent": "the moose it not because we went to the Tetons last summer so we talked a lot about moose last summer .", "words": ["the", "we", "a lot", "to", "last", "so", "moose", "it", "about", "not", "because", "a"]}, {"sent": "its a moose .", "words": ["moose", "a"]}, {"sent": "a moose .", "words": ["moose", "a"]}, {"sent": "and not a bear or a bird or a beaver or a pig or a moose or a opossum or a skunk .", "words": ["bird", "bear", "and", "pig", "moose", "not", "a"]}, {"sent": "no that is a moose .", "words": ["that", "moose", "a", "is"]}, {"sent": "the moose .", "words": ["the", "moose"]}, {"sent": "a moose .", "words": ["moose", "a"]}, {"sent": "thats a moose .", "words": ["moose", "a"]}, {"sent": "raccoons can climb trees though .", "words": ["can", "climb"]}, {"sent": "dont climb on the table .", "words": ["the", "table", "dont", "on", "climb"]}, {"sent": "please dont climb up on it okay .", "words": ["up", "dont", "it", "on", "climb"]}, {"sent": "you can climb onto your bed that way .", "words": ["can", "your", "bed", "you", "climb", "that"]}, {"sent": "dont you think you can climb those steps ?", "words": ["can", "think", "those", "you", "dont", "climb"]}, {"sent": "climb mountains .", "words": ["climb"]}, {"sent": "I cant climb up it .", "words": ["up", "it", "climb"]}, {"sent": "yeah theyre gonna climb up the tree .", "words": ["up", "the", "tree", "climb"]}, {"sent": "you dont want him to climb my leg ?", "words": ["my", "to", "you", "dont", "him", "leg", "climb"]}, {"sent": "she gonna climb the roof maybe she has to fix the roof .", "words": ["the", "to", "fix", "roof", "climb", "she"]}, {"sent": "he get and go and climb up .", "words": ["up", "go", "and", "get", "he", "climb"]}, {"sent": "are you gonna climb in the window ?", "words": ["the", "you", "in", "are", "window", "climb"]}, {"sent": "the elephants gonna climb jump through the flaming hoop .", "words": ["the", "jump", "climb"]}, {"sent": "when we go to TumbleTots right therell be lots and lotsof things like that for you to climb on .", "words": ["for", "go", "we", "to", "be", "like", "and", "you", "that", "on", "climb", "when"]}, {"sent": "so you can climb back in .", "words": ["can", "back", "so", "you", "in", "climb"]}, {"sent": "youve got to climb up havent you ?", "words": ["up", "to", "climb", "you"]}, {"sent": "you cant climb up there .", "words": ["up", "there", "climb", "you"]}, {"sent": "oh are you gonna climb on there ?", "words": ["you", "there", "are", "on", "climb"]}, {"sent": "when you walk down the climb down the step .", "words": ["the", "when", "climb", "you", "down", "walk"]}, {"sent": "if you climb up steps then we say no dont we ?", "words": ["up", "if", "we", "say", "you", "dont", "then", "climb"]}, {"sent": "he likes to climb .", "words": ["to", "climb", "he"]}, {"sent": "did you climb a tree yesterday ?", "words": ["tree", "you", "did", "climb", "a"]}, {"sent": "climb up dont they ?", "words": ["up", "dont", "climb", "they"]}, {"sent": "like the racoon climb up the slide ?", "words": ["the", "up", "slide", "like", "climb"]}, {"sent": "oh so you climb on it up o on top of all the toys to get up here ?", "words": ["up", "all", "the", "of", "so", "to", "you", "get", "here", "it", "on", "climb"]}, {"sent": "you cant climb with the pen in your hand sweetheart .", "words": ["the", "your", "hand", "with", "you", "in", "pen", "climb"]}, {"sent": "oh you did not climb the hill ?", "words": ["the", "you", "did", "climb", "not"]}, {"sent": "climb up there and see the window .", "words": ["up", "the", "and", "there", "see", "window", "climb"]}, {"sent": "climb aboard .", "words": ["climb"]}, {"sent": "and does he does he climb up the webs ?", "words": ["up", "does", "the", "and", "he", "climb"]}, {"sent": "I never saw a giraffe climb at the zoo .", "words": ["the", "zoo", "giraffe", "at", "climb", "a"]}, {"sent": "climb the ladders ?", "words": ["the", "climb"]}, {"sent": "so dont climb up .", "words": ["dont", "so", "climb", "up"]}, {"sent": "we do not climb on tables .", "words": ["do", "we", "on", "climb", "not"]}, {"sent": "is Terence going to climb up the ramp ?", "words": ["up", "the", "is", "to", "climb"]}, {"sent": "lets put lets see if itll climb up .", "words": ["up", "put", "if", "see", "climb"]}, {"sent": "and they climb up on the wall .", "words": ["up", "the", "and", "they", "on", "climb"]}, {"sent": "if we move this over here a bit then you wont hafta climb over Annas feet .", "words": ["if", "we", "you", "here", "this", "then", "climb", "over", "a"]}, {"sent": "they hafta go and climb up that ladder to get that cat down from the tree .", "words": ["up", "the", "tree", "go", "to", "down", "and", "get", "they", "cat", "ladder", "climb", "that"]}, {"sent": "dont climb up there .", "words": ["up", "dont", "climb", "there"]}, {"sent": "Ill hide while you climb .", "words": ["climb", "hide", "you"]}, {"sent": "what and and he can climb in there without getting hurt and he wont get stuck anymore ?", "words": ["can", "what", "hurt", "and", "get", "there", "in", "he", "stuck", "climb"]}, {"sent": "why do you wanna climb over me ?", "words": ["do", "you", "wanna", "me", "climb", "over", "why"]}, {"sent": "climb in .", "words": ["in", "climb"]}, {"sent": "would you like to climb on a motorbike Ethan there .", "words": ["to", "like", "you", "there", "would", "on", "climb", "a"]}, {"sent": "dont climb any higher .", "words": ["dont", "climb", "any"]}, {"sent": "the clown is gonna climb up the ladder like a fireman ?", "words": ["the", "up", "is", "like", "ladder", "climb", "a"]}, {"sent": "what do they climb ?", "words": ["what", "do", "climb", "they"]}, {"sent": "theyre just gonna climb up the ladder .", "words": ["up", "the", "climb", "ladder"]}, {"sent": "a tree to climb so high so high Ill climb up if I dare .", "words": ["up", "tree", "if", "to", "so", "high", "climb", "a"]}, {"sent": "can you climb up ?", "words": ["can", "climb", "up", "you"]}, {"sent": "you got to play with Matt with Maddie and run around and climb trees .", "words": ["to", "play", "with", "and", "you", "run", "around", "climb"]}, {"sent": "but first you hafta climb up so you can reach the roof .", "words": ["up", "can", "the", "so", "first", "you", "roof", "climb", "but"]}, {"sent": "climb on your horse .", "words": ["on", "horse", "climb", "your"]}, {"sent": "shes going to climb .", "words": ["to", "climb"]}, {"sent": "you can climb up by yourself .", "words": ["can", "up", "yourself", "by", "you", "climb"]}, {"sent": "how about we not climb on the table Im just saying .", "words": ["the", "how", "table", "we", "about", "on", "climb", "not"]}, {"sent": "yeah the letters climb the coconut tree .", "words": ["the", "tree", "climb"]}, {"sent": "careful dont climb up there youll fall off .", "words": ["up", "off", "fall", "dont", "there", "careful", "climb"]}, {"sent": "no you mustnt climb up to put it back .", "words": ["up", "put", "back", "to", "you", "it", "climb"]}, {"sent": "Ill climb in your arms and Ill kiss you goodnight .", "words": ["your", "and", "you", "in", "climb", "kiss"]}, {"sent": "now show Jenell where you climb your trees okay ?", "words": ["where", "your", "show", "you", "climb"]}, {"sent": "and off he went looking for a hill to climb to see where he was .", "words": ["off", "where", "for", "to", "was", "and", "he", "see", "climb", "a"]}, {"sent": "or somebody will climb up your ladder ?", "words": ["up", "your", "ladder", "will", "climb"]}, {"sent": "now hes gonna climb the tree .", "words": ["the", "tree", "climb"]}, {"sent": "what did you climb up to get down the slide ?", "words": ["up", "what", "the", "slide", "to", "down", "you", "get", "did", "climb"]}, {"sent": "dont climb on me .", "words": ["on", "dont", "me", "climb"]}, {"sent": "oh yeah you could climb over .", "words": ["could", "climb", "over", "you"]}, {"sent": "and off he went looking for a hill to climb to see where he was .", "words": ["off", "where", "for", "to", "was", "and", "he", "see", "climb", "a"]}, {"sent": "and look and look the elephants can climb it .", "words": ["the", "can", "and", "it", "look", "climb"]}, {"sent": "choose a window to climb into and hop through when you arrive .", "words": ["into", "to", "and", "you", "window", "climb", "when", "a"]}, {"sent": "if you climb up on my if you get down to the from the tree .", "words": ["up", "the", "my", "tree", "if", "down", "to", "you", "get", "on", "climb"]}, {"sent": "you cant climb up there .", "words": ["up", "there", "climb", "you"]}, {"sent": "oh I cant climb up it Thomas .", "words": ["up", "it", "climb"]}, {"sent": "its not a ladder for you to climb .", "words": ["for", "to", "you", "ladder", "climb", "not", "a"]}, {"sent": "well is this little man going to climb up and .", "words": ["up", "is", "to", "little", "and", "this", "climb"]}, {"sent": "Eves gonna climb ?", "words": ["climb"]}, {"sent": "you are ready to climb over huh ?", "words": ["to", "you", "are", "climb", "over"]}, {"sent": "please dont climb on there son Ive asked you hundreds of times not to do it .", "words": ["do", "of", "to", "you", "dont", "there", "it", "on", "climb", "not"]}, {"sent": "you using me to climb ?", "words": ["to", "me", "climb", "you"]}, {"sent": "cant climb into the chair ?", "words": ["the", "climb", "into", "chair"]}, {"sent": "I know but I have to climb over all this stuff .", "words": ["all", "to", "have", "this", "climb", "over", "but"]}, {"sent": "well I you may not climb inside the cupboard .", "words": ["the", "inside", "you", "climb", "not"]}, {"sent": "cant climb up when theres nobody here .", "words": ["up", "climb", "when", "here"]}, {"sent": "youre not going to climb down there Aran .", "words": ["to", "climb", "there", "down", "not"]}, {"sent": "she going to climb up to the top .", "words": ["up", "the", "to", "climb", "she"]}, {"sent": "you can just climb right over cant you .", "words": ["can", "climb", "over", "you"]}, {"sent": "does he climb up that ladder ?", "words": ["up", "does", "he", "ladder", "climb", "that"]}, {"sent": "well climb are you gonna have him climb the ladder ?", "words": ["the", "you", "have", "him", "are", "ladder", "climb"]}, {"sent": "you make him climb around and get on the ladder .", "words": ["the", "you", "and", "get", "him", "ladder", "around", "on", "climb", "make"]}, {"sent": "who else likes to climb ?", "words": ["to", "climb", "who"]}, {"sent": "can you climb up a tree ?", "words": ["can", "up", "tree", "you", "climb", "a"]}, {"sent": "he knew how to climb the monkey bars and pump himself high on the swings .", "words": ["the", "how", "to", "monkey", "and", "he", "high", "on", "climb"]}, {"sent": "we dont climb up there .", "words": ["up", "we", "dont", "there", "climb"]}, {"sent": "this is the flying this is the they climb up and they can dive into the uh .", "words": ["the", "up", "can", "into", "is", "and", "this", "they", "climb"]}, {"sent": "cant climb up there .", "words": ["up", "there", "climb"]}, {"sent": "oh thats Julian with a mouthful of banana .", "words": ["of", "banana", "with", "a"]}, {"sent": "banana yes .", "words": ["banana"]}, {"sent": "banana .", "words": ["banana"]}, {"sent": "banana cookie .", "words": ["cookie", "banana"]}, {"sent": "is that monkey eating a banana ?", "words": ["is", "monkey", "banana", "that", "a"]}, {"sent": "eat a banana Laura .", "words": ["eat", "banana", "a"]}, {"sent": "and banana skins .", "words": ["banana", "and"]}, {"sent": "and banana .", "words": ["banana", "and"]}, {"sent": "or a banana .", "words": ["banana", "a"]}, {"sent": "you want banana .", "words": ["banana", "you"]}, {"sent": "wouldja like some banana muffin ?", "words": ["muffin", "some", "banana", "like"]}, {"sent": "banana .", "words": ["banana"]}, {"sent": "banana darling .", "words": ["banana"]}, {"sent": "banana .", "words": ["banana"]}, {"sent": "banana .", "words": ["banana"]}, {"sent": "what about a banana ?", "words": ["what", "banana", "about", "a"]}, {"sent": "peel a banana .", "words": ["banana", "a"]}, {"sent": "kiwi and fresh banana .", "words": ["banana", "and"]}, {"sent": "would you like a banana or an apple Thomas ?", "words": ["like", "you", "banana", "an", "would", "apple", "a"]}, {"sent": "and b is for banana .", "words": ["banana", "for", "is", "and"]}, {"sent": "Ill make banana bread .", "words": ["banana", "bread", "make"]}, {"sent": "banana .", "words": ["banana"]}, {"sent": "and Thomass Mummy gave you a banana to eat .", "words": ["to", "you", "and", "banana", "eat", "a"]}, {"sent": "theres the banana you like banana .", "words": ["the", "banana", "like", "you"]}, {"sent": "can I have a banana ?", "words": ["can", "banana", "have", "a"]}, {"sent": "a banana .", "words": ["banana", "a"]}, {"sent": "were gonna weigh the banana again ?", "words": ["the", "banana", "were"]}, {"sent": "is the banana in front of the birdie ?", "words": ["the", "is", "of", "banana", "in"]}, {"sent": "how is that banana ?", "words": ["banana", "how", "that", "is"]}, {"sent": "lets eat a little banana .", "words": ["eat", "banana", "little", "a"]}, {"sent": "theres some banana in the bowl but dont eat the banana if you dont want it .", "words": ["the", "bowl", "if", "you", "eat", "banana", "in", "dont", "it", "some", "but"]}, {"sent": "b banana .", "words": ["banana"]}, {"sent": "and then we can have a banana cant we ?", "words": ["can", "we", "and", "have", "banana", "then", "a"]}, {"sent": "I think this looks a bit like a banana as well .", "words": ["think", "like", "this", "banana", "a"]}, {"sent": "she said that someone in a banana suit has a seven .", "words": ["banana", "in", "that", "she", "a"]}, {"sent": "ah my banana .", "words": ["banana", "my"]}, {"sent": "I bet you shed like a banana too you know .", "words": ["like", "you", "banana", "too", "a"]}, {"sent": "wheres the bowl of banana soup ?", "words": ["the", "bowl", "of", "banana", "soup"]}, {"sent": "would you like banana cake ?", "words": ["cake", "like", "you", "banana", "would"]}, {"sent": "go and eat your banana .", "words": ["your", "go", "and", "eat", "banana"]}, {"sent": "is there a banana in there ?", "words": ["is", "there", "in", "banana", "a"]}, {"sent": "and it looks like a green banana but its not nice to eat is it ?", "words": ["is", "to", "nice", "like", "and", "eat", "banana", "it", "green", "not", "but", "a"]}, {"sent": "so it was really nice that she gave you a banana .", "words": ["was", "so", "nice", "you", "banana", "it", "that", "she", "a"]}, {"sent": "sure dont want a banana ?", "words": ["dont", "banana", "a"]}, {"sent": "yeah he can eat the banana .", "words": ["can", "the", "eat", "banana", "he"]}, {"sent": "do you not want your banana ?", "words": ["do", "your", "you", "banana", "not"]}, {"sent": "you eat your banana .", "words": ["eat", "banana", "your", "you"]}, {"sent": "now Daddy is gonna shishi banana and some squash ?", "words": ["banana", "some", "and", "is"]}, {"sent": "Im xxx banana xxx .", "words": ["banana"]}, {"sent": "shall we give the monkey a banana ?", "words": ["the", "we", "monkey", "banana", "give", "a"]}, {"sent": "is he gonna eat that banana ?", "words": ["is", "eat", "banana", "he", "that"]}, {"sent": "an orange and a banana .", "words": ["and", "banana", "an", "orange", "a"]}, {"sent": "um she certainly recognizes the pizza the banana um .", "words": ["the", "banana", "she", "pizza"]}, {"sent": "you can have a banana cookie .", "words": ["can", "cookie", "you", "have", "banana", "a"]}, {"sent": "want a banana ?", "words": ["banana", "a"]}, {"sent": "who wants banana ?", "words": ["banana", "who"]}, {"sent": "I think youve squashed the banana cake aswell since you crawled over the plates but never mind .", "words": ["the", "think", "cake", "you", "banana", "over", "but"]}, {"sent": "have a banana sandwich ?", "words": ["have", "banana", "sandwich", "a"]}, {"sent": "would you like a banana ?", "words": ["like", "you", "banana", "would", "a"]}, {"sent": "banana .", "words": ["banana"]}, {"sent": "now you eat your banana Kathryn .", "words": ["eat", "banana", "your", "you"]}, {"sent": "what a delicious banana .", "words": ["what", "banana", "a"]}, {"sent": "thats a banana .", "words": ["banana", "a"]}, {"sent": "sandwiches a breadstick cheese banana and crisps .", "words": ["banana", "cheese", "and", "a"]}, {"sent": "would you like banana and custard ?", "words": ["like", "and", "you", "banana", "would"]}, {"sent": "beep okay get the lemon now the banana now .", "words": ["get", "the", "banana"]}, {"sent": "no banana .", "words": ["banana"]}, {"sent": "oh Thomas Ive just put my foot on some banana .", "words": ["put", "my", "banana", "foot", "on", "some"]}, {"sent": "theres a banana peel theres a banana peel .", "words": ["banana", "a"]}, {"sent": "can I have a banana please ?", "words": ["can", "banana", "have", "a"]}, {"sent": "and strawberries and a banana .", "words": ["banana", "a", "and"]}, {"sent": "banana .", "words": ["banana"]}, {"sent": "little bit of banana .", "words": ["of", "banana", "little"]}, {"sent": "do you wanna eat your banana ?", "words": ["do", "your", "you", "eat", "banana", "wanna"]}, {"sent": "you want more banana ?", "words": ["banana", "more", "you"]}, {"sent": "I like the banana .", "words": ["the", "banana", "like"]}, {"sent": "would you rather have a banana because its a lot easier to eat Jerry ?", "words": ["a lot", "to", "you", "have", "banana", "eat", "would", "because", "a"]}, {"sent": "a banana ?", "words": ["banana", "a"]}, {"sent": "banana .", "words": ["banana"]}, {"sent": "see all those banana slugs ?", "words": ["all", "see", "those", "banana"]}, {"sent": "you want a banana .", "words": ["banana", "a", "you"]}, {"sent": "Im gonna get that banana .", "words": ["get", "banana", "that"]}, {"sent": "banana .", "words": ["banana"]}, {"sent": "youve got to go shopping before you can have a banana .", "words": ["can", "go", "to", "you", "have", "banana", "a"]}, {"sent": "you felt caca but spitting out banana .", "words": ["banana", "out", "but", "you"]}, {"sent": "like a chicken leg and a banana .", "words": ["like", "and", "banana", "leg", "chicken", "a"]}, {"sent": "banana head .", "words": ["banana", "head"]}, {"sent": "banana ?", "words": ["banana"]}, {"sent": "would you like a banana ?", "words": ["like", "you", "banana", "would", "a"]}, {"sent": "that doesnt look like a banana .", "words": ["like", "banana", "look", "that", "a"]}, {"sent": "you put the banana on top .", "words": ["the", "put", "you", "banana", "on"]}, {"sent": "you want some fresh banana ?", "words": ["some", "banana", "you"]}, {"sent": "what have you done banana head ?", "words": ["what", "head", "you", "have", "banana"]}, {"sent": "hes eating a banana .", "words": ["banana", "a"]}, {"sent": "what shall we make a banana milkshake .", "words": ["what", "we", "banana", "make", "a"]}, {"sent": "heres your cereal and banana .", "words": ["banana", "your", "and", "cereal"]}, {"sent": "think those are banana pieces ?", "words": ["banana", "think", "are", "those"]}, {"sent": "youre eating banana now .", "words": ["banana"]}, {"sent": "in fact you even have that sweater .", "words": ["sweater", "you", "have", "in", "that"]}, {"sent": "your sweater is made outof wool .", "words": ["your", "sweater", "is"]}, {"sent": "so were gonna get a green sweater and then what else are we gonna get because we had some ideas about when grandma and grandpa are here .", "words": ["what", "sweater", "when", "we", "so", "and", "get", "here", "are", "green", "about", "some", "then", "were", "because", "a"]}, {"sent": "oh we got a red sweater .", "words": ["red", "sweater", "a", "we"]}, {"sent": "its all over your sweater .", "words": ["your", "all", "over", "sweater"]}, {"sent": "sweater .", "words": ["sweater"]}, {"sent": "now take this sweater you know with you okay ?", "words": ["sweater", "with", "you", "this", "take"]}, {"sent": "she doesnt have her arms in her sweater .", "words": ["sweater", "have", "in", "her", "she"]}, {"sent": "now lets put your sweater on .", "words": ["on", "put", "your", "sweater"]}, {"sent": "mhm sweater .", "words": ["sweater"]}, {"sent": "what are you going to wear a sweater or an over shirt ?", "words": ["what", "sweater", "to", "shirt", "you", "an", "are", "over", "a"]}, {"sent": "hes wearing a sweater and pants .", "words": ["pants", "sweater", "and", "a"]}, {"sent": "I mean if you put a sweater on on top of that xxx .", "words": ["put", "sweater", "if", "of", "you", "on", "that", "a"]}, {"sent": "thats a sweater .", "words": ["sweater", "a"]}, {"sent": "well get a sweater to wear over it .", "words": ["sweater", "to", "get", "it", "over", "a"]}, {"sent": "and you make a sweater .", "words": ["sweater", "you", "and", "make", "a"]}, {"sent": "put that sweater on .", "words": ["on", "put", "that", "sweater"]}, {"sent": "he has been looking for a warm sweater .", "words": ["for", "he", "sweater", "a"]}, {"sent": "what color is your sweater ?", "words": ["what", "your", "sweater", "is"]}, {"sent": "sweater .", "words": ["sweater"]}, {"sent": "brown sweater or something .", "words": ["sweater", "brown"]}, {"sent": "let me take your sweater off ?", "words": ["off", "your", "sweater", "me", "take"]}, {"sent": "I have a constant battle to keep him outof a sweater .", "words": ["sweater", "to", "have", "him", "a"]}, {"sent": "Cricter also had a warm comfortable bed there he would dream happily under his palm trees see there hes wearing his sweater and what else is he wearing ?", "words": ["what", "sweater", "is", "bed", "and", "there", "under", "he", "would", "his", "see", "a"]}, {"sent": "where does your sweater go ?", "words": ["where", "does", "your", "sweater", "go"]}, {"sent": "okay thats the sweater your gonna wear ?", "words": ["the", "sweater", "your"]}, {"sent": "oh no this is my sweater .", "words": ["this", "my", "sweater", "is"]}, {"sent": "why doesnt she have her sweater on ?", "words": ["sweater", "have", "on", "her", "she", "why"]}, {"sent": "choose you warm sweater .", "words": ["sweater", "you"]}, {"sent": "wheres your sweater ?", "words": ["your", "sweater"]}, {"sent": "hold up the sweater .", "words": ["up", "the", "sweater", "hold"]}, {"sent": "here put your sweater on .", "words": ["put", "your", "sweater", "here", "on"]}, {"sent": "just like when you put on a sweater .", "words": ["put", "sweater", "like", "you", "on", "when", "a"]}, {"sent": "thats her sweater because when we play football the season is the fall right ?", "words": ["the", "sweater", "when", "we", "is", "play", "fall", "her", "because"]}, {"sent": "yeah you even have that sweater hes got on .", "words": ["sweater", "you", "have", "on", "that"]}, {"sent": "did Mummy buy you a sweater ?", "words": ["sweater", "you", "did", "buy", "a"]}, {"sent": "arent you warm in that sweater ?", "words": ["in", "that", "sweater", "you"]}, {"sent": "well I got my sweater on .", "words": ["on", "my", "sweater"]}, {"sent": "no where did mommy put your sweater ?", "words": ["where", "put", "your", "sweater", "did"]}, {"sent": "oh youve got youve got a sweater on too .", "words": ["on", "too", "sweater", "a"]}, {"sent": "oh Mommys wearing a sweater .", "words": ["sweater", "a"]}, {"sent": "pick your sweater up .", "words": ["pick", "up", "your", "sweater"]}, {"sent": "thats your thats not your sweater thats your shirt .", "words": ["shirt", "your", "not", "sweater"]}, {"sent": "thats right Doug has on a blue sweater .", "words": ["on", "blue", "sweater", "a"]}, {"sent": "oh there she is shes dressed in a sweater and a scarf and shoes .", "words": ["sweater", "is", "scarf", "and", "there", "in", "she", "a"]}, {"sent": "whats this sweater from ?", "words": ["this", "sweater"]}, {"sent": "let see if this sweater still fits you .", "words": ["sweater", "if", "you", "this", "see"]}, {"sent": "but you havent got a warm woolly sweater this morning have you ?", "words": ["sweater", "you", "have", "this", "but", "a"]}, {"sent": "take her sweater off ?", "words": ["off", "her", "take", "sweater"]}, {"sent": "thats thats a sweater ?", "words": ["sweater", "a"]}, {"sent": "you got your sweater on .", "words": ["on", "your", "sweater", "you"]}, {"sent": "and heres her sweater .", "words": ["her", "sweater", "and"]}, {"sent": "put your sweater on .", "words": ["on", "put", "your", "sweater"]}, {"sent": "you have a green sweater on but you have a white turtleneck on .", "words": ["sweater", "you", "have", "green", "on", "white", "but", "a"]}, {"sent": "lets take that sweater off .", "words": ["off", "that", "take", "sweater"]}, {"sent": "get your sweater on .", "words": ["get", "on", "your", "sweater"]}, {"sent": "sweater .", "words": ["sweater"]}, {"sent": "xxx Christmas xxx xxx Christmas sweater .", "words": ["sweater"]}, {"sent": "well youre gonna put your sweater on then well put this on over it .", "words": ["put", "your", "sweater", "this", "it", "on", "then", "over"]}, {"sent": "Doug what color is Carolines sweater ?", "words": ["what", "sweater", "is"]}, {"sent": "thats PBBear putting his sweater on .", "words": ["his", "on", "sweater"]}, {"sent": "Leah has a pretty sweater on doesnt she ?", "words": ["sweater", "pretty", "on", "she", "a"]}, {"sent": "do you want your sweater back on ?", "words": ["do", "your", "sweater", "back", "you", "on"]}, {"sent": "sweater .", "words": ["sweater"]}, {"sent": "okay Im just gonna ask you to clip that back onto your sweater .", "words": ["your", "sweater", "back", "to", "you", "that"]}, {"sent": "but now Dandelion thought he really should wear something more elegant than a sweater to the party .", "words": ["the", "sweater", "to", "party", "more", "he", "but", "a"]}, {"sent": "yeah I had a sweater on yesterday .", "words": ["on", "sweater", "a"]}, {"sent": "Nina where does your sweater go ?", "words": ["where", "does", "your", "sweater", "go"]}, {"sent": "pretty red sweater .", "words": ["red", "pretty", "sweater"]}, {"sent": "sweater okay .", "words": ["sweater"]}, {"sent": "no lets keep our sweater on sweetie .", "words": ["on", "sweater", "our"]}, {"sent": "were gonna go out to the zoo so we hafta keep our sweater on .", "words": ["the", "zoo", "sweater", "go", "we", "to", "so", "our", "out", "on", "were"]}, {"sent": "put a s uh I dont know if you need a sweater on under that coat .", "words": ["put", "need", "if", "sweater", "you", "dont", "under", "on", "that", "coat", "a"]}, {"sent": "and today I have on a shirt and a sweater some of the time .", "words": ["the", "sweater", "of", "shirt", "and", "have", "on", "some", "a"]}, {"sent": "thats xxx sweater .", "words": ["sweater"]}, {"sent": "under my sweater ?", "words": ["under", "my", "sweater"]}, {"sent": "pull your sweater out Tish .", "words": ["out", "your", "sweater", "pull"]}, {"sent": "what color of a sweater have you got on Sarah ?", "words": ["what", "sweater", "of", "you", "have", "on", "a"]}, {"sent": "whoopsie its on your sweater .", "words": ["on", "your", "sweater"]}, {"sent": "thats a sweater .", "words": ["sweater", "a"]}, {"sent": "with that sweater .", "words": ["that", "with", "sweater"]}, {"sent": "awright Brett get your sweater .", "words": ["get", "your", "sweater"]}, {"sent": "whos that sweater for ?", "words": ["for", "that", "sweater"]}, {"sent": "my sister has a sweater like I mean a coat like dat .", "words": ["my", "sweater", "like", "coat", "a"]}, {"sent": "youre gonna hafta take that thing off Carl so Angela can put on his sweater so she can comb .", "words": ["off", "can", "put", "sweater", "so", "comb", "his", "on", "take", "that", "she"]}, {"sent": "is your sweater blue ?", "words": ["blue", "your", "sweater", "is"]}, {"sent": "did he have a sweater aswell ?", "words": ["sweater", "have", "he", "did", "a"]}, {"sent": "Mommy has gotta put her sweater on .", "words": ["on", "put", "her", "sweater"]}, {"sent": "wanna wear your new sweater from Miss xxx ?", "words": ["new", "your", "sweater", "wanna"]}, {"sent": "up onto my sweater .", "words": ["up", "my", "sweater"]}, {"sent": "she was wearing the sweater we gave her .", "words": ["the", "sweater", "we", "was", "her", "she"]}, {"sent": "put the sweater back on you like that .", "words": ["the", "put", "sweater", "back", "like", "you", "on", "that"]}, {"sent": "its a sweater .", "words": ["sweater", "a"]}, {"sent": "what color is Manuelas sweater ?", "words": ["what", "sweater", "is"]}, {"sent": "you have a blue sweater at home .", "words": ["sweater", "blue", "have", "you", "at", "home", "a"]}, {"sent": "you have a red sweater too .", "words": ["red", "sweater", "you", "have", "too", "a"]}, {"sent": "youre going to drive a motorcycle ?", "words": ["drive", "to", "motorcycle", "a"]}, {"sent": "motorcycle .", "words": ["motorcycle"]}, {"sent": "your motorcycle crashed .", "words": ["motorcycle", "your"]}, {"sent": "look at the mommy rides the motorcycle .", "words": ["the", "motorcycle", "at", "look"]}, {"sent": "oh theres a man on a motorcycle .", "words": ["on", "motorcycle", "a"]}, {"sent": "now he cant ride the motorcycle anymore ?", "words": ["the", "motorcycle", "he", "ride"]}, {"sent": "Cody broke his motorcycle ?", "words": ["his", "motorcycle"]}, {"sent": "put the baby on the motorcycle and go to the store .", "words": ["the", "motorcycle", "put", "go", "to", "store", "and", "on"]}, {"sent": "well you dont hafta have your motorcycle .", "words": ["motorcycle", "your", "you", "have", "dont"]}, {"sent": "I put the motorcycle down .", "words": ["the", "motorcycle", "put", "down"]}, {"sent": "motorcycle .", "words": ["motorcycle"]}, {"sent": "xxx motorcycle .", "words": ["motorcycle"]}, {"sent": "whoa there goes the motorcycle through the tunnel .", "words": ["the", "there", "motorcycle"]}, {"sent": "motorcycle is right here .", "words": ["here", "motorcycle", "is"]}, {"sent": "yeah theres a motorcycle see two wheels and a seat .", "words": ["see", "motorcycle", "and", "a"]}, {"sent": "kahah have my book my motorcycle ?", "words": ["have", "book", "my", "motorcycle"]}, {"sent": "is he allowed to put that motorcycle into the store ?", "words": ["the", "motorcycle", "put", "into", "is", "to", "store", "he", "that"]}, {"sent": "motorcycle handles ?", "words": ["motorcycle"]}, {"sent": "Ill ride you in your motorcycle one more time .", "words": ["motorcycle", "your", "you", "in", "more", "ride"]}, {"sent": "and a motorcycle .", "words": ["motorcycle", "a", "and"]}, {"sent": "wow look the motorcycle is spinning its wheels xxx .", "words": ["the", "motorcycle", "is", "look"]}, {"sent": "a motorcycle .", "words": ["motorcycle", "a"]}, {"sent": "motorcycle man did say uh oh .", "words": ["motorcycle", "say", "did"]}, {"sent": "see this one motorcycle .", "words": ["motorcycle", "see", "this"]}, {"sent": "does Lyle have a motorcycle ?", "words": ["have", "does", "motorcycle", "a"]}, {"sent": "heres a motorcycle .", "words": ["motorcycle", "a"]}, {"sent": "youre right its a motorcycle .", "words": ["motorcycle", "a"]}, {"sent": "he cant have a motorcycle on the boat .", "words": ["the", "motorcycle", "have", "he", "on", "boat", "a"]}, {"sent": "wheres the motorcycle say ?", "words": ["the", "motorcycle", "say"]}, {"sent": "the bear is riding a motorcycle .", "words": ["the", "motorcycle", "is", "bear", "a"]}, {"sent": "motorcycle .", "words": ["motorcycle"]}, {"sent": "you got a motorcycle at home .", "words": ["motorcycle", "you", "at", "a", "home"]}, {"sent": "motorcycle .", "words": ["motorcycle"]}, {"sent": "oh the big red motorcycle .", "words": ["the", "motorcycle", "red", "big"]}, {"sent": "you got a motorcycle at Leannes house ?", "words": ["motorcycle", "you", "house", "at", "a"]}, {"sent": "motorcycle .", "words": ["motorcycle"]}, {"sent": "hell chase that motorcycle .", "words": ["chase", "motorcycle", "that"]}, {"sent": "that wants to be a big barn for motorcycle chicken ?", "words": ["motorcycle", "for", "to", "be", "that", "chicken", "big", "a"]}, {"sent": "why dont you draw a picture of Bill on the motorcycle ?", "words": ["the", "motorcycle", "of", "draw", "picture", "you", "dont", "on", "why", "a"]}, {"sent": "wheres my motorcycle ?", "words": ["motorcycle", "my"]}, {"sent": "raggedy Andy will pick up the motorcycle .", "words": ["the", "up", "motorcycle", "pick", "will"]}, {"sent": "wheres youre motorcycle at Leannes house ?", "words": ["motorcycle", "house", "at"]}, {"sent": "Drew and I were following a police motorcycle .", "words": ["motorcycle", "were", "a", "and"]}, {"sent": "is that something to do with the motorcycle ones ?", "words": ["the", "do", "motorcycle", "is", "to", "with", "that"]}, {"sent": "did Nina drive the motorcycle ?", "words": ["drive", "the", "motorcycle", "did"]}, {"sent": "cant mommy drive a motorcycle ?", "words": ["drive", "motorcycle", "a"]}, {"sent": "got a car and a motorcycle .", "words": ["motorcycle", "car", "and", "a"]}, {"sent": "there goes a motorcycle huh ?", "words": ["motorcycle", "there", "a"]}, {"sent": "you wanna play with the motorcycle ?", "words": ["the", "motorcycle", "play", "with", "you", "wanna"]}, {"sent": "oh its a motorcycle .", "words": ["motorcycle", "a"]}, {"sent": "did you ever see a monkey on a motorcycle ?", "words": ["motorcycle", "monkey", "you", "did", "on", "see", "a"]}, {"sent": "who rides a motorcycle ?", "words": ["motorcycle", "a", "who"]}, {"sent": "motorcycle .", "words": ["motorcycle"]}, {"sent": "and then he goes back on his motorcycle .", "words": ["motorcycle", "back", "and", "his", "he", "on", "then"]}, {"sent": "thats a motorcycle .", "words": ["motorcycle", "a"]}, {"sent": "we never had a motorcycle in our stuff huh ?", "words": ["motorcycle", "we", "our", "in", "a"]}, {"sent": "oh so the motorcycle can tow the car .", "words": ["the", "can", "motorcycle", "so", "car"]}, {"sent": "that doesnt look like a motorcycle .", "words": ["motorcycle", "like", "look", "that", "a"]}, {"sent": "hey wait a minute this is my motorcycle .", "words": ["motorcycle", "my", "wait", "is", "this", "a"]}, {"sent": "motorcycle .", "words": ["motorcycle"]}, {"sent": "motorcycle can ju guy can jump over the whole thing .", "words": ["can", "motorcycle", "the", "jump", "over"]}, {"sent": "motorcycle .", "words": ["motorcycle"]}, {"sent": "motorcycle ?", "words": ["motorcycle"]}, {"sent": "did you get your motorcycle driver ?", "words": ["motorcycle", "your", "you", "get", "did"]}, {"sent": "motor sounds motorcycle typeis your trike is your tricycle a motor cycle ?", "words": ["motorcycle", "your", "is", "tricycle", "a"]}, {"sent": "there goes the motorcycle .", "words": ["the", "there", "motorcycle"]}, {"sent": "a motorcycle ?", "words": ["motorcycle", "a"]}, {"sent": "xxx play motorcycle ?", "words": ["motorcycle", "play"]}, {"sent": "tell them about the motorcycle you rode on .", "words": ["the", "motorcycle", "them", "you", "about", "on"]}, {"sent": "went on a motorcycle ?", "words": ["on", "motorcycle", "a"]}, {"sent": "Daddy doesnt ride a motorcycle .", "words": ["motorcycle", "ride", "a"]}, {"sent": "and the motorcycle knocked over the fire hydrant .", "words": ["the", "motorcycle", "over", "and"]}, {"sent": "now what is the motorcycle doing in the pool ?", "words": ["the", "what", "motorcycle", "is", "pool", "in"]}, {"sent": "a motorcycle .", "words": ["motorcycle", "a"]}, {"sent": "no I dont think it is a motorcycle .", "words": ["motorcycle", "think", "is", "dont", "it", "a"]}, {"sent": "cant you put him back on your motorcycle ?", "words": ["motorcycle", "put", "your", "back", "you", "him", "on"]}, {"sent": "xxx xxx motorcycle .", "words": ["motorcycle"]}, {"sent": "how does the front wheels to this motorcycle turn ?", "words": ["the", "does", "how", "motorcycle", "to", "this"]}, {"sent": "xxx ambulance xxx policeman fell out of his motorcycle .", "words": ["of", "his", "motorcycle", "out"]}, {"sent": "where is that Bobby and that motorcycle .", "words": ["where", "motorcycle", "is", "and", "that"]}, {"sent": "thats a motorcycle .", "words": ["motorcycle", "a"]}, {"sent": "motorcycle .", "words": ["motorcycle"]}, {"sent": "theres Peter in the motorcycle with Bill .", "words": ["the", "motorcycle", "in", "with"]}, {"sent": "poor guy better stay off the motorcycle huh ?", "words": ["off", "better", "the", "motorcycle", "poor", "stay"]}, {"sent": "oh the motorcycle crashed again .", "words": ["the", "motorcycle"]}, {"sent": "I gotta make sure my motorcycle works .", "words": ["motorcycle", "my", "make"]}, {"sent": "we cant find your helmet you know your bike helmet right your motorcycle helmet ?", "words": ["motorcycle", "your", "we", "you", "find"]}, {"sent": "Sean Hayes right you say you gonna let me take your motorcycle home and you say yes .", "words": ["motorcycle", "your", "say", "you", "and", "me", "take", "home"]}, {"sent": "maybe its a motorcycle .", "words": ["motorcycle", "a"]}, {"sent": "theres the motorcycle .", "words": ["the", "motorcycle"]}, {"sent": "Alan Alan drives the mower like a motorcycle and its very loud .", "words": ["the", "motorcycle", "like", "and", "loud", "a"]}, {"sent": "Im going to take a ride around the townhouse on your motorcycle .", "words": ["the", "motorcycle", "your", "to", "ride", "around", "on", "take", "a"]}, {"sent": "whats he doing on that motorcycle ?", "words": ["on", "motorcycle", "that", "he"]}, {"sent": "wheres my motorcycle ?", "words": ["motorcycle", "my"]}, {"sent": "huh here comes motorcycle man going under the bridge .", "words": ["here", "motorcycle", "under", "the"]}, {"sent": "is that the motorcycle ?", "words": ["the", "motorcycle", "that", "is"]}, {"sent": "were gonna make the motorcycle go faster .", "words": ["the", "motorcycle", "go", "make", "were"]}, {"sent": "on Bobbys motorcycle ?", "words": ["on", "motorcycle"]}, {"sent": "is there a guy riding a motorcycle ?", "words": ["motorcycle", "there", "a", "is"]}, {"sent": "sidewalk .", "words": ["sidewalk"]}, {"sent": "what do you do with sidewalk chalk ?", "words": ["what", "do", "chalk", "sidewalk", "with", "you"]}, {"sent": "get on that sidewalk .", "words": ["get", "on", "that", "sidewalk"]}, {"sent": "on the sidewalk .", "words": ["on", "the", "sidewalk"]}, {"sent": "he walks on the sidewalk .", "words": ["on", "the", "he", "sidewalk"]}, {"sent": "he made this sidewalk and look at what this pig did .", "words": ["what", "sidewalk", "and", "pig", "this", "at", "he", "did", "look"]}, {"sent": "lying on the sidewalk .", "words": ["on", "the", "sidewalk"]}, {"sent": "oh youre putting so many different things on the sidewalk .", "words": ["on", "so", "sidewalk", "the"]}, {"sent": "it is just a line on the sidewalk ?", "words": ["the", "sidewalk", "is", "it", "on", "a"]}, {"sent": "lets make a line on the sidewalk down there .", "words": ["the", "sidewalk", "there", "on", "down", "make", "a"]}, {"sent": "maybe like where the sidewalk ends now .", "words": ["where", "the", "sidewalk", "like"]}, {"sent": "no they shouldnt throw it on the sidewalk at all should they ?", "words": ["the", "all", "throw", "sidewalk", "they", "it", "at", "on"]}, {"sent": "pretend thats a sidewalk ?", "words": ["pretend", "sidewalk", "a"]}, {"sent": "looks like sidewalk too .", "words": ["sidewalk", "like", "too"]}, {"sent": "it looks like a sidewalk .", "words": ["sidewalk", "it", "like", "a"]}, {"sent": "lets stand back and get on the sidewalk .", "words": ["the", "sidewalk", "back", "stand", "and", "get", "on"]}, {"sent": "what do you do when you have snow all over the sidewalk ?", "words": ["the", "what", "do", "all", "sidewalk", "snow", "you", "have", "over", "when"]}, {"sent": "well well see the neighborhood doesnt like the fact that his business has extended out to the sidewalk where its very dangerous to walk .", "words": ["the", "where", "sidewalk", "to", "like", "walk", "out", "his", "see", "that"]}, {"sent": "wanna go for a walk with me down the sidewalk ?", "words": ["the", "for", "sidewalk", "go", "with", "wanna", "me", "down", "walk", "a"]}, {"sent": "sidewalk yes .", "words": ["sidewalk"]}, {"sent": "oh think the sidewalk will talk to you ?", "words": ["the", "think", "sidewalk", "to", "you", "talk", "will"]}, {"sent": "xxx make a sidewalk ?", "words": ["make", "sidewalk", "a"]}, {"sent": "sometimes it goes up in the sidewalk .", "words": ["up", "the", "sidewalk", "in", "it"]}, {"sent": "that grow across the sidewalk .", "words": ["the", "that", "sidewalk"]}, {"sent": "and chalk for the sidewalk .", "words": ["the", "chalk", "for", "sidewalk", "and"]}, {"sent": "the truck was on the sidewalk .", "words": ["the", "sidewalk", "was", "truck", "on"]}, {"sent": "so everyone got onto the sidewalk .", "words": ["the", "so", "sidewalk"]}, {"sent": "Hendrikas walking on the sidewalk .", "words": ["on", "the", "sidewalk"]}, {"sent": "is this the road or the sidewalk ?", "words": ["the", "this", "sidewalk", "is"]}, {"sent": "its okay to be on the sidewalk but the street is bad .", "words": ["the", "sidewalk", "is", "to", "be", "bad", "on", "street", "but"]}, {"sent": "go back on the sidewalk .", "words": ["the", "sidewalk", "back", "go", "on"]}, {"sent": "theres no needta come up on the sidewalk .", "words": ["up", "the", "on", "sidewalk"]}, {"sent": "a red brick sidewalk .", "words": ["red", "sidewalk", "a"]}, {"sent": "careful theres a lady on the sidewalk .", "words": ["the", "sidewalk", "careful", "on", "a"]}, {"sent": "do you wanna go outside with Grandma and color on the sidewalk ?", "words": ["the", "do", "sidewalk", "go", "outside", "with", "you", "and", "wanna", "on"]}, {"sent": "I see so you keep the dog on the sidewalk .", "words": ["the", "sidewalk", "so", "you", "on", "see", "dog"]}, {"sent": "first he walked along a crack in the sidewalk .", "words": ["the", "sidewalk", "first", "in", "he", "a"]}, {"sent": "get on that sidewalk .", "words": ["get", "on", "that", "sidewalk"]}, {"sent": "a pair of pants lying there on the sidewalk .", "words": ["the", "sidewalk", "of", "there", "pants", "on", "a"]}, {"sent": "this is the sidewalk .", "words": ["the", "this", "sidewalk", "is"]}, {"sent": "I think so too one day in a sidewalk cafe Madame Beaudeaux told her friend .", "words": ["think", "sidewalk", "so", "in", "too", "her", "a"]}, {"sent": "is this the road or the sidewalk ?", "words": ["the", "this", "sidewalk", "is"]}, {"sent": "outside on the sidewalk ?", "words": ["on", "outside", "sidewalk", "the"]}, {"sent": "were you on the sidewalk ?", "words": ["the", "sidewalk", "you", "on", "were"]}, {"sent": "on the sidewalk .", "words": ["on", "the", "sidewalk"]}, {"sent": "shes walking on the sidewalk .", "words": ["on", "the", "sidewalk"]}, {"sent": "theres the sidewalk .", "words": ["the", "sidewalk"]}, {"sent": "um lets see people were selling lotsof other things on the sidewalk too .", "words": ["the", "sidewalk", "other", "too", "on", "see", "were"]}, {"sent": "a sidewalk .", "words": ["sidewalk", "a"]}, {"sent": "it was great so Sully was on the sidewalk and then ah Mike drove off in the car but then he wheels fell off right ?", "words": ["the", "off", "sidewalk", "was", "so", "car", "and", "in", "it", "he", "on", "then", "but"]}, {"sent": "youre going to have a sidewalk full of cups ?", "words": ["sidewalk", "full", "of", "to", "have", "a"]}, {"sent": "well lets find the sidewalk .", "words": ["the", "sidewalk", "find"]}, {"sent": "xxx what do you think you wanna try where the sidewalk ends ?", "words": ["where", "what", "do", "think", "the", "sidewalk", "you", "wanna", "try"]}, {"sent": "you have alotof things on the sidewalk dont you ?", "words": ["the", "sidewalk", "you", "have", "dont", "on"]}, {"sent": "hes on the sidewalk doo doo doo doo doo doo doo doo doo doo doo .", "words": ["on", "the", "sidewalk"]}, {"sent": "okay go on the sidewalk let me see .", "words": ["the", "sidewalk", "go", "see", "on", "me"]}, {"sent": "sidewalk right ?", "words": ["sidewalk"]}, {"sent": "hes going on the sidewalk ?", "words": ["on", "the", "sidewalk"]}, {"sent": "people on the sidewalk .", "words": ["on", "the", "sidewalk"]}, {"sent": "I wish Martin could have come though he would have liked seeing the live lobsters on the sidewalk .", "words": ["the", "sidewalk", "wish", "have", "he", "would", "on", "could"]}, {"sent": "what do you have on the sidewalk ?", "words": ["the", "what", "do", "sidewalk", "you", "have", "on"]}, {"sent": "you pretending its a sidewalk now ?", "words": ["sidewalk", "a", "you"]}, {"sent": "let you do what on the sidewalk ?", "words": ["the", "do", "what", "sidewalk", "you", "on"]}, {"sent": "everyone get on the sidewalk he said .", "words": ["the", "sidewalk", "get", "he", "on"]}, {"sent": "someone was selling little sweet potato pies right on the sidewalk and we each got one .", "words": ["the", "sidewalk", "each", "we", "was", "little", "and", "on", "potato"]}, {"sent": "why do you put water on the sidewalk ?", "words": ["the", "do", "put", "sidewalk", "you", "water", "on", "why"]}, {"sent": "where shall we build the sidewalk ?", "words": ["where", "the", "sidewalk", "we", "build"]}, {"sent": "Im gonna walk down the sidewalk .", "words": ["the", "down", "walk", "sidewalk"]}, {"sent": "hes making a poopie in the sidewalk .", "words": ["the", "in", "sidewalk", "a"]}, {"sent": "on the sidewalk .", "words": ["on", "the", "sidewalk"]}, {"sent": "maybe this afternoon we can go out and take your bike out and ride on the sidewalk would that be fun ?", "words": ["can", "the", "your", "sidewalk", "go", "we", "be", "ride", "and", "this", "that", "out", "would", "on", "take"]}, {"sent": "Matty was walking on the sidewalk I think he was actually running and then what happened ?", "words": ["the", "what", "think", "sidewalk", "was", "and", "he", "on", "then"]}, {"sent": "thats the sidewalk thats right .", "words": ["the", "sidewalk"]}, {"sent": "thats the sidewalk .", "words": ["the", "sidewalk"]}, {"sent": "hello mike mommy called were home mike ran down the sidewalk to the car daddy got out first he gave mike an extra high boost into the air then he turned to mommy and little pat .", "words": ["the", "sidewalk", "into", "to", "car", "first", "and", "little", "an", "he", "out", "high", "then", "down", "were", "home"]}, {"sent": "oh what would have happened if you wouldve wrestled on the sidewalk ?", "words": ["the", "what", "if", "sidewalk", "you", "have", "would", "on"]}, {"sent": "go back down the sidewalk .", "words": ["the", "sidewalk", "back", "go", "down"]}, {"sent": "its on the sidewalk .", "words": ["on", "the", "sidewalk"]}, {"sent": "here we can get the line but see the fish cant go in the air up on the sidewalk here because he cant breathe then fish can only breathe in the water .", "words": ["the", "can", "up", "water", "sidewalk", "we", "go", "because", "here", "get", "in", "he", "fish", "on", "see", "then", "but"]}, {"sent": "people useta play marbles on the sidewalk Gabriella .", "words": ["on", "the", "play", "sidewalk"]}, {"sent": "get on get on the sidewalk .", "words": ["get", "the", "on", "sidewalk"]}, {"sent": "is this the sidewalk ?", "words": ["the", "this", "sidewalk", "is"]}, {"sent": "a cat on the sidewalk too .", "words": ["the", "sidewalk", "cat", "too", "on", "a"]}, {"sent": "if hes walking so slowly its because its slippery on the sidewalk probably .", "words": ["the", "if", "sidewalk", "so", "on", "because"]}, {"sent": "oh they were fixing the sidewalk right ?", "words": ["the", "they", "were", "sidewalk"]}, {"sent": "thats the sidewalk yeah .", "words": ["the", "sidewalk"]}, {"sent": "did we see somebody with umbrellas the other day on the sidewalk ?", "words": ["the", "sidewalk", "we", "with", "other", "did", "on", "see"]}, {"sent": "youre going to put your books on the sidewalk ?", "words": ["the", "put", "your", "sidewalk", "to", "on"]}, {"sent": "you put water on the sidewalk ?", "words": ["the", "put", "sidewalk", "you", "water", "on"]}, {"sent": "three idiots walking up and down the sidewalk .", "words": ["up", "the", "sidewalk", "and", "down"]}, {"sent": "and we went for a walk on the sidewalk .", "words": ["the", "for", "sidewalk", "we", "and", "on", "walk", "a"]}, {"sent": "your whole house is going on the sidewalk .", "words": ["the", "your", "sidewalk", "is", "house", "on"]}, {"sent": "good are you the airplane doctor ?", "words": ["the", "airplane", "you", "are", "good"]}, {"sent": "take the airplane in to mommy in the bathroom .", "words": ["the", "airplane", "to", "in", "bathroom", "take"]}, {"sent": "then well slide down and well go really fast faster than an airplane .", "words": ["fast", "slide", "go", "airplane", "and", "an", "then", "down"]}, {"sent": "the airplane .", "words": ["the", "airplane"]}, {"sent": "we would steer it around and run all the way around the driveway like that see his airplane ?", "words": ["the", "all", "we", "airplane", "like", "would", "and", "run", "it", "around", "his", "see", "that"]}, {"sent": "the airplane is upside down .", "words": ["the", "airplane", "down", "is"]}, {"sent": "he made a paper airplane .", "words": ["airplane", "paper", "he", "a"]}, {"sent": "here comes the airplane .", "words": ["here", "the", "airplane"]}, {"sent": "an airplane ?", "words": ["airplane", "an"]}, {"sent": "you know what an airplane is .", "words": ["what", "is", "airplane", "you", "an"]}, {"sent": "I need him to fly the airplane .", "words": ["the", "need", "airplane", "to", "him"]}, {"sent": "yeah thats a airplane .", "words": ["airplane", "a"]}, {"sent": "I think youre gonna make an airplane .", "words": ["airplane", "think", "make", "an"]}, {"sent": "no thats not an airplane .", "words": ["airplane", "an", "not"]}, {"sent": "airplane ?", "words": ["airplane"]}, {"sent": "no Im gonna take an airplane .", "words": ["airplane", "an", "take"]}, {"sent": "airplane .", "words": ["airplane"]}, {"sent": "that is airplane .", "words": ["airplane", "that", "is"]}, {"sent": "what is the airplane doing ?", "words": ["the", "what", "airplane", "is"]}, {"sent": "airplane .", "words": ["airplane"]}, {"sent": "see the airplane ?", "words": ["the", "see", "airplane"]}, {"sent": "the airplane ?", "words": ["the", "airplane"]}, {"sent": "why dont you be the helicopter and Ill be the airplane ?", "words": ["the", "helicopter", "airplane", "be", "you", "and", "dont", "why"]}, {"sent": "no it doesnt fit on that airplane .", "words": ["airplane", "fit", "it", "on", "that"]}, {"sent": "people going in that airplane .", "words": ["airplane", "in", "that"]}, {"sent": "airplane on the roof .", "words": ["airplane", "the", "on", "roof"]}, {"sent": "what does the airplane do ?", "words": ["the", "what", "do", "does", "airplane"]}, {"sent": "over to the ah airplane seat ?", "words": ["the", "to", "over", "airplane"]}, {"sent": "thats a red airplane yeah .", "words": ["red", "airplane", "a"]}, {"sent": "can you say airplane ?", "words": ["can", "airplane", "say", "you"]}, {"sent": "and when are you gonna go on the airplane ?", "words": ["the", "go", "airplane", "you", "and", "are", "on", "when"]}, {"sent": "whats in the airplane ?", "words": ["the", "airplane", "in"]}, {"sent": "no its not an airplane .", "words": ["airplane", "an", "not"]}, {"sent": "airplane .", "words": ["airplane"]}, {"sent": "airplane I dont hear an airplane .", "words": ["airplane", "dont", "an", "hear"]}, {"sent": "airplane .", "words": ["airplane"]}, {"sent": "who are you gonna go with on the airplane ?", "words": ["the", "go", "airplane", "with", "you", "are", "who", "on"]}, {"sent": "it turns like an airplane ?", "words": ["airplane", "an", "it", "like"]}, {"sent": "gonna riding in a airplane .", "words": ["airplane", "in", "a"]}, {"sent": "airplane heehee .", "words": ["airplane"]}, {"sent": "oh Linda lets see that airplane .", "words": ["airplane", "see", "that"]}, {"sent": "you going on an airplane tomorrow ?", "words": ["on", "airplane", "an", "you"]}, {"sent": "do you have a stewardess on that airplane .", "words": ["do", "airplane", "you", "have", "on", "that", "a"]}, {"sent": "is there an airplane at the desert museum huh ?", "words": ["the", "is", "airplane", "there", "an", "at"]}, {"sent": "how am I sposta build the airplane wings when you pull it off ?", "words": ["the", "off", "how", "airplane", "you", "pull", "it", "build", "am", "when"]}, {"sent": "get in the airplane .", "words": ["get", "the", "in", "airplane"]}, {"sent": "why dont you take me for a ride in your airplane can you do that ?", "words": ["can", "do", "for", "your", "airplane", "ride", "you", "dont", "in", "that", "me", "take", "why", "a"]}, {"sent": "you gonna tell Jenell all about the airplane ?", "words": ["the", "all", "airplane", "you", "about"]}, {"sent": "to drink on the airplane .", "words": ["the", "drink", "airplane", "to", "on"]}, {"sent": "you know what we are going to see grandma and grandpa tomorrow were going to take an airplane tomorrow .", "words": ["what", "we", "airplane", "to", "you", "and", "an", "are", "see", "take", "were"]}, {"sent": "lets get out of the airplane .", "words": ["the", "of", "airplane", "get", "out"]}, {"sent": "thats not an airplane what is that ?", "words": ["what", "is", "airplane", "an", "that", "not"]}, {"sent": "get the airplane .", "words": ["get", "the", "airplane"]}, {"sent": "airplane ?", "words": ["airplane"]}, {"sent": "tell Linda about the airplane .", "words": ["the", "airplane", "about"]}, {"sent": "yeah were gonna make the airplane gift shop .", "words": ["the", "airplane", "make", "were"]}, {"sent": "heres an airplane .", "words": ["airplane", "an"]}, {"sent": "airplane .", "words": ["airplane"]}, {"sent": "because then it makes the airplane float .", "words": ["the", "airplane", "it", "then", "because"]}, {"sent": "an oldfashioned upstairs airplane .", "words": ["airplane", "an"]}, {"sent": "airplane ?", "words": ["airplane"]}, {"sent": "have you ever been on an airplane ?", "words": ["airplane", "you", "have", "an", "on"]}, {"sent": "can we turn on the airplane ?", "words": ["can", "the", "we", "airplane", "on"]}, {"sent": "how about the airplane ?", "words": ["the", "airplane", "how", "about"]}, {"sent": "right you heard that airplane .", "words": ["airplane", "that", "you"]}, {"sent": "theres already an airplane here .", "words": ["airplane", "an", "here"]}, {"sent": "thats the airplane .", "words": ["the", "airplane"]}, {"sent": "airplane yes .", "words": ["airplane"]}, {"sent": "how does an airplane come down ?", "words": ["does", "how", "airplane", "an", "down"]}, {"sent": "we took an airplane from here to here .", "words": ["we", "airplane", "to", "here", "an"]}, {"sent": "somebodys flying an airplane over there ?", "words": ["airplane", "there", "an", "over"]}, {"sent": "this is an airplane .", "words": ["airplane", "this", "an", "is"]}, {"sent": "who was that nice man that we met on the airplane wearing a fancy hat ?", "words": ["the", "we", "airplane", "was", "nice", "hat", "who", "on", "that", "a"]}, {"sent": "I dont see the people in the airplane .", "words": ["the", "airplane", "dont", "in", "see"]}, {"sent": "airplane .", "words": ["airplane"]}, {"sent": "baby airplane .", "words": ["airplane"]}, {"sent": "show Linda the book that the stewardess gave you on the airplane .", "words": ["the", "show", "airplane", "you", "on", "book", "that"]}, {"sent": "hey wait a minute you cant put a truck an airplane in a truck .", "words": ["wait", "put", "airplane", "truck", "you", "an", "in", "a"]}, {"sent": "hand me the airplane the big airplane .", "words": ["the", "hand", "airplane", "me", "big"]}, {"sent": "theres your airplane .", "words": ["airplane", "your"]}, {"sent": "and then they took they actually took an airplane to London where Paddington lives and then they took a train to France where Danielle used to live .", "words": ["where", "train", "airplane", "to", "and", "they", "an", "then", "a"]}, {"sent": "yeah tell me about that airplane .", "words": ["airplane", "me", "that", "about"]}, {"sent": "oh you see an airplane over there .", "words": ["airplane", "you", "there", "an", "see", "over"]}, {"sent": "the gambler wants to fly the airplane .", "words": ["the", "to", "airplane"]}, {"sent": "airplane .", "words": ["airplane"]}, {"sent": "see you drive the suitcases over to the airplane then you open the back up and you put these in the airplane .", "words": ["the", "up", "put", "these", "back", "airplane", "to", "open", "you", "drive", "and", "in", "see", "then", "over"]}, {"sent": "thats the tail of the airplane .", "words": ["the", "of", "airplane"]}, {"sent": "airplane .", "words": ["airplane"]}, {"sent": "and there is the airplane .", "words": ["the", "is", "airplane", "and", "there"]}, {"sent": "bring the airplane back here for landing .", "words": ["the", "for", "back", "airplane", "bring", "here"]}, {"sent": "Im going to take an airplane ride .", "words": ["airplane", "to", "ride", "an", "take"]}, {"sent": "airplane yes .", "words": ["airplane"]}, {"sent": "enjoyed the airplane ?", "words": ["the", "airplane"]}, {"sent": "this is like an airplane Baloos airplane that lands on the water .", "words": ["the", "is", "airplane", "like", "this", "an", "water", "on", "that"]}, {"sent": "thats an airplane .", "words": ["airplane", "an"]}, {"sent": "I cant drink outa that airplane .", "words": ["airplane", "that", "drink"]}, {"sent": "wouldnt that be nice if you could take go up in the airplane ?", "words": ["up", "the", "if", "go", "airplane", "be", "nice", "you", "in", "take", "could", "that"]}, {"sent": "airplane up there .", "words": ["airplane", "there", "up"]}, {"sent": "airplane .", "words": ["airplane"]}, {"sent": "its your bathtub ?", "words": ["your", "bathtub"]}, {"sent": "oh good she fits in the bathtub .", "words": ["the", "in", "bathtub", "good", "she"]}, {"sent": "its a crack in the bathtub .", "words": ["the", "in", "bathtub", "a"]}, {"sent": "Ive got my rubber ducky in the bathtub .", "words": ["the", "in", "my", "bathtub"]}, {"sent": "oh okay mommys watching television in the bathtub .", "words": ["the", "in", "bathtub"]}, {"sent": "but she realized if she bangs one toy against the edge of the real bathtub it makes noise .", "words": ["the", "if", "of", "toy", "it", "bathtub", "she", "but"]}, {"sent": "theres a dinosaur in the bathtub .", "words": ["the", "in", "bathtub", "a"]}, {"sent": "an a bathtub letter a .", "words": ["an", "bathtub", "a"]}, {"sent": "the elephant is in the bathtub .", "words": ["the", "is", "elephant", "in", "bathtub"]}, {"sent": "Babykinss bathtub doesnt .", "words": ["bathtub"]}, {"sent": "you just clip it on the bathtub .", "words": ["the", "you", "it", "bathtub", "on"]}, {"sent": "you can slide a bathtub I guess .", "words": ["can", "slide", "you", "bathtub", "a"]}, {"sent": "and does that look like a bathtub to you ?", "words": ["does", "to", "like", "and", "you", "bathtub", "look", "that", "a"]}, {"sent": "you wanna get in the bathtub ?", "words": ["the", "you", "get", "in", "bathtub", "wanna"]}, {"sent": "so what do you think Emma should this be the bathtub here ?", "words": ["the", "what", "do", "think", "so", "be", "you", "here", "this", "bathtub"]}, {"sent": "yeah and you guys just swim in the bathtub .", "words": ["the", "swim", "you", "and", "in", "bathtub"]}, {"sent": "whats this part of the bathtub ?", "words": ["of", "this", "bathtub", "the"]}, {"sent": "you oughta be in the bathtub before seven thirty .", "words": ["the", "be", "you", "in", "bathtub"]}, {"sent": "Marky do you wanna go peepee in the bathtub ?", "words": ["the", "do", "go", "you", "in", "bathtub", "wanna"]}, {"sent": "hes in the bathtub in the bath hes in the bathroom hes in the bathroom .", "words": ["the", "bathroom", "in", "bathtub"]}, {"sent": "well guess what theres the bathtub .", "words": ["the", "what", "bathtub"]}, {"sent": "youre in Yayas bathtub .", "words": ["in", "bathtub"]}, {"sent": "hes in the bathtub .", "words": ["the", "in", "bathtub"]}, {"sent": "its time those guys yikes they better be getting in the bathtub .", "words": ["the", "better", "those", "be", "they", "in", "bathtub"]}, {"sent": "what does Ernie always get in the bathtub with ?", "words": ["the", "what", "does", "with", "get", "in", "bathtub"]}, {"sent": "put them on the bathtub .", "words": ["the", "put", "them", "bathtub", "on"]}, {"sent": "in the bathtub do you stick them somewhere ?", "words": ["the", "do", "stick", "them", "you", "in", "bathtub"]}, {"sent": "say bathtub .", "words": ["say", "bathtub"]}, {"sent": "and the bathtub .", "words": ["the", "bathtub", "and"]}, {"sent": "its almost time to get in the bathtub .", "words": ["the", "to", "get", "in", "bathtub"]}, {"sent": "my mother slipped in the bathtub ?", "words": ["the", "in", "my", "bathtub"]}, {"sent": "he filled the bathtub with water .", "words": ["the", "with", "he", "bathtub", "water"]}, {"sent": "he also has some bubbles to play with in the bathtub .", "words": ["the", "to", "play", "with", "in", "he", "bathtub", "some", "bubbles"]}, {"sent": "did you have fun in the bathtub ?", "words": ["the", "you", "have", "in", "bathtub", "did"]}, {"sent": "are you gonna have a sink and a shower and a bathtub ?", "words": ["shower", "you", "have", "and", "are", "sink", "bathtub", "a"]}, {"sent": "do you splash in the bathtub ?", "words": ["the", "do", "you", "in", "splash", "bathtub"]}, {"sent": "the bathtub .", "words": ["the", "bathtub"]}, {"sent": "like he likes to do it in the bathtub and stuff like that .", "words": ["the", "do", "to", "like", "and", "in", "it", "he", "bathtub", "that"]}, {"sent": "wow bathtub and a bed all in one ?", "words": ["all", "bed", "and", "in", "bathtub", "a"]}, {"sent": "youre in the bathtub .", "words": ["the", "in", "bathtub"]}, {"sent": "oh that is a bathtub now .", "words": ["that", "bathtub", "a", "is"]}, {"sent": "he can float in the bathtub .", "words": ["can", "the", "in", "he", "bathtub"]}, {"sent": "oh mommys in the bathtub now ?", "words": ["the", "in", "bathtub"]}, {"sent": "what else is in the bathtub ?", "words": ["the", "what", "is", "in", "bathtub"]}, {"sent": "theres not a bathtub in there .", "words": ["there", "in", "bathtub", "not", "a"]}, {"sent": "do you like to go in the bathtub ?", "words": ["the", "do", "go", "to", "like", "you", "in", "bathtub"]}, {"sent": "no you play with this plane in the bathtub .", "words": ["the", "play", "with", "you", "this", "in", "bathtub"]}, {"sent": "now what do we have in the bathtub ?", "words": ["the", "what", "do", "we", "have", "in", "bathtub"]}, {"sent": "do you want to get out of the bathtub now ?", "words": ["the", "do", "of", "to", "you", "get", "bathtub", "want to", "out"]}, {"sent": "Ive got a bathtub just like that .", "words": ["that", "bathtub", "like", "a"]}, {"sent": "no it goes beside the bathtub doesnt it ?", "words": ["the", "beside", "it", "bathtub"]}, {"sent": "youre going to read the story to her in the bathtub or while shes getting dry ?", "words": ["the", "to", "dry", "in", "read", "bathtub", "story", "her"]}, {"sent": "is she in the bathtub ?", "words": ["the", "is", "in", "bathtub", "she"]}, {"sent": "Flounder you can only play with in the pool or in the bathtub .", "words": ["can", "the", "play", "with", "pool", "you", "in", "bathtub"]}, {"sent": "in the bathtub ?", "words": ["the", "in", "bathtub"]}, {"sent": "in Lorraines bathtub .", "words": ["in", "bathtub"]}, {"sent": "did you ever tell Jenell who you swim like in the bathtub ?", "words": ["the", "swim", "like", "you", "in", "bathtub", "did", "who"]}, {"sent": "the bathtub ?", "words": ["the", "bathtub"]}, {"sent": "can you give me the boy in the bathtub ?", "words": ["can", "the", "you", "in", "bathtub", "me", "give"]}, {"sent": "are you gonna get in the bathtub with me ?", "words": ["the", "with", "you", "get", "in", "are", "bathtub", "me"]}, {"sent": "well we just started using a big bathtub with you .", "words": ["we", "with", "you", "bathtub", "big", "a"]}, {"sent": "oh youre right thats a bathtub letter .", "words": ["bathtub", "a"]}, {"sent": "no he still has a little bathtub because hes still a little baby .", "words": ["little", "he", "bathtub", "because", "a"]}, {"sent": "Why is the bathtub coming up ?", "words": ["the", "up", "bathtub", "is"]}, {"sent": "nothing much going on up here in this bathtub .", "words": ["up", "here", "this", "in", "bathtub", "on", "much"]}, {"sent": "the bathtub ?", "words": ["the", "bathtub"]}, {"sent": "I gotta pay for more seeds but it was your idea to put them in the bathtub .", "words": ["the", "put", "for", "your", "was", "to", "them", "more", "it", "in", "bathtub", "but"]}, {"sent": "was it something about some people that who were in swimming or something when you were in the bathtub you were telling me the story .", "words": ["the", "when", "was", "me", "you", "in", "it", "bathtub", "about", "who", "some", "story", "that", "were"]}, {"sent": "oh its the bathtub .", "words": ["the", "bathtub"]}, {"sent": "a bathtub .", "words": ["bathtub", "a"]}, {"sent": "in our bathtub dont we ?", "words": ["we", "our", "dont", "in", "bathtub"]}, {"sent": "what are you doing theres your little tushie and youre playing in the bathtub in your little swimming pool and youre absolutely naked that pretty silly xxx .", "words": ["the", "what", "your", "pool", "little", "you", "and", "in", "are", "bathtub", "pretty", "that"]}, {"sent": "no standing in the bathtub .", "words": ["the", "in", "bathtub"]}, {"sent": "no hes not in the bathtub .", "words": ["the", "in", "bathtub", "not"]}, {"sent": "is she playing in the bathtub ?", "words": ["the", "is", "in", "bathtub", "she"]}, {"sent": "hes taking a nap in the bathtub now I will ask you please dont ever take a nap in the bathtub .", "words": ["the", "you", "dont", "in", "bathtub", "will", "take", "a"]}, {"sent": "thats the bathtub .", "words": ["the", "bathtub"]}, {"sent": "your bathtub does thats true .", "words": ["does", "your", "bathtub"]}, {"sent": "did you tell Daddy about floating in the bathtub last night .", "words": ["the", "last", "you", "in", "bathtub", "did", "about"]}, {"sent": "she doesnt want the bathtub .", "words": ["the", "bathtub", "she"]}, {"sent": "that one right there by the bathtub .", "words": ["the", "by", "there", "bathtub", "that"]}, {"sent": "that are glued to the bathtub so that .", "words": ["the", "to", "so", "are", "bathtub", "that"]}, {"sent": "Nomi you wanna put Georgie in the bathtub .", "words": ["the", "put", "you", "in", "bathtub", "wanna"]}, {"sent": "I think it goes to the bathtub .", "words": ["the", "think", "to", "it", "bathtub"]}, {"sent": "out of the bathtub ?", "words": ["of", "the", "bathtub", "out"]}, {"sent": "oh I see we had a misunderstanding I was just sposta get water out of the bathtub .", "words": ["the", "we", "of", "was", "get", "bathtub", "out", "water", "see", "a"]}, {"sent": "well you can Im gonna clean you up and then Ill bring her over and get you a bathtub .", "words": ["can", "up", "clean", "bring", "and", "you", "get", "bathtub", "over", "then", "her", "a"]}, {"sent": "was it in the bathtub ?", "words": ["the", "was", "in", "it", "bathtub"]}, {"sent": "would she like to rinse her off in that special bathtub ?", "words": ["off", "to", "like", "in", "that", "bathtub", "would", "her", "she"]}, {"sent": "into the bathtub .", "words": ["the", "bathtub", "into"]}, {"sent": "its a bathtub .", "words": ["bathtub", "a"]}, {"sent": "do you make juice in the bathtub ?", "words": ["the", "do", "you", "in", "bathtub", "juice", "make"]}, {"sent": "you gonna get in the bathtub ?", "words": ["the", "you", "get", "in", "bathtub"]}, {"sent": "that says bathtub .", "words": ["that", "bathtub"]}, {"sent": "you blow them in the bathtub too .", "words": ["the", "them", "you", "in", "bathtub", "too", "blow"]}, {"sent": "what does she do when she sits in the bathtub ?", "words": ["the", "what", "do", "does", "in", "bathtub", "when", "she"]}, {"sent": "do you think the horse can fit in the bathtub ?", "words": ["the", "do", "think", "can", "fit", "you", "in", "bathtub", "horse"]}, {"sent": "thats a boy in the bathtub .", "words": ["the", "in", "bathtub", "a"]}, {"sent": "Missus Brown hugged him harder and kissed his warm cheek .", "words": ["his", "cheek", "him", "and"]}, {"sent": "your other cheek ?", "words": ["cheek", "your", "other"]}, {"sent": "hes licking his cheek .", "words": ["his", "cheek"]}, {"sent": "shall I bite your cheek ?", "words": ["bite", "cheek", "your"]}, {"sent": "chikada chikada chikada cheek .", "words": ["cheek"]}, {"sent": "and on your cheek .", "words": ["on", "cheek", "your", "and"]}, {"sent": "can I have a kiss on the cheek ?", "words": ["can", "the", "cheek", "have", "on", "kiss", "a"]}, {"sent": "cheek ?", "words": ["cheek"]}, {"sent": "this is her cheek .", "words": ["cheek", "this", "her", "is"]}, {"sent": "wash the other cheek .", "words": ["the", "cheek", "wash", "other"]}, {"sent": "put the shaving cream on your cheek ?", "words": ["the", "put", "your", "cheek", "on"]}, {"sent": "lick your cheek .", "words": ["your", "cheek", "lick"]}, {"sent": "thats her cheek .", "words": ["cheek", "her"]}, {"sent": "its stuck in his cheek isnt it ?", "words": ["cheek", "in", "it", "stuck", "his"]}, {"sent": "chikada chikada chikada cheek .", "words": ["cheek"]}, {"sent": "has it left a mark on Mummys cheek ?", "words": ["on", "cheek", "it", "a"]}, {"sent": "cheek there .", "words": ["cheek", "there"]}, {"sent": "chikada chikada chikada cheek .", "words": ["cheek"]}, {"sent": "and your cheek and now we can get .", "words": ["can", "your", "we", "cheek", "and", "get"]}, {"sent": "over your head and across your cheek and your mouth .", "words": ["your", "cheek", "head", "mouth", "and", "over"]}, {"sent": "whats this on your cheek ?", "words": ["on", "this", "your", "cheek"]}, {"sent": "tickle tickle on the cheek .", "words": ["on", "the", "cheek", "tickle"]}, {"sent": "on your cheek ?", "words": ["on", "cheek", "your"]}, {"sent": "I thought you were putting in on my cheek its not gonna stick anymore .", "words": ["my", "stick", "cheek", "you", "in", "on", "were", "not"]}, {"sent": "come here heres your cheek .", "words": ["here", "cheek", "your"]}, {"sent": "when you were a baby you useta squish this right against your face and leave little dots on your cheek .", "words": ["your", "face", "cheek", "you", "and", "this", "little", "on", "when", "were", "a"]}, {"sent": "what about cheek ?", "words": ["cheek", "what", "about"]}, {"sent": "on your cheek .", "words": ["on", "cheek", "your"]}, {"sent": "what about my my cheek ?", "words": ["cheek", "what", "my", "about"]}, {"sent": "Mrs Bear hugged him harder and kissed his warm cheek .", "words": ["his", "cheek", "him", "and"]}, {"sent": "wash cheek .", "words": ["cheek", "wash"]}, {"sent": "oh that is very silly you look like you have a big swollen cheek .", "words": ["is", "cheek", "like", "you", "have", "big", "look", "that", "a"]}, {"sent": "cheek .", "words": ["cheek"]}, {"sent": "she knows nose mouth eyes cheek um .", "words": ["cheek", "mouth", "she", "nose"]}, {"sent": "you wanna put some put some peam on your cheek ?", "words": ["put", "your", "cheek", "you", "wanna", "on", "some"]}, {"sent": "like your cheek is chapped from the cold my lip is chapped from the cold .", "words": ["the", "my", "your", "is", "cheek", "like", "cold"]}, {"sent": "skunks cheek .", "words": ["cheek"]}, {"sent": "cheek .", "words": ["cheek"]}, {"sent": "but it broke apart and I never got her a new one but whenever shes sees my keys just do anything to get hold of them Lately weve noticed that she offers her cheek for a kiss .", "words": ["hold", "of", "cheek", "her", "it", "that", "she", "for", "to", "and", "keys", "new", "a", "do", "my", "them", "get", "kiss", "but"]}, {"sent": "thats the doctors cheek .", "words": ["the", "cheek"]}, {"sent": "are you being cheek .", "words": ["cheek", "are", "you"]}, {"sent": "apart from some cheek .", "words": ["cheek", "some"]}, {"sent": "you cheek whos a cheeky little boy eh ?", "words": ["cheek", "little", "a", "you"]}, {"sent": "no I dont want the sticker on my cheek .", "words": ["the", "my", "cheek", "dont", "on"]}, {"sent": "it was your cheek .", "words": ["cheek", "was", "it", "your"]}, {"sent": "thats now on my cheek so Im going to .", "words": ["my", "cheek", "so", "to", "on"]}, {"sent": "whats cheeky cheeky cheek ?", "words": ["cheek"]}, {"sent": "come here wheres your cheek ?", "words": ["here", "cheek", "your"]}, {"sent": "shell say cheek and shell point to her elbow .", "words": ["cheek", "to", "say", "and", "her"]}, {"sent": "whats in your cheek are you a hamster ?", "words": ["your", "cheek", "you", "in", "are", "a"]}, {"sent": "Thomass left cheek .", "words": ["cheek"]}, {"sent": "is it on your cheek ?", "words": ["your", "is", "cheek", "it", "on"]}, {"sent": "and your cheek .", "words": ["cheek", "your", "and"]}, {"sent": "do her cheek .", "words": ["cheek", "do", "her"]}, {"sent": "let me see if I can put it on your cheek .", "words": ["can", "put", "your", "if", "cheek", "me", "it", "on", "see"]}, {"sent": "I like to kiss your cheek .", "words": ["your", "cheek", "to", "like", "kiss"]}, {"sent": "and you cut your lip and your mouth and your cheek .", "words": ["your", "cut", "cheek", "mouth", "you", "and"]}, {"sent": "chikada chikada chikada cheek .", "words": ["cheek"]}, {"sent": "sweetie are you actually gonna swallow that corned beef in your mouth or are you just gonna keep it stored in your cheek ?", "words": ["your", "cheek", "mouth", "you", "in", "are", "it", "that"]}, {"sent": "do you have a bruise on your cheek ?", "words": ["do", "your", "cheek", "you", "have", "on", "a"]}, {"sent": "you believe in turning the other cheek I see n en that whole thing .", "words": ["the", "cheek", "other", "you", "in", "see", "that"]}, {"sent": "where is her cheek ?", "words": ["where", "cheek", "her", "is"]}, {"sent": "kiss you on the cheek .", "words": ["the", "cheek", "you", "on", "kiss"]}, {"sent": "this cheek .", "words": ["cheek", "this"]}, {"sent": "you bit your cheek ?", "words": ["cheek", "your", "you"]}, {"sent": "can I take this cheek right here ?", "words": ["can", "cheek", "here", "this", "take"]}, {"sent": "do you wanna show Manuela how we learned to give a big kiss on Mommys cheek with your lips ?", "words": ["do", "how", "your", "we", "show", "to", "cheek", "with", "you", "on", "wanna", "kiss", "give", "big", "a"]}, {"sent": "come here and Ill bite your cheek .", "words": ["your", "cheek", "and", "here", "bite"]}, {"sent": "Sarah what did you do to your cheek ?", "words": ["what", "do", "your", "cheek", "to", "you", "did"]}, {"sent": "thats a bit of a cheek .", "words": ["of", "cheek", "a"]}, {"sent": "you got it all in your cheek il here .", "words": ["all", "your", "cheek", "you", "here", "in", "it"]}, {"sent": "thats my cheek thats my ear .", "words": ["cheek", "my", "ear"]}, {"sent": "and shes holding her cheek isnt she ?", "words": ["cheek", "her", "she", "and"]}, {"sent": "chikada chikada chikada cheek .", "words": ["cheek"]}, {"sent": "you better clean off your cheek .", "words": ["clean", "better", "off", "your", "cheek", "you"]}, {"sent": "poor Sam she said and hugged him harder and kissed his warm cheek .", "words": ["cheek", "and", "poor", "him", "his", "she"]}, {"sent": "youre a rosy cheek girl .", "words": ["cheek", "a"]}, {"sent": "cheek and whats this ?", "words": ["cheek", "this", "and"]}, {"sent": "Thomass right cheek .", "words": ["cheek"]}, {"sent": "cheeks nose eyes ears mouth cheek .", "words": ["cheek", "mouth", "nose"]}, {"sent": "and your nose and your cheek .", "words": ["nose", "cheek", "your", "and"]}, {"sent": "now we still need part of your cheek .", "words": ["need", "your", "we", "of", "cheek"]}, {"sent": "a rosy cheek .", "words": ["cheek", "a"]}, {"sent": "feel the skin on your cheek .", "words": ["the", "cheek", "on", "your"]}, {"sent": "I wonder is part of your cheek here ?", "words": ["your", "is", "of", "cheek", "here"]}, {"sent": "here you wanna put it on my cheek ?", "words": ["put", "my", "cheek", "you", "here", "it", "wanna", "on"]}, {"sent": "youre going to put the sticker on my cheek ?", "words": ["the", "put", "my", "cheek", "to", "on"]}, {"sent": "and kissed your cheek .", "words": ["cheek", "your", "and"]}, {"sent": "got your cheek .", "words": ["cheek", "your"]}, {"sent": "is your cheek rosy ?", "words": ["cheek", "your", "is"]}, {"sent": "is that your cheek ?", "words": ["your", "cheek", "that", "is"]}, {"sent": "chikada chikada chikada cheek .", "words": ["cheek"]}, {"sent": "chikada chikada chikada cheek .", "words": ["cheek"]}, {"sent": "got your cheek .", "words": ["cheek", "your"]}, {"sent": "chikada chikada chikada cheek .", "words": ["cheek"]}, {"sent": "ooh can I give the king a little kiss on the cheek ?", "words": ["can", "the", "cheek", "little", "on", "give", "kiss", "a"]}, {"sent": "thats a couch .", "words": ["couch", "a"]}, {"sent": "oh I think I already put the couch in I think that was one I already put away .", "words": ["the", "think", "put", "was", "couch", "in", "away", "that"]}, {"sent": "wanna put the couch in ?", "words": ["the", "put", "couch", "in", "wanna"]}, {"sent": "you gonna put it on the couch ?", "words": ["the", "put", "couch", "you", "it", "on"]}, {"sent": "you were spitting on the couch .", "words": ["the", "couch", "you", "on", "were"]}, {"sent": "I think I saw it land on the couch .", "words": ["the", "think", "couch", "it", "on"]}, {"sent": "why you getting off the couch ?", "words": ["off", "the", "couch", "you", "why"]}, {"sent": "there should try trace wherever that light thats plugged into because theres outlet I know its plugged in so theres gotta be is it behind the couch ?", "words": ["the", "try", "into", "is", "light", "so", "be", "behind", "couch", "there", "in", "it", "that", "because"]}, {"sent": "because if we lay on the couch then they cant take your picture can they ?", "words": ["the", "can", "your", "if", "we", "picture", "couch", "they", "on", "then", "take", "because"]}, {"sent": "xxx come here dear let go of de couch .", "words": ["here", "of", "couch", "go"]}, {"sent": "but youre laying on the couch .", "words": ["on", "the", "couch", "but"]}, {"sent": "honey you left Lala and Pluto here on the couch .", "words": ["the", "couch", "and", "here", "you", "on"]}, {"sent": "you know Sweets lets put this on the table instead of the couch because I dont wanna get Playdoh stuck all over the couch okay ?", "words": ["the", "all", "put", "table", "of", "couch", "you", "get", "this", "dont", "stuck", "wanna", "on", "over", "because"]}, {"sent": "can you promise not to pee on the couch ?", "words": ["can", "the", "to", "couch", "you", "on", "not"]}, {"sent": "hey look at this couch .", "words": ["this", "at", "couch", "look"]}, {"sent": "couch ?", "words": ["couch"]}, {"sent": "a couch ?", "words": ["couch", "a"]}, {"sent": "yeah and how did we set up the couch so it would be very comfortable ?", "words": ["up", "the", "how", "we", "so", "be", "couch", "and", "it", "did", "would"]}, {"sent": "yeah Dingos car is on the couch .", "words": ["the", "is", "car", "couch", "on"]}, {"sent": "look they can stick on the couch .", "words": ["can", "the", "stick", "couch", "they", "look", "on"]}, {"sent": "lets lie on the couch .", "words": ["on", "the", "couch"]}, {"sent": "I was on the couch .", "words": ["on", "was", "couch", "the"]}, {"sent": "daddy he just fell off the couch .", "words": ["off", "the", "he", "couch"]}, {"sent": "couch .", "words": ["couch"]}, {"sent": "is that the couch ?", "words": ["the", "that", "couch", "is"]}, {"sent": "and we were sitting on the couch and she mustve gotten an eyelash or something in her eye .", "words": ["the", "we", "her", "couch", "eye", "and", "an", "in", "on", "were", "she"]}, {"sent": "a couch yes .", "words": ["couch", "a"]}, {"sent": "is there a dress anywhere in this room on the couch is there a dress on the couch ?", "words": ["the", "is", "couch", "there", "in", "this", "room", "on", "dress", "a"]}, {"sent": "that is a couch .", "words": ["that", "couch", "a", "is"]}, {"sent": "alright lets go sit on the couch and look .", "words": ["the", "go", "couch", "and", "look", "on", "sit"]}, {"sent": "youre jumping on the couch .", "words": ["on", "the", "couch"]}, {"sent": "its spilt no the couch has it ?", "words": ["the", "it", "couch"]}, {"sent": "okay you wanna sit on the couch ?", "words": ["the", "couch", "you", "wanna", "on", "sit"]}, {"sent": "I can build if you bring them here to me on the couch .", "words": ["can", "the", "if", "to", "them", "bring", "couch", "you", "here", "build", "on", "me"]}, {"sent": "over by the couch .", "words": ["the", "over", "couch", "by"]}, {"sent": "yeah you can climb on the couch .", "words": ["can", "the", "couch", "you", "on", "climb"]}, {"sent": "he wants to get the couch xxx .", "words": ["the", "to", "couch", "get", "he"]}, {"sent": "put it on the couch so we dont forget it okay ?", "words": ["the", "put", "we", "so", "couch", "dont", "it", "on"]}, {"sent": "instead of on the couch .", "words": ["of", "the", "on", "couch"]}, {"sent": "wow you have you have so many toys on your couch .", "words": ["your", "so", "couch", "you", "have", "on"]}, {"sent": "well maybe its under the couch I saw it yesterday .", "words": ["the", "under", "it", "couch"]}, {"sent": "Honey I dont want you to fall off the couch .", "words": ["off", "the", "to", "fall", "couch", "you", "dont"]}, {"sent": "couch couch couch .", "words": ["couch"]}, {"sent": "you wanna get up on the couch ?", "words": ["up", "the", "couch", "you", "get", "wanna", "on"]}, {"sent": "oh youre gonna sit on the couch .", "words": ["on", "the", "sit", "couch"]}, {"sent": "what d the couch ?", "words": ["the", "what", "couch"]}, {"sent": "can you make a pee first on your potty so you dont make a pee on the couch ?", "words": ["can", "the", "your", "so", "first", "couch", "you", "dont", "potty", "on", "make", "a"]}, {"sent": "Sweetie lets take your shoes off if youre gonna climb on the couch .", "words": ["off", "the", "your", "if", "couch", "on", "climb", "take"]}, {"sent": "you making a mess out of my couch man .", "words": ["my", "of", "couch", "you", "out", "a"]}, {"sent": "whats this family doing sitting on the couch ?", "words": ["on", "this", "couch", "the"]}, {"sent": "on the couch .", "words": ["on", "the", "couch"]}, {"sent": "he was lying on the couch then .", "words": ["the", "was", "couch", "he", "on", "then"]}, {"sent": "this is the couch William .", "words": ["the", "this", "couch", "is"]}, {"sent": "sometimes it needs to go under the couch stop it .", "words": ["the", "go", "to", "stop", "couch", "under", "it"]}, {"sent": "Ill wait until they go and then Ill take a nap on the couch .", "words": ["the", "wait", "go", "couch", "and", "they", "on", "then", "take", "a"]}, {"sent": "do you wanna sit on the couch ?", "words": ["the", "do", "couch", "you", "wanna", "on", "sit"]}, {"sent": "theres a I think theres a pencil on the couch .", "words": ["the", "think", "couch", "pencil", "on", "a"]}, {"sent": "by the couch ?", "words": ["the", "couch", "by"]}, {"sent": "on a couch .", "words": ["on", "couch", "a"]}, {"sent": "on couches yes except this couch is pretty full its got alotof people on it .", "words": ["full", "is", "couch", "this", "it", "pretty", "on"]}, {"sent": "you wanna read on the couch ?", "words": ["the", "couch", "you", "read", "wanna", "on"]}, {"sent": "she doesnt say cow she says couch .", "words": ["couch", "say", "she", "cow"]}, {"sent": "oh on the couch you mean ?", "words": ["on", "the", "couch", "you"]}, {"sent": "where does the couch go ?", "words": ["where", "does", "the", "go", "couch"]}, {"sent": "ah not on the couch .", "words": ["on", "the", "couch", "not"]}, {"sent": "sitting on the couch and she said he just came charging at her .", "words": ["the", "couch", "and", "he", "at", "on", "her", "she"]}, {"sent": "get down off the couch ?", "words": ["the", "off", "couch", "get", "down"]}, {"sent": "the couch is looks funny doesnt it .", "words": ["the", "it", "couch", "is"]}, {"sent": "look under the couch .", "words": ["the", "under", "couch", "look"]}, {"sent": "youre gonna sit there up on the couch .", "words": ["up", "the", "couch", "there", "on", "sit"]}, {"sent": "mom is gonna go lay on the couch .", "words": ["the", "go", "is", "couch", "on"]}, {"sent": "its under the couch .", "words": ["the", "under", "couch"]}, {"sent": "theres the couch .", "words": ["the", "couch"]}, {"sent": "how about the ones that went under the couch right here ?", "words": ["the", "how", "couch", "here", "under", "about", "that"]}, {"sent": "well Frasers sitting right there on the couch .", "words": ["on", "there", "couch", "the"]}, {"sent": "alright well Im gonna be downstairs and Im gonna sit in front of sit on the couch and read this book .", "words": ["the", "of", "be", "couch", "and", "this", "in", "read", "on", "book", "sit"]}, {"sent": "sitting up on the couch ?", "words": ["up", "the", "on", "couch"]}, {"sent": "okay Henry youre up on the couch .", "words": ["up", "the", "on", "couch"]}, {"sent": "come here sit on the couch please .", "words": ["the", "couch", "here", "on", "sit"]}, {"sent": "yeah that looks just like something like a couch .", "words": ["couch", "that", "like", "a"]}, {"sent": "no the car is on the couch .", "words": ["the", "is", "car", "couch", "on"]}, {"sent": "Joe is the kuh is the couch open ?", "words": ["the", "couch", "open", "is"]}, {"sent": "you got off the couch before .", "words": ["off", "the", "couch", "you"]}, {"sent": "what do you do on that couch ?", "words": ["what", "do", "couch", "you", "on", "that"]}, {"sent": "its hard to get it all the way up on the couch .", "words": ["the", "all", "up", "to", "couch", "get", "it", "hard", "on"]}, {"sent": "you want ta lie down on the couch and put your head on the pillow ?", "words": ["the", "put", "your", "head", "couch", "you", "and", "on", "down", "pillow"]}, {"sent": "lookit here is a couch .", "words": ["here", "couch", "a", "is"]}, {"sent": "you think its a couch ?", "words": ["think", "couch", "a", "you"]}, {"sent": "right by the couch .", "words": ["the", "couch", "by"]}, {"sent": "you saw one on the couch ?", "words": ["on", "the", "couch", "you"]}, {"sent": "sit on the couch .", "words": ["on", "the", "sit", "couch"]}, {"sent": "well you can lie on the couch .", "words": ["can", "the", "couch", "you", "on"]}, {"sent": "how did you get on the telephone if youre sitting on our couch ?", "words": ["the", "how", "if", "couch", "you", "get", "our", "did", "on", "telephone"]}, {"sent": "okay why dont you bring that to the couch Mommy has xxx a little more wine to wipe doo doo doo doo doo doo .", "words": ["the", "to", "bring", "couch", "you", "little", "dont", "more", "wipe", "that", "why", "a"]}, {"sent": "the one that you go to sleep on the couch ?", "words": ["the", "go", "sleep", "to", "couch", "you", "on", "that"]}, {"sent": "feel free to sit on the couch .", "words": ["the", "to", "couch", "on", "sit"]}, {"sent": "its on the couch .", "words": ["on", "the", "couch"]}, {"sent": "which is strange because you willnt drink anything else with a taste will you ?", "words": ["drink", "is", "with", "taste", "you", "will", "which", "because", "a"]}, {"sent": "does it taste good ?", "words": ["does", "it", "taste", "good"]}, {"sent": "that doesnt taste good .", "words": ["that", "taste", "good"]}, {"sent": "that taste good ?", "words": ["that", "taste", "good"]}, {"sent": "that wouldnt taste very nice would it ?", "words": ["nice", "taste", "it", "would", "that"]}, {"sent": "I taste of jelly babies ?", "words": ["of", "jelly", "taste"]}, {"sent": "what does yours taste like ?", "words": ["what", "does", "taste", "like"]}, {"sent": "how did the toys taste ?", "words": ["the", "taste", "how", "did"]}, {"sent": "can I have a taste ?", "words": ["can", "have", "taste", "a"]}, {"sent": "does it taste like potatoes ?", "words": ["does", "it", "taste", "like"]}, {"sent": "does it taste good ?", "words": ["does", "it", "taste", "good"]}, {"sent": "you taste of liquorice .", "words": ["of", "taste", "you"]}, {"sent": "what does it taste like ?", "words": ["what", "does", "taste", "like", "it"]}, {"sent": "but then after youve swallowed it it doesnt taste bitter does it ?", "words": ["does", "taste", "it", "then", "but"]}, {"sent": "whats that taste of ?", "words": ["of", "that", "taste"]}, {"sent": "doesnt taste very nice uhn does it ?", "words": ["does", "nice", "taste", "it"]}, {"sent": "hows that taste ?", "words": ["that", "taste"]}, {"sent": "how does that taste ?", "words": ["does", "how", "that", "taste"]}, {"sent": "xxx the beans taste good .", "words": ["the", "beans", "taste", "good"]}, {"sent": "do they taste good or bad ?", "words": ["do", "taste", "bad", "they", "good"]}, {"sent": "and then you taste it .", "words": ["taste", "you", "and", "it", "then"]}, {"sent": "does it taste good ?", "words": ["does", "it", "taste", "good"]}, {"sent": "okay just one little taste .", "words": ["taste", "little"]}, {"sent": "did it taste nicer than when you used the spoon ?", "words": ["the", "taste", "you", "it", "did", "spoon", "when"]}, {"sent": "does the apple taste good ?", "words": ["the", "does", "taste", "good", "apple"]}, {"sent": "do they taste good ?", "words": ["do", "they", "taste", "good"]}, {"sent": "here taste it .", "words": ["here", "it", "taste"]}, {"sent": "they taste awful dont they ?", "words": ["dont", "they", "taste"]}, {"sent": "what does it taste like ?", "words": ["what", "does", "taste", "like", "it"]}, {"sent": "oh you wanna taste it ?", "words": ["it", "taste", "wanna", "you"]}, {"sent": "what does it taste like Mia ?", "words": ["what", "does", "taste", "like", "it"]}, {"sent": "so you dont like the taste of it youre gonna throw it out ?", "words": ["the", "throw", "of", "so", "like", "taste", "you", "dont", "it", "out"]}, {"sent": "hows that waffle taste ?", "words": ["that", "taste"]}, {"sent": "no I mean does it taste very much different than the others taste ?", "words": ["the", "does", "taste", "it", "much"]}, {"sent": "mm they taste nice dont they ?", "words": ["dont", "they", "nice", "taste"]}, {"sent": "does it taste good ?", "words": ["does", "it", "taste", "good"]}, {"sent": "taste good huh ?", "words": ["taste", "good"]}, {"sent": "does it taste good ?", "words": ["does", "it", "taste", "good"]}, {"sent": "youll taste it I mean you salt it before you taste it .", "words": ["salt", "it", "taste", "you"]}, {"sent": "do you wanna taste it too ?", "words": ["do", "taste", "you", "it", "wanna", "too"]}, {"sent": "well hows it taste ?", "words": ["it", "taste"]}, {"sent": "does the apple taste good ?", "words": ["the", "does", "taste", "good", "apple"]}, {"sent": "they dont even know what fish fingers taste like .", "words": ["what", "taste", "like", "dont", "they", "fish"]}, {"sent": "shoes taste good .", "words": ["taste", "good"]}, {"sent": "can I taste one ?", "words": ["can", "taste"]}, {"sent": "Alex Im not sure this oatmeal is gonna taste too good .", "words": ["is", "taste", "this", "good", "too", "not"]}, {"sent": "yeah does that taste good ?", "words": ["does", "that", "taste", "good"]}, {"sent": "that didnt taste good .", "words": ["that", "taste", "good"]}, {"sent": "does it taste better when you eat it directly from the box ?", "words": ["the", "better", "does", "box", "taste", "you", "eat", "it", "when"]}, {"sent": "do you wanna just taste a little bit ?", "words": ["do", "taste", "you", "little", "wanna", "a"]}, {"sent": "and what does it taste like ?", "words": ["what", "does", "taste", "like", "and", "it"]}, {"sent": "does it taste good does the strawberry taste nice ?", "words": ["the", "does", "strawberry", "nice", "taste", "it", "good"]}, {"sent": "hows it taste .", "words": ["it", "taste"]}, {"sent": "have a did you taste it ?", "words": ["taste", "you", "have", "it", "did", "a"]}, {"sent": "give her a taste a little bite .", "words": ["taste", "little", "bite", "give", "her", "a"]}, {"sent": "can you taste the glue .", "words": ["can", "the", "taste", "you", "glue"]}, {"sent": "taste it .", "words": ["it", "taste"]}, {"sent": "does that taste like dairy queen ?", "words": ["does", "that", "taste", "like"]}, {"sent": "ooh these cornflakes taste good dont they ?", "words": ["these", "taste", "dont", "they", "good"]}, {"sent": "Awww give Awww a taste Awww .", "words": ["give", "taste", "a"]}, {"sent": "you think you might be willing to taste this ?", "words": ["think", "to", "be", "taste", "you", "this"]}, {"sent": "you put it in your mouth and taste it .", "words": ["put", "your", "mouth", "taste", "you", "and", "in", "it"]}, {"sent": "does that taste like fish ?", "words": ["does", "taste", "like", "fish", "that"]}, {"sent": "how does it taste ?", "words": ["does", "how", "it", "taste"]}, {"sent": "taste good ?", "words": ["taste", "good"]}, {"sent": "how it taste ?", "words": ["how", "it", "taste"]}, {"sent": "xxx taste .", "words": ["taste"]}, {"sent": "taste it .", "words": ["it", "taste"]}, {"sent": "do they taste so good ?", "words": ["do", "so", "taste", "they", "good"]}, {"sent": "you wanna taste it ?", "words": ["it", "taste", "wanna", "you"]}, {"sent": "how does that taste cake taste ?", "words": ["does", "how", "cake", "taste", "that"]}, {"sent": "oh no the bee doesnt taste very good .", "words": ["the", "bee", "taste", "good"]}, {"sent": "er er right doesnt custard pudding taste good ?", "words": ["pudding", "taste", "good"]}, {"sent": "does that taste good ?", "words": ["does", "that", "taste", "good"]}, {"sent": "mm does it taste good ?", "words": ["does", "it", "taste", "good"]}, {"sent": "they always taste better when theyre warm these rolls dont they ?", "words": ["better", "these", "taste", "they", "dont", "when"]}, {"sent": "uh it might taste pretty good too .", "words": ["taste", "it", "pretty", "good", "too"]}, {"sent": "can I taste it ?", "words": ["can", "it", "taste"]}, {"sent": "okay does that taste good ?", "words": ["does", "that", "taste", "good"]}, {"sent": "did it taste good ?", "words": ["taste", "it", "did", "good"]}, {"sent": "does that taste good ?", "words": ["does", "that", "taste", "good"]}, {"sent": "how does it taste with butter on your meatloaf ?", "words": ["does", "how", "your", "with", "taste", "it", "butter", "on"]}, {"sent": "would you like to taste this toast ?", "words": ["toast", "to", "taste", "like", "you", "this", "would"]}, {"sent": "and its sposta pour uh taste better .", "words": ["better", "pour", "taste", "and"]}, {"sent": "youre right it does taste good .", "words": ["does", "it", "taste", "good"]}, {"sent": "do they taste good ?", "words": ["do", "they", "taste", "good"]}, {"sent": "does the worms taste delicious snake ?", "words": ["the", "does", "taste"]}, {"sent": "not gonna taste very nice no .", "words": ["taste", "nice", "not"]}, {"sent": "this taste so weird .", "words": ["this", "so", "taste"]}, {"sent": "does that taste good ?", "words": ["does", "that", "taste", "good"]}, {"sent": "taste like orange .", "words": ["orange", "taste", "like"]}, {"sent": "taste it ?", "words": ["it", "taste"]}, {"sent": "doesnt taste very good its its plastic .", "words": ["taste", "good"]}, {"sent": "taste .", "words": ["taste"]}, {"sent": "um but I dont know if she just didnt like the taste of them .", "words": ["the", "if", "of", "them", "like", "taste", "dont", "she", "but"]}, {"sent": "yeah why dont you taste it and see ?", "words": ["taste", "you", "and", "dont", "it", "see", "why"]}, {"sent": "does the apple juice taste good ?", "words": ["the", "does", "taste", "good", "apple", "juice"]}, {"sent": "did you taste these ?", "words": ["taste", "these", "did", "you"]}, {"sent": "does that taste good ?", "words": ["does", "that", "taste", "good"]}, {"sent": "that ant is running around .", "words": ["around", "that", "ant", "is"]}, {"sent": "an ant eater .", "words": ["an", "ant"]}, {"sent": "anyway the circus bugs all promised to come back and visit us at our little ant hill .", "words": ["the", "all", "back", "to", "and", "our", "us", "little", "ant", "at"]}, {"sent": "a little ant hopping .", "words": ["ant", "little", "a"]}, {"sent": "well the sign says dont go in there cause theres ant traps in there .", "words": ["the", "go", "dont", "in", "there", "ant"]}, {"sent": "wheres the wheres the little ant ?", "words": ["the", "ant", "little"]}, {"sent": "thats beautiful ant Eleanor .", "words": ["ant"]}, {"sent": "no ant had ever left ant island before .", "words": ["ant"]}, {"sent": "ant .", "words": ["ant"]}, {"sent": "the ant went home to her house ?", "words": ["the", "to", "ant", "house", "her", "home"]}, {"sent": "ant hill .", "words": ["ant"]}, {"sent": "the ant was on the table it was on the table .", "words": ["the", "table", "was", "it", "ant", "on"]}, {"sent": "an ant is bigger than you ?", "words": ["an", "ant", "you", "is"]}, {"sent": "yes one ant and one elephant .", "words": ["elephant", "ant", "and"]}, {"sent": "another ant .", "words": ["another", "ant"]}, {"sent": "they were unloading an ant farm .", "words": ["they", "an", "were", "ant"]}, {"sent": "ant .", "words": ["ant"]}, {"sent": "its an ant .", "words": ["an", "ant"]}, {"sent": "whos bigger you or an ant ?", "words": ["an", "ant", "you"]}, {"sent": "what does the ant do ?", "words": ["the", "what", "do", "does", "ant"]}, {"sent": "its an ant .", "words": ["an", "ant"]}, {"sent": "but Max let the ants run into his pocket instead of into the ant farm .", "words": ["the", "into", "of", "run", "ant", "his", "but"]}, {"sent": "how is an ant and a butterfly snap ?", "words": ["butterfly", "how", "is", "and", "an", "ant", "a"]}, {"sent": "whered you see the ant ?", "words": ["the", "see", "ant", "you"]}, {"sent": "hows your ant bite ?", "words": ["bite", "your", "ant"]}, {"sent": "check me out Im an ant ?", "words": ["me", "an", "ant", "out"]}, {"sent": "ant .", "words": ["ant"]}, {"sent": "the red ant ?", "words": ["the", "red", "ant"]}, {"sent": "did you see an ant ?", "words": ["you", "an", "ant", "did", "see"]}, {"sent": "and you have a ant bite on your back ?", "words": ["your", "back", "you", "have", "and", "on", "ant", "bite", "a"]}, {"sent": "oh the ant went down did it ?", "words": ["the", "it", "ant", "did", "down"]}, {"sent": "ant to sit on the block ?", "words": ["the", "block", "to", "ant", "on", "sit"]}, {"sent": "but Max let the ants run into his pocket instead of the ant farm .", "words": ["the", "into", "of", "run", "ant", "his", "but"]}, {"sent": "where did you see ant ?", "words": ["where", "you", "ant", "did", "see"]}, {"sent": "we have ant .", "words": ["have", "ant", "we"]}, {"sent": "your ant farm ants have escaped Max said Ruby .", "words": ["have", "your", "ant"]}, {"sent": "oh ant went home to her house ?", "words": ["to", "house", "ant", "her", "home"]}, {"sent": "did you see an ant ?", "words": ["you", "an", "ant", "did", "see"]}, {"sent": "I saw a leaf cutter ant bite off a leaf and carry it all the way back to its nest .", "words": ["off", "all", "the", "back", "to", "and", "carry", "it", "ant", "bite", "a"]}, {"sent": "one day we heard the grasshoppers arriving so we all ran inside the ant hill Flick was left outside with his invention .", "words": ["the", "all", "we", "so", "inside", "was", "outside", "with", "ant", "his"]}, {"sent": "home oh the ant went home ?", "words": ["the", "ant", "home"]}, {"sent": "ant eater ant eater ?", "words": ["ant"]}, {"sent": "ant .", "words": ["ant"]}, {"sent": "its the ant hole ?", "words": ["the", "ant"]}, {"sent": "an ant .", "words": ["an", "ant"]}, {"sent": "ant you can read it to me .", "words": ["can", "to", "you", "it", "ant", "read", "me"]}, {"sent": "that is an ant .", "words": ["an", "that", "ant", "is"]}, {"sent": "xxx ooh thats a big one thats a ant .", "words": ["big", "ant", "a"]}, {"sent": "for ant ant and bee ?", "words": ["for", "ant", "bee", "and"]}, {"sent": "an ant .", "words": ["an", "ant"]}, {"sent": "there ya go ant .", "words": ["there", "ant", "go"]}, {"sent": "ant .", "words": ["ant"]}, {"sent": "yes they do ants live in the ant hill .", "words": ["the", "do", "they", "in", "ant"]}, {"sent": "isnt he a good ant ?", "words": ["ant", "he", "good", "a"]}, {"sent": "is this an ant eater .", "words": ["this", "an", "ant", "is"]}, {"sent": "look a is for ant .", "words": ["for", "is", "ant", "look", "a"]}, {"sent": "theres theres an ant .", "words": ["an", "ant"]}, {"sent": "no that is an ant .", "words": ["an", "that", "ant", "is"]}, {"sent": "now what were you sposta do with that ant hill ?", "words": ["what", "do", "with", "you", "that", "ant", "were"]}, {"sent": "the ant does what ?", "words": ["the", "does", "what", "ant"]}, {"sent": "take away his g l and make him an ant .", "words": ["and", "him", "away", "make", "an", "ant", "his", "take"]}, {"sent": "ant .", "words": ["ant"]}, {"sent": "an ant .", "words": ["an", "ant"]}, {"sent": "can you say ant eater ?", "words": ["can", "say", "ant", "you"]}, {"sent": "from the crawling ant to the leaping antelope .", "words": ["the", "to", "ant"]}, {"sent": "an ant .", "words": ["an", "ant"]}, {"sent": "ant .", "words": ["ant"]}, {"sent": "your ant farms ants have escaped said Ruby .", "words": ["have", "your", "ant"]}, {"sent": "one day an ant was caught in the rain where can I hide ?", "words": ["the", "where", "can", "rain", "hide", "was", "an", "in", "ant"]}, {"sent": "you see our ant hill used to be in a ton of trouble .", "words": ["of", "to", "be", "our", "you", "in", "ant", "see", "a"]}, {"sent": "ant .", "words": ["ant"]}, {"sent": "is the ant under the leaves ?", "words": ["the", "under", "ant", "is"]}, {"sent": "and the ant jumping in to the water will make ripples .", "words": ["the", "to", "and", "in", "ant", "water", "will", "make"]}, {"sent": "a for ant .", "words": ["for", "ant", "a"]}, {"sent": "you ant it to go down .", "words": ["go", "to", "you", "it", "ant", "down"]}, {"sent": "ant ants angry ants advance .", "words": ["ant"]}, {"sent": "do you wanna put the ant on the carpet ?", "words": ["the", "do", "put", "you", "ant", "wanna", "on"]}, {"sent": "and what was inside the ant hill ?", "words": ["the", "what", "was", "inside", "and", "ant"]}, {"sent": "that would be an ant there right ?", "words": ["be", "there", "an", "ant", "would", "that"]}, {"sent": "a carpenter ant .", "words": ["ant", "a"]}, {"sent": "ant hill .", "words": ["ant"]}, {"sent": "that is an ant .", "words": ["an", "that", "ant", "is"]}, {"sent": "thats a big ant .", "words": ["big", "ant", "a"]}, {"sent": "ant .", "words": ["ant"]}, {"sent": "home oh the ant went home ?", "words": ["the", "ant", "home"]}, {"sent": "its an ant ?", "words": ["an", "ant"]}, {"sent": "well you know I didnt really go in the ant hill very much because remember the tunnels were very small I could not fit in there .", "words": ["the", "were", "go", "fit", "you", "there", "in", "ant", "could", "much", "not", "because"]}, {"sent": "a is for ant .", "words": ["for", "ant", "is", "a"]}, {"sent": "an ant .", "words": ["an", "ant"]}, {"sent": "ant .", "words": ["ant"]}, {"sent": "for ant .", "words": ["for", "ant"]}, {"sent": "an ant .", "words": ["an", "ant"]}, {"sent": "yeah what else ant what else ?", "words": ["what", "ant"]}, {"sent": "did the ant get ya ?", "words": ["the", "get", "ant", "did"]}, {"sent": "thats the bug I think hes an ant .", "words": ["the", "bug", "think", "an", "ant"]}, {"sent": "wheres the a for ant ?", "words": ["the", "for", "ant", "a"]}, {"sent": "and another block .", "words": ["another", "block", "and"]}, {"sent": "you found a lock block did you ?", "words": ["block", "did", "a", "you"]}, {"sent": "block what color is it ?", "words": ["what", "it", "block", "is"]}, {"sent": "it is a it is a block like a block .", "words": ["block", "is", "like", "it", "a"]}, {"sent": "you have block look .", "words": ["have", "look", "block", "you"]}, {"sent": "look what this block says .", "words": ["what", "block", "this", "look"]}, {"sent": "this block almost look like a treadmill if it was this way .", "words": ["block", "if", "was", "like", "this", "it", "look", "a"]}, {"sent": "Seths block .", "words": ["block"]}, {"sent": "with a block ?", "words": ["block", "with", "a"]}, {"sent": "tell him block .", "words": ["him", "block"]}, {"sent": "Im gonna try to block you right down here .", "words": ["block", "to", "you", "here", "down", "try", "try to"]}, {"sent": "hes sitting on the block .", "words": ["on", "the", "block"]}, {"sent": "want another block ?", "words": ["another", "block"]}, {"sent": "oh your block towers oh .", "words": ["block", "your"]}, {"sent": "what oh to make it stand up they just put something in it they put a block or something under it where itll stay up .", "words": ["up", "what", "put", "where", "block", "to", "stand", "they", "in", "it", "under", "stay", "make", "a"]}, {"sent": "is that a blue block ?", "words": ["block", "is", "blue", "that", "a"]}, {"sent": "the little block .", "words": ["the", "block", "little"]}, {"sent": "Ronald you was in the block .", "words": ["the", "block", "was", "you", "in"]}, {"sent": "thats not a block .", "words": ["block", "not", "a"]}, {"sent": "thats a big block alright .", "words": ["block", "big", "a"]}, {"sent": "can you put a person on top of the block ?", "words": ["can", "the", "put", "block", "of", "you", "on", "a"]}, {"sent": "on a block .", "words": ["on", "block", "a"]}, {"sent": "can I put this inside your block ?", "words": ["can", "put", "your", "block", "inside", "this"]}, {"sent": "I gotta look fer a block .", "words": ["block", "a", "look"]}, {"sent": "heres a block .", "words": ["block", "a"]}, {"sent": "its a block .", "words": ["block", "a"]}, {"sent": "that block is standing up ?", "words": ["up", "block", "that", "is"]}, {"sent": "its a block .", "words": ["block", "a"]}, {"sent": "do you want another block ?", "words": ["another", "do", "block", "you"]}, {"sent": "do you have a block ?", "words": ["do", "block", "you", "have", "a"]}, {"sent": "there is another block .", "words": ["another", "there", "block", "is"]}, {"sent": "its a block .", "words": ["block", "a"]}, {"sent": "this block ?", "words": ["this", "block"]}, {"sent": "no more block .", "words": ["more", "block"]}, {"sent": "if he gets there you better block him .", "words": ["better", "block", "if", "you", "there", "him", "he"]}, {"sent": "a block .", "words": ["block", "a"]}, {"sent": "a block ?", "words": ["block", "a"]}, {"sent": "here is my block .", "words": ["here", "my", "block", "is"]}, {"sent": "and thats a big block .", "words": ["block", "big", "a", "and"]}, {"sent": "you like that block better ?", "words": ["better", "block", "like", "you", "that"]}, {"sent": "with big block like that .", "words": ["block", "with", "like", "big", "that"]}, {"sent": "uhhuh blue block .", "words": ["block", "blue"]}, {"sent": "Ill make some more block towers .", "words": ["some", "more", "block", "make"]}, {"sent": "just like this is a curved block .", "words": ["block", "is", "like", "this", "a"]}, {"sent": "want the block ?", "words": ["the", "block"]}, {"sent": "and how does she play with block is it .", "words": ["does", "how", "block", "is", "play", "with", "and", "it", "she"]}, {"sent": "my block .", "words": ["my", "block"]}, {"sent": "oh it is a block you know its a cedar block .", "words": ["block", "is", "you", "it", "a"]}, {"sent": "you want the block ?", "words": ["the", "block", "you"]}, {"sent": "right I dont know where that last block has gone but you can do Dinosorter now .", "words": ["where", "can", "do", "block", "last", "you", "dont", "that", "but"]}, {"sent": "another block ?", "words": ["another", "block"]}, {"sent": "teddy bears gonna sit inside block can sit there ?", "words": ["can", "block", "inside", "there", "sit"]}, {"sent": "where is the green block ?", "words": ["where", "the", "block", "is", "green"]}, {"sent": "wheres that block ?", "words": ["block", "that"]}, {"sent": "and the block xxx .", "words": ["the", "block", "and"]}, {"sent": "you just put one block on top of the other like this .", "words": ["the", "put", "block", "of", "like", "you", "other", "this", "on"]}, {"sent": "wheres the green block ?", "words": ["the", "block", "green"]}, {"sent": "its like a block .", "words": ["block", "like", "a"]}, {"sent": "I have a block .", "words": ["have", "block", "a"]}, {"sent": "so go with somebody too around your block .", "words": ["around", "your", "block", "go", "so", "with", "too"]}, {"sent": "you wanna block me ?", "words": ["me", "block", "wanna", "you"]}, {"sent": "what are you gonna do with that block honey ?", "words": ["what", "do", "block", "with", "you", "are", "that"]}, {"sent": "Ill come next to the mounting block so you can climb on .", "words": ["the", "can", "block", "to", "so", "you", "on", "climb"]}, {"sent": "he is gonna eat that block .", "words": ["block", "is", "eat", "he", "that"]}, {"sent": "no thats a red block .", "words": ["red", "block", "a"]}, {"sent": "he had only gone to ride around the block a few times .", "words": ["the", "block", "to", "he", "around", "ride", "a"]}, {"sent": "so nine plus nine blocks plus one block leaves how many ?", "words": ["so", "how", "block"]}, {"sent": "Mummys got the block .", "words": ["the", "block"]}, {"sent": "what letter do you see on that block ?", "words": ["what", "do", "block", "you", "on", "see", "that"]}, {"sent": "what block what color block is that ?", "words": ["what", "that", "block", "is"]}, {"sent": "xxx by my xxx oo this is a block .", "words": ["my", "block", "is", "by", "this", "a"]}, {"sent": "put the block on top .", "words": ["the", "put", "block", "on"]}, {"sent": "I see thats a blue block .", "words": ["see", "blue", "block", "a"]}, {"sent": "block .", "words": ["block"]}, {"sent": "wheres the green block ?", "words": ["the", "block", "green"]}, {"sent": "oh thats a very big block okay now well make a big tunnel .", "words": ["block", "big", "make", "a"]}, {"sent": "remember the block ?", "words": ["the", "block"]}, {"sent": "and theres another block over there .", "words": ["block", "and", "another", "there", "over"]}, {"sent": "that block ?", "words": ["block", "that"]}, {"sent": "some yellow block .", "words": ["some", "block", "yellow"]}, {"sent": "can you get that block for me please ?", "words": ["can", "for", "block", "you", "get", "me", "that"]}, {"sent": "are you sawing the block ?", "words": ["the", "are", "block", "you"]}, {"sent": "click clock and block .", "words": ["block", "and", "clock"]}, {"sent": "the unique color of CoppertoneKids color block .", "words": ["the", "of", "block"]}, {"sent": "thats a half block .", "words": ["block", "a"]}, {"sent": "thats a building block to the house .", "words": ["the", "block", "to", "house", "a"]}, {"sent": "eh block thats right block .", "words": ["block"]}, {"sent": "can you put this block up ?", "words": ["can", "up", "put", "block", "you", "this"]}, {"sent": "Ill get you your block that you dropped .", "words": ["block", "your", "you", "get", "that"]}, {"sent": "the block xxx its like a a block wall .", "words": ["the", "block", "like", "a"]}, {"sent": "thats a red block on the bottom .", "words": ["red", "the", "bottom", "block", "on", "a"]}, {"sent": "where is the block ?", "words": ["where", "the", "block", "is"]}, {"sent": "block area .", "words": ["block"]}, {"sent": "I dont know why you would say that about our block you know .", "words": ["block", "say", "you", "our", "dont", "about", "would", "that", "why"]}, {"sent": "I wanna put it in a green block .", "words": ["put", "block", "in", "it", "wanna", "green", "a"]}, {"sent": "ah here put the block in the cup .", "words": ["the", "cup", "put", "block", "here", "in"]}, {"sent": "found the final block for the shape sorter downstairs .", "words": ["the", "for", "block"]}, {"sent": "no I think that rabbit is pushing a block isnt he ?", "words": ["think", "block", "is", "he", "that", "a"]}, {"sent": "whats the matter pumpkin ?", "words": ["the", "pumpkin"]}, {"sent": "theres a pumpkin right there .", "words": ["there", "pumpkin", "a"]}, {"sent": "do you see the pumpkin anywhere on this page ?", "words": ["the", "do", "you", "this", "on", "see", "pumpkin"]}, {"sent": "is he a happy pumpkin ?", "words": ["happy", "is", "he", "pumpkin", "a"]}, {"sent": "when you cut a pumpkin made a funny face .", "words": ["cut", "face", "you", "pumpkin", "when", "a"]}, {"sent": "did we have a pumpkin at Halloween ?", "words": ["we", "have", "did", "at", "pumpkin", "a"]}, {"sent": "pumpkin .", "words": ["pumpkin"]}, {"sent": "pumpkin patch and thats how we get to the .", "words": ["the", "how", "we", "to", "and", "get", "pumpkin"]}, {"sent": "why dont you want the pumpkin to be made into a pumpkin pie ?", "words": ["the", "into", "to", "be", "you", "dont", "pumpkin", "why", "a"]}, {"sent": "peter peter pumpkin eater had a wife and couldnt keep her he put her in a pumpkin shell and there he kept her very well .", "words": ["put", "and", "there", "in", "he", "pumpkin", "her", "a"]}, {"sent": "that cover do you think we should throw that pumpkin away ?", "words": ["do", "think", "throw", "we", "you", "away", "cover", "pumpkin", "that"]}, {"sent": "how is your little pumpkin doing ?", "words": ["how", "your", "is", "little", "pumpkin"]}, {"sent": "whats this pumpkin ?", "words": ["this", "pumpkin"]}, {"sent": "its a pumpkin ?", "words": ["pumpkin", "a"]}, {"sent": "pumpkin yay .", "words": ["pumpkin"]}, {"sent": "the pumpkin patch .", "words": ["the", "pumpkin"]}, {"sent": "youre gonna be a big fat pumpkin huh ?", "words": ["be", "pumpkin", "big", "a"]}, {"sent": "Peter Peter pumpkin eater had a wife and couldnt keep her .", "words": ["pumpkin", "her", "and", "a"]}, {"sent": "whos sitting on your pumpkin ?", "words": ["on", "pumpkin", "your"]}, {"sent": "is it a big pumpkin or a little pumpkin ?", "words": ["is", "little", "it", "pumpkin", "big", "a"]}, {"sent": "pumpkin .", "words": ["pumpkin"]}, {"sent": "it was his own pumpkin huh ?", "words": ["his", "was", "pumpkin", "it"]}, {"sent": "oh that must be a halloween card with a pumpkin on it .", "words": ["be", "with", "it", "on", "pumpkin", "that", "a"]}, {"sent": "thats right pumpkin pie .", "words": ["pumpkin"]}, {"sent": "there is a pumpkin .", "words": ["there", "pumpkin", "a", "is"]}, {"sent": "hey thats a pumpkin huh ?", "words": ["pumpkin", "a"]}, {"sent": "do you wanna cut our pumpkin and put our gourds in ?", "words": ["do", "put", "cut", "our", "you", "and", "in", "wanna", "pumpkin"]}, {"sent": "yeah pumpkin thats right its pumpkins .", "words": ["pumpkin"]}, {"sent": "Peter Peter pumpkin eater had a wife and couldnt keep her .", "words": ["pumpkin", "her", "and", "a"]}, {"sent": "look at the size of that pumpkin .", "words": ["the", "look", "of", "at", "pumpkin", "that"]}, {"sent": "a french pumpkin ?", "words": ["pumpkin", "a"]}, {"sent": "hello my pumpkin .", "words": ["my", "pumpkin"]}, {"sent": "that is a pumpkin right .", "words": ["pumpkin", "that", "a", "is"]}, {"sent": "it has melted and it has run out of the candle down the front of the pumpkin .", "words": ["the", "of", "and", "run", "it", "out", "pumpkin", "down"]}, {"sent": "are you gonna go to the pumpkin farm today ?", "words": ["the", "go", "to", "you", "are", "pumpkin"]}, {"sent": "oh theres a two pumpkin and theres a one pumpkin and this is a pumpkin too .", "words": ["is", "and", "this", "too", "pumpkin", "a"]}, {"sent": "pumpkin .", "words": ["pumpkin"]}, {"sent": "the pumpkin patch was where we saw the emu .", "words": ["the", "where", "we", "was", "pumpkin"]}, {"sent": "well draw an orange pumpkin .", "words": ["an", "draw", "pumpkin", "orange"]}, {"sent": "one pumpkin .", "words": ["pumpkin"]}, {"sent": "not gonna tell you dont wanna tell the story about your pumpkin ?", "words": ["the", "your", "you", "dont", "pumpkin", "wanna", "about", "story", "not"]}, {"sent": "is that a pumpkin ?", "words": ["pumpkin", "that", "a", "is"]}, {"sent": "you want your pumpkin to be red ?", "words": ["red", "your", "to", "be", "you", "pumpkin"]}, {"sent": "thats a pumpkin candle .", "words": ["pumpkin", "a"]}, {"sent": "Gar where is the pumpkin ?", "words": ["where", "the", "pumpkin", "is"]}, {"sent": "silly little pumpkin .", "words": ["pumpkin", "little"]}, {"sent": "got sticky eyes havent you pumpkin ?", "words": ["sticky", "pumpkin", "you"]}, {"sent": "is that the kind of face youre gonna have on your pumpkin ?", "words": ["the", "your", "face", "is", "of", "have", "on", "pumpkin", "that"]}, {"sent": "a pumpkin .", "words": ["pumpkin", "a"]}, {"sent": "comere pumpkin .", "words": ["pumpkin"]}, {"sent": "hello pumpkin .", "words": ["pumpkin"]}, {"sent": "is there a pumpkin there ?", "words": ["there", "pumpkin", "a", "is"]}, {"sent": "which one would you like pumpkin ?", "words": ["like", "you", "would", "pumpkin", "which"]}, {"sent": "oh pumpkin youre getting tired ?", "words": ["pumpkin", "tired"]}, {"sent": "the big pumpkin hugs the little one ?", "words": ["the", "pumpkin", "big", "little"]}, {"sent": "do you remember you showed me a pumpkin there ?", "words": ["do", "you", "there", "me", "pumpkin", "a"]}, {"sent": "Mummy is gonna draw an orange pumpkin for you okay ?", "words": ["for", "is", "draw", "you", "an", "orange", "pumpkin"]}, {"sent": "can you see the pumpkin lantern on the table ?", "words": ["can", "the", "table", "you", "on", "see", "pumpkin"]}, {"sent": "my little pumpkin head .", "words": ["my", "pumpkin", "head", "little"]}, {"sent": "woof woof woof woof woof like pumpkin .", "words": ["pumpkin", "like"]}, {"sent": "night night pumpkin .", "words": ["pumpkin"]}, {"sent": "okay pumpkin ?", "words": ["pumpkin"]}, {"sent": "that is the pumpkin .", "words": ["the", "pumpkin", "that", "is"]}, {"sent": "pumpkin .", "words": ["pumpkin"]}, {"sent": "but then what happened to the pumpkin ?", "words": ["the", "what", "to", "pumpkin", "then", "but"]}, {"sent": "there is a pumpkin .", "words": ["there", "pumpkin", "a", "is"]}, {"sent": "oh Ethan be careful pumpkin .", "words": ["be", "pumpkin", "careful"]}, {"sent": "more laughs little pumpkin more more more .", "words": ["more", "pumpkin", "little"]}, {"sent": "it looks like the baby pumpkin that you have .", "words": ["the", "like", "you", "have", "it", "pumpkin", "that"]}, {"sent": "oh pumpkin are we gonna have a hard time today with this microphone ?", "words": ["we", "with", "have", "this", "are", "hard", "pumpkin", "a"]}, {"sent": "Erin hasnt seen the baby pumpkin .", "words": ["the", "pumpkin"]}, {"sent": "oh pumpkin .", "words": ["pumpkin"]}, {"sent": "wanna draw a pumpkin ?", "words": ["pumpkin", "draw", "wanna", "a"]}, {"sent": "hey pumpkin .", "words": ["pumpkin"]}, {"sent": "pumpkin .", "words": ["pumpkin"]}, {"sent": "my little shaking pumpkin .", "words": ["my", "pumpkin", "little"]}, {"sent": "I dunno where the purple backpack is pumpkin .", "words": ["where", "the", "pumpkin", "is"]}, {"sent": "what did you see at the pumpkin patch ?", "words": ["the", "what", "you", "did", "at", "see", "pumpkin"]}, {"sent": "well just do the pumpkin one Thomas .", "words": ["the", "do", "pumpkin"]}, {"sent": "why are you over here pumpkin ?", "words": ["you", "here", "are", "pumpkin", "over", "why"]}, {"sent": "or pumpkin pie ?", "words": ["pumpkin"]}, {"sent": "xxx look tired pumpkin .", "words": ["pumpkin", "tired", "look"]}, {"sent": "a pumpkin face .", "words": ["pumpkin", "face", "a"]}, {"sent": "okay put it right on the pumpkin pies .", "words": ["the", "put", "it", "on", "pumpkin"]}, {"sent": "the pumpkin pumpkin pumpkins .", "words": ["the", "pumpkin"]}, {"sent": "and pumpkins ooh you know I think Nancy is growing a very large pumpkin we hafta keep our eye on that .", "words": ["think", "is", "we", "eye", "you", "and", "our", "on", "pumpkin", "that", "a"]}, {"sent": "I think it was a pumpkin .", "words": ["think", "was", "it", "pumpkin", "a"]}, {"sent": "and the pumpkin sprout grew into a pumpkin plant .", "words": ["the", "plant", "into", "and", "pumpkin", "a"]}, {"sent": "but he saved six pumpkin seeds for planting in the springtime .", "words": ["the", "for", "in", "he", "pumpkin", "but"]}, {"sent": "thats a pumpkin .", "words": ["pumpkin", "a"]}, {"sent": "remember you have one on your pumpkin ?", "words": ["your", "you", "have", "on", "pumpkin"]}, {"sent": "look it pumpkin .", "words": ["pumpkin", "it", "look"]}, {"sent": "little pumpkin .", "words": ["pumpkin", "little"]}, {"sent": "whats the pumpkin in ?", "words": ["the", "in", "pumpkin"]}, {"sent": "pumpkin .", "words": ["pumpkin"]}, {"sent": "come here pumpkin .", "words": ["here", "pumpkin"]}, {"sent": "pumpkin .", "words": ["pumpkin"]}, {"sent": "Mey look at this pumpkin right here .", "words": ["look", "here", "this", "at", "pumpkin"]}, {"sent": "pumpkin .", "words": ["pumpkin"]}, {"sent": "candy is yummy .", "words": ["candy", "is"]}, {"sent": "candy I know does LittleJohn eat candy ?", "words": ["eat", "does", "candy"]}, {"sent": "thats always a problem we have to figure out whos gonna stay home to hand out candy .", "words": ["home", "we", "hand", "to", "have", "out", "stay", "candy", "a"]}, {"sent": "oh he gave you some candy .", "words": ["some", "candy", "he", "you"]}, {"sent": "which ones the candy cane ?", "words": ["the", "candy", "which"]}, {"sent": "the scarecrow says we can find candy behind two pumpkins that match .", "words": ["the", "can", "we", "behind", "find", "candy", "that"]}, {"sent": "and you get some candy .", "words": ["you", "get", "and", "some", "candy"]}, {"sent": "that looks like a candy cane .", "words": ["candy", "that", "like", "a"]}, {"sent": "its just little chocolate candy .", "words": ["candy", "chocolate", "little"]}, {"sent": "a big one with a lot of candy .", "words": ["of", "a lot", "with", "candy", "big", "a"]}, {"sent": "you got candy ?", "words": ["candy", "you"]}, {"sent": "shes just like candy .", "words": ["candy", "like"]}, {"sent": "need more candy there .", "words": ["there", "more", "candy", "need"]}, {"sent": "I got my candy .", "words": ["my", "candy"]}, {"sent": "candy .", "words": ["candy"]}, {"sent": "I mean a candy bar .", "words": ["candy", "a"]}, {"sent": "does gerbil eat candy Julia ?", "words": ["eat", "does", "candy"]}, {"sent": "stick of candy .", "words": ["of", "stick", "candy"]}, {"sent": "oh he likes candy xxx .", "words": ["candy", "he"]}, {"sent": "any more candy ?", "words": ["more", "candy", "any"]}, {"sent": "has candy on top .", "words": ["on", "candy"]}, {"sent": "or do you hafta say something to get candy .", "words": ["do", "to", "say", "you", "get", "candy"]}, {"sent": "thats a funny looking candy cane to me .", "words": ["to", "me", "candy", "a"]}, {"sent": "to buy a stick of candy .", "words": ["stick", "of", "to", "candy", "buy", "a"]}, {"sent": "Marie Tony cant have anymore candy .", "words": ["have", "candy"]}, {"sent": "so you said there was candy where was the candy was the candy taped to the egg ?", "words": ["where", "the", "egg", "was", "so", "to", "you", "there", "candy"]}, {"sent": "oh Bonkers candy ?", "words": ["candy"]}, {"sent": "you had candy ?", "words": ["candy", "you"]}, {"sent": "my candy fell out her mouth .", "words": ["my", "mouth", "out", "candy", "her"]}, {"sent": "mommy go get the candy now .", "words": ["get", "the", "candy", "go"]}, {"sent": "he bought me a candy .", "words": ["me", "candy", "he", "a"]}, {"sent": "you used it for candy .", "words": ["candy", "it", "for", "you"]}, {"sent": "a candy bone yeah .", "words": ["candy", "a"]}, {"sent": "candy bars ?", "words": ["candy"]}, {"sent": "where did you find these candy canes ?", "words": ["where", "these", "you", "did", "find", "candy"]}, {"sent": "I go to candy cane .", "words": ["to", "candy", "go"]}, {"sent": "xxx candy .", "words": ["candy"]}, {"sent": "yeah thats candy .", "words": ["candy"]}, {"sent": "stick of candy .", "words": ["of", "stick", "candy"]}, {"sent": "what kind of candy do you think the dolly would like ?", "words": ["the", "what", "do", "think", "of", "like", "you", "would", "candy"]}, {"sent": "candy ?", "words": ["candy"]}, {"sent": "when we got candy and went trickortreating ?", "words": ["candy", "when", "and", "we"]}, {"sent": "Easter candy .", "words": ["candy"]}, {"sent": "whered my candy bar go ?", "words": ["my", "candy", "go"]}, {"sent": "no cotton candy ?", "words": ["candy"]}, {"sent": "is candy supper food ?", "words": ["food", "candy", "is"]}, {"sent": "candy ?", "words": ["candy"]}, {"sent": "theres that candy for them .", "words": ["candy", "that", "for", "them"]}, {"sent": "what kind of candy ?", "words": ["of", "what", "candy"]}, {"sent": "its the baker see hes making all these things candy and cookies pie cake mm doesnt that look good ?", "words": ["the", "all", "these", "cake", "and", "good", "look", "see", "candy", "that"]}, {"sent": "you are a candy king .", "words": ["candy", "are", "a", "you"]}, {"sent": "xxx eating your candy .", "words": ["candy", "your"]}, {"sent": "there was some candy in there .", "words": ["was", "there", "in", "some", "candy"]}, {"sent": "oh I bet SantaClaus came last night and put candy canes in the tree .", "words": ["the", "put", "tree", "last", "and", "in", "candy"]}, {"sent": "other people scrambled to get all the candy that they can and they put it in a little bag right ?", "words": ["the", "all", "can", "put", "to", "other", "get", "they", "and", "it", "in", "little", "candy", "that", "a"]}, {"sent": "your candy .", "words": ["candy", "your"]}, {"sent": "she clearly was sneaking candy bars for breakfast that I was not aware of .", "words": ["for", "of", "was", "not", "candy", "that", "she"]}, {"sent": "lets go get some candy in Alaska .", "words": ["go", "get", "in", "some", "candy"]}, {"sent": "how do you get the candy buddy ?", "words": ["the", "do", "how", "you", "get", "candy"]}, {"sent": "candy floss .", "words": ["candy"]}, {"sent": "I think you know what kind of candy you are .", "words": ["what", "think", "of", "you", "are", "candy"]}, {"sent": "candy corn .", "words": ["candy", "corn"]}, {"sent": "theres a song actually Thomas about a candy man .", "words": ["candy", "about", "a"]}, {"sent": "do you want a piece of candy right now ?", "words": ["do", "of", "you", "candy", "a"]}, {"sent": "is your candy good ?", "words": ["candy", "your", "good", "is"]}, {"sent": "its not candy .", "words": ["candy", "not"]}, {"sent": "you havent had candy in a whi .", "words": ["in", "candy", "a", "you"]}, {"sent": "because marshmallows are candy .", "words": ["candy", "are", "because"]}, {"sent": "try to eat up your candy and stuff .", "words": ["up", "your", "to", "and", "eat", "candy", "try", "try to"]}, {"sent": "oh candy bar .", "words": ["candy"]}, {"sent": "candy ?", "words": ["candy"]}, {"sent": "Gam do you want some candy ?", "words": ["do", "candy", "some", "you"]}, {"sent": "Ill make some candy cane ornaments .", "words": ["some", "candy", "make"]}, {"sent": "ya well go grocery shopping in a little bit and well get some candy .", "words": ["go", "and", "little", "get", "in", "some", "candy", "a"]}, {"sent": "Peter thats stale old candy left over from Halloween thats not good .", "words": ["old", "good", "candy", "over", "not"]}, {"sent": "how about a piece of candy ?", "words": ["how", "of", "about", "candy", "a"]}, {"sent": "but look he had all this candy and stuff all around him see theyre all mad at him .", "words": ["all", "around", "mad", "and", "this", "him", "at", "he", "look", "see", "candy", "but"]}, {"sent": "I got my candy .", "words": ["my", "candy"]}, {"sent": "no put the candy up .", "words": ["the", "put", "candy", "up"]}, {"sent": "candy basket ?", "words": ["candy", "basket"]}, {"sent": "would you like some dinosaur candy ?", "words": ["like", "you", "would", "some", "candy"]}, {"sent": "he bought me a xxx candy .", "words": ["me", "candy", "he", "a"]}, {"sent": "are you eating yummy candy ?", "words": ["candy", "are", "you"]}, {"sent": "look at that candy necklace .", "words": ["necklace", "look", "at", "candy", "that"]}, {"sent": "some candy ?", "words": ["some", "candy"]}, {"sent": "yeah this is nice I like this candy .", "words": ["is", "nice", "like", "this", "candy"]}, {"sent": "would ya what would you like to do one of these on page eight a candy a little candy heres a caramel .", "words": ["what", "do", "these", "of", "to", "like", "you", "little", "would", "on", "candy", "a"]}, {"sent": "xxx Mommy needs one more sucking candy .", "words": ["more", "candy"]}, {"sent": "I have some candy .", "words": ["have", "some", "candy"]}, {"sent": "is your candy bar in your hand ?", "words": ["your", "hand", "is", "in", "candy"]}, {"sent": "candy ?", "words": ["candy"]}, {"sent": "we hafta follow the candy trail through the spooky forest past the .", "words": ["the", "candy", "we"]}, {"sent": "but she doesnt eat cookies she doesnt eat cake she doesnt eat icecream or candy .", "words": ["cake", "eat", "candy", "she", "but"]}, {"sent": "Yeah but can can you go up dressed like that to their door to get candy ?", "words": ["can", "up", "their", "go", "to", "like", "you", "get", "candy", "that", "but", "door"]}, {"sent": "youre bringing her a candy .", "words": ["candy", "her", "a"]}, {"sent": "well theyre little flowers that we planted called something like candy tuft .", "words": ["we", "like", "little", "candy", "that"]}, {"sent": "ew eat some candy .", "words": ["eat", "some", "candy"]}, {"sent": "oh the candy canes too sticky to have while youre playing with the doggy .", "words": ["the", "sticky", "to", "with", "have", "too", "candy"]}, {"sent": "eat daet candy ?", "words": ["eat", "candy"]}, {"sent": "to buy a stick of candy .", "words": ["stick", "of", "to", "candy", "buy", "a"]}, {"sent": "heres a belly button .", "words": ["belly button", "button", "a"]}, {"sent": "wheres your belly button ?", "words": ["belly button", "your", "button"]}, {"sent": "there it is hes got a very cute belly button .", "words": ["is", "cute", "there", "belly button", "it", "button", "a"]}, {"sent": "right on the belly button .", "words": ["on", "the", "button", "belly button"]}, {"sent": "mhm look he has a belly button .", "words": ["belly button", "he", "look", "button", "a"]}, {"sent": "hes going to bite your belly button .", "words": ["your", "to", "belly button", "bite", "button"]}, {"sent": "its the same its the its the same with the flap but its um wheres babys belly button and so you look different parts like wheres babys head and its under the so .", "words": ["the", "so", "head", "with", "like", "and", "you", "under", "belly button", "same", "look", "button", "but"]}, {"sent": "this is the same as where is babys belly button .", "words": ["the", "where", "is", "this", "belly button", "same", "button"]}, {"sent": "oh your belly button .", "words": ["belly button", "your", "button"]}, {"sent": "saw your belly button .", "words": ["belly button", "your", "button"]}, {"sent": "I didnt I didnt get your belly button so dont hurt me with that .", "words": ["your", "hurt", "so", "with", "get", "dont", "that", "belly button", "me", "button"]}, {"sent": "everybody start getting Robbies belly button .", "words": ["belly button", "button"]}, {"sent": "ahhah everybody get Robbies belly button everybody build a tunnel .", "words": ["get", "belly button", "build", "button", "a"]}, {"sent": "hi belly button .", "words": ["belly button", "button"]}, {"sent": "wheres her belly button .", "words": ["belly button", "her", "button"]}, {"sent": "you call it your belly button .", "words": ["your", "you", "belly button", "it", "button"]}, {"sent": "thats my thats my belly button .", "words": ["belly button", "my", "button"]}, {"sent": "where is your belly button ?", "words": ["where", "your", "is", "belly button", "button"]}, {"sent": "thats your belly button .", "words": ["belly button", "your", "button"]}, {"sent": "its your belly button with your finger in .", "words": ["your", "with", "finger", "in", "belly button", "button"]}, {"sent": "is that a belly button ?", "words": ["is", "belly button", "that", "button", "a"]}, {"sent": "where is your belly button .", "words": ["where", "your", "is", "belly button", "button"]}, {"sent": "dry belly button .", "words": ["belly button", "dry", "button"]}, {"sent": "hes gonna get your belly button .", "words": ["get", "belly button", "your", "button"]}, {"sent": "try to eat my belly button .", "words": ["my", "button", "to", "eat", "belly button", "try", "try to"]}, {"sent": "I can see Hucklecats belly button .", "words": ["can", "see", "button", "belly button"]}, {"sent": "hello little naked one is that your little belly button ?", "words": ["your", "is", "little", "belly button", "that", "button"]}, {"sent": "oh theres your belly button .", "words": ["belly button", "your", "button"]}, {"sent": "and gives that little guys belly a kiss right in the middle of his belly button tickle tickle .", "words": ["the", "button", "of", "tickle", "little", "and", "belly button", "that", "in", "his", "kiss", "a"]}, {"sent": "and gives little guy a kiss right in the belly button .", "words": ["the", "button", "little", "and", "belly button", "in", "kiss", "a"]}, {"sent": "belly button around .", "words": ["belly button", "button", "around"]}, {"sent": "wheres your belly button I just saw that belly button .", "words": ["belly button", "that", "your", "button"]}, {"sent": "I seem to have a tennis ball in my belly button .", "words": ["my", "to", "have", "ball", "in", "belly button", "button", "a"]}, {"sent": "I boinked you in your belly button .", "words": ["your", "you", "belly button", "in", "button"]}, {"sent": "and she has her own belly button yeah .", "words": ["button", "and", "belly button", "her", "she"]}, {"sent": "I think maybe you could use a little more food in your belly button .", "words": ["think", "your", "little", "you", "belly button", "more", "food", "in", "could", "button", "a"]}, {"sent": "no do you dont have a belly button ?", "words": ["do", "you", "have", "dont", "belly button", "button", "a"]}, {"sent": "how about that William doesnt have a belly button .", "words": ["how", "button", "have", "belly button", "about", "that", "a"]}, {"sent": "William doesnt have a belly button .", "words": ["have", "belly button", "button", "a"]}, {"sent": "belly button yes .", "words": ["belly button", "button"]}, {"sent": "your belly button is right there yeah .", "words": ["your", "is", "there", "belly button", "button"]}, {"sent": "wheres your belly button ?", "words": ["belly button", "your", "button"]}, {"sent": "this is your belly button right here .", "words": ["your", "is", "here", "this", "belly button", "button"]}, {"sent": "your belly button ?", "words": ["belly button", "your", "button"]}, {"sent": "screw your belly button in .", "words": ["belly button", "in", "your", "button"]}, {"sent": "wheres your belly button ?", "words": ["belly button", "your", "button"]}, {"sent": "I see your belly button .", "words": ["see", "your", "belly button", "button"]}, {"sent": "uh I wanna kiss your belly button .", "words": ["your", "belly button", "wanna", "kiss", "button"]}, {"sent": "can I kiss your belly button ?", "words": ["can", "your", "button", "belly button", "kiss"]}, {"sent": "we also have theres one by the same person about your belly button .", "words": ["the", "your", "we", "by", "have", "belly button", "same", "about", "button"]}, {"sent": "we have wheres babys belly button ?", "words": ["have", "belly button", "button", "we"]}, {"sent": "this is the same as babys belly button .", "words": ["the", "is", "this", "belly button", "same", "button"]}, {"sent": "not the same as babys belly button but close .", "words": ["the", "belly button", "same", "close", "button", "not", "but"]}, {"sent": "I actually thought hed love this one because we have where is babys belly button .", "words": ["where", "we", "is", "have", "this", "belly button", "love", "button", "because"]}, {"sent": "wheres your belly button .", "words": ["belly button", "your", "button"]}, {"sent": "Im going to have your belly button .", "words": ["your", "to", "have", "belly button", "button"]}, {"sent": "xxx belly button .", "words": ["belly button", "button"]}, {"sent": "belly button is up .", "words": ["up", "belly button", "button", "is"]}, {"sent": "belly button .", "words": ["belly button", "button"]}, {"sent": "right above my belly button .", "words": ["above", "belly button", "my", "button"]}, {"sent": "at his belly button .", "words": ["his", "belly button", "button", "at"]}, {"sent": "ah haha get his belly button .", "words": ["get", "his", "button", "belly button"]}, {"sent": "belly button .", "words": ["belly button", "button"]}, {"sent": "everybody get Robbies belly button .", "words": ["get", "belly button", "button"]}, {"sent": "hey everybody get Robbies belly button .", "words": ["get", "belly button", "button"]}, {"sent": "mister quickies belly button .", "words": ["belly button", "button"]}, {"sent": "Im gonna poop all over your belly button .", "words": ["all", "your", "button", "belly button", "over"]}, {"sent": "I see your belly button .", "words": ["see", "your", "belly button", "button"]}, {"sent": "wheres the belly button ?", "words": ["the", "belly button", "button"]}, {"sent": "Thomass belly button .", "words": ["belly button", "button"]}, {"sent": "it sounded like you said you were going to kiss Mummys belly button .", "words": ["button", "to", "like", "you", "belly button", "it", "kiss", "were"]}, {"sent": "his belly button ?", "words": ["his", "belly button", "button"]}, {"sent": "or does that go on your belly button ?", "words": ["does", "your", "button", "go", "belly button", "on", "that"]}, {"sent": "is that your belly button ?", "words": ["your", "button", "is", "belly button", "that"]}, {"sent": "its your belly button .", "words": ["belly button", "your", "button"]}, {"sent": "xxx shes got a belly button top on .", "words": ["on", "belly button", "button", "a"]}, {"sent": "this budleyley has a belly button .", "words": ["this", "button", "belly button", "a"]}, {"sent": "does he have a belly button ?", "words": ["does", "have", "belly button", "he", "button", "a"]}, {"sent": "a belly button ?", "words": ["belly button", "button", "a"]}, {"sent": "show me where is your belly button .", "words": ["where", "your", "is", "show", "belly button", "me", "button"]}, {"sent": "where is your belly button .", "words": ["where", "your", "is", "belly button", "button"]}, {"sent": "gonna dry his belly button .", "words": ["his", "belly button", "dry", "button"]}, {"sent": "in the closet ?", "words": ["the", "in", "closet"]}, {"sent": "its in the back of the closet Abe .", "words": ["the", "back", "of", "closet", "in"]}, {"sent": "Im gonna check in the closet .", "words": ["the", "in", "closet"]}, {"sent": "oh Thomas lets not open the closet because now he wants to go in there .", "words": ["the", "go", "open", "to", "closet", "there", "in", "he", "not", "because"]}, {"sent": "and the sneakers went into Maxs closet and the windup bugs went into the bug box .", "words": ["the", "bug", "box", "into", "closet", "and"]}, {"sent": "I think the puppy didnt know there were so many things in the closet that were going to come falling out when he opened the door .", "words": ["the", "think", "when", "so", "closet", "to", "there", "in", "that", "puppy", "he", "out", "were", "door"]}, {"sent": "cocoa get outof the closet please .", "words": ["get", "the", "closet"]}, {"sent": "okay we will go in the closet .", "words": ["the", "go", "we", "closet", "in", "will"]}, {"sent": "thats a closet .", "words": ["closet", "a"]}, {"sent": "and another room for a closet .", "words": ["for", "closet", "and", "another", "room", "a"]}, {"sent": "Naima youre in the closet .", "words": ["the", "in", "closet"]}, {"sent": "remember whats the show we were watching and they had the boy hadta live in the closet ?", "words": ["the", "we", "show", "closet", "and", "they", "in", "were"]}, {"sent": "yeah it must be in the closet somewhere .", "words": ["the", "be", "closet", "in", "it"]}, {"sent": "you are going in the closet ?", "words": ["the", "closet", "you", "in", "are"]}, {"sent": "do we have a blue broom in the closet ?", "words": ["the", "do", "we", "closet", "blue", "have", "broom", "in", "a"]}, {"sent": "well leave it in the closet until tomorrow .", "words": ["the", "in", "it", "closet"]}, {"sent": "yep I looked in the closet and there was all these xxx .", "words": ["the", "all", "these", "was", "closet", "and", "there", "in"]}, {"sent": "and whats in the closet ?", "words": ["the", "closet", "in", "and"]}, {"sent": "theres a monkey in the closet .", "words": ["the", "monkey", "closet", "in", "a"]}, {"sent": "that looks like a little closet yeah .", "words": ["closet", "like", "little", "that", "a"]}, {"sent": "xxx you closed that closet ball .", "words": ["ball", "closet", "that", "you"]}, {"sent": "look in the closet .", "words": ["the", "closet", "in", "look"]}, {"sent": "in the closet .", "words": ["the", "in", "closet"]}, {"sent": "your new coats in the closet .", "words": ["the", "your", "closet", "in", "new"]}, {"sent": "maybe its in your room in the closet .", "words": ["the", "your", "closet", "in", "room"]}, {"sent": "are they in the closet ?", "words": ["the", "closet", "they", "in", "are"]}, {"sent": "is Maisey in the closet ?", "words": ["the", "closet", "in", "is"]}, {"sent": "oh thats a good idea but I dont think the closet opens .", "words": ["the", "think", "closet", "dont", "good", "but", "a"]}, {"sent": "put it back in the closet and use these drawers for clothes .", "words": ["the", "these", "put", "for", "back", "closet", "and", "in", "it"]}, {"sent": "whos in the closet ?", "words": ["the", "in", "closet"]}, {"sent": "tv is back in the closet .", "words": ["the", "back", "is", "closet", "in", "tv"]}, {"sent": "into the closet ?", "words": ["the", "closet", "into"]}, {"sent": "closet yeah no were not gonna go inside .", "words": ["go", "closet", "inside", "were", "not"]}, {"sent": "should we keep it up in the closet so it doesnt break ?", "words": ["up", "the", "we", "so", "closet", "in", "it", "break"]}, {"sent": "I suppose theres another nightmare in my closet but my beds not big enough for three .", "words": ["my", "for", "closet", "another", "in", "big", "not", "but"]}, {"sent": "look in the closet .", "words": ["the", "closet", "in", "look"]}, {"sent": "look what I found in the closet .", "words": ["the", "what", "closet", "in", "look"]}, {"sent": "is it in the closet ?", "words": ["the", "is", "closet", "in", "it"]}, {"sent": "Im putting this box back in the closet .", "words": ["the", "box", "back", "closet", "this", "in"]}, {"sent": "another nightmare in the closet ?", "words": ["another", "the", "in", "closet"]}, {"sent": "god I made a mess of this closet .", "words": ["of", "this", "closet", "a"]}, {"sent": "wheres the closet ?", "words": ["the", "closet"]}, {"sent": "its probably in your closet .", "words": ["in", "your", "closet"]}, {"sent": "Mamas hung ya in the closet .", "words": ["the", "in", "closet"]}, {"sent": "theyre in the closet .", "words": ["the", "in", "closet"]}, {"sent": "no its not in the closet .", "words": ["the", "in", "not", "closet"]}, {"sent": "in the closet .", "words": ["the", "in", "closet"]}, {"sent": "can you go into his closet and get a burp cloth please or you know what theres even one I think underneath his gym out there theres a burp cloth .", "words": ["can", "what", "think", "into", "go", "closet", "you", "and", "get", "there", "out", "his", "a"]}, {"sent": "oh theyre in the closet in the hall .", "words": ["the", "in", "closet"]}, {"sent": "back in the closet .", "words": ["the", "in", "back", "closet"]}, {"sent": "Im gonna put these sheets in the closet where they belong .", "words": ["the", "these", "put", "where", "closet", "they", "in"]}, {"sent": "okay you put it in the closet .", "words": ["the", "put", "closet", "you", "in", "it"]}, {"sent": "you trying to shut the closet door ?", "words": ["the", "to", "closet", "you", "door"]}, {"sent": "and Eves coat is in the closet too right .", "words": ["the", "is", "closet", "and", "in", "too", "coat"]}, {"sent": "is he in the closet ?", "words": ["the", "is", "closet", "in", "he"]}, {"sent": "are you in the closet ?", "words": ["the", "closet", "you", "in", "are"]}, {"sent": "maybe your soldiers in the closet .", "words": ["the", "in", "your", "closet"]}, {"sent": "okay then it can go back in the closet .", "words": ["can", "the", "back", "go", "closet", "in", "it", "then"]}, {"sent": "is it in your closet ?", "words": ["your", "is", "closet", "in", "it"]}, {"sent": "um lock your closet .", "words": ["closet", "your"]}, {"sent": "whos that hiding in the closet ?", "words": ["the", "in", "that", "closet"]}, {"sent": "theyre not in the closet .", "words": ["the", "in", "not", "closet"]}, {"sent": "whos in that closet ?", "words": ["in", "that", "closet"]}, {"sent": "put the necklaces back in the closet .", "words": ["the", "put", "back", "closet", "in"]}, {"sent": "you want some balls that are in that closet ?", "words": ["closet", "you", "in", "are", "some", "that"]}, {"sent": "who wants to live in a closet ?", "words": ["to", "closet", "in", "who", "a"]}, {"sent": "in the closet .", "words": ["the", "in", "closet"]}, {"sent": "what made you suspect that he was in the closet ?", "words": ["the", "what", "was", "closet", "you", "in", "he", "that"]}, {"sent": "do you think Sams in the closet ?", "words": ["the", "do", "think", "closet", "you", "in"]}, {"sent": "okay lets see well maybe well look in the closet and see .", "words": ["the", "closet", "and", "in", "look", "see"]}, {"sent": "its in the closet .", "words": ["the", "in", "closet"]}, {"sent": "well go in the closet do you wanna go in the closet ?", "words": ["the", "do", "go", "closet", "you", "in", "wanna"]}, {"sent": "oh you wanna go in the closet and hide .", "words": ["the", "hide", "go", "closet", "you", "and", "in", "wanna"]}, {"sent": "theres a monkey in the closet .", "words": ["the", "monkey", "closet", "in", "a"]}, {"sent": "gotta go put this in the closet .", "words": ["the", "put", "go", "closet", "this", "in"]}, {"sent": "is he in the closet ?", "words": ["the", "is", "closet", "in", "he"]}, {"sent": "you looking in the closet for the tv ?", "words": ["the", "for", "closet", "you", "in", "tv"]}, {"sent": "I put them in a box on this end of the closet ?", "words": ["the", "put", "box", "of", "closet", "them", "this", "in", "on", "a"]}, {"sent": "I want you outof your closet .", "words": ["closet", "your", "you"]}, {"sent": "they all go in the closet uhhuh .", "words": ["the", "all", "go", "closet", "they", "in"]}, {"sent": "oh yes theyre in the closet .", "words": ["the", "in", "closet"]}, {"sent": "theyre in the closet .", "words": ["the", "in", "closet"]}, {"sent": "no not that closet your closet .", "words": ["your", "closet", "that", "not"]}, {"sent": "should we get oh my gosh what happened to your closet ?", "words": ["what", "my", "your", "we", "to", "closet", "get"]}, {"sent": "oh boy lets warn everyone looking on the other side of the camera what your closet looks like .", "words": ["the", "what", "camera", "your", "of", "closet", "like", "other", "on"]}, {"sent": "do you remember what you were looking for inside ah inside your closet before ?", "words": ["do", "what", "for", "your", "inside", "closet", "you", "were"]}, {"sent": "were not going in the closet anymore .", "words": ["the", "closet", "in", "were", "not"]}, {"sent": "theres a closet with clothes .", "words": ["closet", "with", "a"]}, {"sent": "is Lucy in the closet maybe shed like a bath .", "words": ["the", "is", "closet", "like", "in", "a"]}, {"sent": "well shall we put them in the closet ?", "words": ["the", "put", "we", "closet", "them", "in"]}, {"sent": "somebody in the closet ?", "words": ["the", "in", "closet"]}, {"sent": "where is he cry is he in the closet ?", "words": ["where", "the", "is", "cry", "closet", "in", "he"]}, {"sent": "in the closet .", "words": ["the", "in", "closet"]}, {"sent": "are they in your closet ?", "words": ["your", "closet", "they", "in", "are"]}, {"sent": "in the closet ?", "words": ["the", "in", "closet"]}, {"sent": "did you shut Mommy in the closet ?", "words": ["the", "closet", "you", "in", "did"]}, {"sent": "then Ruby saw the green stuff whats oozing out from under your closet door asked Ruby .", "words": ["the", "your", "closet", "under", "out", "green", "then", "door"]}, {"sent": "need the firetruck .", "words": ["the", "need", "firetruck"]}, {"sent": "did you hear a firetruck ?", "words": ["hear", "you", "did", "firetruck", "a"]}, {"sent": "here is a firetruck .", "words": ["here", "firetruck", "a", "is"]}, {"sent": "theres the firetruck .", "words": ["the", "firetruck"]}, {"sent": "firetruck thats right .", "words": ["firetruck"]}, {"sent": "the firetruck ?", "words": ["the", "firetruck"]}, {"sent": "are you a firetruck ?", "words": ["firetruck", "are", "a", "you"]}, {"sent": "so its like a cherry picker firetruck isnt it ?", "words": ["so", "like", "it", "firetruck", "a"]}, {"sent": "that one looks like the firetruck one doesnt it ?", "words": ["the", "like", "it", "firetruck", "that"]}, {"sent": "a firetruck .", "words": ["firetruck", "a"]}, {"sent": "is that the right person for the firetruck ?", "words": ["the", "for", "is", "firetruck", "that"]}, {"sent": "oh theres the firetruck .", "words": ["the", "firetruck"]}, {"sent": "there is a firetruck .", "words": ["there", "firetruck", "a", "is"]}, {"sent": "wheres the firetruck ?", "words": ["the", "firetruck"]}, {"sent": "where is that firetruck ?", "words": ["where", "firetruck", "that", "is"]}, {"sent": "look in the pages and find Nan a firetruck .", "words": ["the", "look", "and", "in", "firetruck", "find", "a"]}, {"sent": "thats a firetruck .", "words": ["firetruck", "a"]}, {"sent": "sounds like a firetruck to me .", "words": ["to", "like", "firetruck", "me", "a"]}, {"sent": "is that the right thing for the firetruck ?", "words": ["the", "for", "is", "firetruck", "that"]}, {"sent": "for this superduper firetruck were making here ?", "words": ["for", "here", "this", "firetruck", "were"]}, {"sent": "theres one bigger firetruck .", "words": ["firetruck"]}, {"sent": "is that the firetruck ?", "words": ["the", "firetruck", "that", "is"]}, {"sent": "he goes on your firetruck .", "words": ["on", "firetruck", "he", "your"]}, {"sent": "you know like in your big firetruck ?", "words": ["your", "like", "you", "in", "firetruck", "big"]}, {"sent": "thats an oldfashioned firetruck .", "words": ["an", "firetruck"]}, {"sent": "you were talking about a firetruck very good .", "words": ["you", "about", "firetruck", "good", "were", "a"]}, {"sent": "is your firetruck missing ?", "words": ["firetruck", "your", "is"]}, {"sent": "that looks like a firetruck to me .", "words": ["to", "like", "firetruck", "me", "that", "a"]}, {"sent": "he has these puzzles youknow that have airplane firetruck youknow you take them out of their spots .", "words": ["these", "their", "airplane", "of", "them", "you", "have", "he", "out", "firetruck", "take", "that"]}, {"sent": "what about the screws on your firetruck .", "words": ["the", "what", "your", "about", "firetruck", "on"]}, {"sent": "ooh there just went a firetruck sweetie but we missed it .", "words": ["we", "there", "it", "firetruck", "but", "a"]}, {"sent": "wanna mix it up with a firetruck ?", "words": ["up", "with", "it", "wanna", "firetruck", "a"]}, {"sent": "where is the firetruck ?", "words": ["where", "the", "firetruck", "is"]}, {"sent": "lookit there is a big long firetruck .", "words": ["is", "long", "there", "firetruck", "big", "a"]}, {"sent": "should we make the firetruck ?", "words": ["the", "firetruck", "make", "we"]}, {"sent": "oh its an airplane firetruck ?", "words": ["airplane", "an", "firetruck"]}, {"sent": "thats not the people for the firetruck ?", "words": ["the", "for", "not", "firetruck"]}, {"sent": "no a firetruck .", "words": ["firetruck", "a"]}, {"sent": "go look behind your firetruck on the floor .", "words": ["the", "look", "your", "go", "behind", "firetruck", "on"]}, {"sent": "I thought you said they belonged on your firetruck on your tow truck .", "words": ["your", "truck", "you", "they", "firetruck", "on"]}, {"sent": "you hear the firetruck ?", "words": ["the", "hear", "firetruck", "you"]}, {"sent": "oh they can ride in the firetruck ?", "words": ["can", "the", "they", "in", "firetruck", "ride"]}, {"sent": "xxx it looks more like a firetruck like that lets see .", "words": ["like", "more", "it", "firetruck", "see", "that", "a"]}, {"sent": "so if you help me find the firetruck .", "words": ["the", "help", "if", "so", "you", "firetruck", "find", "me"]}, {"sent": "arent they gonna fall off when the firetruck goes up ?", "words": ["off", "the", "up", "fall", "they", "firetruck", "when"]}, {"sent": "ah firetruck firetruck www .", "words": ["firetruck"]}, {"sent": "is that airplane firetruck driving over the letters ?", "words": ["the", "is", "airplane", "that", "firetruck", "over"]}, {"sent": "it has a firetruck inside and theres the firehouse dog waiting for the next fire .", "words": ["the", "for", "inside", "and", "it", "firetruck", "dog", "a"]}, {"sent": "is that a firetruck ?", "words": ["firetruck", "that", "a", "is"]}, {"sent": "hes riding the firetruck .", "words": ["the", "firetruck"]}, {"sent": "do you see the firetruck ?", "words": ["the", "do", "you", "firetruck", "see"]}, {"sent": "ah firetruck firetruck .", "words": ["firetruck"]}, {"sent": "put some wheels on the firetruck ?", "words": ["the", "put", "firetruck", "on", "some"]}, {"sent": "yeah when we take the dumper off it looks more like a firetruck .", "words": ["the", "off", "when", "we", "like", "more", "it", "firetruck", "take", "a"]}, {"sent": "its a firetruck .", "words": ["firetruck", "a"]}, {"sent": "thats the firetruck .", "words": ["the", "firetruck"]}, {"sent": "theres a firetruck .", "words": ["firetruck", "a"]}, {"sent": "sometimes lights flash on a firetruck and that could be the lights flashing right ?", "words": ["the", "be", "and", "firetruck", "on", "could", "that", "a"]}, {"sent": "why is that an airplane firetruck ?", "words": ["is", "airplane", "an", "firetruck", "that", "why"]}, {"sent": "firetruck that belongs to the set .", "words": ["the", "to", "that", "firetruck"]}, {"sent": "Ill park the firetruck in the garage .", "words": ["the", "garage", "in", "park", "firetruck"]}, {"sent": "you saw firetruck .", "words": ["firetruck", "you"]}, {"sent": "what does a firetruck do ?", "words": ["what", "do", "does", "firetruck", "a"]}, {"sent": "ah look at the firetruck .", "words": ["the", "firetruck", "at", "look"]}, {"sent": "who whered did you get sit in a firetruck ?", "words": ["you", "get", "in", "firetruck", "did", "who", "sit", "a"]}, {"sent": "is that a firetruck ?", "words": ["firetruck", "that", "a", "is"]}, {"sent": "where is the firetruck ?", "words": ["where", "the", "firetruck", "is"]}, {"sent": "a firetruck and theyre squirting .", "words": ["firetruck", "and", "a"]}, {"sent": "fireman piece its the firetruck .", "words": ["the", "firetruck"]}, {"sent": "the what firetruck ?", "words": ["the", "what", "firetruck"]}, {"sent": "xxx the firetruck there last night ?", "words": ["the", "there", "last", "firetruck"]}, {"sent": "look a special kind of firetruck .", "words": ["of", "firetruck", "a", "look"]}, {"sent": "how do you make a firetruck ?", "words": ["do", "how", "you", "firetruck", "make", "a"]}, {"sent": "its a firetruck .", "words": ["firetruck", "a"]}, {"sent": "you saw the firetruck and the fighters ?", "words": ["the", "firetruck", "and", "you"]}, {"sent": "where is the red firetruck ?", "words": ["where", "the", "red", "is", "firetruck"]}, {"sent": "hes a firetruck right RevinEvan .", "words": ["firetruck", "a"]}, {"sent": "thats a firetruck .", "words": ["firetruck", "a"]}, {"sent": "here you drive the firetruck .", "words": ["the", "you", "here", "drive", "firetruck"]}, {"sent": "okay you gotta put the firehydrant on the firetruck thats a good idea .", "words": ["the", "put", "you", "good", "firetruck", "on", "a"]}, {"sent": "and there thats where the firetruck goes he stays in there .", "words": ["where", "the", "and", "there", "in", "he", "firetruck"]}, {"sent": "say her firetruck .", "words": ["her", "say", "firetruck"]}, {"sent": "do you see the firetruck on there ?", "words": ["the", "do", "you", "there", "firetruck", "on", "see"]}, {"sent": "yes firetruck .", "words": ["firetruck"]}, {"sent": "is that a firetruck Hin ?", "words": ["firetruck", "that", "a", "is"]}, {"sent": "hey does he get a firetruck ?", "words": ["does", "get", "he", "firetruck", "a"]}, {"sent": "oo need some people for the firetruck Sean ?", "words": ["the", "for", "need", "firetruck", "some"]}, {"sent": "where is the red firetruck ?", "words": ["where", "the", "red", "is", "firetruck"]}, {"sent": "I wanna put in the firetruck in there .", "words": ["the", "put", "there", "in", "wanna", "firetruck"]}, {"sent": "yeah its like a cherry picker firetruck isnt it ?", "words": ["it", "firetruck", "like", "a"]}, {"sent": "is that a firetruck ?", "words": ["firetruck", "that", "a", "is"]}, {"sent": "you can fix the screws in your firetruck .", "words": ["can", "the", "your", "fix", "you", "in", "firetruck"]}, {"sent": "where do you think the firetruck should go ?", "words": ["where", "do", "think", "the", "go", "you", "firetruck"]}, {"sent": "are you scareda the firetruck ?", "words": ["the", "firetruck", "are", "you"]}, {"sent": "we walk right by the firetruck house every day .", "words": ["the", "we", "by", "house", "every", "firetruck", "walk"]}, {"sent": "its a firetruck .", "words": ["firetruck", "a"]}, {"sent": "I thought you put a firetruck in the firemens house .", "words": ["the", "put", "you", "in", "house", "firetruck", "a"]}, {"sent": "heres a man for your firetruck .", "words": ["firetruck", "for", "your", "a"]}, {"sent": "does the stove get hot or cold ?", "words": ["the", "does", "get", "hot", "stove", "cold"]}, {"sent": "thats a stove .", "words": ["stove", "a"]}, {"sent": "well do you wanna bring the pretend stove out here so we can cook dinner on it ?", "words": ["the", "do", "can", "we", "so", "bring", "you", "here", "cook", "it", "stove", "wanna", "out", "on", "pretend"]}, {"sent": "oh you wanna put it on the stove ?", "words": ["the", "put", "you", "it", "stove", "wanna", "on"]}, {"sent": "lets get the er stove .", "words": ["get", "the", "stove"]}, {"sent": "is that a stove ?", "words": ["that", "stove", "a", "is"]}, {"sent": "I got some on the stove ?", "words": ["on", "some", "stove", "the"]}, {"sent": "well we need the stove huh ?", "words": ["the", "need", "stove", "we"]}, {"sent": "Ill move the stove a little bit .", "words": ["the", "stove", "little", "a"]}, {"sent": "yeah she seemed to know what the stove was and know that .", "words": ["the", "what", "to", "was", "and", "stove", "that", "she"]}, {"sent": "on the stove ?", "words": ["on", "the", "stove"]}, {"sent": "who put this in the stove Bernard Barry ?", "words": ["the", "put", "this", "in", "stove", "who"]}, {"sent": "stick it on the stove .", "words": ["the", "stick", "it", "stove", "on"]}, {"sent": "we need the stove uhn dont we ?", "words": ["the", "need", "we", "dont", "stove"]}, {"sent": "yeah you put it on the stove .", "words": ["the", "put", "you", "it", "stove", "on"]}, {"sent": "are you gonna put it on top of the stove ?", "words": ["the", "put", "of", "you", "it", "are", "stove", "on"]}, {"sent": "and thats a stove .", "words": ["stove", "a", "and"]}, {"sent": "this is the play stove isnt it ?", "words": ["the", "is", "play", "this", "it", "stove"]}, {"sent": "oh yeah turn the stove on .", "words": ["the", "on", "stove"]}, {"sent": "xxx stick it on the stove ?", "words": ["the", "stick", "it", "stove", "on"]}, {"sent": "you gonna put that on the stove ?", "words": ["the", "put", "you", "stove", "on", "that"]}, {"sent": "put this on the stove .", "words": ["the", "put", "this", "stove", "on"]}, {"sent": "oh thats a stove .", "words": ["stove", "a"]}, {"sent": "turn the stove on .", "words": ["the", "on", "stove"]}, {"sent": "put it on the stove .", "words": ["the", "put", "it", "stove", "on"]}, {"sent": "yeah its on the stove .", "words": ["on", "the", "stove"]}, {"sent": "he does have a stove at home .", "words": ["does", "have", "he", "stove", "at", "home", "a"]}, {"sent": "wanna put the pot on the stove ?", "words": ["the", "put", "stove", "wanna", "on"]}, {"sent": "is that a stove ?", "words": ["that", "stove", "a", "is"]}, {"sent": "we put it on the stove of the kitchen dont we ?", "words": ["the", "put", "we", "of", "dont", "it", "stove", "kitchen", "on"]}, {"sent": "I dont want you near the stove .", "words": ["the", "dont", "stove", "you"]}, {"sent": "when I cook something on the stove what do I sometimes fry it in ?", "words": ["the", "what", "do", "cook", "it", "in", "stove", "on", "when"]}, {"sent": "yeah we dont cut the stove while the stoves on .", "words": ["the", "cut", "we", "dont", "stove", "on"]}, {"sent": "you have kitchen and stove at home ?", "words": ["and", "have", "you", "stove", "kitchen", "at", "home"]}, {"sent": "on top of the stove please .", "words": ["on", "of", "stove", "the"]}, {"sent": "can you put your stool over by the stove and well put these on to cook .", "words": ["can", "the", "put", "these", "your", "to", "you", "by", "and", "cook", "stove", "on", "over"]}, {"sent": "I found a stove to do cooking on .", "words": ["do", "to", "stove", "on", "a"]}, {"sent": "its the stove .", "words": ["the", "stove"]}, {"sent": "another stove .", "words": ["another", "stove"]}, {"sent": "is the stove hot ?", "words": ["the", "hot", "stove", "is"]}, {"sent": "down the side of the stove ?", "words": ["the", "of", "down", "stove"]}, {"sent": "you gonna cook your juice on the stove ?", "words": ["the", "your", "you", "cook", "stove", "on", "juice"]}, {"sent": "this is a Eva stove .", "words": ["this", "stove", "a", "is"]}, {"sent": "Ellie let us borrow the stove .", "words": ["the", "us", "stove"]}, {"sent": "I hafta stir whats on the stove first so it doesnt burn .", "words": ["the", "so", "first", "it", "stove", "on"]}, {"sent": "I could make some for you though on the stove .", "words": ["the", "for", "you", "stove", "on", "some", "could", "make"]}, {"sent": "gonna put it on the stove .", "words": ["the", "put", "it", "stove", "on"]}, {"sent": "Daniel do you have a stove in your bed in your school at the alliance ?", "words": ["the", "do", "your", "school", "bed", "you", "have", "in", "stove", "at", "a"]}, {"sent": "yeah she has the toy food and she has like the little sink or yeah the stove and refrigerator with a bunch of play food so .", "words": ["the", "refrigerator", "of", "so", "play", "with", "like", "and", "little", "toy", "food", "sink", "stove", "she", "a"]}, {"sent": "hurry turn off the stove .", "words": ["off", "the", "hurry", "stove"]}, {"sent": "and the little stove top .", "words": ["the", "stove", "little", "and"]}, {"sent": "I hafta go back to the stove because I actually have something on the stove .", "words": ["the", "back", "go", "to", "have", "stove", "on", "because"]}, {"sent": "but its interesting because they know what to do at the stove so they know what .", "words": ["the", "what", "do", "to", "so", "they", "stove", "but", "at", "because"]}, {"sent": "thats not the stove .", "words": ["the", "not", "stove"]}, {"sent": "that sounds like the stove ?", "words": ["the", "that", "like", "stove"]}, {"sent": "its a stove .", "words": ["stove", "a"]}, {"sent": "puts the kettle on the stove .", "words": ["the", "on", "stove"]}, {"sent": "mm it was on the stove so it might be hot .", "words": ["the", "was", "so", "be", "hot", "it", "stove", "on"]}, {"sent": "you like to play with your the stove ?", "words": ["the", "your", "to", "play", "with", "like", "you", "stove"]}, {"sent": "put it on the stove okay .", "words": ["the", "put", "it", "stove", "on"]}, {"sent": "look we have a stove over here mommy didnt see that .", "words": ["we", "have", "here", "that", "stove", "look", "see", "over", "a"]}, {"sent": "it looks like a stove put that away where that came from too .", "words": ["where", "put", "like", "it", "away", "stove", "too", "that", "a"]}, {"sent": "can you put it on the stove ?", "words": ["can", "the", "put", "you", "it", "stove", "on"]}, {"sent": "I dont really see a stove and a place to cook do you ?", "words": ["do", "to", "and", "you", "dont", "cook", "stove", "see", "a"]}, {"sent": "if you put the stove in itll be the kitchen .", "words": ["the", "put", "if", "be", "you", "in", "stove", "kitchen"]}, {"sent": "see the stove ?", "words": ["the", "see", "stove"]}, {"sent": "lets take this off the stove .", "words": ["off", "the", "this", "stove", "take"]}, {"sent": "oh thats the cooking bit wheres the cooking stove ?", "words": ["the", "stove"]}, {"sent": "thats a stove .", "words": ["stove", "a"]}, {"sent": "a stove top kitchen .", "words": ["stove", "kitchen", "a"]}, {"sent": "except thats a pretend stove huh so that ones not hot .", "words": ["so", "hot", "stove", "pretend", "that", "not", "a"]}, {"sent": "why dont you xxx and make a stove .", "words": ["and", "you", "dont", "stove", "make", "why", "a"]}, {"sent": "can you put it on the stove ?", "words": ["can", "the", "put", "you", "it", "stove", "on"]}, {"sent": "heres a stove .", "words": ["stove", "a"]}, {"sent": "look you have a little stove over there to bake on .", "words": ["to", "little", "have", "you", "there", "stove", "look", "on", "over", "a"]}, {"sent": "here Ill put this on the stove too .", "words": ["the", "put", "here", "this", "stove", "too", "on"]}, {"sent": "can I make a pie on the stove ?", "words": ["can", "the", "stove", "on", "make", "a"]}, {"sent": "next time dont leave your stove on .", "words": ["on", "dont", "your", "stove"]}, {"sent": "this is the stove see ?", "words": ["the", "is", "this", "stove", "see"]}, {"sent": "in the stove .", "words": ["the", "in", "stove"]}, {"sent": "and the pots and pans actually we havent done it but theyre sposta be you can cook them on the stove but theyre like little child sized ones .", "words": ["the", "can", "we", "be", "them", "like", "and", "you", "little", "cook", "it", "stove", "on", "but"]}, {"sent": "and she was mainly picking out stuff and cooking it for other people and we showed her how to work the stove and .", "words": ["the", "how", "for", "work", "we", "was", "to", "and", "other", "it", "stove", "out", "her", "she"]}, {"sent": "on the stove ?", "words": ["on", "the", "stove"]}, {"sent": "you have a stove here .", "words": ["you", "have", "here", "stove", "a"]}, {"sent": "I got a new stove after the last time .", "words": ["the", "last", "stove", "new", "a"]}, {"sent": "we dont cook peanut butter on the stove .", "words": ["the", "we", "dont", "cook", "butter", "stove", "peanut butter", "on"]}, {"sent": "look at the little stove .", "words": ["the", "look", "little", "stove", "at"]}, {"sent": "you gonna cook from the back of the stove ?", "words": ["the", "back", "of", "you", "cook", "stove"]}, {"sent": "I have some dal cooking on the stove I kinda need to stir it do you wanna come with me ?", "words": ["the", "do", "need", "need to", "to", "me", "with", "you", "have", "it", "stove", "wanna", "on", "some"]}, {"sent": "going to put it in your on your um stove and cook it ?", "words": ["put", "your", "to", "and", "in", "it", "cook", "stove", "on"]}, {"sent": "pots on the stove .", "words": ["on", "the", "stove"]}, {"sent": "stove ?", "words": ["stove"]}, {"sent": "I think your pots a little bit big for your stove but that should work .", "words": ["think", "for", "your", "work", "little", "that", "stove", "big", "but", "a"]}, {"sent": "um he was interested by I guess the sound this one the stove was making the little timer .", "words": ["the", "was", "by", "little", "this", "he", "stove"]}, {"sent": "I say a stove at school .", "words": ["school", "say", "stove", "at", "a"]}, {"sent": "yyy and you said he does have a stove like this at home ?", "words": ["does", "like", "you", "have", "and", "this", "he", "stove", "at", "home", "a"]}, {"sent": "yeah use that for the stove .", "words": ["the", "for", "that", "stove"]}, {"sent": "you turned on the stove .", "words": ["on", "the", "stove", "you"]}, {"sent": "he has a stove on his back .", "words": ["back", "his", "he", "stove", "on", "a"]}, {"sent": "you gonna cook ?", "words": ["cook", "you"]}, {"sent": "so you said she watches you cook at home ?", "words": ["so", "you", "cook", "at", "she", "home"]}, {"sent": "here put your pot on the stove and cook it .", "words": ["the", "put", "your", "and", "here", "cook", "it", "stove", "on"]}, {"sent": "I hafta pretend to cook him so how do we play it whatd you need in order to play the game ?", "words": ["the", "do", "how", "game", "need", "we", "to", "so", "play", "you", "cook", "him", "it", "in", "pretend"]}, {"sent": "did you cook it ?", "words": ["cook", "it", "did", "you"]}, {"sent": "you gonna cook something for us ?", "words": ["us", "cook", "for", "you"]}, {"sent": "you wanna cook some peas I think thats a good idea .", "words": ["think", "peas", "you", "cook", "wanna", "good", "some", "a"]}, {"sent": "cook some dinner ?", "words": ["some", "cook"]}, {"sent": "we hafta cook it first .", "words": ["cook", "it", "first", "we"]}, {"sent": "just pretend cook hes not really .", "words": ["pretend", "cook", "not"]}, {"sent": "cook .", "words": ["cook"]}, {"sent": "you cook it then .", "words": ["cook", "it", "then", "you"]}, {"sent": "got to cook Harriet .", "words": ["to", "cook"]}, {"sent": "do you wanna cook some hot dog ?", "words": ["do", "you", "cook", "hot", "wanna", "some", "dog"]}, {"sent": "or do you just wanna cook for someone else ?", "words": ["do", "for", "you", "cook", "wanna"]}, {"sent": "put them in the pan and cook it ?", "words": ["the", "put", "them", "and", "in", "cook", "it"]}, {"sent": "you sure people cook grapes though ?", "words": ["cook", "grapes", "you"]}, {"sent": "how do you cook eggs ?", "words": ["do", "how", "cook", "you"]}, {"sent": "you can help Mama cook .", "words": ["can", "help", "cook", "you"]}, {"sent": "you have the spoons like when we cook with daddy ?", "words": ["the", "we", "with", "like", "you", "have", "cook", "when"]}, {"sent": "shes just not a very good cook .", "words": ["cook", "not", "good", "a"]}, {"sent": "oh I dont think they cook them .", "words": ["think", "them", "dont", "cook", "they"]}, {"sent": "oh are you the cook now ?", "words": ["the", "cook", "are", "you"]}, {"sent": "you hafta cook with a pot .", "words": ["cook", "with", "a", "you"]}, {"sent": "okay you cook dinner for Dolly and Mommy now .", "words": ["cook", "for", "and", "you"]}, {"sent": "my husband is the cook .", "words": ["the", "my", "cook", "is"]}, {"sent": "you gonna cook corn ?", "words": ["cook", "corn", "you"]}, {"sent": "put the um put the um the chicken in there and then we leave it to cook .", "words": ["the", "put", "we", "to", "and", "there", "in", "it", "cook", "chicken", "then"]}, {"sent": "Im gonna cook it in the hot water and then it will get soft enough to eat .", "words": ["the", "soft", "to", "and", "get", "eat", "in", "cook", "it", "hot", "water", "will", "then"]}, {"sent": "you gonna cook that too ?", "words": ["too", "cook", "that", "you"]}, {"sent": "Ill hafta cook them first .", "words": ["cook", "first", "them"]}, {"sent": "so when I learn how to cook Ill know where to find everything .", "words": ["where", "how", "to", "so", "cook", "find", "when"]}, {"sent": "I dont think did she try to cook on it at all though ?", "words": ["all", "think", "to", "dont", "cook", "it", "did", "at", "on", "try to", "try", "she"]}, {"sent": "yeah we can both cook them .", "words": ["can", "cook", "them", "we"]}, {"sent": "maybe you could cook something on the oven for Mommy ?", "words": ["the", "for", "you", "cook", "on", "could", "oven"]}, {"sent": "he said thats um what you cook in that funny smelling stuff my daddys friends .", "words": ["what", "my", "you", "cook", "in", "he", "that"]}, {"sent": "do you wanna cook the pig ?", "words": ["the", "do", "you", "pig", "cook", "wanna"]}, {"sent": "Im gonna cook it .", "words": ["cook", "it"]}, {"sent": "oh you gonna cook ?", "words": ["cook", "you"]}, {"sent": "you cant cook anymore ?", "words": ["cook", "you"]}, {"sent": "see Dad is a good cook you see .", "words": ["is", "you", "cook", "good", "see", "a"]}, {"sent": "did you cook dinner ?", "words": ["cook", "did", "you"]}, {"sent": "you gonna cook mommy some muffins ?", "words": ["some", "cook", "you"]}, {"sent": "Im going to cook it .", "words": ["to", "cook", "it"]}, {"sent": "whats here lets take the shoe out of the pot I dont think that we should cook the shoe .", "words": ["the", "think", "we", "of", "shoe", "here", "dont", "that", "cook", "out", "take"]}, {"sent": "uh Cyrus Momma cook .", "words": ["cook"]}, {"sent": "does she like watch you cook anything specific ?", "words": ["does", "watch", "like", "you", "cook", "she"]}, {"sent": "you gonna cook some dinner ?", "words": ["some", "cook", "you"]}, {"sent": "are you going to cook it ?", "words": ["to", "you", "cook", "are", "it"]}, {"sent": "yeah oh look you can cook here .", "words": ["can", "you", "here", "cook", "look"]}, {"sent": "I think were gonna cook something with .", "words": ["think", "cook", "were", "with"]}, {"sent": "he still loves to cook .", "words": ["to", "cook", "he"]}, {"sent": "and pretend to cook food and open and close cupboards and move things around .", "words": ["open", "to", "and", "close", "cook", "food", "around", "pretend"]}, {"sent": "sausages would start to cook .", "words": ["to", "cook", "would"]}, {"sent": "going to cook some fish Harry ?", "words": ["to", "cook", "fish", "some"]}, {"sent": "have you got a frying pan to cook them ?", "words": ["to", "them", "you", "have", "cook", "a"]}, {"sent": "should we cook the octopus ?", "words": ["the", "cook", "we"]}, {"sent": "cook in the oven ?", "words": ["the", "in", "cook", "oven"]}, {"sent": "well Im sposta come and help cook dinner ?", "words": ["help", "cook", "and"]}, {"sent": "no Im getting ready to cook dinner .", "words": ["to", "cook"]}, {"sent": "is it exciting helping to cook ?", "words": ["to", "cook", "it", "is"]}, {"sent": "can you cook both of them now ?", "words": ["can", "of", "them", "you", "cook"]}, {"sent": "can they cook at the beach now ?", "words": ["can", "the", "they", "cook", "beach", "at"]}, {"sent": "you really dont want me to cook more penne pasta ?", "words": ["to", "you", "dont", "cook", "more", "me"]}, {"sent": "now we can cook the green beans .", "words": ["can", "the", "we", "green beans", "cook", "beans", "green"]}, {"sent": "what should you cook me for dinner ?", "words": ["what", "for", "you", "cook", "me"]}, {"sent": "gonna cook a turkey in there ?", "words": ["there", "in", "cook", "turkey", "a"]}, {"sent": "okay we needta cook it up .", "words": ["up", "cook", "it", "we"]}, {"sent": "I think it looks like cook .", "words": ["cook", "think", "it", "like"]}, {"sent": "Eleanor shall I cook your dinner for you ?", "words": ["cook", "for", "your", "you"]}, {"sent": "my girls like to cook invisible food .", "words": ["my", "to", "like", "cook", "food"]}, {"sent": "look you can cook .", "words": ["can", "cook", "you", "look"]}, {"sent": "do you wanna cook them all again Wes ?", "words": ["do", "all", "them", "you", "cook", "wanna"]}, {"sent": "what you gonna cook ?", "words": ["what", "cook", "you"]}, {"sent": "where is the wheres the cook ?", "words": ["where", "the", "cook", "is"]}, {"sent": "and uh she likes to help cook for real too .", "words": ["help", "for", "to", "and", "cook", "too", "she"]}, {"sent": "cut up pieces of string to cook ?", "words": ["up", "cut", "of", "to", "cook"]}, {"sent": "cook it on the stove .", "words": ["the", "cook", "it", "stove", "on"]}, {"sent": "I cook ?", "words": ["cook"]}, {"sent": "youve got to cook it first .", "words": ["to", "cook", "it", "first"]}, {"sent": "are you gonna cook the orange juice ?", "words": ["the", "you", "cook", "are", "orange", "juice"]}, {"sent": "whatre you gonna cook for me ?", "words": ["me", "cook", "for", "you"]}, {"sent": "are you going to cook it for us ?", "words": ["for", "to", "you", "us", "cook", "are", "it"]}, {"sent": "you know and then xxx this friend of mine told me about this magic spoon cook book so .", "words": ["of", "so", "and", "you", "this", "mine", "cook", "spoon", "about", "book", "me", "then"]}, {"sent": "but we do cook at home .", "words": ["do", "we", "cook", "at", "but", "home"]}, {"sent": "okay so what kinds of things did I pretend to cook you up to be ?", "words": ["up", "what", "of", "so", "to", "be", "you", "cook", "did", "pretend"]}, {"sent": "whenever they were hungry they said cook little pot cook .", "words": ["hungry", "little", "they", "cook", "were"]}, {"sent": "yeah we do good with the xxx cook book .", "words": ["the", "do", "we", "with", "cook", "good", "book"]}, {"sent": "youve got to cook it first ?", "words": ["to", "cook", "it", "first"]}, {"sent": "what are you gonna cook for Mommy Im hungry .", "words": ["what", "for", "hungry", "you", "cook", "are"]}, {"sent": "mm youre a good cook .", "words": ["cook", "good", "a"]}, {"sent": "you wanna cook ?", "words": ["cook", "wanna", "you"]}, {"sent": "dont cook it thats gross .", "words": ["dont", "cook", "it"]}, {"sent": "what do you think shell cook for tea Dan ?", "words": ["what", "do", "think", "for", "you", "cook"]}, {"sent": "can you cook me some more food ?", "words": ["can", "you", "cook", "more", "food", "me", "some"]}, {"sent": "are you gonna cook them something on the stove ?", "words": ["the", "them", "you", "cook", "are", "stove", "on"]}, {"sent": "thats exactly how you eat corn at home .", "words": ["how", "you", "eat", "corn", "at", "home"]}, {"sent": "now is that corn too hot to eat ?", "words": ["is", "to", "eat", "hot", "corn", "too", "that"]}, {"sent": "corn carrots .", "words": ["carrots", "corn"]}, {"sent": "corn .", "words": ["corn"]}, {"sent": "what do we put on our corn flakes ?", "words": ["what", "do", "put", "we", "our", "corn", "on"]}, {"sent": "ah what I wanted I thought we had some gonna make some can muffins cause that goes well with fish though I was outof corn muffin .", "words": ["can", "what", "muffin", "we", "was", "with", "that", "corn", "fish", "some", "make"]}, {"sent": "Helen you have more corn on here and you hafta eat your fish and you hafta eat the spinach .", "words": ["the", "your", "you", "have", "here", "more", "and", "eat", "corn", "fish", "on"]}, {"sent": "corn on the cob .", "words": ["on", "the", "corn"]}, {"sent": "see all that corn on there ?", "words": ["all", "there", "corn", "on", "see", "that"]}, {"sent": "oh you got some corn ?", "words": ["some", "corn", "you"]}, {"sent": "or you could put that butter on your corn .", "words": ["put", "your", "you", "butter", "corn", "on", "could", "that"]}, {"sent": "corn .", "words": ["corn"]}, {"sent": "well we could pretend this was corn because it is yellow like corn .", "words": ["we", "is", "was", "like", "this", "it", "corn", "yellow", "pretend", "because", "could"]}, {"sent": "popcorn comes from corn .", "words": ["popcorn", "corn"]}, {"sent": "corn flakes .", "words": ["corn"]}, {"sent": "xxx eat you corn curl .", "words": ["eat", "corn", "you"]}, {"sent": "Im not sure if horses are allowed to eat corn cobs or not .", "words": ["if", "to", "eat", "are", "corn", "not"]}, {"sent": "I want another corn .", "words": ["another", "corn"]}, {"sent": "corn .", "words": ["corn"]}, {"sent": "where is the corn ?", "words": ["where", "the", "corn", "is"]}, {"sent": "do you see any corn ?", "words": ["do", "any", "you", "corn", "see"]}, {"sent": "does he like to eat corn too ?", "words": ["does", "to", "like", "eat", "he", "corn", "too"]}, {"sent": "er you havent done corn .", "words": ["corn", "you"]}, {"sent": "Id Id like to have some corn please .", "words": ["to", "like", "have", "corn", "some"]}, {"sent": "Jimmy crack corn I dont care Jimmy crack corn I dont care Jimmy crack corn I dont care oh masters gone away .", "words": ["dont", "away", "corn"]}, {"sent": "corn .", "words": ["corn"]}, {"sent": "by corn turkeys by ?", "words": ["corn", "by"]}, {"sent": "I do work theyre carrying corn .", "words": ["do", "work", "corn"]}, {"sent": "there there are more pieces of corn .", "words": ["of", "there", "more", "are", "corn"]}, {"sent": "and corn on the cob .", "words": ["on", "the", "corn", "and"]}, {"sent": "there is no more corn candy but here is a pumpkin .", "words": ["is", "here", "there", "more", "corn", "pumpkin", "candy", "but", "a"]}, {"sent": "um she probably dont know what corn is because I dont cook it that much .", "words": ["what", "is", "much", "dont", "cook", "it", "corn", "that", "she", "because"]}, {"sent": "its corn .", "words": ["corn"]}, {"sent": "you dont like corn ?", "words": ["dont", "like", "corn", "you"]}, {"sent": "thats some good corn .", "words": ["some", "good", "corn"]}, {"sent": "corn on the cob ?", "words": ["on", "the", "corn"]}, {"sent": "you put butter on corn .", "words": ["put", "you", "butter", "corn", "on"]}, {"sent": "Im sorry not the corn .", "words": ["the", "not", "corn"]}, {"sent": "there it is you found the corn hooray .", "words": ["the", "is", "you", "there", "it", "corn"]}, {"sent": "and then eat the corn .", "words": ["the", "and", "eat", "corn", "then"]}, {"sent": "corn ?", "words": ["corn"]}, {"sent": "well thats what most corn looks at looks like Sarah .", "words": ["what", "like", "corn", "at"]}, {"sent": "could I have some corn on the cob ?", "words": ["the", "have", "corn", "on", "some", "could"]}, {"sent": "in fact well heat Daddy up some corn I think .", "words": ["up", "think", "in", "corn", "some"]}, {"sent": "that corn on the cob ?", "words": ["on", "the", "that", "corn"]}, {"sent": "corn is a tall leafy plant grown for its plump yellow cobs of grain .", "words": ["for", "is", "of", "corn", "yellow", "plant", "a"]}, {"sent": "Roman did you eat corn on Thanksgiving ?", "words": ["you", "eat", "did", "corn", "on"]}, {"sent": "corn .", "words": ["corn"]}, {"sent": "frozen corn .", "words": ["corn"]}, {"sent": "he loves corn on the cob .", "words": ["on", "the", "he", "corn"]}, {"sent": "we go to the corn and I buy xxx huh ?", "words": ["the", "go", "we", "to", "and", "corn", "buy"]}, {"sent": "no thats corn .", "words": ["corn"]}, {"sent": "black crows on the yellow corn .", "words": ["the", "black", "corn", "on", "yellow"]}, {"sent": "theyre having some nice corn cobs .", "words": ["some", "nice", "corn"]}, {"sent": "see if you can find a can that has pictures of corn on it ?", "words": ["can", "if", "of", "you", "it", "corn", "find", "on", "see", "that", "a"]}, {"sent": "can I have some corn ?", "words": ["can", "some", "have", "corn"]}, {"sent": "Jimmy crack corn I dont care .", "words": ["dont", "corn"]}, {"sent": "thats the corn .", "words": ["the", "corn"]}, {"sent": "dont put the corn in your mouth .", "words": ["the", "put", "your", "mouth", "dont", "in", "corn"]}, {"sent": "do you want some cold corn ?", "words": ["do", "you", "corn", "some", "cold"]}, {"sent": "good bite of corn and peas ?", "words": ["of", "peas", "and", "good", "corn", "bite"]}, {"sent": "you had corn for lunch ?", "words": ["for", "corn", "you"]}, {"sent": "uh he knows corn .", "words": ["he", "corn"]}, {"sent": "corn .", "words": ["corn"]}, {"sent": "see that corn right there ?", "words": ["see", "that", "corn", "there"]}, {"sent": "theres corn .", "words": ["corn"]}, {"sent": "corn yum .", "words": ["corn"]}, {"sent": "corn .", "words": ["corn"]}, {"sent": "the sheeps in the meadow cows in the corn .", "words": ["the", "in", "corn"]}, {"sent": "pretend youre biting the corn .", "words": ["the", "pretend", "corn"]}, {"sent": "they eat corn .", "words": ["eat", "they", "corn"]}, {"sent": "corn .", "words": ["corn"]}, {"sent": "thats a corn cob yeah .", "words": ["corn", "a"]}, {"sent": "thats why we dont have corn on the cob .", "words": ["the", "we", "have", "dont", "corn", "on", "why"]}, {"sent": "an ear of corn .", "words": ["of", "an", "corn", "ear"]}, {"sent": "I want another corn .", "words": ["another", "corn"]}, {"sent": "gonna have corn and some fish ?", "words": ["and", "have", "corn", "fish", "some"]}, {"sent": "an ear of corn has a cob right in the middle of it .", "words": ["the", "of", "an", "in", "it", "corn", "ear", "a"]}, {"sent": "Nomi do you want corn flakes or Crispies or Cheerios ?", "words": ["do", "corn", "you"]}, {"sent": "ooh corn .", "words": ["corn"]}, {"sent": "thank you Francis for cooking corn and chocolate .", "words": ["for", "chocolate", "and", "you", "corn"]}, {"sent": "thats corn .", "words": ["corn"]}, {"sent": "he carried he brought out the corn flakes he helped to set the table .", "words": ["the", "table", "to", "he", "out", "corn"]}, {"sent": "how many corn ?", "words": ["how", "corn"]}, {"sent": "corn and car ?", "words": ["car", "corn", "and"]}, {"sent": "goes with the corn on your chin .", "words": ["the", "your", "with", "chin", "corn", "on"]}, {"sent": "he still calls corn dogs burgers .", "words": ["he", "corn"]}, {"sent": "you like corn on the cob .", "words": ["the", "like", "you", "corn", "on"]}, {"sent": "thats corn .", "words": ["corn"]}, {"sent": "Nebraskas got lotsof corn .", "words": ["corn"]}, {"sent": "oh look theres mice in the corn seed .", "words": ["the", "in", "corn", "look"]}, {"sent": "caramel corn hey way to go kid we do make caramel corn with peanuts .", "words": ["do", "we", "go", "to", "with", "corn", "make"]}, {"sent": "corn ?", "words": ["corn"]}, {"sent": "corn .", "words": ["corn"]}, {"sent": "here give me some corn on the cob too .", "words": ["the", "here", "corn", "too", "on", "me", "give", "some"]}, {"sent": "oh your squash and corn .", "words": ["your", "corn", "and"]}, {"sent": "popping corn .", "words": ["corn"]}, {"sent": "I know you know some other songs you know Jimmy crack corn song ?", "words": ["some", "other", "corn", "you"]}, {"sent": "theres a little corn man a bunch of grapes .", "words": ["of", "little", "grapes", "corn", "a"]}, {"sent": "couldnt work out if that was corn flour or icing sugar .", "words": ["work", "if", "was", "out", "corn", "that"]}, {"sent": "well put it on the conveyor belt .", "words": ["the", "put", "belt", "it", "on"]}, {"sent": "like a belt .", "words": ["belt", "like", "a"]}, {"sent": "wow he has a ring on his belt .", "words": ["belt", "his", "he", "on", "a"]}, {"sent": "daddys belt .", "words": ["belt"]}, {"sent": "heres a belt .", "words": ["belt", "a"]}, {"sent": "he needs a belt huh ?", "words": ["he", "belt", "a"]}, {"sent": "show Lois your belt .", "words": ["show", "your", "belt"]}, {"sent": "lets see I have a lil belt .", "words": ["have", "see", "belt", "a"]}, {"sent": "and if youre gonna wear a belt then you put this through here through the hooks .", "words": ["the", "put", "if", "belt", "and", "you", "this", "here", "then", "a"]}, {"sent": "but yours is a definite belt isnt it ?", "words": ["is", "belt", "it", "but", "a"]}, {"sent": "oh thats a nice belt .", "words": ["nice", "belt", "a"]}, {"sent": "AnnMarie has a belt but you have no belt .", "words": ["belt", "you", "have", "but", "a"]}, {"sent": "and this is a belt .", "words": ["is", "belt", "and", "this", "a"]}, {"sent": "that belt ?", "words": ["that", "belt"]}, {"sent": "its his belt again isnt it ?", "words": ["his", "it", "belt"]}, {"sent": "I mean or the uh t tool belt .", "words": ["the", "belt"]}, {"sent": "xxx will you help me pick out my belt Violet ?", "words": ["help", "my", "pick", "belt", "you", "out", "will", "me"]}, {"sent": "lets have a red belt shall we ?", "words": ["red", "we", "belt", "have", "a"]}, {"sent": "oh he doesnt have the seat belt on yet ?", "words": ["the", "belt", "have", "he", "on"]}, {"sent": "did you tie a big belt around them ?", "words": ["belt", "them", "you", "did", "around", "big", "a"]}, {"sent": "youre eating mommys belt honey other people hafta wear that belt too .", "words": ["too", "that", "belt", "other"]}, {"sent": "you lost yer tool belt .", "words": ["belt", "you"]}, {"sent": "belt on otherwise he might fall out .", "words": ["belt", "fall", "he", "out", "on"]}, {"sent": "Im wearing my hard hat and tool belt full of tools ready for work .", "words": ["my", "for", "work", "full", "of", "belt", "and", "hard", "hat"]}, {"sent": "tool belt see you wear it around you like this .", "words": ["belt", "like", "you", "this", "it", "around", "see"]}, {"sent": "red belt .", "words": ["red", "belt"]}, {"sent": "belt .", "words": ["belt"]}, {"sent": "I think its a tool belt .", "words": ["think", "belt", "a"]}, {"sent": "just in case you fall off you had better put your belt on .", "words": ["off", "better", "put", "your", "belt", "fall", "you", "in", "on"]}, {"sent": "let me let me just fasten the belt please .", "words": ["the", "me", "belt"]}, {"sent": "Ben made that belt .", "words": ["that", "belt"]}, {"sent": "that the baby belt .", "words": ["the", "that", "belt"]}, {"sent": "thats a belt .", "words": ["belt", "a"]}, {"sent": "do you wanna put on your seat belt or should I ?", "words": ["do", "put", "your", "belt", "you", "wanna", "on"]}, {"sent": "is that a belt or a scar ?", "words": ["that", "belt", "a", "is"]}, {"sent": "a pink belt .", "words": ["belt", "a"]}, {"sent": "you two will go in one seat belt .", "words": ["go", "belt", "you", "in", "will"]}, {"sent": "take off your safety belt .", "words": ["off", "take", "belt", "your"]}, {"sent": "is that the belt of the jacket ?", "words": ["the", "is", "of", "belt", "that", "jacket"]}, {"sent": "mm now he needs some tools on his tool belt doesnt he ?", "words": ["belt", "his", "he", "on", "some"]}, {"sent": "I gotta put my tool belt on .", "words": ["on", "put", "belt", "my"]}, {"sent": "gosh hes got a big belt .", "words": ["big", "belt", "a"]}, {"sent": "a gravel pit belt .", "words": ["belt", "a"]}, {"sent": "do you think your belt will fit in this toolbox ?", "words": ["do", "think", "your", "fit", "belt", "you", "this", "in", "will"]}, {"sent": "belt buckle ?", "words": ["belt"]}, {"sent": "so you hafta wear a belt ?", "words": ["so", "belt", "a", "you"]}, {"sent": "a tool belt and you put your .", "words": ["put", "your", "belt", "you", "and", "a"]}, {"sent": "where is the belt ?", "words": ["where", "the", "belt", "is"]}, {"sent": "there you have a belt .", "words": ["belt", "you", "have", "there", "a"]}, {"sent": "this is your belt .", "words": ["this", "your", "belt", "is"]}, {"sent": "BobtheBuilder has got his belt .", "words": ["his", "belt"]}, {"sent": "what how come you hafta take his belt off ?", "words": ["off", "what", "how", "belt", "you", "his", "take"]}, {"sent": "you were too busy unloading them from the shopping trolley onto the conveyer belt .", "words": ["the", "them", "belt", "you", "too", "were"]}, {"sent": "did he have his seat belt on ?", "words": ["belt", "have", "on", "he", "did", "his"]}, {"sent": "oh well pretend that he has a seat belt .", "words": ["belt", "he", "pretend", "that", "a"]}, {"sent": "and look shes wearing a belt just like BobtheBuilder isnt she ?", "words": ["belt", "like", "and", "look", "she", "a"]}, {"sent": "with a BobTheBuilder belt .", "words": ["with", "belt", "a"]}, {"sent": "Im a get duh belt .", "words": ["get", "belt", "a"]}, {"sent": "mm Im gonna make a tool belt after I color in .", "words": ["in", "make", "belt", "a"]}, {"sent": "Joe theres your belt where is it ?", "words": ["where", "your", "is", "belt", "it"]}, {"sent": "thats right you did you did get in a belt buckle .", "words": ["belt", "you", "get", "in", "did", "a"]}, {"sent": "is this a belt ?", "words": ["this", "belt", "a", "is"]}, {"sent": "Wendys wearing his belt .", "words": ["his", "belt"]}, {"sent": "Mommys got Daddys belt .", "words": ["belt"]}, {"sent": "and that belt that BobTheBuilders wearing has tools all the way round it doesnt it ?", "words": ["the", "all", "belt", "and", "it", "that"]}, {"sent": "and theyve got a belt .", "words": ["belt", "a", "and"]}, {"sent": "could you find my belt that you pulled off my top xxx ?", "words": ["off", "my", "belt", "you", "find", "could", "that"]}, {"sent": "its just like a belt .", "words": ["belt", "like", "a"]}, {"sent": "you mean you didnt have your seat your belt on all this time sheesh .", "words": ["all", "your", "belt", "you", "have", "this", "on"]}, {"sent": "do you like that belt ?", "words": ["do", "belt", "like", "you", "that"]}, {"sent": "no its not your belt darling .", "words": ["your", "not", "belt"]}, {"sent": "this sleeve makes a nice belt dont you think Miss Honey says .", "words": ["think", "nice", "belt", "you", "dont", "this", "a"]}, {"sent": "I dont know I got it from my girlfriend when she moved and she says that the belt doesnt work sometimes .", "words": ["the", "my", "work", "belt", "and", "dont", "that", "it", "when", "she"]}, {"sent": "yeah Jwwws got a belt on today hasnt he ?", "words": ["on", "he", "belt", "a"]}, {"sent": "let me just get my belt .", "words": ["get", "me", "my", "belt"]}, {"sent": "the belt is called a cummerbund .", "words": ["the", "belt", "a", "is"]}, {"sent": "the asteroid belt is located between Mars and Jupiter .", "words": ["the", "belt", "and", "is"]}, {"sent": "youre gonna close the belt ?", "words": ["the", "close", "belt"]}, {"sent": "belt ?", "words": ["belt"]}, {"sent": "where dat belt at ?", "words": ["where", "belt", "at"]}, {"sent": "this is your tool belt your tool belt look .", "words": ["your", "is", "belt", "this", "look"]}, {"sent": "thats my belt to keep my back together .", "words": ["to", "my", "belt", "back"]}, {"sent": "thats right put him on aw belt .", "words": ["on", "put", "belt", "him"]}, {"sent": "do you like that belt ?", "words": ["do", "belt", "like", "you", "that"]}, {"sent": "what have you got in a belt ?", "words": ["what", "belt", "you", "have", "in", "a"]}, {"sent": "little belt .", "words": ["belt", "little"]}, {"sent": "one on her belt .", "words": ["on", "her", "belt"]}, {"sent": "it looks like a fancy belt .", "words": ["it", "belt", "like", "a"]}, {"sent": "maybe on your belt you have four because my belt sure because he his belt sure doesnt have four .", "words": ["my", "your", "belt", "you", "have", "his", "he", "on", "because"]}, {"sent": "youll hafta show Kate your tool belt .", "words": ["show", "your", "belt"]}, {"sent": "xxx the asteroid belt .", "words": ["the", "belt"]}, {"sent": "get your belt off .", "words": ["get", "your", "off", "belt"]}, {"sent": "going along the conveyor belt .", "words": ["the", "belt"]}, {"sent": "I think you well you got your white belt .", "words": ["think", "your", "belt", "you", "white"]}, {"sent": "kinda like a belt .", "words": ["belt", "like", "a"]}, {"sent": "thats er his belt darling .", "words": ["his", "belt"]}, {"sent": "are you putting it on the conveyor belt .", "words": ["the", "belt", "you", "it", "are", "on"]}, {"sent": "yeah can you see his tool belt ?", "words": ["can", "belt", "you", "his", "see"]}, {"sent": "do you have a belt ?", "words": ["do", "belt", "you", "have", "a"]}, {"sent": "youre putting it under my belt .", "words": ["under", "my", "it", "belt"]}, {"sent": "Ill have this this as pudding .", "words": ["have", "this", "pudding"]}, {"sent": "no you eat your pudding .", "words": ["eat", "pudding", "your", "you"]}, {"sent": "no you have pudding for lunch .", "words": ["have", "pudding", "for", "you"]}, {"sent": "aw okay eat your pudding .", "words": ["eat", "pudding", "your"]}, {"sent": "Christmas pudding .", "words": ["pudding"]}, {"sent": "yes getting pudding on it .", "words": ["on", "pudding", "it"]}, {"sent": "I think the little girls gonna have some pudding .", "words": ["the", "think", "little", "have", "some", "pudding"]}, {"sent": "thats called pudding what you had for dinner .", "words": ["what", "pudding", "for", "you"]}, {"sent": "you wanna eat some pudding ?", "words": ["you", "eat", "wanna", "some", "pudding"]}, {"sent": "Violet do you like butterscotch pudding ?", "words": ["do", "pudding", "like", "you"]}, {"sent": "is this his pudding ?", "words": ["his", "this", "pudding", "is"]}, {"sent": "and a chocolate muffin for pudding .", "words": ["muffin", "for", "chocolate", "and", "pudding", "a"]}, {"sent": "is that for my pudding ?", "words": ["my", "for", "is", "pudding", "that"]}, {"sent": "oh ice cream for the pudding .", "words": ["the", "ice cream", "for", "ice", "pudding"]}, {"sent": "yeah doesnt it look like chocolate pudding or chocolate syrup ?", "words": ["chocolate", "like", "it", "look", "pudding"]}, {"sent": "is that for pudding ?", "words": ["pudding", "for", "that", "is"]}, {"sent": "well there isnt pudding Thomas you had cornflakes and I offered you toasted fruit bread .", "words": ["bread", "and", "you", "there", "pudding"]}, {"sent": "youre getting to be a right wee pudding lately .", "words": ["to", "pudding", "be", "a"]}, {"sent": "I like eating my butterscotch pudding slowly and then it lasts longer .", "words": ["my", "then", "like", "and", "it", "pudding"]}, {"sent": "because whos a Christmas pudding ?", "words": ["pudding", "because", "a"]}, {"sent": "the beat is what Mummy does when shes making batter like a Yorkshire pudding pancakes .", "words": ["the", "what", "does", "is", "like", "pudding", "when", "a"]}, {"sent": "did you grandma make you christmas pudding ?", "words": ["pudding", "make", "did", "you"]}, {"sent": "youll have your pudding now .", "words": ["have", "pudding", "your"]}, {"sent": "and then the first rule is when Im making a summer pudding .", "words": ["the", "is", "first", "and", "pudding", "then", "when", "a"]}, {"sent": "whatre you going to have for pudding ?", "words": ["for", "to", "you", "have", "pudding"]}, {"sent": "a fish a pudding and some pears and oranges .", "words": ["and", "fish", "some", "pudding", "a"]}, {"sent": "I thought you wanted to take this pudding home .", "words": ["to", "you", "this", "pudding", "take", "home"]}, {"sent": "pudding you just suck it down .", "words": ["down", "pudding", "it", "you"]}, {"sent": "roast beef and Yorkshire pudding .", "words": ["pudding", "and"]}, {"sent": "and would you like a Yorkshire pudding aswell ?", "words": ["like", "you", "and", "would", "pudding", "a"]}, {"sent": "sausages on top of vanilla rice pudding .", "words": ["on", "of", "pudding", "vanilla"]}, {"sent": "what would you like for pudding ?", "words": ["what", "for", "like", "you", "would", "pudding"]}, {"sent": "didnt you have any christmas pudding then ?", "words": ["any", "then", "you", "have", "pudding"]}, {"sent": "pudding .", "words": ["pudding"]}, {"sent": "Noel is a Christmas pudding uhn isnt he ?", "words": ["pudding", "he", "a", "is"]}, {"sent": "cheese for pudding ?", "words": ["pudding", "for", "cheese"]}, {"sent": "what do you want for your pudding ?", "words": ["what", "do", "for", "your", "you", "pudding"]}, {"sent": "black pudding ?", "words": ["black", "pudding"]}, {"sent": "pudding .", "words": ["pudding"]}, {"sent": "cant have pudding if you dont eat your main course .", "words": ["your", "if", "you", "have", "dont", "eat", "pudding"]}, {"sent": "hey listen how about some pudding .", "words": ["how", "listen", "about", "some", "pudding"]}, {"sent": "pudding for breakfast is it ?", "words": ["it", "pudding", "for", "is"]}, {"sent": "its for your pudding .", "words": ["pudding", "for", "your"]}, {"sent": "did you tell Jeannine what you yesterday for your pudding at home ?", "words": ["what", "for", "your", "you", "did", "at", "pudding", "home"]}, {"sent": "and then they gave you chocolate pudding with chocolate sprinkles .", "words": ["chocolate", "with", "you", "and", "they", "pudding", "then"]}, {"sent": "you like chocolate pudding ?", "words": ["pudding", "chocolate", "like", "you"]}, {"sent": "pudding .", "words": ["pudding"]}, {"sent": "would you like some more pudding ?", "words": ["like", "you", "more", "would", "some", "pudding"]}, {"sent": "would you like some pudding ?", "words": ["like", "you", "would", "some", "pudding"]}, {"sent": "remember that from last night pudding ?", "words": ["pudding", "last", "that"]}, {"sent": "pudding .", "words": ["pudding"]}, {"sent": "you want some pudding ?", "words": ["some", "pudding", "you"]}, {"sent": "when we went to supper last night did you have pudding ?", "words": ["we", "to", "last", "you", "have", "did", "pudding", "when"]}, {"sent": "and then you had chocolate muffin for pudding .", "words": ["muffin", "for", "chocolate", "you", "and", "pudding", "then"]}, {"sent": "would you like some chocolate pudding ?", "words": ["chocolate", "like", "you", "would", "some", "pudding"]}, {"sent": "what sort of pudding are we gonna have ?", "words": ["what", "we", "of", "have", "are", "pudding"]}, {"sent": "oh you havent seen whats for pudding .", "words": ["pudding", "for", "you"]}, {"sent": "have I got that for my pudding ?", "words": ["my", "for", "have", "pudding", "that"]}, {"sent": "and some pudding .", "words": ["some", "pudding", "and"]}, {"sent": "you wanna have a little pudding ?", "words": ["little", "have", "you", "wanna", "pudding", "a"]}, {"sent": "thats for pudding isnt it ?", "words": ["it", "pudding", "for"]}, {"sent": "more pudding ?", "words": ["more", "pudding"]}, {"sent": "thats how he likes to eat his pudding .", "words": ["how", "to", "eat", "he", "his", "pudding"]}, {"sent": "giant Yorkshire pudding .", "words": ["pudding"]}, {"sent": "xxx pudding .", "words": ["pudding"]}, {"sent": "its a chocolate um a Christmas pudding .", "words": ["pudding", "chocolate", "a"]}, {"sent": "I think the Christmas pudding was a bit too rich for you .", "words": ["the", "think", "for", "was", "you", "too", "pudding", "a"]}, {"sent": "thats Eves pudding .", "words": ["pudding"]}, {"sent": "do you want some pudding today Father ?", "words": ["do", "pudding", "some", "you"]}, {"sent": "are you making Yorkshire pudding ?", "words": ["pudding", "are", "you"]}, {"sent": "pudding .", "words": ["pudding"]}, {"sent": "what sort of pudding is ?", "words": ["of", "what", "pudding", "is"]}, {"sent": "oh well then Ill give all the pudding .", "words": ["the", "all", "then", "pudding", "give"]}, {"sent": "what am I having for pudding ?", "words": ["what", "pudding", "for", "am"]}, {"sent": "Ive nothing for pudding .", "words": ["pudding", "for"]}, {"sent": "thats a pudding ?", "words": ["pudding", "a"]}, {"sent": "Christmas pudding .", "words": ["pudding"]}, {"sent": "Christmas pudding .", "words": ["pudding"]}, {"sent": "a Christmas pudding .", "words": ["pudding", "a"]}, {"sent": "well you can have a sweetie but thats your pudding .", "words": ["can", "your", "you", "have", "pudding", "but", "a"]}, {"sent": "whatre you gonna do with the pudding then ?", "words": ["the", "do", "with", "then", "you", "pudding"]}, {"sent": "youre getting to be a big heavy pudding .", "words": ["to", "be", "pudding", "heavy", "big", "a"]}, {"sent": "Yorkshire pudding .", "words": ["pudding"]}, {"sent": "it looks like a Christmas pudding does it ?", "words": ["does", "like", "it", "pudding", "a"]}, {"sent": "youve just gone through the Yorkshire pudding .", "words": ["the", "pudding"]}, {"sent": "its sposta be a Christmas pudding with ice .", "words": ["be", "with", "ice", "pudding", "a"]}, {"sent": "pudding .", "words": ["pudding"]}, {"sent": "more pudding .", "words": ["more", "pudding"]}, {"sent": "are you gonna cook her some some pudding ?", "words": ["you", "cook", "are", "some", "pudding", "her"]}, {"sent": "Georgie Porgie pudding and pie .", "words": ["pudding", "and"]}, {"sent": "thats pudding .", "words": ["pudding"]}, {"sent": "wheres my pudding ?", "words": ["my", "pudding"]}, {"sent": "get the custard pudding .", "words": ["get", "the", "pudding"]}, {"sent": "whats for pudding ?", "words": ["pudding", "for"]}, {"sent": "I got pudding mix out .", "words": ["pudding", "out"]}, {"sent": "with banana for pudding and orange juice to drink .", "words": ["for", "drink", "to", "with", "and", "banana", "orange", "pudding", "juice"]}, {"sent": "yeah what about this flag ?", "words": ["flag", "what", "about", "this"]}, {"sent": "see the flag ?", "words": ["the", "see", "flag"]}, {"sent": "Daddy wants to get a flag as well .", "words": ["get", "to", "flag", "a"]}, {"sent": "and that red thing here is the flag .", "words": ["red", "the", "is", "flag", "and", "here", "that"]}, {"sent": "and wave the flag ?", "words": ["the", "flag", "and"]}, {"sent": "theres a flag isnt it ?", "words": ["flag", "it", "a"]}, {"sent": "wave the flag .", "words": ["the", "flag"]}, {"sent": "thats a fat cat with a flag of Denmark .", "words": ["of", "flag", "with", "cat", "a"]}, {"sent": "a flag was that ?", "words": ["was", "flag", "that", "a"]}, {"sent": "blow the flag ?", "words": ["the", "flag", "blow"]}, {"sent": "you thought it was a flag at first but then we told you it was an umbrella .", "words": ["we", "flag", "was", "first", "you", "an", "it", "at", "then", "but", "a"]}, {"sent": "Im having a green flag .", "words": ["flag", "green", "a"]}, {"sent": "the flag pole you see .", "words": ["the", "flag", "see", "you"]}, {"sent": "very good a flag .", "words": ["flag", "good", "a"]}, {"sent": "thats his flag .", "words": ["his", "flag"]}, {"sent": "the big flag .", "words": ["the", "flag", "big"]}, {"sent": "a flag and a bag .", "words": ["flag", "and", "a"]}, {"sent": "I wonder if that flag is brand new ?", "words": ["if", "is", "flag", "new", "that"]}, {"sent": "hes hid behind a flag .", "words": ["flag", "behind", "a"]}, {"sent": "see if hell let you have that flag .", "words": ["if", "flag", "you", "have", "see", "that"]}, {"sent": "wheres that flag ?", "words": ["flag", "that"]}, {"sent": "what do you say when you see the flag ?", "words": ["the", "what", "do", "flag", "say", "you", "see", "when"]}, {"sent": "youve got a flag havent you ?", "words": ["flag", "you", "a"]}, {"sent": "is that a flag ?", "words": ["flag", "that", "a", "is"]}, {"sent": "oh a flag hold it this way .", "words": ["hold", "flag", "this", "it", "a"]}, {"sent": "around the flag raising .", "words": ["the", "flag", "around"]}, {"sent": "can you see the house with the flag ?", "words": ["can", "the", "flag", "with", "you", "house", "see"]}, {"sent": "do you mean the flag ?", "words": ["the", "do", "flag", "you"]}, {"sent": "flag .", "words": ["flag"]}, {"sent": "the flag is going backwards .", "words": ["the", "flag", "is"]}, {"sent": "flag .", "words": ["flag"]}, {"sent": "I pledge allegiance to the flag of the United States .", "words": ["the", "to", "flag", "of"]}, {"sent": "your flag ?", "words": ["flag", "your"]}, {"sent": "it was this that reminded me it looks like the lolly stick that you had in Marks and Spencers I think this is actually off a flag isnt it ?", "words": ["the", "off", "stick", "think", "is", "was", "flag", "like", "you", "and", "this", "in", "it", "me", "that", "a"]}, {"sent": "a flag .", "words": ["flag", "a"]}, {"sent": "Adam we forgot to bring that flag home from nursery school .", "words": ["we", "flag", "to", "school", "bring", "that", "home"]}, {"sent": "blow the whistle and wave the flag .", "words": ["the", "flag", "blow", "and"]}, {"sent": "do you march like in school with your flag ?", "words": ["do", "your", "school", "flag", "with", "like", "you", "in"]}, {"sent": "a flag .", "words": ["flag", "a"]}, {"sent": "its a flag .", "words": ["flag", "a"]}, {"sent": "give me the flag .", "words": ["the", "me", "give", "flag"]}, {"sent": "a little flag isnt it ?", "words": ["flag", "it", "little", "a"]}, {"sent": "look at the flag Stephen .", "words": ["the", "flag", "at", "look"]}, {"sent": "theres that flag there and have we got a flag pole ?", "words": ["we", "flag", "and", "have", "there", "that", "a"]}, {"sent": "mm is that a little flag ?", "words": ["is", "flag", "little", "that", "a"]}, {"sent": "we can look at the flag xxx .", "words": ["can", "the", "we", "flag", "at", "look"]}, {"sent": "yep a flag .", "words": ["flag", "a"]}, {"sent": "and I want a flag on xxx .", "words": ["on", "flag", "a", "and"]}, {"sent": "theres a flag .", "words": ["flag", "a"]}, {"sent": "um thats a flag .", "words": ["flag", "a"]}, {"sent": "flag .", "words": ["flag"]}, {"sent": "so after we had been to music lesson we went into that shop in Didsbury and bought a flag didnt we ?", "words": ["into", "we", "to", "so", "flag", "and", "in", "that", "a"]}, {"sent": "a little flag .", "words": ["flag", "little", "a"]}, {"sent": "wave the flag ?", "words": ["the", "flag"]}, {"sent": "flag yes I know .", "words": ["flag"]}, {"sent": "youre going to get another McDonalds flag arent we ?", "words": ["we", "to", "flag", "get", "another"]}, {"sent": "thats a flag .", "words": ["flag", "a"]}, {"sent": "here are you gonna do that flag ?", "words": ["do", "flag", "you", "here", "are", "that"]}, {"sent": "Im its the flag of Romania .", "words": ["the", "flag", "of"]}, {"sent": "thats the American flag .", "words": ["the", "flag"]}, {"sent": "thats a flag like on a castle ?", "words": ["on", "flag", "like", "a"]}, {"sent": "oh hes up there near the flag .", "words": ["up", "there", "flag", "the"]}, {"sent": "no no no the flag .", "words": ["the", "flag"]}, {"sent": "and what about the guard here waving his flag ?", "words": ["the", "what", "flag", "and", "here", "about", "his"]}, {"sent": "thats the flag .", "words": ["the", "flag"]}, {"sent": "after you had waved your flag at the train we saw another one on the bridge didnt we ?", "words": ["the", "your", "train", "we", "flag", "you", "another", "at", "on"]}, {"sent": "Daddy wants to get a flag as well .", "words": ["get", "to", "flag", "a"]}, {"sent": "its got a little flag hasnt it ?", "words": ["flag", "it", "little", "a"]}, {"sent": "does the guard usually wave the flag Thomas on the train ?", "words": ["the", "does", "train", "flag", "on"]}, {"sent": "the guard who waves his flag .", "words": ["the", "his", "flag", "who"]}, {"sent": "thats the flag .", "words": ["the", "flag"]}, {"sent": "and the FatControllers waving his flag isnt he ?", "words": ["the", "flag", "and", "he", "his"]}, {"sent": "what kind of flag ?", "words": ["of", "what", "flag"]}, {"sent": "and thats the flag on the train waving just like you saw on Friday .", "words": ["the", "train", "flag", "like", "and", "you", "on"]}, {"sent": "hes holding a flag .", "words": ["flag", "a"]}, {"sent": "thats a little flag .", "words": ["flag", "little", "a"]}, {"sent": "thats a little flag on the boat .", "words": ["the", "flag", "little", "on", "boat", "a"]}, {"sent": "go and eat a flag ?", "words": ["go", "flag", "and", "eat", "a"]}, {"sent": "but if the wind is if that a the flag were right here and the wind was coming from over here it would go prr see ?", "words": ["the", "were", "if", "a", "is", "go", "flag", "was", "and", "here", "it", "would", "over", "see", "that", "but", "wind"]}, {"sent": "you could wave the train off with my flag said the engine driver .", "words": ["the", "off", "my", "train", "flag", "with", "you", "could"]}, {"sent": "you mean your fingers a flag ?", "words": ["flag", "your", "a", "you"]}, {"sent": "well have an orange flag pole as well .", "words": ["have", "flag", "an", "orange"]}, {"sent": "thats a flag int it ?", "words": ["flag", "it", "a"]}, {"sent": "a flag yeah you know we hafta put the booklet in too .", "words": ["the", "put", "we", "flag", "you", "in", "too", "a"]}, {"sent": "flag .", "words": ["flag"]}, {"sent": "thats a flag .", "words": ["flag", "a"]}, {"sent": "if its red and white its the England flag .", "words": ["red", "the", "if", "flag", "and", "white"]}, {"sent": "its a picture of a flag .", "words": ["of", "flag", "picture", "a"]}, {"sent": "where did you get that flag from ?", "words": ["where", "flag", "you", "get", "did", "that"]}, {"sent": "I dont think you said wave the flag at first did you ?", "words": ["the", "think", "flag", "first", "you", "dont", "did", "at"]}, {"sent": "you said blow the whistle and something the flag .", "words": ["the", "flag", "and", "you", "blow"]}, {"sent": "Jeff put the flag down a minute .", "words": ["the", "put", "flag", "down", "a"]}, {"sent": "and theres a flag on top of the truck isnt there ?", "words": ["the", "of", "flag", "truck", "and", "there", "on", "a"]}, {"sent": "teddy stood on the platform and waved the flag as the train pulled away .", "words": ["the", "train", "flag", "and", "away", "on"]}, {"sent": "is somebody waving the flag and blowing the whistle ?", "words": ["the", "flag", "and", "is"]}, {"sent": "flag .", "words": ["flag"]}, {"sent": "is it a flag ?", "words": ["flag", "it", "a", "is"]}, {"sent": "maybe its a flag of some sort .", "words": ["of", "flag", "some", "a"]}, {"sent": "a nice clean kitchen with a flag floor and wooden beams .", "words": ["clean", "flag", "nice", "with", "and", "kitchen", "a"]}, {"sent": "maybe she can sit down on the bench over here .", "words": ["can", "the", "here", "bench", "on", "sit", "down", "over", "she"]}, {"sent": "I did undo the bench .", "words": ["the", "did", "bench"]}, {"sent": "the work bench .", "words": ["the", "work", "bench"]}, {"sent": "Im not sure oh I think thats bench .", "words": ["think", "not", "bench"]}, {"sent": "you need a bench ?", "words": ["bench", "need", "a", "you"]}, {"sent": "yeah we have a little work bench .", "words": ["work", "we", "little", "have", "bench", "a"]}, {"sent": "well put mummy on the bench .", "words": ["on", "the", "put", "bench"]}, {"sent": "the bench is red .", "words": ["the", "red", "is", "bench"]}, {"sent": "youre doing a bench again are you ?", "words": ["bench", "are", "you", "a"]}, {"sent": "and you made Jo and myself sit on a bench didnt you ?", "words": ["you", "and", "myself", "bench", "on", "sit", "a"]}, {"sent": "say on the bench for me .", "words": ["the", "for", "say", "bench", "on", "me"]}, {"sent": "what other tools were there on that bench ?", "words": ["what", "other", "there", "that", "bench", "on", "were"]}, {"sent": "and theres a garden bench here with somebodys sun hat .", "words": ["sun", "with", "and", "here", "garden", "bench", "hat", "a"]}, {"sent": "thats the bench .", "words": ["the", "bench"]}, {"sent": "bench .", "words": ["bench"]}, {"sent": "and the person and the person can sit on the bench .", "words": ["the", "can", "and", "bench", "on", "sit"]}, {"sent": "bench .", "words": ["bench"]}, {"sent": "she can sit on the bench though .", "words": ["can", "the", "bench", "on", "sit", "she"]}, {"sent": "this ones gonna sit over here on the bench and watch .", "words": ["the", "watch", "and", "here", "this", "bench", "on", "sit", "over"]}, {"sent": "and a bench .", "words": ["bench", "a", "and"]}, {"sent": "wheres the bench ?", "words": ["the", "bench"]}, {"sent": "they saw Godilucks in the bench .", "words": ["the", "they", "in", "bench"]}, {"sent": "she says is there a bench for me to sit on ?", "words": ["for", "is", "to", "there", "bench", "on", "me", "sit", "she", "a"]}, {"sent": "but um and she really liked the uh the tool bench I think .", "words": ["the", "think", "and", "bench", "she", "but"]}, {"sent": "how about the bench ?", "words": ["the", "how", "about", "bench"]}, {"sent": "nother bench nother bench .", "words": ["bench"]}, {"sent": "and whoevers wearing the hat has just popped it down on the bench for a few moments .", "words": ["the", "for", "and", "on", "it", "bench", "hat", "down", "a"]}, {"sent": "do you see a bench ?", "words": ["do", "you", "bench", "see", "a"]}, {"sent": "on the bench .", "words": ["on", "the", "bench"]}, {"sent": "sat on the bench .", "words": ["on", "the", "bench"]}, {"sent": "thats another bench .", "words": ["another", "bench"]}, {"sent": "a work bench .", "words": ["bench", "work", "a"]}, {"sent": "Im gonna go sit on the bench Im a lazy teacher .", "words": ["the", "go", "bench", "on", "sit", "a"]}, {"sent": "its not a bench .", "words": ["bench", "not", "a"]}, {"sent": "the slots in the bench can be the ticket machine .", "words": ["the", "can", "be", "in", "bench"]}, {"sent": "come and sit back down on your bench .", "words": ["your", "back", "and", "bench", "on", "sit", "down"]}, {"sent": "he just got a tool bench for Christmas .", "words": ["bench", "for", "he", "a"]}, {"sent": "he was teasing them that they were gonna sleep at the playground and Thomas and Catherine were gonna sleep on the slide and daddy was gonna sleep on the bench .", "words": ["the", "bench", "slide", "sleep", "was", "them", "and", "they", "he", "at", "on", "that", "were"]}, {"sent": "and who sits on the bench ?", "words": ["the", "bench", "and", "who", "on"]}, {"sent": "and have a rest on the bench .", "words": ["the", "and", "have", "bench", "on", "a"]}, {"sent": "where shall I draw the bench then ?", "words": ["where", "the", "draw", "bench", "then"]}, {"sent": "you put the table up there and the bench ?", "words": ["the", "up", "put", "table", "you", "and", "there", "bench"]}, {"sent": "a bench ?", "words": ["bench", "a"]}, {"sent": "sit on the bench please .", "words": ["on", "the", "sit", "bench"]}, {"sent": "jump over the little person run around the big person and then sit on the bench .", "words": ["the", "bench", "sit", "little", "and", "jump", "run", "big", "around", "on", "then", "over"]}, {"sent": "more bench gone ?", "words": ["more", "bench"]}, {"sent": "the the bench the chair ?", "words": ["the", "chair", "bench"]}, {"sent": "shes under the bench thats nice and theres some other people outside .", "words": ["the", "outside", "nice", "and", "other", "under", "bench", "some"]}, {"sent": "I didnt do the bench .", "words": ["the", "do", "bench"]}, {"sent": "well pretend there yare put your work here yare no put your work bench on there you be a worker .", "words": ["put", "your", "work", "be", "you", "here", "there", "bench", "on", "pretend", "a"]}, {"sent": "a picnic table and a bench .", "words": ["bench", "table", "and", "a"]}, {"sent": "well dont put your knees down there on the bench .", "words": ["the", "put", "your", "dont", "there", "bench", "on", "down"]}, {"sent": "see the lady sitting on the bench ?", "words": ["the", "see", "on", "bench"]}, {"sent": "alright Ill find the play bench .", "words": ["the", "play", "bench", "find"]}, {"sent": "thats a bench .", "words": ["bench", "a"]}, {"sent": "everybody sat on the bench apart from daddy .", "words": ["on", "the", "bench"]}, {"sent": "are you gonna come over here on your bench ?", "words": ["your", "you", "here", "are", "bench", "on", "over"]}, {"sent": "yah he has um he has a little bench that you can hammer .", "words": ["can", "hammer", "little", "you", "he", "bench", "that", "a"]}, {"sent": "you can touch the bench .", "words": ["can", "the", "touch", "you", "bench"]}, {"sent": "on the bench ?", "words": ["on", "the", "bench"]}, {"sent": "she jumps over the little person runs around the big person and then sits on the bench .", "words": ["the", "bench", "little", "and", "big", "around", "on", "then", "over", "she"]}, {"sent": "we cant get the bench out of the way sweetie because its stuck there .", "words": ["the", "we", "of", "get", "there", "stuck", "out", "bench", "because"]}, {"sent": "where wheres your bench ?", "words": ["where", "your", "bench"]}, {"sent": "sit down on the bench ?", "words": ["the", "bench", "on", "sit", "down"]}, {"sent": "does she sit on that bench ?", "words": ["does", "bench", "on", "sit", "that", "she"]}, {"sent": "this is a bench its a wooden bench it doesnt bounce .", "words": ["is", "this", "it", "bench", "a"]}, {"sent": "er do you want the bench ?", "words": ["the", "do", "bench", "you"]}, {"sent": "well if youre gonna take the file in there look you can take your hammer in there but if youre gonna take the file in there I want you to take your truck out here to your work bench .", "words": ["the", "can", "hammer", "bench", "your", "if", "work", "to", "truck", "you", "here", "there", "in", "out", "look", "take", "but"]}, {"sent": "this bench .", "words": ["this", "bench"]}, {"sent": "what bench ?", "words": ["what", "bench"]}, {"sent": "we bought some sausage rolls and we sat on a bench outside the church .", "words": ["the", "we", "outside", "church", "and", "bench", "on", "some", "a"]}, {"sent": "and thats the little bench where you sit .", "words": ["the", "where", "little", "and", "you", "bench", "sit"]}, {"sent": "okay I thought it was a bench .", "words": ["bench", "was", "it", "a"]}, {"sent": "put it on the bench where you can xxx .", "words": ["the", "where", "put", "can", "you", "it", "bench", "on"]}, {"sent": "the house and the bench .", "words": ["the", "house", "bench", "and"]}, {"sent": "youre sitting on the bench .", "words": ["on", "the", "bench"]}, {"sent": "on the park bench ?", "words": ["on", "the", "park", "bench"]}, {"sent": "a bench ?", "words": ["bench", "a"]}, {"sent": "oh theres your tool bench over there go and see what you can fix .", "words": ["can", "what", "your", "go", "fix", "and", "you", "there", "bench", "see", "over"]}, {"sent": "it could go besides its a bench for the kitchen table .", "words": ["the", "for", "table", "go", "it", "kitchen", "bench", "could", "a"]}, {"sent": "whats mummy got on the bench ?", "words": ["on", "the", "bench"]}, {"sent": "youre youre what for a bench ?", "words": ["bench", "what", "for", "a"]}, {"sent": "yeah lets stay right here and play with your work bench .", "words": ["your", "work", "play", "with", "and", "here", "bench", "stay"]}, {"sent": "speaking of hiding Jilly do you see this little bench ?", "words": ["do", "of", "you", "little", "this", "bench", "see"]}, {"sent": "you want the play bench .", "words": ["the", "play", "bench", "you"]}, {"sent": "when we get there Mummy usually sits on a bench doesnt she ?", "words": ["we", "get", "there", "bench", "on", "when", "she", "a"]}, {"sent": "say on the bench .", "words": ["on", "the", "say", "bench"]}, {"sent": "we sat another time on a bench when we were walking round the lake your shoelace came undone and we sat on a bench .", "words": ["the", "your", "we", "and", "another", "bench", "on", "when", "were", "a"]}, {"sent": "on the bench ?", "words": ["on", "the", "bench"]}, {"sent": "theres music in the bench .", "words": ["the", "in", "bench"]}, {"sent": "thats not a bench .", "words": ["bench", "not", "a"]}, {"sent": "you youve got a tool bench .", "words": ["bench", "a", "you"]}, {"sent": "yeah he has a little work bench .", "words": ["work", "little", "he", "bench", "a"]}, {"sent": "oh are you sitting on the bench ?", "words": ["the", "you", "are", "bench", "on"]}, {"sent": "a bench and a little bench .", "words": ["bench", "little", "and", "a"]}, {"sent": "cookie monster .", "words": ["cookie"]}, {"sent": "uhhuh youve got your cookie .", "words": ["cookie", "your"]}, {"sent": "here you want ta cookie ?", "words": ["here", "cookie", "you"]}, {"sent": "may I please have another cookie Catherine ?", "words": ["have", "cookie", "another"]}, {"sent": "you finished with you pau with that cookie .", "words": ["cookie", "that", "with", "you"]}, {"sent": "what kind of cookie is that ?", "words": ["what", "cookie", "is", "of", "that"]}, {"sent": "yyy a cookie ?", "words": ["cookie", "a"]}, {"sent": "cookie .", "words": ["cookie"]}, {"sent": "oh I think I see a cookie over there .", "words": ["cookie", "think", "there", "see", "over", "a"]}, {"sent": "cookie cookie .", "words": ["cookie"]}, {"sent": "its a cookie .", "words": ["cookie", "a"]}, {"sent": "and heres a cookie .", "words": ["cookie", "a", "and"]}, {"sent": "cookie .", "words": ["cookie"]}, {"sent": "but were were not gonna have a cookie at the same time were playing with playdough thats the thing .", "words": ["the", "cookie", "with", "have", "same", "at", "were", "not", "but", "a"]}, {"sent": "what does cookie monster like to eat ?", "words": ["cookie", "what", "does", "to", "like", "eat"]}, {"sent": "pretend cookie .", "words": ["cookie", "pretend"]}, {"sent": "hey Carwin heres your cookie .", "words": ["cookie", "your"]}, {"sent": "gimme a cookie cookie cookie .", "words": ["cookie", "a"]}, {"sent": "cookie .", "words": ["cookie"]}, {"sent": "it is cookie .", "words": ["cookie", "it", "is"]}, {"sent": "Mommy get a cookie .", "words": ["get", "cookie", "a"]}, {"sent": "look at your cookie and my cookie .", "words": ["cookie", "my", "look", "your", "and", "at"]}, {"sent": "well what do you put in for the cookie ?", "words": ["the", "what", "do", "put", "for", "cookie", "you", "in"]}, {"sent": "Gaston wants a cookie ?", "words": ["cookie", "a"]}, {"sent": "oh mommys gonna eat the cookie .", "words": ["eat", "the", "cookie"]}, {"sent": "I love cookies and I love cookie ow .", "words": ["cookie", "love", "and"]}, {"sent": "cookie .", "words": ["cookie"]}, {"sent": "hi Francis can I have some cookie ?", "words": ["can", "some", "have", "cookie"]}, {"sent": "did you eat a gingerbread boy cookie ?", "words": ["cookie", "you", "eat", "did", "a"]}, {"sent": "want your cookie ?", "words": ["cookie", "your"]}, {"sent": "you love that little cookie is that exciting ?", "words": ["cookie", "is", "little", "you", "love", "that"]}, {"sent": "heres the cookie .", "words": ["the", "cookie"]}, {"sent": "you want a cookie ?", "words": ["cookie", "a", "you"]}, {"sent": "okay bye bye cookie .", "words": ["cookie"]}, {"sent": "now Im gonna put my cookie back together and Im gonna roll my cookie into a funny long cookie .", "words": ["cookie", "put", "my", "into", "back", "long", "and", "a"]}, {"sent": "whats in the cookie ?", "words": ["the", "cookie", "in"]}, {"sent": "mm cookie yum yum .", "words": ["cookie"]}, {"sent": "is the cookie momnster where is the cookie monster ?", "words": ["the", "cookie", "where", "is"]}, {"sent": "I want some cookie .", "words": ["cookie", "some"]}, {"sent": "you want a cookie ?", "words": ["cookie", "a", "you"]}, {"sent": "no I dont want that cookie .", "words": ["cookie", "dont", "that"]}, {"sent": "you found a cookie .", "words": ["cookie", "a", "you"]}, {"sent": "put your cookie down and go pass him the can .", "words": ["the", "cookie", "put", "can", "your", "go", "and", "him", "down"]}, {"sent": "Ill go get you another a cookie okay ?", "words": ["cookie", "go", "you", "get", "another", "a"]}, {"sent": "because I only because I on I only had one cookie .", "words": ["on", "cookie", "because"]}, {"sent": "cookie .", "words": ["cookie"]}, {"sent": "cookie .", "words": ["cookie"]}, {"sent": "no you finish eating your cookie .", "words": ["cookie", "your", "finish", "you"]}, {"sent": "want a cookie ?", "words": ["cookie", "a"]}, {"sent": "oh gotta have a cookie .", "words": ["have", "cookie", "a"]}, {"sent": "its just a cookie monster .", "words": ["cookie", "a"]}, {"sent": "thats a cookie .", "words": ["cookie", "a"]}, {"sent": "say cookie .", "words": ["cookie", "say"]}, {"sent": "its a cookie but not right now okay ?", "words": ["cookie", "not", "but", "a"]}, {"sent": "and she does it with every word she said cookie last week and we said and we said can you say cookie again baby and she said dede .", "words": ["can", "does", "cookie", "we", "last", "with", "say", "and", "you", "it", "every", "she"]}, {"sent": "cookie ?", "words": ["cookie"]}, {"sent": "give Sandra a cookie kiss .", "words": ["cookie", "give", "kiss", "a"]}, {"sent": "would Joseph like a cookie ?", "words": ["cookie", "like", "a", "would"]}, {"sent": "well you better sit down if you want some cookie .", "words": ["better", "cookie", "if", "you", "some", "sit", "down"]}, {"sent": "heres the Pooh cookie cutter .", "words": ["the", "cookie"]}, {"sent": "can you give me a pretend cookie ?", "words": ["can", "cookie", "you", "pretend", "me", "give", "a"]}, {"sent": "you want the cookie instead and thats all youre interested in is the cracker ?", "words": ["the", "cookie", "all", "is", "cracker", "and", "you", "in"]}, {"sent": "I want a cookie okay .", "words": ["cookie", "a"]}, {"sent": "you want a cookie ?", "words": ["cookie", "a", "you"]}, {"sent": "you know what you can have a camel cookie but I would like you to have some fruit too .", "words": ["can", "what", "cookie", "to", "like", "you", "have", "would", "some", "too", "but", "a"]}, {"sent": "she picked up the cookie briefly .", "words": ["up", "the", "cookie", "she"]}, {"sent": "and a cookie .", "words": ["cookie", "a", "and"]}, {"sent": "I had only I have only one cookie .", "words": ["have", "cookie"]}, {"sent": "cookie ?", "words": ["cookie"]}, {"sent": "what was the cookie for ?", "words": ["the", "what", "cookie", "for", "was"]}, {"sent": "wait until you finish eating your cookie .", "words": ["cookie", "wait", "your", "finish", "you"]}, {"sent": "well it sure is nice recording you when you got your mouth full of cookie all the time .", "words": ["the", "cookie", "all", "your", "full", "is", "of", "nice", "mouth", "you", "it", "when"]}, {"sent": "yeah you got a cookie .", "words": ["cookie", "a", "you"]}, {"sent": "cookie should we make a sandwich ?", "words": ["cookie", "we", "make", "sandwich", "a"]}, {"sent": "oh your cookie is coming apart all over Daddys bed .", "words": ["all", "cookie", "your", "is", "bed", "over"]}, {"sent": "ha thats a cookie .", "words": ["cookie", "a"]}, {"sent": "mm that cookie looks yummy is it good ?", "words": ["cookie", "is", "it", "good", "that"]}, {"sent": "cookie thats right .", "words": ["cookie"]}, {"sent": "xxx does kitty cat get a cookie ?", "words": ["does", "cookie", "get", "cat", "kitty", "a"]}, {"sent": "you want a cookie ?", "words": ["cookie", "a", "you"]}, {"sent": "now Im gonna eat half of my cookie .", "words": ["eat", "of", "my", "cookie"]}, {"sent": "cookie .", "words": ["cookie"]}, {"sent": "see theres some juice and some cookie .", "words": ["cookie", "and", "see", "juice", "some"]}, {"sent": "I guess in his in his toy box he has like a stuffed cookie monster or and an Elmo .", "words": ["cookie", "box", "like", "and", "toy", "in", "he", "an", "his", "a"]}, {"sent": "cookie .", "words": ["cookie"]}, {"sent": "do you have a cookie ?", "words": ["do", "cookie", "you", "have", "a"]}, {"sent": "you trying to eat that cookie ?", "words": ["cookie", "to", "you", "eat", "that"]}, {"sent": "you dont get a lot of those but you know cookie monster likes cookies .", "words": ["cookie", "those", "of", "a lot", "you", "get", "dont", "but", "a"]}, {"sent": "and the cookie tree ?", "words": ["the", "cookie", "tree", "and"]}, {"sent": "does the cookie monster have a tongue ?", "words": ["the", "does", "cookie", "have", "tongue", "a"]}, {"sent": "a makebelieve cookie .", "words": ["cookie", "a"]}, {"sent": "youre getting cookie crumbs all over the floor .", "words": ["all", "cookie", "over", "the"]}, {"sent": "will your Mommy xxx xxx cookie baby ?", "words": ["will", "cookie", "your"]}, {"sent": "whose cookie is it ?", "words": ["cookie", "it", "is"]}, {"sent": "stay there stay in the kitchen to eat your cookie .", "words": ["the", "cookie", "your", "to", "eat", "there", "in", "kitchen", "stay"]}, {"sent": "okay put the boy down see if we can find a cookie ?", "words": ["the", "can", "put", "cookie", "if", "we", "find", "see", "down", "a"]}, {"sent": "ya gonna eat your cookie ?", "words": ["eat", "cookie", "your"]}, {"sent": "thats your cookie .", "words": ["cookie", "your"]}, {"sent": "the which cookie Nomi ?", "words": ["the", "cookie", "which"]}, {"sent": "you want a napkin ?", "words": ["napkin", "a", "you"]}, {"sent": "take it out and put it on the napkin .", "words": ["the", "put", "and", "it", "out", "on", "take", "napkin"]}, {"sent": "give Momma your napkin .", "words": ["give", "your", "napkin"]}, {"sent": "what do you do with a napkin ?", "words": ["what", "do", "with", "you", "napkin", "a"]}, {"sent": "heres a napkin for ya .", "words": ["for", "napkin", "a"]}, {"sent": "theres a napkin for you .", "words": ["for", "napkin", "you", "a"]}, {"sent": "do you wanna put your apple on a napkin ?", "words": ["do", "put", "your", "you", "wanna", "on", "apple", "napkin", "a"]}, {"sent": "well heres a clean napkin if you need it okay ?", "words": ["clean", "need", "if", "you", "it", "napkin", "a"]}, {"sent": "let me get you a napkin wipe your mouth .", "words": ["your", "mouth", "you", "get", "me", "wipe", "napkin", "a"]}, {"sent": "napkin ?", "words": ["napkin"]}, {"sent": "Im not moving your napkin .", "words": ["napkin", "your", "not"]}, {"sent": "get the napkin under it .", "words": ["the", "get", "under", "it", "napkin"]}, {"sent": "now why dont you put the napkin away ?", "words": ["the", "put", "you", "dont", "away", "napkin", "why"]}, {"sent": "napkin .", "words": ["napkin"]}, {"sent": "napkin .", "words": ["napkin"]}, {"sent": "thats a napkin .", "words": ["napkin", "a"]}, {"sent": "I didnt see you touch your napkin with your hands and take them off those hands off those piano keys and get them off your pants and shirt and come back and use your napkin .", "words": ["off", "your", "those", "back", "shirt", "with", "them", "touch", "you", "and", "get", "pants", "see", "keys", "take", "napkin"]}, {"sent": "heres you a napkin .", "words": ["napkin", "a", "you"]}, {"sent": "and well give you another napkin to wipe your face and hands .", "words": ["your", "face", "to", "you", "another", "and", "give", "wipe", "napkin"]}, {"sent": "theres a napkin there .", "words": ["there", "napkin", "a"]}, {"sent": "xxx napkin xxx .", "words": ["napkin"]}, {"sent": "oh I tell you what we can skip the napkin tonight okay ?", "words": ["can", "what", "the", "we", "you", "napkin"]}, {"sent": "heres a napkin .", "words": ["napkin", "a"]}, {"sent": "you bring me the napkin too .", "words": ["the", "bring", "you", "too", "me", "napkin"]}, {"sent": "napkin ?", "words": ["napkin"]}, {"sent": "this is my napkin Roland .", "words": ["this", "my", "napkin", "is"]}, {"sent": "take the spoon out and put it on the napkin .", "words": ["the", "put", "and", "it", "spoon", "out", "on", "take", "napkin"]}, {"sent": "use your napkin .", "words": ["your", "napkin"]}, {"sent": "theres a napkin .", "words": ["napkin", "a"]}, {"sent": "napkin ?", "words": ["napkin"]}, {"sent": "Robert throw your napkin away .", "words": ["napkin", "away", "your", "throw"]}, {"sent": "we find a napkin too .", "words": ["we", "find", "too", "napkin", "a"]}, {"sent": "okay lets go throw the napkin away and well go to your room .", "words": ["the", "your", "throw", "go", "to", "and", "away", "room", "napkin"]}, {"sent": "Steve is to big to get a napkin .", "words": ["is", "to", "get", "big", "napkin", "a"]}, {"sent": "whats that on the napkin ?", "words": ["on", "the", "that", "napkin"]}, {"sent": "are they napkins or are they napkin ?", "words": ["they", "are", "napkin"]}, {"sent": "monkey do you want do you wanna give George the napkin ?", "words": ["the", "do", "monkey", "you", "wanna", "give", "napkin"]}, {"sent": "wipe it up with your napkin .", "words": ["up", "your", "with", "it", "wipe", "napkin"]}, {"sent": "do you want a napkin ?", "words": ["do", "napkin", "a", "you"]}, {"sent": "now we put our napkin in our lap .", "words": ["put", "we", "our", "in", "napkin"]}, {"sent": "napkin .", "words": ["napkin"]}, {"sent": "yeah its a napkin .", "words": ["napkin", "a"]}, {"sent": "well youll hafta find uh something smaller like a cloth napkin .", "words": ["napkin", "like", "a", "find"]}, {"sent": "bye napkin .", "words": ["napkin"]}, {"sent": "is Lala on the napkin ?", "words": ["on", "the", "napkin", "is"]}, {"sent": "thats a napkin .", "words": ["napkin", "a"]}, {"sent": "get your napkin .", "words": ["get", "your", "napkin"]}, {"sent": "whats a hamster napkin ?", "words": ["napkin", "a"]}, {"sent": "oh you want a piece of napkin ?", "words": ["of", "napkin", "a", "you"]}, {"sent": "oh heres a napkin right here .", "words": ["here", "napkin", "a"]}, {"sent": "give me the napkin .", "words": ["the", "me", "give", "napkin"]}, {"sent": "and then you can it on the napkin .", "words": ["can", "the", "you", "and", "it", "on", "then", "napkin"]}, {"sent": "can you get a napkin for yourself ?", "words": ["can", "for", "yourself", "you", "get", "napkin", "a"]}, {"sent": "would you like a napkin and wipe your mouth a little bit ?", "words": ["your", "mouth", "like", "you", "and", "little", "would", "wipe", "napkin", "a"]}, {"sent": "did I give you a napkin ?", "words": ["you", "did", "give", "napkin", "a"]}, {"sent": "ah Mommys got a napkin .", "words": ["napkin", "a"]}, {"sent": "napkin .", "words": ["napkin"]}, {"sent": "olive and feta do you want me to get a napkin so you can wipe that off your hand ?", "words": ["can", "do", "off", "your", "hand", "to", "so", "you", "and", "get", "me", "wipe", "that", "napkin", "a"]}, {"sent": "hey ei Mrs Missus Blasingame Blaesingeim tell Derwood gimme that napkin .", "words": ["that", "napkin"]}, {"sent": "mhm shes gonna get a napkin .", "words": ["get", "napkin", "a"]}, {"sent": "you want a napkin .", "words": ["napkin", "a", "you"]}, {"sent": "Mommy needs to put a napkin down on the table .", "words": ["the", "put", "table", "to", "on", "down", "napkin", "a"]}, {"sent": "do you want a little napkin for that ?", "words": ["do", "for", "little", "you", "that", "napkin", "a"]}, {"sent": "that wasnt a plate that was a napkin .", "words": ["plate", "was", "that", "napkin", "a"]}, {"sent": "its a napkin .", "words": ["napkin", "a"]}, {"sent": "why would you take her a napkin ?", "words": ["you", "would", "take", "her", "napkin", "why", "a"]}, {"sent": "heres your napkin .", "words": ["your", "napkin"]}, {"sent": "can I have a napkin ?", "words": ["can", "have", "napkin", "a"]}, {"sent": "with a new napkin .", "words": ["new", "with", "napkin", "a"]}, {"sent": "here a napkin .", "words": ["here", "napkin", "a"]}, {"sent": "and theres your napkin .", "words": ["your", "napkin", "and"]}, {"sent": "ah Jason needs a a napkin .", "words": ["napkin", "a"]}, {"sent": "and then thats the napkin and then thats a little turkey napkin ring .", "words": ["the", "and", "little", "turkey", "then", "napkin", "a"]}, {"sent": "oh the napkin will get soaked .", "words": ["the", "get", "will", "napkin"]}, {"sent": "well xxx maybe a napkin .", "words": ["napkin", "a"]}, {"sent": "I took your napkin .", "words": ["your", "napkin"]}, {"sent": "this your napkin ?", "words": ["this", "your", "napkin"]}, {"sent": "not on the napkin .", "words": ["on", "the", "napkin", "not"]}, {"sent": "Im gonna throw this wet napkin away xxx .", "words": ["wet", "throw", "this", "away", "napkin"]}, {"sent": "he dont have any napkin .", "words": ["any", "have", "dont", "he", "napkin"]}, {"sent": "what do you think is underneath that little napkin ?", "words": ["what", "do", "think", "is", "you", "little", "that", "napkin"]}, {"sent": "do you want a napkin to wipe your hands on ?", "words": ["do", "your", "to", "you", "on", "wipe", "napkin", "a"]}, {"sent": "where is the napkin Linda ?", "words": ["where", "the", "napkin", "is"]}, {"sent": "wheres your napkin ?", "words": ["your", "napkin"]}, {"sent": "a napkin .", "words": ["napkin", "a"]}, {"sent": "where can I find a napkin ?", "words": ["where", "can", "find", "napkin", "a"]}, {"sent": "shall we shall we all have a napkin ?", "words": ["all", "we", "have", "napkin", "a"]}, {"sent": "heres your napkin .", "words": ["your", "napkin"]}, {"sent": "pick it up put it in a napkin .", "words": ["up", "put", "pick", "in", "it", "napkin", "a"]}, {"sent": "you need a napkin ?", "words": ["need", "napkin", "a", "you"]}, {"sent": "would you like a napkin .", "words": ["like", "you", "would", "napkin", "a"]}, {"sent": "xxx back of my napkin .", "words": ["of", "my", "napkin", "back"]}, {"sent": "what do you think this napkin is for ?", "words": ["what", "do", "think", "for", "is", "you", "this", "napkin"]}, {"sent": "wipe it up with your napkin and eat this good breakfast .", "words": ["up", "your", "with", "and", "eat", "this", "it", "good", "wipe", "napkin"]}, {"sent": "dont eat the napkin silly .", "words": ["eat", "dont", "napkin", "the"]}, {"sent": "give me your napkin .", "words": ["me", "give", "your", "napkin"]}, {"sent": "did you take something off the tray ?", "words": ["off", "the", "you", "did", "tray", "take"]}, {"sent": "lemme put your tray up .", "words": ["up", "put", "your", "tray"]}, {"sent": "youre finished cleaning the tray ?", "words": ["the", "tray"]}, {"sent": "wash the tray yes ?", "words": ["the", "wash", "tray"]}, {"sent": "and put all your teaset back in the tray .", "words": ["the", "all", "put", "your", "back", "and", "in", "tray"]}, {"sent": "and I pinched a tray for the potatoes and all .", "words": ["the", "all", "for", "and", "tray", "a"]}, {"sent": "tray .", "words": ["tray"]}, {"sent": "tray ?", "words": ["tray"]}, {"sent": "you wanna get us one more tray ?", "words": ["you", "get", "us", "more", "wanna", "tray"]}, {"sent": "oh now its very watery tray .", "words": ["tray"]}, {"sent": "we might need a tray .", "words": ["tray", "need", "a", "we"]}, {"sent": "that was good cleaning up your tray .", "words": ["up", "your", "was", "good", "tray", "that"]}, {"sent": "maybe lets move your shoe off the tray thatll help .", "words": ["off", "the", "help", "your", "shoe", "tray"]}, {"sent": "you get a tray .", "words": ["get", "tray", "a", "you"]}, {"sent": "here comes Chi bringing our tray from the cafeteria .", "words": ["here", "the", "tray", "our"]}, {"sent": "well put that liner on the tray .", "words": ["the", "put", "tray", "on", "that"]}, {"sent": "I think we need a tray to set it on and a little bit of water .", "words": ["think", "need", "we", "of", "to", "and", "little", "it", "water", "on", "tray", "a"]}, {"sent": "youll hafta put it on a nice tray with a flower down there .", "words": ["put", "flower", "nice", "with", "there", "it", "on", "down", "tray", "a"]}, {"sent": "Joseph why do you keep falling over and bonking your head on the tray ?", "words": ["the", "do", "your", "head", "and", "you", "tray", "on", "over", "why"]}, {"sent": "you shouldnt have hit your head into that tray .", "words": ["your", "into", "head", "you", "have", "tray", "hit", "that"]}, {"sent": "and then Im gonna wipe your tray ?", "words": ["your", "and", "then", "wipe", "tray"]}, {"sent": "if you make made some biscuits and one of them got up off the baking tray and ran off through the open door and down the street .", "words": ["up", "off", "the", "if", "of", "open", "street", "them", "and", "you", "tray", "some", "down", "make", "door"]}, {"sent": "wheres your tray ?", "words": ["your", "tray"]}, {"sent": "drying off the tray is good .", "words": ["off", "the", "is", "good", "tray"]}, {"sent": "there is your tray .", "words": ["there", "tray", "your", "is"]}, {"sent": "Purdies tray ?", "words": ["tray"]}, {"sent": "let Mommy wash your tray off .", "words": ["your", "wash", "off", "tray"]}, {"sent": "do ya want the tray ?", "words": ["the", "do", "tray"]}, {"sent": "we hafta clean your tray .", "words": ["clean", "tray", "your", "we"]}, {"sent": "shall we pop that on the baking tray ?", "words": ["the", "pop", "we", "tray", "on", "that"]}, {"sent": "right now we put that on the tray as well .", "words": ["the", "put", "we", "tray", "on", "that"]}, {"sent": "Im cleaning off your tray .", "words": ["off", "your", "tray"]}, {"sent": "I think we need the tray .", "words": ["the", "think", "need", "we", "tray"]}, {"sent": "we need a tray .", "words": ["tray", "need", "a", "we"]}, {"sent": "I think Naima this tray will work for this puzzle .", "words": ["think", "for", "work", "this", "puzzle", "will", "tray"]}, {"sent": "right there on your tray .", "words": ["on", "there", "your", "tray"]}, {"sent": "or perhaps it was a yellow and blue highchair with a nice yellow tray .", "words": ["was", "nice", "with", "and", "blue", "it", "yellow", "tray", "a"]}, {"sent": "look got a tray .", "words": ["tray", "a", "look"]}, {"sent": "I thought it was a feeding tray for the sheep .", "words": ["the", "for", "was", "it", "sheep", "tray", "a"]}, {"sent": "would you like to reach a tray please ?", "words": ["to", "like", "you", "would", "tray", "a"]}, {"sent": "its like a little secret tray isnt it ?", "words": ["like", "little", "it", "tray", "a"]}, {"sent": "he eats off his tray right now so hes not plates go flying right so we dont mess with plates .", "words": ["off", "go", "we", "so", "with", "dont", "he", "his", "tray", "not"]}, {"sent": "okay tray .", "words": ["tray"]}, {"sent": "let me move these that youve iced to the back of the baking tray and then well put the ones we havent iced at the front .", "words": ["the", "these", "put", "back", "we", "of", "to", "and", "at", "tray", "me", "then", "that"]}, {"sent": "shall we go into the kitchen and see if we can find nice tray ?", "words": ["the", "can", "into", "if", "go", "we", "nice", "and", "kitchen", "find", "see", "tray"]}, {"sent": "lookit theres a tray and food .", "words": ["food", "tray", "and", "a"]}, {"sent": "well clean off this xxx tray okay ?", "words": ["clean", "this", "tray", "off"]}, {"sent": "weve got to put it on the baking tray first .", "words": ["the", "put", "to", "first", "it", "on", "tray"]}, {"sent": "no no we dont need it on the baking tray love .", "words": ["the", "need", "we", "dont", "it", "love", "on", "tray"]}, {"sent": "if youd like to pull that tray out .", "words": ["if", "to", "like", "pull", "out", "tray", "that"]}, {"sent": "now will you put that piece of paper back onto the tray please .", "words": ["the", "put", "back", "of", "paper", "you", "tray", "will", "that"]}, {"sent": "I dont think Purdie would be very happy if we used her tray .", "words": ["think", "tray", "if", "happy", "we", "be", "dont", "would", "her"]}, {"sent": "Mommys cleaning your tray .", "words": ["your", "tray"]}, {"sent": "and pop it on the tray like that .", "words": ["the", "pop", "like", "and", "that", "it", "on", "tray"]}, {"sent": "because were going to need the tray when we open the box .", "words": ["the", "need", "when", "box", "we", "open", "to", "tray", "were", "because"]}, {"sent": "and let me wipe your tray .", "words": ["your", "and", "me", "wipe", "tray"]}, {"sent": "what can you see on the tray Thomas ?", "words": ["can", "what", "the", "you", "on", "see", "tray"]}, {"sent": "my own little tray .", "words": ["my", "tray", "little"]}, {"sent": "we could do with finding a tray for them all to stand in .", "words": ["do", "all", "for", "we", "to", "stand", "with", "them", "in", "could", "tray", "a"]}, {"sent": "put the tray .", "words": ["the", "put", "tray"]}, {"sent": "then I think you put it into a big tray dont you ?", "words": ["put", "think", "into", "you", "dont", "it", "tray", "then", "big", "a"]}, {"sent": "time to clean up your tray .", "words": ["clean", "up", "your", "to", "tray"]}, {"sent": "did you just get the shower tray in ?", "words": ["the", "shower", "you", "get", "in", "did", "tray"]}, {"sent": "now let me just think where I could get a tray from .", "words": ["where", "think", "get", "me", "could", "tray", "a"]}, {"sent": "water tray ?", "words": ["tray", "water"]}, {"sent": "the tray ?", "words": ["the", "tray"]}, {"sent": "thats Papas tray .", "words": ["tray"]}, {"sent": "dont think uh she needs to put them in a tray to get them where shes taking them .", "words": ["where", "think", "put", "to", "them", "get", "dont", "in", "tray", "she", "a"]}, {"sent": "what can you see on the tray ?", "words": ["can", "what", "the", "you", "on", "see", "tray"]}, {"sent": "put your tray away .", "words": ["put", "away", "your", "tray"]}, {"sent": "and then put them in the tray .", "words": ["the", "put", "them", "and", "in", "then", "tray"]}, {"sent": "theres a tray behind you .", "words": ["you", "tray", "behind", "a"]}, {"sent": "I need help now from the man with the bun cases and the bun tray .", "words": ["the", "help", "need", "with", "and", "tray"]}, {"sent": "would you like a tray mum ?", "words": ["like", "you", "would", "tray", "a"]}, {"sent": "should I get a tray here for your cracker ?", "words": ["for", "your", "cracker", "get", "here", "tray", "a"]}, {"sent": "oh youre making a lot of noise as the money hits the back of the tray though isnt it ?", "words": ["the", "money", "back", "of", "a lot", "it", "tray", "a"]}, {"sent": "this is a little tray containing all the letters .", "words": ["the", "all", "is", "little", "this", "tray", "a"]}, {"sent": "in your tray ?", "words": ["in", "your", "tray"]}, {"sent": "but if Purdie knocks some food off her plate it goes onto the tray underneath so we can just pick it up with a fork .", "words": ["off", "the", "can", "up", "plate", "if", "we", "pick", "so", "with", "food", "it", "fork", "tray", "some", "her", "but", "a"]}, {"sent": "thats right put it on the tray .", "words": ["the", "put", "it", "on", "tray"]}, {"sent": "isnt look hes not in your play tray is he ?", "words": ["your", "is", "play", "in", "he", "look", "tray", "not"]}, {"sent": "right shall we take take all this then on the tray and go and sit down at the table with it ?", "words": ["the", "all", "table", "we", "go", "sit", "with", "and", "this", "at", "it", "tray", "on", "then", "down", "take"]}, {"sent": "actually we dont need the tray .", "words": ["the", "need", "we", "dont", "tray"]}, {"sent": "Mommys gonna clean up the tray now .", "words": ["clean", "the", "up", "tray"]}, {"sent": "well heres another tray .", "words": ["another", "tray"]}, {"sent": "we needta scrub this tray .", "words": ["this", "tray", "we"]}, {"sent": "so that needs to go on the tray doesnt it ?", "words": ["the", "go", "to", "so", "it", "tray", "on", "that"]}, {"sent": "theres a lot of water here let me take the tray and pour the water off .", "words": ["the", "off", "tray", "of", "a lot", "pour", "and", "here", "water", "me", "take", "a"]}, {"sent": "dump everything onto your tray so you can pick it apart .", "words": ["can", "your", "pick", "so", "you", "it", "dump", "tray"]}, {"sent": "in the xxx tray havent you ?", "words": ["the", "in", "tray", "you"]}, {"sent": "lets put the tray across .", "words": ["the", "put", "tray"]}, {"sent": "tip and flip play tray .", "words": ["play", "tray", "and"]}, {"sent": "yeah Mommys cleaning the tray .", "words": ["the", "tray"]}, {"sent": "now heres the tray and Ill get you some kiwi .", "words": ["the", "you", "get", "and", "some", "tray"]}, {"sent": "thats a pussy cat tray isnt it ?", "words": ["cat", "it", "tray", "a"]}, {"sent": "how about onto a tray ?", "words": ["how", "tray", "about", "a"]}, {"sent": "both got a the grey tray in this .", "words": ["the", "this", "in", "tray", "a"]}, {"sent": "okay take a bite of your vanilla wafer .", "words": ["your", "of", "vanilla", "bite", "take", "a"]}, {"sent": "vanilla ?", "words": ["vanilla"]}, {"sent": "put some ice cream in the blender some powdered milk some frozen orange juice concentrate and a few drops of vanilla and some ice cubes and its all beat up .", "words": ["the", "milk", "put", "all", "up", "ice cream", "of", "vanilla", "and", "in", "orange", "ice", "some", "juice", "a"]}, {"sent": "and a touch of vanilla .", "words": ["of", "vanilla", "touch", "and", "a"]}, {"sent": "vanilla shake .", "words": ["vanilla", "shake"]}, {"sent": "mm vanilla .", "words": ["vanilla"]}, {"sent": "chocolate and vanilla shake your daddy likes chocolate mommy likes vanilla .", "words": ["your", "chocolate", "vanilla", "shake", "and"]}, {"sent": "special kind of vanilla shake with jammy beans .", "words": ["of", "vanilla", "shake", "with", "beans"]}, {"sent": "Mommy loves vanilla milkshakes too .", "words": ["vanilla", "too"]}, {"sent": "I like vanilla milkshakes .", "words": ["vanilla", "like"]}, {"sent": "vanilla wow .", "words": ["vanilla"]}, {"sent": "vanilla .", "words": ["vanilla"]}, {"sent": "vanilla .", "words": ["vanilla"]}, {"sent": "we have regular white ice cream vanilla ice cream with chocolate syrup .", "words": ["ice cream", "we", "chocolate", "vanilla", "with", "have", "ice", "white"]}, {"sent": "you want a vanilla wafer ?", "words": ["vanilla", "a", "you"]}, {"sent": "vanilla is usually white what flavor is it ?", "words": ["what", "is", "vanilla", "it", "white"]}, {"sent": "maybe French vanilla .", "words": ["vanilla"]}, {"sent": "vanilla .", "words": ["vanilla"]}, {"sent": "I like vanilla ones .", "words": ["vanilla", "like"]}, {"sent": "do you like vanilla ?", "words": ["do", "vanilla", "like", "you"]}, {"sent": "is that vanilla ?", "words": ["that", "vanilla", "is"]}, {"sent": "vanilla yoghurt is something to eat .", "words": ["eat", "to", "vanilla", "is"]}, {"sent": "no this theres no vanilla this isnt vanilla fudge .", "words": ["this", "vanilla"]}, {"sent": "you also like cupcakes vanilla and strawberry right ?", "words": ["strawberry", "vanilla", "like", "and", "you"]}, {"sent": "it says vanilla icecream on it though .", "words": ["on", "it", "vanilla"]}, {"sent": "you have strawberry and heres vanilla .", "words": ["strawberry", "vanilla", "and", "have", "you"]}, {"sent": "they have vanilla ants in the health food stores .", "words": ["the", "vanilla", "have", "they", "in", "food"]}, {"sent": "I like to eat vanilla yoghurt .", "words": ["eat", "to", "vanilla", "like"]}, {"sent": "you had a nice strawberry and vanilla yoghurt didnt you ?", "words": ["strawberry", "nice", "vanilla", "and", "you", "a"]}, {"sent": "were gonna put some vanilla in .", "words": ["put", "vanilla", "in", "some", "were"]}, {"sent": "now its just plain vanilla .", "words": ["vanilla"]}, {"sent": "its like a chocolate pop tart with a lil vanilla cream on the side its like the oreo of pop tarts .", "words": ["the", "pop", "of", "chocolate", "with", "like", "vanilla", "on", "a"]}, {"sent": "here she can have the vanilla one .", "words": ["can", "the", "vanilla", "here", "have", "she"]}, {"sent": "jammy seeds and then vanilla icecream .", "words": ["then", "vanilla", "and"]}, {"sent": "was it a vanilla ice cream ?", "words": ["ice cream", "was", "vanilla", "it", "ice", "a"]}, {"sent": "vanilla .", "words": ["vanilla"]}, {"sent": "you have vanilla .", "words": ["have", "vanilla", "you"]}, {"sent": "was it a vanilla cupcake ?", "words": ["was", "it", "vanilla", "a"]}, {"sent": "is it vanilla ice cream ?", "words": ["ice cream", "is", "vanilla", "it", "ice"]}, {"sent": "I think it is French vanilla .", "words": ["think", "it", "vanilla", "is"]}, {"sent": "how about if you eat your vanilla wafer ?", "words": ["how", "your", "if", "vanilla", "you", "eat", "about"]}, {"sent": "vanilla beans are black .", "words": ["black", "beans", "vanilla", "are"]}, {"sent": "vanilla .", "words": ["vanilla"]}, {"sent": "thats vanilla .", "words": ["vanilla"]}, {"sent": "this ones vanilla .", "words": ["this", "vanilla"]}, {"sent": "taste like vanilla ice cream .", "words": ["ice cream", "vanilla", "taste", "like", "ice"]}, {"sent": "come on Sam said Stella lets go outside is the snow cold asked Sam is it hard and icy snow is as cold as vanilla icecream and soft as baby rabbit fur said Stella .", "words": ["the", "soft", "is", "go", "outside", "snow", "vanilla", "and", "it", "hard", "on", "cold"]}, {"sent": "icecream this one is vanilla .", "words": ["this", "vanilla", "is"]}, {"sent": "you mean vanilla ice ?", "words": ["ice", "vanilla", "you"]}, {"sent": "is it a vanilla milkshake ?", "words": ["it", "vanilla", "a", "is"]}, {"sent": "thats vanilla ice cream .", "words": ["ice", "ice cream", "vanilla"]}, {"sent": "bread Cheerios crackers vanilla wafers .", "words": ["bread", "vanilla"]}, {"sent": "would you like some more vanilla wafers ?", "words": ["vanilla", "like", "you", "more", "would", "some"]}, {"sent": "want the vanilla one too ?", "words": ["the", "vanilla", "too"]}, {"sent": "think vanilla milkshake is good .", "words": ["think", "vanilla", "good", "is"]}, {"sent": "I have lemon or vanilla .", "words": ["have", "vanilla"]}, {"sent": "vanilla .", "words": ["vanilla"]}, {"sent": "Cheerios crackers vanilla wafers .", "words": ["vanilla"]}, {"sent": "you like vanilla ?", "words": ["vanilla", "like", "you"]}, {"sent": "I think its got vanilla in it actually .", "words": ["in", "think", "it", "vanilla"]}, {"sent": "I chose vanilla and my mother had mango .", "words": ["my", "vanilla", "and"]}, {"sent": "um vanilla cabbage is very healthy .", "words": ["vanilla", "is"]}, {"sent": "and vanilla ?", "words": ["vanilla", "and"]}, {"sent": "vanilla .", "words": ["vanilla"]}, {"sent": "Papa bought some vanilla yes because we didnt have any .", "words": ["we", "vanilla", "any", "have", "some", "because"]}, {"sent": "is it vanilla ?", "words": ["it", "vanilla", "is"]}, {"sent": "or we could do chocolate hearts with vanilla frosting .", "words": ["do", "we", "chocolate", "with", "vanilla", "could"]}, {"sent": "you want vanilla or strawberry ?", "words": ["strawberry", "vanilla", "you"]}, {"sent": "was it vanilla cake ?", "words": ["was", "it", "vanilla", "cake"]}, {"sent": "thats vanilla icecream .", "words": ["vanilla"]}, {"sent": "you want the vanilla one too ?", "words": ["the", "too", "vanilla", "you"]}, {"sent": "uhhuh it is vanilla .", "words": ["it", "vanilla", "is"]}, {"sent": "he likes chocolate ice cream better than vanilla .", "words": ["better", "ice cream", "chocolate", "vanilla", "he", "ice"]}, {"sent": "alrighty put our happy vanilla back up here .", "words": ["up", "put", "back", "happy", "vanilla", "our", "here"]}, {"sent": "vanilla .", "words": ["vanilla"]}, {"sent": "ooh its a chocolate donut really it looks like a vanilla donut .", "words": ["donut", "chocolate", "vanilla", "like", "it", "a"]}, {"sent": "xxx vanilla ?", "words": ["vanilla"]}, {"sent": "George was scooping all the icecream into a bowl he was just about to put some vanilla in when Mister Herb came out George thats not what my customer ordered he said .", "words": ["the", "all", "bowl", "put", "what", "into", "my", "was", "to", "vanilla", "in", "he", "about", "out", "some", "when", "not", "a"]}, {"sent": "I think Im gonna have some vanilla .", "words": ["have", "some", "think", "vanilla"]}, {"sent": "thats vanilla ice cream .", "words": ["ice", "ice cream", "vanilla"]}, {"sent": "vanilla mhm like vanilla .", "words": ["vanilla", "like"]}, {"sent": "snow is as cold as vanilla ice cream answered Stella and as soft as baby rabbit fur .", "words": ["ice cream", "soft", "is", "snow", "vanilla", "and", "ice", "cold"]}, {"sent": "is it vanilla icecream or strawberry icecream or chocolate ?", "words": ["strawberry", "is", "chocolate", "vanilla", "it"]}, {"sent": "very nice vanilla icecream .", "words": ["nice", "vanilla"]}, {"sent": "its a vanilla icecream .", "words": ["vanilla", "a"]}, {"sent": "vanilla wafer .", "words": ["vanilla"]}, {"sent": "vanilla ?", "words": ["vanilla"]}, {"sent": "its vanilla .", "words": ["vanilla"]}, {"sent": "hey Nathaniel why dont you make some vanilla cabbage .", "words": ["vanilla", "you", "dont", "some", "make", "why"]}, {"sent": "thats called vanilla cabbage .", "words": ["vanilla"]}, {"sent": "alright you want chocolate hearts with vanilla frosting .", "words": ["chocolate", "with", "vanilla", "you"]}, {"sent": "vanilla yoghurt is something to eat .", "words": ["eat", "to", "vanilla", "is"]}, {"sent": "I like vanilla ones .", "words": ["vanilla", "like"]}, {"sent": "thats fake vanilla ice cream .", "words": ["ice", "ice cream", "vanilla"]}, {"sent": "chocolate or vanilla .", "words": ["chocolate", "vanilla"]}, {"sent": "do you like peanut butter ?", "words": ["do", "like", "you", "peanut butter", "butter"]}, {"sent": "look at what they have at their house peanut butter .", "words": ["what", "look", "their", "have", "they", "house", "butter", "peanut butter", "at"]}, {"sent": "we have peanut butter .", "words": ["have", "peanut butter", "butter", "we"]}, {"sent": "Im not really a big peanut butter fan .", "words": ["peanut butter", "butter", "big", "not", "a"]}, {"sent": "how about a spoon of peanut butter ?", "words": ["how", "of", "spoon", "about", "butter", "peanut butter", "a"]}, {"sent": "you know what makes peanut butter ?", "words": ["what", "peanut butter", "butter", "you"]}, {"sent": "put it put the peanut butter on the toast ?", "words": ["the", "put", "toast", "it", "butter", "peanut butter", "on"]}, {"sent": "this is peanut butter .", "words": ["this", "peanut butter", "butter", "is"]}, {"sent": "toast with jam and peanut butter .", "words": ["toast", "with", "and", "peanut butter", "butter"]}, {"sent": "do you want peanut butter on these ?", "words": ["do", "these", "you", "butter", "peanut butter", "on"]}, {"sent": "I dont want you on peanut butter bread either .", "words": ["bread", "you", "dont", "butter", "peanut butter", "on"]}, {"sent": "um the peanut butter xxx like the soup containers he liked .", "words": ["the", "like", "he", "butter", "peanut butter", "soup"]}, {"sent": "peanut butter oh .", "words": ["butter", "peanut butter"]}, {"sent": "I got you peanut butter and jelly .", "words": ["jelly", "and", "you", "peanut butter", "butter"]}, {"sent": "or peanut butter and rye bread ?", "words": ["bread", "peanut butter", "butter", "and"]}, {"sent": "I hope they have peanut butter today .", "words": ["have", "they", "butter", "peanut butter"]}, {"sent": "but not as fun as peanut butter .", "words": ["butter", "not", "but", "peanut butter"]}, {"sent": "no peanut butter ?", "words": ["butter", "peanut butter"]}, {"sent": "hm peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "at least we made some peanut butter .", "words": ["we", "peanut butter", "butter", "at", "some"]}, {"sent": "do you want a grilled peanut butter and cheese and honey ?", "words": ["do", "and", "you", "butter", "peanut butter", "cheese", "a"]}, {"sent": "Im gonna put that pizza on the peanut butter .", "words": ["the", "put", "pizza", "butter", "peanut butter", "on", "that"]}, {"sent": "well thats peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "mommy will put some on the peanut butter .", "words": ["the", "put", "on", "butter", "peanut butter", "will", "some"]}, {"sent": "probably is peanut butter and jelly sandwiches .", "words": ["jelly", "is", "and", "peanut butter", "butter"]}, {"sent": "you need peanut butter on your cornbread ?", "words": ["need", "your", "you", "butter", "peanut butter", "on"]}, {"sent": "like hell eat bagels with peanut butter .", "words": ["with", "like", "eat", "butter", "peanut butter"]}, {"sent": "peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "do you ever put peanut butter on your hamburger ?", "words": ["do", "put", "your", "you", "hamburger", "butter", "peanut butter", "on"]}, {"sent": "come finish your peanut butter .", "words": ["butter", "your", "finish", "peanut butter"]}, {"sent": "where is peanut butter ?", "words": ["where", "peanut butter", "butter", "is"]}, {"sent": "you wanna use the spoon to put the peanut butter on the bread ?", "words": ["the", "put", "bread", "to", "you", "spoon", "wanna", "butter", "on", "peanut butter"]}, {"sent": "do you like peanut butter ?", "words": ["do", "like", "you", "peanut butter", "butter"]}, {"sent": "peanut butter ?", "words": ["butter", "peanut butter"]}, {"sent": "thats peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "ya I like peanut butter .", "words": ["butter", "like", "peanut butter"]}, {"sent": "peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "but its just a little bit sweeter than regular peanut butter .", "words": ["little", "peanut butter", "butter", "but", "a"]}, {"sent": "peanut butter too ?", "words": ["peanut butter", "butter", "too"]}, {"sent": "peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "that yummy you got peanut butter all over your face dah .", "words": ["all", "your", "face", "you", "that", "butter", "peanut butter", "over"]}, {"sent": "thats a jar of peanut butter .", "words": ["of", "peanut butter", "butter", "jar", "a"]}, {"sent": "peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "do you know how do you like to eat peanut butter ?", "words": ["do", "how", "to", "like", "you", "eat", "butter", "peanut butter"]}, {"sent": "have you ever made peanut butter ?", "words": ["have", "peanut butter", "butter", "you"]}, {"sent": "xxx your peanut butter chips look .", "words": ["peanut butter", "your", "butter", "look"]}, {"sent": "oh okay do you want peanut butter and jelly ?", "words": ["jelly", "do", "and", "you", "butter", "peanut butter"]}, {"sent": "peanut butter and jelly .", "words": ["jelly", "peanut butter", "butter", "and"]}, {"sent": "some peanut butter .", "words": ["some", "butter", "peanut butter"]}, {"sent": "peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "its peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "oh peanut butter you want a peanut butter sandwich .", "words": ["you", "peanut butter", "butter", "sandwich", "a"]}, {"sent": "peanut butter ?", "words": ["butter", "peanut butter"]}, {"sent": "do you guys want plain Ritz crackers or do you want me to put some peanut butter sandwich Ritz crackers in there ?", "words": ["do", "put", "to", "you", "there", "in", "butter", "peanut butter", "me", "sandwich", "some"]}, {"sent": "peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "do you want the peanut butter ?", "words": ["the", "do", "you", "peanut butter", "butter"]}, {"sent": "you wanna put the peanut butter in there ?", "words": ["the", "put", "you", "there", "in", "butter", "wanna", "peanut butter"]}, {"sent": "peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "peanut butter mayonnaise mustard ?", "words": ["butter", "peanut butter"]}, {"sent": "do they like peanut butter and jelly sandwiches ?", "words": ["jelly", "do", "like", "and", "they", "butter", "peanut butter"]}, {"sent": "pea peanut butter sandwich .", "words": ["butter", "sandwich", "peanut butter"]}, {"sent": "peanut butter ?", "words": ["butter", "peanut butter"]}, {"sent": "and I will go get you some crackers with peanut butter okay ?", "words": ["go", "with", "you", "get", "and", "butter", "peanut butter", "will", "some"]}, {"sent": "heres your peanut butter spoon .", "words": ["your", "butter", "spoon", "peanut butter"]}, {"sent": "ah yeah thats peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "get the peanut butter out and put it on the waffle .", "words": ["the", "put", "and", "get", "it", "butter", "out", "peanut butter", "on"]}, {"sent": "I want a peanut butter and jelly sandwich now .", "words": ["jelly", "and", "butter", "peanut butter", "sandwich", "a"]}, {"sent": "peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "hm I think youre sitting on the peanut butter .", "words": ["the", "think", "butter", "peanut butter", "on"]}, {"sent": "do you want a peanut butter and jelly sandwich ?", "words": ["jelly", "do", "and", "you", "butter", "peanut butter", "sandwich", "a"]}, {"sent": "peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "yup theres the peanut butter .", "words": ["the", "butter", "peanut butter"]}, {"sent": "you want a peanut butter and jelly sandwich ?", "words": ["jelly", "and", "you", "butter", "peanut butter", "sandwich", "a"]}, {"sent": "Violet you want some more peanut butter and jelly ?", "words": ["jelly", "and", "you", "more", "butter", "peanut butter", "some"]}, {"sent": "peanut butter and jelly sandwich ?", "words": ["jelly", "and", "peanut butter", "butter", "sandwich"]}, {"sent": "you want the rest of that peanut butter .", "words": ["the", "of", "you", "butter", "peanut butter", "that"]}, {"sent": "and peanut butter .", "words": ["peanut butter", "butter", "and"]}, {"sent": "it says peanut butter .", "words": ["it", "butter", "peanut butter"]}, {"sent": "hes done with the peanut butter .", "words": ["the", "with", "butter", "peanut butter"]}, {"sent": "Im going to have a peanut butter apple .", "words": ["to", "have", "butter", "peanut butter", "apple", "a"]}, {"sent": "peanut butter right ?", "words": ["butter", "peanut butter"]}, {"sent": "you wanna dip that in the peanut butter ?", "words": ["the", "you", "in", "butter", "wanna", "peanut butter", "that"]}, {"sent": "and back to the peanut butter .", "words": ["the", "back", "to", "and", "butter", "peanut butter"]}, {"sent": "peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "some peanut butter for the bee ?", "words": ["the", "for", "bee", "butter", "peanut butter", "some"]}, {"sent": "how about on the table by the peanut butter .", "words": ["the", "how", "table", "by", "butter", "about", "peanut butter", "on"]}, {"sent": "peanut butter .", "words": ["butter", "peanut butter"]}, {"sent": "two all beef etis sede sedai pickle onion on a sesame .", "words": ["on", "all", "a", "pickle"]}, {"sent": "you pickle you .", "words": ["pickle", "you"]}, {"sent": "pickle .", "words": ["pickle"]}, {"sent": "do you want the pickle ?", "words": ["the", "do", "pickle", "you"]}, {"sent": "thats a pickle train .", "words": ["pickle", "train", "a"]}, {"sent": "would you like to buy a pickle ?", "words": ["to", "like", "you", "pickle", "would", "buy", "a"]}, {"sent": "and pickle is sour .", "words": ["pickle", "is", "and"]}, {"sent": "I had cheese and pickle the last time .", "words": ["the", "last", "pickle", "and", "cheese"]}, {"sent": "my pickle .", "words": ["my", "pickle"]}, {"sent": "this is some pickle .", "words": ["this", "pickle", "some", "is"]}, {"sent": "pickle ?", "words": ["pickle"]}, {"sent": "thats not a pickle train .", "words": ["pickle", "train", "not", "a"]}, {"sent": "a peck of pickle peppers picked he .", "words": ["of", "pickle", "he", "a"]}, {"sent": "hold the pickle hold the lettuce special orders dyon upset us .", "words": ["the", "us", "hold", "pickle"]}, {"sent": "thats a pickle train .", "words": ["pickle", "train", "a"]}, {"sent": "its pickle .", "words": ["pickle"]}, {"sent": "its pickle .", "words": ["pickle"]}, {"sent": "its a pickle .", "words": ["pickle", "a"]}, {"sent": "or a pickle ?", "words": ["pickle", "a"]}, {"sent": "you still have my pickle ?", "words": ["have", "my", "pickle", "you"]}, {"sent": "PeterPiper picked a peck of pickle peppers .", "words": ["of", "pickle", "a"]}, {"sent": "if PeterPiper picked a peck of pickle peppers wheres the peck of pickle peppers PeterPiper picked ?", "words": ["the", "if", "of", "pickle", "a"]}, {"sent": "Im in a terrible pickle she said .", "words": ["in", "pickle", "she", "a"]}, {"sent": "oh look pickle slices .", "words": ["pickle", "look"]}, {"sent": "if PeterPiper picked a peck of pickle peppers wheres the peck of pickle peppers PeterPiper picked ?", "words": ["the", "if", "of", "pickle", "a"]}, {"sent": "a pickle ?", "words": ["pickle", "a"]}, {"sent": "oh is he making the pickle hurt ?", "words": ["the", "is", "hurt", "pickle", "he"]}, {"sent": "on saturday he ate through one piece of chocolate cake one icecream cone one pickle one slice of swiss cheese one slice of salami one lollipop one .", "words": ["cake", "of", "chocolate", "pickle", "lollipop", "he", "on", "cheese"]}, {"sent": "thats another pickle train .", "words": ["another", "train", "pickle"]}, {"sent": "thats a pickle .", "words": ["pickle", "a"]}, {"sent": "it really does smell like a pickle doesnt it ?", "words": ["does", "like", "pickle", "it", "a"]}, {"sent": "its time we flew said ickle me pickle me tickle me too .", "words": ["we", "tickle", "pickle", "too", "me"]}, {"sent": "come and be a pickle over here .", "words": ["be", "pickle", "and", "here", "over", "a"]}, {"sent": "a peck of pickle peppers picked he .", "words": ["of", "pickle", "he", "a"]}, {"sent": "Ive got that off but thats the pickle .", "words": ["off", "the", "pickle", "that", "but"]}, {"sent": "Peter Piper picked a peck of pickle peppers a peck of pickle .", "words": ["of", "pickle", "a"]}, {"sent": "see the pickle .", "words": ["the", "see", "pickle"]}, {"sent": "pickle .", "words": ["pickle"]}, {"sent": "well you are a little pickle .", "words": ["pickle", "little", "you", "are", "a"]}, {"sent": "give him a pickle ?", "words": ["give", "pickle", "him", "a"]}, {"sent": "eating the pickle .", "words": ["the", "pickle"]}, {"sent": "hooray what fun its time we flew said ickle me pickle me tickle me too .", "words": ["what", "we", "tickle", "pickle", "too", "me"]}, {"sent": "where did all the pickle peppers go ?", "words": ["where", "all", "the", "go", "pickle", "did"]}, {"sent": "peppers Peter Piper picked if Peter Piper picked a peck of pickle peppers wheres the peck of pickle peppers Peter Piper picked ?", "words": ["the", "if", "of", "pickle", "a"]}, {"sent": "one pickle .", "words": ["pickle"]}, {"sent": "okay PeterPiper picked a peck of pickle peppers .", "words": ["of", "pickle", "a"]}, {"sent": "its not pickle .", "words": ["not", "pickle"]}, {"sent": "do you want some pickle on your toast ?", "words": ["do", "your", "toast", "pickle", "you", "on", "some"]}, {"sent": "a pickle .", "words": ["pickle", "a"]}, {"sent": "Im going to put some pickle in mine .", "words": ["put", "to", "pickle", "mine", "in", "some"]}, {"sent": "remember we saw a pickle car and then we saw a carrot car ?", "words": ["we", "car", "pickle", "and", "then", "a"]}, {"sent": "Margie I have but I Ma I left my one pickle .", "words": ["have", "my", "but", "pickle"]}, {"sent": "if Peter Piper picked a peck of pickle peppers wheres the peck of pickle peppers Peter Piper picked ?", "words": ["the", "if", "of", "pickle", "a"]}, {"sent": "cheese and pickle sandwich .", "words": ["pickle", "cheese", "sandwich", "and"]}, {"sent": "pickle .", "words": ["pickle"]}, {"sent": "oh yes a pickle is a vegetable .", "words": ["pickle", "is", "a"]}, {"sent": "the pickle fork ?", "words": ["the", "fork", "pickle"]}, {"sent": "wheres the pickle ?", "words": ["the", "pickle"]}, {"sent": "pickle eater .", "words": ["pickle"]}, {"sent": "thats a pickle train .", "words": ["pickle", "train", "a"]}, {"sent": "a pickle .", "words": ["pickle", "a"]}, {"sent": "hamburger pickle .", "words": ["pickle", "hamburger"]}, {"sent": "but wasnt the pickle I was looking for .", "words": ["the", "for", "was", "pickle", "but"]}, {"sent": "you wanna pickle ?", "words": ["pickle", "wanna", "you"]}, {"sent": "one pickle .", "words": ["pickle"]}, {"sent": "thats not a pickle train .", "words": ["pickle", "train", "not", "a"]}, {"sent": "I thought she said pickle .", "words": ["she", "pickle"]}, {"sent": "did you have a pickle ?", "words": ["pickle", "have", "you", "did", "a"]}, {"sent": "its pickle .", "words": ["pickle"]}, {"sent": "thats pickle darling .", "words": ["pickle"]}, {"sent": "one pickle .", "words": ["pickle"]}, {"sent": "you are in a state in a pickle this afternoon arent you ?", "words": ["pickle", "you", "this", "in", "are", "a"]}, {"sent": "cookie pickle .", "words": ["cookie", "pickle"]}, {"sent": "a pickle .", "words": ["pickle", "a"]}, {"sent": "youve got pickle juice all over your chin .", "words": ["all", "your", "chin", "pickle", "juice", "over"]}, {"sent": "thats the pickle I think .", "words": ["the", "think", "pickle"]}, {"sent": "you are a little pickle ever since you got up at six oclock this morning .", "words": ["up", "pickle", "little", "you", "this", "are", "at", "a"]}, {"sent": "thats a pickle .", "words": ["pickle", "a"]}, {"sent": "pickle juice from last weekend .", "words": ["last", "juice", "pickle"]}, {"sent": "a pickle car .", "words": ["car", "pickle", "a"]}, {"sent": "Ill remove our pickle dishes if you remove your chameleon .", "words": ["your", "if", "pickle", "you", "our"]}, {"sent": "Mey would you like to buy a pickle ?", "words": ["to", "like", "you", "pickle", "would", "buy", "a"]}, {"sent": "ooh watch out because theres a little pickle juice on the plate we dont wanna spill it on you .", "words": ["the", "watch", "plate", "we", "little", "pickle", "dont", "you", "it", "spill", "out", "wanna", "on", "juice", "because", "a"]}, {"sent": "a pickle train .", "words": ["pickle", "train", "a"]}, {"sent": "Patricia had a pickle for an appetizer .", "words": ["pickle", "for", "an", "a"]}, {"sent": "Peter Piper picked a peck of pickle peppers .", "words": ["of", "pickle", "a"]}, {"sent": "hippopotamus you get a pickle from the pickle barrel .", "words": ["the", "pickle", "get", "you", "a"]}, {"sent": "are you being a pickle ?", "words": ["pickle", "are", "a", "you"]}, {"sent": "pickle ?", "words": ["pickle"]}, {"sent": "thats a big pickle .", "words": ["pickle", "big", "a"]}, {"sent": "I gave you a pickle to eat .", "words": ["to", "pickle", "eat", "you", "a"]}, {"sent": "she likes pickle ?", "words": ["she", "pickle"]}, {"sent": "is that a pickle ?", "words": ["pickle", "that", "a", "is"]}, {"sent": "or pickle .", "words": ["pickle"]}, {"sent": "look Mummys going to have some pickle on her sandwich .", "words": ["to", "pickle", "have", "look", "on", "some", "her", "sandwich"]}, {"sent": "did you drop a pickle on the dirty floor ?", "words": ["the", "pickle", "you", "drop", "did", "dirty", "on", "a"]}, {"sent": "thats pickle darling .", "words": ["pickle"]}, {"sent": "you dont wanna put them in the trash .", "words": ["the", "put", "trash", "them", "you", "dont", "in", "wanna"]}, {"sent": "did you throw it in the trash basket ?", "words": ["the", "throw", "trash", "you", "in", "it", "basket", "did"]}, {"sent": "why dont you come over here and dump it in the trash .", "words": ["the", "trash", "you", "here", "dont", "and", "it", "in", "dump", "over", "why"]}, {"sent": "and did we trash it ?", "words": ["we", "trash", "and", "it", "did"]}, {"sent": "right in the trash how about that ?", "words": ["the", "how", "trash", "in", "about", "that"]}, {"sent": "you been in my trash can .", "words": ["can", "my", "trash", "you", "in"]}, {"sent": "put those envelopes in the trash .", "words": ["the", "put", "those", "trash", "in"]}, {"sent": "the rabbit was thrown on the trash heap for burning .", "words": ["the", "for", "was", "trash", "on"]}, {"sent": "well put some trash in here then well push it out .", "words": ["put", "push", "trash", "here", "in", "it", "out", "some", "then"]}, {"sent": "they come to get the trash .", "words": ["the", "to", "trash", "get", "they"]}, {"sent": "what kind of cup was that that you could put it in the trash can ?", "words": ["cup", "what", "put", "the", "can", "of", "was", "trash", "you", "in", "it", "could", "that"]}, {"sent": "this Easter egg is from last year Max into the trash bucket it goes .", "words": ["the", "egg", "into", "is", "bucket", "last", "trash", "this", "it"]}, {"sent": "put that in the trash can .", "words": ["the", "can", "put", "trash", "in", "that"]}, {"sent": "into the trash Max said Ruby .", "words": ["the", "trash", "into"]}, {"sent": "in the trash he went .", "words": ["the", "in", "trash", "he"]}, {"sent": "I dont care if that trash aint full .", "words": ["if", "full", "trash", "dont", "that"]}, {"sent": "well it was allgone and I took the middle called the cob and I threw it in the trash .", "words": ["the", "was", "trash", "and", "in", "it"]}, {"sent": "so you put him in the trash can .", "words": ["the", "can", "put", "so", "trash", "you", "in", "him"]}, {"sent": "look at all that trash .", "words": ["all", "look", "trash", "at", "that"]}, {"sent": "can you go put it in the trash ?", "words": ["can", "the", "put", "go", "trash", "you", "in", "it"]}, {"sent": "trash ?", "words": ["trash"]}, {"sent": "in the trash can ?", "words": ["the", "can", "in", "trash"]}, {"sent": "are you peeking in the trash ?", "words": ["the", "trash", "you", "in", "are"]}, {"sent": "dont move the trash .", "words": ["the", "dont", "trash"]}, {"sent": "thats the trash .", "words": ["the", "trash"]}, {"sent": "the trash .", "words": ["the", "trash"]}, {"sent": "yup trash .", "words": ["trash"]}, {"sent": "we go to the trash can with it .", "words": ["the", "can", "go", "we", "to", "trash", "with", "it"]}, {"sent": "this is trash .", "words": ["this", "trash", "is"]}, {"sent": "he went through the trash barrels .", "words": ["the", "trash", "he"]}, {"sent": "trash can looks full .", "words": ["can", "trash", "full"]}, {"sent": "here xxx all the trash pushes out .", "words": ["the", "all", "trash", "here", "out"]}, {"sent": "trash needs to go out .", "words": ["to", "trash", "out", "go"]}, {"sent": "oh hes gonna go put his napkin in the trash .", "words": ["the", "put", "go", "trash", "in", "his", "napkin"]}, {"sent": "should I just leave my trash right here ?", "words": ["here", "my", "trash"]}, {"sent": "push all the trash out .", "words": ["the", "all", "push", "trash", "out"]}, {"sent": "put it in the trash if youre finished with it .", "words": ["the", "put", "if", "trash", "with", "in", "it"]}, {"sent": "put the trash together .", "words": ["the", "put", "trash"]}, {"sent": "well not in my trash please thankyou .", "words": ["in", "trash", "not", "my"]}, {"sent": "no no no sweetie thats trash .", "words": ["trash"]}, {"sent": "yeah we took all the trash out to the garage .", "words": ["the", "all", "we", "garage", "to", "trash", "out"]}, {"sent": "not in the trash .", "words": ["the", "in", "trash", "not"]}, {"sent": "say trash .", "words": ["trash", "say"]}, {"sent": "you can put it in the trash .", "words": ["can", "the", "put", "trash", "you", "in", "it"]}, {"sent": "this weekend were going to have a trash session .", "words": ["to", "trash", "have", "this", "were", "a"]}, {"sent": "here lets put it in the trash can because itll make a mess on the floor .", "words": ["the", "can", "put", "trash", "here", "in", "it", "on", "make", "because", "a"]}, {"sent": "throw them in the trash ?", "words": ["the", "throw", "trash", "them", "in"]}, {"sent": "say trash .", "words": ["trash", "say"]}, {"sent": "dont you want ta put that in the trash can ?", "words": ["the", "can", "put", "trash", "you", "dont", "in", "that"]}, {"sent": "no theres no trash can mommy will hold the trash .", "words": ["can", "the", "hold", "trash", "will"]}, {"sent": "we threw it away in the trash .", "words": ["the", "we", "trash", "in", "it", "away"]}, {"sent": "and Mommy cut the bottom outof a trash basket like that one right there .", "words": ["the", "bottom", "cut", "trash", "like", "and", "there", "basket", "that", "a"]}, {"sent": "go put your pop in the trash please .", "words": ["the", "put", "pop", "your", "go", "trash", "in"]}, {"sent": "thats right when you were little you didnt know how to empty trash .", "words": ["how", "to", "trash", "little", "you", "empty", "when", "were"]}, {"sent": "a recycling truck carries away trash but this trash will not go to a dump .", "words": ["go", "to", "trash", "truck", "this", "away", "will", "dump", "not", "but", "a"]}, {"sent": "get out of the trash can Allen .", "words": ["the", "can", "of", "trash", "get", "out"]}, {"sent": "kitty is in the trash can .", "words": ["the", "can", "is", "trash", "in", "kitty"]}, {"sent": "Im going to take the trash out .", "words": ["the", "to", "trash", "out", "take"]}, {"sent": "wanna put the paper in the trash can ?", "words": ["the", "can", "put", "paper", "trash", "in", "wanna"]}, {"sent": "and and then sometimes she drives the trash truck .", "words": ["the", "trash", "truck", "and", "then", "she"]}, {"sent": "put it in the trash .", "words": ["the", "put", "trash", "in", "it"]}, {"sent": "isnt that strange to live in a trash can ?", "words": ["can", "to", "trash", "in", "that", "a"]}, {"sent": "thats gonna go in the trash .", "words": ["the", "in", "trash", "go"]}, {"sent": "I was taking the trash out .", "words": ["the", "was", "trash", "out"]}, {"sent": "put it in the trash ?", "words": ["the", "put", "trash", "in", "it"]}, {"sent": "ah ah get outof that trash .", "words": ["get", "trash", "that"]}, {"sent": "*SI1: wait wait a minute its its tuesday though theres no its not trash day .", "words": ["wait", "trash", "not", "a"]}, {"sent": "oh thank you for putting the trash away .", "words": ["the", "for", "trash", "you", "away"]}, {"sent": "you dont wanna take all the trash out .", "words": ["the", "all", "trash", "you", "dont", "wanna", "out", "take"]}, {"sent": "I gave you a banana this morning and you only ate half and you threw the other half in the trash which I did not want you to do .", "words": ["the", "do", "to", "trash", "and", "you", "banana", "this", "other", "in", "did", "which", "not", "a"]}, {"sent": "dont touch the trash .", "words": ["the", "dont", "trash", "touch"]}, {"sent": "thats trash .", "words": ["trash"]}, {"sent": "were gonna hafta throw that in the trash .", "words": ["the", "throw", "trash", "in", "that", "were"]}, {"sent": "yeah but that has to Daddy has to clean that out and throw it in the trash .", "words": ["clean", "the", "throw", "to", "trash", "and", "in", "it", "out", "that", "but"]}, {"sent": "this is a mm trash truck .", "words": ["is", "trash", "truck", "this", "a"]}, {"sent": "little trash or little rash ?", "words": ["trash", "little"]}, {"sent": "but thats not a trash truck .", "words": ["trash", "truck", "not", "but", "a"]}, {"sent": "you can put this trash in here now .", "words": ["can", "put", "trash", "you", "here", "this", "in"]}, {"sent": "the trash goes in there and then you pull the little things and out comes the trash like this .", "words": ["the", "trash", "like", "and", "you", "there", "in", "pull", "little", "this", "out", "then"]}, {"sent": "we make alotof dirty diapers so we need trash bags .", "words": ["need", "we", "so", "trash", "dirty", "make"]}, {"sent": "crash went the trash .", "words": ["the", "trash"]}, {"sent": "trash is dirty .", "words": ["trash", "dirty", "is"]}, {"sent": "we hafta take the trash out pretty soon .", "words": ["the", "we", "trash", "pretty", "out", "take"]}, {"sent": "theres a trash can .", "words": ["can", "trash", "a"]}, {"sent": "oh I have the trash over here so .", "words": ["the", "so", "trash", "have", "here", "over"]}, {"sent": "in a trash .", "words": ["in", "trash", "a"]}, {"sent": "Mommy is gonna throw all the dirt in the trash .", "words": ["the", "all", "throw", "is", "trash", "in"]}, {"sent": "I was like well throw it in the trash so I threw it away but he never put it in his mouth .", "words": ["the", "put", "throw", "was", "so", "trash", "mouth", "like", "in", "it", "away", "he", "his", "but"]}, {"sent": "I thought this trash can opened .", "words": ["can", "this", "trash"]}, {"sent": "now put it in the trash .", "words": ["the", "put", "trash", "in", "it"]}, {"sent": "xxx the trash .", "words": ["the", "trash"]}, {"sent": "a trash can .", "words": ["can", "trash", "a"]}, {"sent": "its the trash .", "words": ["the", "trash"]}, {"sent": "hey zo theres nothing in the trash can that you could possibly want .", "words": ["the", "can", "trash", "you", "in", "could", "that"]}, {"sent": "put your diaper in the trash can .", "words": ["the", "can", "put", "your", "diaper", "trash", "in"]}, {"sent": "thats for the trash .", "words": ["the", "for", "trash"]}, {"sent": "do I know what trash is ?", "words": ["do", "what", "trash", "is"]}, {"sent": "it looks like hes eating cheerios dunna it ?", "words": ["cheerios", "it", "like"]}, {"sent": "would you like some cheerios ?", "words": ["like", "cheerios", "you", "would", "some"]}, {"sent": "yummy yummy yummy I wonder if Thomas and Catherine would like a little cheerios too when they get home .", "words": ["if", "home", "like", "and", "little", "they", "get", "would", "cheerios", "too", "when", "a"]}, {"sent": "hes eating cheerios ?", "words": ["cheerios"]}, {"sent": "want me to get you some cheerios to give to the cow ?", "words": ["the", "to", "you", "get", "cheerios", "me", "give", "cow", "some"]}, {"sent": "cheerios .", "words": ["cheerios"]}, {"sent": "like he knows what his what the cheerios box looks like I dont know if its because Im saying cheerios .", "words": ["the", "what", "box", "if", "like", "dont", "he", "his", "cheerios", "because"]}, {"sent": "she likes cheerios a lot .", "words": ["a lot", "cheerios", "she", "a"]}, {"sent": "you had cheerios .", "words": ["cheerios", "you"]}, {"sent": "yeah they are wearing cheerios youre right .", "words": ["they", "are", "cheerios"]}, {"sent": "and your cheerios ?", "words": ["cheerios", "your", "and"]}, {"sent": "you ate all that big breakfast this morning french toast and cheerios .", "words": ["all", "toast", "you", "and", "this", "that", "cheerios", "big"]}, {"sent": "would you like some cheerios ?", "words": ["like", "cheerios", "you", "would", "some"]}, {"sent": "can you say cheerios ?", "words": ["can", "cheerios", "say", "you"]}, {"sent": "want some more cheerios ?", "words": ["some", "more", "cheerios"]}, {"sent": "there are your cheerios .", "words": ["there", "are", "your", "cheerios"]}, {"sent": "I found some cheerios .", "words": ["some", "cheerios"]}, {"sent": "no cheerios .", "words": ["cheerios"]}, {"sent": "no cheerios .", "words": ["cheerios"]}, {"sent": "you spilled all your cheerios .", "words": ["all", "cheerios", "your", "you"]}, {"sent": "you spilled all your cheerios .", "words": ["all", "cheerios", "your", "you"]}, {"sent": "you wanna put some cheerios in the box ?", "words": ["the", "put", "box", "cheerios", "you", "in", "wanna", "some"]}, {"sent": "cheerios in the box .", "words": ["the", "cheerios", "in", "box"]}, {"sent": "want some more cheerios ?", "words": ["some", "more", "cheerios"]}, {"sent": "or cheerios ?", "words": ["cheerios"]}, {"sent": "youre eating all of Eeyores cheerios .", "words": ["of", "all", "cheerios"]}, {"sent": "want some more cheerios ?", "words": ["some", "more", "cheerios"]}, {"sent": "you want some cheerios ?", "words": ["some", "cheerios", "you"]}, {"sent": "should we get some cheerios ?", "words": ["get", "some", "cheerios", "we"]}, {"sent": "cheerios .", "words": ["cheerios"]}, {"sent": "Im gonna get you some different kind of cheerios here .", "words": ["of", "cheerios", "get", "you", "here", "some"]}, {"sent": "cereal cheerios .", "words": ["cheerios", "cereal"]}, {"sent": "eat cheerios .", "words": ["eat", "cheerios"]}, {"sent": "crackers oyster crackers goldfish cheese puffs cheerios cookies ?", "words": ["cheerios", "cheese"]}, {"sent": "and you probably have some cheerios and Oatios too .", "words": ["cheerios", "have", "you", "and", "too", "some"]}, {"sent": "would you like some cheerios ?", "words": ["like", "cheerios", "you", "would", "some"]}, {"sent": "okay you want cheerios with bananas ?", "words": ["cheerios", "with", "you"]}, {"sent": "I felt like having cheerios today .", "words": ["cheerios", "like"]}, {"sent": "other than uh as table foods go hell eat pieces of bread and cheerios but .", "words": ["bread", "table", "go", "of", "other", "eat", "and", "cheerios", "but"]}, {"sent": "cheerios maybe I I keep thinking they would .", "words": ["cheerios", "they", "would"]}, {"sent": "and she eats puffs and cheerios .", "words": ["cheerios", "she", "and"]}, {"sent": "like any pops or cheerios or anything like that he wont have anything to do with it .", "words": ["do", "to", "any", "like", "with", "have", "it", "he", "cheerios", "that"]}, {"sent": "she eats like cheerios and fruit puffs and stuff like that .", "words": ["like", "and", "cheerios", "that", "she"]}, {"sent": "um I definitely think they they look realistic but um when she eats peas theyre like little cheerios you know she can pick them up with xxx and do them in their spread out so um Im not sure she associated that with what peas are .", "words": ["what", "think", "like", "when", "not", "can", "their", "peas", "with", "are", "cheerios", "that", "she", "up", "pick", "so", "little", "you", "and", "do", "them", "they", "in", "out", "look", "but"]}, {"sent": "thats a little cereal box just like you eat your cheerios out of .", "words": ["box", "your", "of", "like", "little", "you", "eat", "out", "cheerios", "cereal", "a"]}, {"sent": "oh you had cheerios this morning .", "words": ["cheerios", "this", "you"]}, {"sent": "cheerios ?", "words": ["cheerios"]}, {"sent": "like cheerios ?", "words": ["cheerios", "like"]}, {"sent": "are they cheerios ?", "words": ["they", "are", "cheerios"]}, {"sent": "cheerios cereal cheerios cereal .", "words": ["cheerios", "cereal"]}, {"sent": "oh cheerios .", "words": ["cheerios"]}, {"sent": "cheerios .", "words": ["cheerios"]}, {"sent": "cheerios are gonegone .", "words": ["cheerios", "are"]}, {"sent": "here is your juice and your sandwich and your cheerios .", "words": ["your", "is", "and", "here", "cheerios", "juice", "sandwich"]}, {"sent": "oh cheerios ?", "words": ["cheerios"]}, {"sent": "the cheerios are gonegone .", "words": ["the", "cheerios", "are"]}, {"sent": "cheerios .", "words": ["cheerios"]}, {"sent": "diaper diaper diaper .", "words": ["diaper"]}, {"sent": "thats on your diaper pail right ?", "words": ["on", "diaper", "your"]}, {"sent": "oh Mommas going to diaper the baby .", "words": ["diaper", "to", "the"]}, {"sent": "say diaper diaper diaper .", "words": ["diaper", "say"]}, {"sent": "shall I change your diaper ?", "words": ["diaper", "your"]}, {"sent": "you need a clean diaper and clean .", "words": ["clean", "need", "diaper", "you", "and", "a"]}, {"sent": "let us go get a diaper .", "words": ["go", "diaper", "get", "us", "a"]}, {"sent": "gonna do your diaper .", "words": ["diaper", "do", "your"]}, {"sent": "would you like a new diaper ?", "words": ["diaper", "like", "you", "would", "new", "a"]}, {"sent": "cant even get any babys diaper xxx .", "words": ["get", "diaper", "any"]}, {"sent": "we gotta change your diaper .", "words": ["diaper", "your", "we"]}, {"sent": "thats the dirty diaper .", "words": ["the", "diaper", "dirty"]}, {"sent": "here give me the diaper bag .", "words": ["the", "diaper", "here", "me", "give"]}, {"sent": "we put on diaper ?", "words": ["on", "diaper", "put", "we"]}, {"sent": "Daddy is gonna get a diaper .", "words": ["get", "diaper", "a", "is"]}, {"sent": "I think we needta go change your diaper .", "words": ["think", "your", "go", "diaper", "we"]}, {"sent": "you need a clean diaper .", "words": ["clean", "need", "diaper", "you", "a"]}, {"sent": "now let me go get the diaper .", "words": ["the", "go", "diaper", "get", "me"]}, {"sent": "lets go change the diaper .", "words": ["the", "diaper", "go"]}, {"sent": "lets put your diaper on .", "words": ["diaper", "put", "your", "on"]}, {"sent": "that diaper looks so bad .", "words": ["diaper", "so", "that", "bad"]}, {"sent": "it is a bad diaper .", "words": ["is", "diaper", "bad", "it", "a"]}, {"sent": "lemme just tuck this in where your diaper is .", "words": ["where", "your", "is", "diaper", "this", "in"]}, {"sent": "and then were gonna change your diaper .", "words": ["your", "diaper", "and", "then", "were"]}, {"sent": "let us go change your diaper okay ?", "words": ["diaper", "us", "your", "go"]}, {"sent": "well give you a few minutes and then well change your diaper .", "words": ["your", "diaper", "then", "and", "you", "give", "a"]}, {"sent": "and you also got a stinky diaper .", "words": ["diaper", "a", "you", "and"]}, {"sent": "I guess we have to put your diaper back on .", "words": ["put", "your", "back", "we", "diaper", "to", "have", "on"]}, {"sent": "no diaper on eh ?", "words": ["diaper", "on"]}, {"sent": "which which dolly needs a diaper ?", "words": ["diaper", "which", "a"]}, {"sent": "lets get your your diaper on .", "words": ["get", "diaper", "on", "your"]}, {"sent": "find the diaper .", "words": ["the", "diaper", "find"]}, {"sent": "I change your diaper ?", "words": ["diaper", "your"]}, {"sent": "you been saying sore when we changed your diaper .", "words": ["your", "we", "diaper", "you", "when"]}, {"sent": "you cant crawl around in diaper because it is a cold day .", "words": ["is", "diaper", "you", "in", "it", "around", "cold", "because", "a"]}, {"sent": "xxx his diaper .", "words": ["his", "diaper"]}, {"sent": "because I think you should have a clean diaper on when you go nightnight .", "words": ["clean", "think", "go", "diaper", "you", "have", "on", "when", "because", "a"]}, {"sent": "diaper diaper .", "words": ["diaper"]}, {"sent": "say diaper .", "words": ["diaper", "say"]}, {"sent": "diaper .", "words": ["diaper"]}, {"sent": "I wanna put your diaper on .", "words": ["put", "your", "diaper", "wanna", "on"]}, {"sent": "diaper .", "words": ["diaper"]}, {"sent": "lets put the diaper on now .", "words": ["the", "diaper", "put", "on"]}, {"sent": "okay lets finish changing your diaper first .", "words": ["diaper", "first", "your", "finish"]}, {"sent": "oh you didnt even wanna play with diaper .", "words": ["diaper", "play", "with", "you", "wanna"]}, {"sent": "I just need to get you a diaper Ill be right back .", "words": ["need", "back", "diaper", "to", "be", "need to", "you", "get", "a"]}, {"sent": "why did you throw the diaper on the floor ?", "words": ["the", "throw", "diaper", "you", "did", "on", "why"]}, {"sent": "oo you messed up your diaper real .", "words": ["up", "diaper", "your", "you"]}, {"sent": "just hold the diaper .", "words": ["the", "diaper", "hold"]}, {"sent": "I thought you had a dirty diaper and you dont .", "words": ["diaper", "and", "you", "dont", "dirty", "a"]}, {"sent": "you do not you do not even know that is a diaper .", "words": ["do", "is", "diaper", "you", "that", "not", "a"]}, {"sent": "lets lets lets get the diaper on .", "words": ["get", "the", "on", "diaper"]}, {"sent": "I bet that you need your diaper changed .", "words": ["your", "need", "diaper", "you", "that"]}, {"sent": "first were gonna put your diaper on .", "words": ["put", "your", "diaper", "first", "on", "were"]}, {"sent": "this diaper is gross .", "words": ["diaper", "this", "is"]}, {"sent": "okay lets change the diaper .", "words": ["the", "diaper"]}, {"sent": "okay now leave your diaper alone now .", "words": ["diaper", "your"]}, {"sent": "sleeping wearing a diaper .", "words": ["diaper", "a"]}, {"sent": "sit on Daddys face with a wet diaper .", "words": ["wet", "face", "diaper", "with", "on", "sit", "a"]}, {"sent": "do we need a diaper ?", "words": ["do", "need", "we", "diaper", "a"]}, {"sent": "I look down at my diaper .", "words": ["my", "look", "diaper", "at", "down"]}, {"sent": "and Mommys gonna change your diaper .", "words": ["diaper", "your", "and"]}, {"sent": "off the diaper .", "words": ["off", "the", "diaper"]}, {"sent": "were gonna put a diaper on you in a minute huh ?", "words": ["put", "diaper", "you", "in", "on", "were", "a"]}, {"sent": "and um she likes to tuck the pig into the blanket and she talks to them and sometimes OinkOink is crying or sometimes he needs a new diaper or sometimes hes happy .", "words": ["the", "into", "happy", "is", "diaper", "to", "them", "pig", "and", "he", "blanket", "new", "she", "a"]}, {"sent": "its okay its just a diaper .", "words": ["diaper", "a"]}, {"sent": "change your diaper .", "words": ["diaper", "your"]}, {"sent": "put a diaper on .", "words": ["diaper", "put", "on", "a"]}, {"sent": "is she wearing a diaper ?", "words": ["diaper", "she", "a", "is"]}, {"sent": "Im gonna change your diaper .", "words": ["diaper", "your"]}, {"sent": "your diaper is about to come apart .", "words": ["your", "is", "diaper", "to", "about"]}, {"sent": "when I was a baby I had a diaper round my little tummy .", "words": ["my", "diaper", "was", "little", "tummy", "when", "a"]}, {"sent": "lets lets change your diaper .", "words": ["diaper", "your"]}, {"sent": "well I she gets well she probably should have her diaper changed .", "words": ["have", "diaper", "her", "she"]}, {"sent": "let me get your diaper off .", "words": ["off", "your", "diaper", "get", "me"]}, {"sent": "nobody likes to fight to put on a diaper .", "words": ["put", "diaper", "to", "on", "a"]}, {"sent": "he needs a diaper on ?", "words": ["diaper", "on", "he", "a"]}, {"sent": "lets get your diaper and then you can play .", "words": ["can", "your", "diaper", "play", "and", "get", "you", "then"]}, {"sent": "xxx diaper .", "words": ["diaper"]}, {"sent": "she doesnt have a diaper on .", "words": ["diaper", "have", "on", "she", "a"]}, {"sent": "you wear a diaper ?", "words": ["diaper", "a", "you"]}, {"sent": "yeah hes sleeping with his little diaper on .", "words": ["diaper", "with", "little", "on", "his"]}, {"sent": "do you think that baby doll has a pooped diaper also ?", "words": ["do", "think", "diaper", "doll", "you", "that", "a"]}, {"sent": "yeah we gotta change your diaper .", "words": ["diaper", "your", "we"]}, {"sent": "we gotta go up and change your diaper .", "words": ["up", "your", "go", "we", "diaper", "and"]}, {"sent": "we gotta put your diaper on .", "words": ["put", "your", "we", "diaper", "on"]}, {"sent": "do you wanna hold that for a second while I change your diaper ?", "words": ["do", "for", "your", "hold", "diaper", "you", "wanna", "that", "a"]}, {"sent": "heres a diaper .", "words": ["diaper", "a"]}, {"sent": "theres a baby wearing a diaper .", "words": ["diaper", "a"]}, {"sent": "but you dont have a diaper on .", "words": ["diaper", "you", "have", "dont", "on", "but", "a"]}, {"sent": "you want me to change your diaper and get your peejays on ?", "words": ["your", "diaper", "to", "you", "and", "get", "on", "me"]}, {"sent": "little bunnys baby sister goes in her diaper .", "words": ["diaper", "in", "her", "little"]}, {"sent": "oh that was a full diaper .", "words": ["full", "diaper", "was", "that", "a"]}, {"sent": "get get a diaper Nina .", "words": ["get", "diaper", "a"]}, {"sent": "okay now we need to get diaper .", "words": ["need", "we", "diaper", "to", "need to", "get"]}, {"sent": "now we gotta get you a new diaper .", "words": ["we", "diaper", "you", "get", "new", "a"]}, {"sent": "youre in their little baby swimming pool when you were a little baby in your diaper .", "words": ["your", "their", "diaper", "pool", "little", "you", "in", "when", "were", "a"]}, {"sent": "yeah you have meatball all over your chin .", "words": ["all", "your", "chin", "you", "have", "over"]}, {"sent": "oh now youre getting my chin all dirty .", "words": ["all", "my", "dirty", "chin"]}, {"sent": "yeah its her chin yeah .", "words": ["her", "chin"]}, {"sent": "one misty moisty morning when cloudy was the weather I chanced to meet an old man clothed all in leather with a cap under his chin .", "words": ["the", "old", "all", "was", "to", "with", "chin", "under", "an", "in", "his", "when", "a"]}, {"sent": "didja hit your chin ?", "words": ["hit", "your", "chin"]}, {"sent": "this stuff on your chin .", "words": ["on", "this", "your", "chin"]}, {"sent": "look at Crankys funny chin .", "words": ["at", "chin", "look"]}, {"sent": "yeah look at the mirror with your chinnie chin chin here .", "words": ["the", "look", "your", "with", "chin", "here", "at"]}, {"sent": "youre putting Daddys chin on Naimas chin .", "words": ["on", "chin"]}, {"sent": "thats a man scratching his chin .", "words": ["his", "chin", "a"]}, {"sent": "so it wont fall down without tying it to your chin .", "words": ["your", "to", "so", "fall", "chin", "it", "down"]}, {"sent": "and my chin .", "words": ["my", "chin", "and"]}, {"sent": "give him a chin ?", "words": ["give", "chin", "him", "a"]}, {"sent": "put this under your chin .", "words": ["put", "your", "chin", "this", "under"]}, {"sent": "you put it under Dobbins chin and see if Dobbin goes yellow .", "words": ["put", "if", "yellow", "chin", "you", "and", "under", "it", "see"]}, {"sent": "what about your chin ?", "words": ["what", "your", "chin", "about"]}, {"sent": "chin .", "words": ["chin"]}, {"sent": "chin .", "words": ["chin"]}, {"sent": "chin up .", "words": ["up", "chin"]}, {"sent": "say chin .", "words": ["say", "chin"]}, {"sent": "did you hurt your chin ?", "words": ["your", "hurt", "chin", "you", "did"]}, {"sent": "chin .", "words": ["chin"]}, {"sent": "is that your chin ?", "words": ["your", "that", "chin", "is"]}, {"sent": "and his chin .", "words": ["his", "chin", "and"]}, {"sent": "dont hurt her chin .", "words": ["hurt", "dont", "her", "chin"]}, {"sent": "not by the hair on my chinnie chin chin .", "words": ["the", "my", "chin", "by", "on", "hair", "not"]}, {"sent": "a yellow strap to go under your chin .", "words": ["your", "go", "to", "chin", "under", "yellow", "a"]}, {"sent": "under your chin or round the back .", "words": ["the", "your", "back", "chin", "under"]}, {"sent": "hes got two bumps on his chin .", "words": ["on", "his", "chin"]}, {"sent": "theres another hair underneath your chin .", "words": ["another", "hair", "your", "chin"]}, {"sent": "and a big chin ?", "words": ["big", "chin", "a", "and"]}, {"sent": "cheeks chin lips gums teeth .", "words": ["chin"]}, {"sent": "okay xxx lets get your mouth chin and cheeks here .", "words": ["your", "mouth", "chin", "and", "get", "here"]}, {"sent": "you had a funny chin when you were a baby didnt you ?", "words": ["when", "chin", "you", "were", "a"]}, {"sent": "chin chin chin .", "words": ["chin"]}, {"sent": "you put your chin right into that crummy cookie .", "words": ["cookie", "put", "your", "into", "chin", "you", "that"]}, {"sent": "that is your chin .", "words": ["your", "that", "chin", "is"]}, {"sent": "wash Nathaniels chin .", "words": ["wash", "chin"]}, {"sent": "so he was lonely and she says chin up chin up .", "words": ["up", "was", "so", "chin", "and", "he", "she"]}, {"sent": "do your chin .", "words": ["do", "your", "chin"]}, {"sent": "chin chin chin ?", "words": ["chin"]}, {"sent": "your chin hurts does it ?", "words": ["does", "it", "your", "chin"]}, {"sent": "one more chin wipe .", "words": ["more", "wipe", "chin"]}, {"sent": "watch your watch your chin on that door .", "words": ["watch", "your", "chin", "on", "that", "door"]}, {"sent": "no not by the hair on my chinnie chin chin .", "words": ["the", "my", "chin", "by", "on", "hair", "not"]}, {"sent": "it didnt cut your chin .", "words": ["it", "your", "cut", "chin"]}, {"sent": "is it Eve chin or Eves chin .", "words": ["it", "chin", "is"]}, {"sent": "chin .", "words": ["chin"]}, {"sent": "and your chinnie chin chin .", "words": ["your", "chin", "and"]}, {"sent": "here lets put some cream under your chin .", "words": ["put", "your", "chin", "here", "under", "some"]}, {"sent": "right up to the chin chin chin .", "words": ["up", "to", "chin", "the"]}, {"sent": "where is your chin .", "words": ["where", "your", "chin", "is"]}, {"sent": "where was your chin ?", "words": ["where", "was", "your", "chin"]}, {"sent": "thats your chin .", "words": ["your", "chin"]}, {"sent": "mm Im gonna bite your chin gonna eat your chin .", "words": ["bite", "your", "chin", "eat"]}, {"sent": "wheres your chin ?", "words": ["your", "chin"]}, {"sent": "xxx you have a yellow chin .", "words": ["chin", "you", "have", "yellow", "a"]}, {"sent": "come here let me wipe your chin .", "words": ["your", "chin", "here", "me", "wipe"]}, {"sent": "Cheerio hanging off your chin .", "words": ["off", "your", "chin"]}, {"sent": "whats happened to your chin ?", "words": ["to", "your", "chin"]}, {"sent": "theres his chin .", "words": ["his", "chin"]}, {"sent": "where is your chin ?", "words": ["where", "your", "chin", "is"]}, {"sent": "you chin ?", "words": ["chin", "you"]}, {"sent": "what if I kissed your little chin ?", "words": ["what", "your", "if", "chin", "little"]}, {"sent": "under my chin .", "words": ["under", "my", "chin"]}, {"sent": "chin up .", "words": ["up", "chin"]}, {"sent": "dribble chin .", "words": ["chin"]}, {"sent": "she says chin up chin up .", "words": ["up", "chin", "she"]}, {"sent": "I kissed your chin .", "words": ["your", "chin"]}, {"sent": "youve got marks all underneath your chin .", "words": ["all", "your", "chin"]}, {"sent": "I keep buying all these creams for your chin and then I keep losing em .", "words": ["all", "these", "for", "your", "chin", "and", "then"]}, {"sent": "a chin .", "words": ["chin", "a"]}, {"sent": "are you gonna put it on your own chin ?", "words": ["put", "your", "chin", "you", "it", "are", "on"]}, {"sent": "chin up chin up and then I forget .", "words": ["up", "then", "chin", "and"]}, {"sent": "on the chin ?", "words": ["on", "the", "chin"]}, {"sent": "you want a tissue for your chin ?", "words": ["for", "your", "tissue", "chin", "you", "a"]}, {"sent": "I think hes just rubbing his chin on the tree brunch isnt he ?", "words": ["the", "think", "tree", "chin", "on", "he", "his"]}, {"sent": "wipe your chin then .", "words": ["then", "wipe", "your", "chin"]}, {"sent": "if I put that under your chin it should make it glow yellow .", "words": ["put", "your", "if", "chin", "under", "it", "make", "yellow", "that"]}, {"sent": "thats your chin .", "words": ["your", "chin"]}, {"sent": "lemme see your chin .", "words": ["see", "your", "chin"]}, {"sent": "okay you got your own chin .", "words": ["your", "chin", "you"]}, {"sent": "not by the hair of my chinnie chin chin .", "words": ["the", "my", "of", "chin", "by", "hair", "not"]}, {"sent": "no no not by the hair of my chinnie chin chin .", "words": ["the", "my", "of", "chin", "by", "hair", "not"]}, {"sent": "say chin .", "words": ["say", "chin"]}, {"sent": "shall we put some on your chin ?", "words": ["put", "your", "we", "chin", "on", "some"]}, {"sent": "wipe your chin .", "words": ["wipe", "your", "chin"]}, {"sent": "he did fix your chin ?", "words": ["your", "fix", "chin", "he", "did"]}, {"sent": "lets get that cheese off of your chin .", "words": ["off", "your", "of", "chin", "get", "that", "cheese"]}, {"sent": "chin chin .", "words": ["chin"]}, {"sent": "tickling me under the chin ?", "words": ["the", "me", "under", "chin"]}, {"sent": "chin up please .", "words": ["up", "chin"]}, {"sent": "did you have green beans yesterday ?", "words": ["green beans", "you", "have", "beans", "did", "green"]}, {"sent": "some green beans .", "words": ["some", "green beans", "beans", "green"]}, {"sent": "youve got green beans on your nose .", "words": ["your", "green beans", "beans", "green", "on", "nose"]}, {"sent": "cooking green beans .", "words": ["green beans", "beans", "green"]}, {"sent": "green beans .", "words": ["green beans", "beans", "green"]}, {"sent": "what about some green beans ?", "words": ["what", "green beans", "beans", "green", "about", "some"]}, {"sent": "maybe theyre green beans .", "words": ["green beans", "beans", "green"]}, {"sent": "or green beans .", "words": ["green beans", "beans", "green"]}, {"sent": "you like the green beans they look real huh ?", "words": ["the", "like", "green beans", "you", "they", "beans", "green", "look"]}, {"sent": "you play with that with your green beans .", "words": ["your", "play", "with", "green beans", "you", "beans", "green", "that"]}, {"sent": "green beans ?", "words": ["green beans", "beans", "green"]}, {"sent": "maybe we can make some green beans .", "words": ["can", "we", "green beans", "beans", "green", "some", "make"]}, {"sent": "well okay Ill fix your green beans ?", "words": ["your", "fix", "green beans", "beans", "green"]}, {"sent": "is there any green beans there ?", "words": ["is", "any", "green beans", "there", "beans", "green"]}, {"sent": "Daddys eating green beans .", "words": ["green beans", "beans", "green"]}, {"sent": "you gonna cook the green beans ?", "words": ["the", "green beans", "you", "cook", "beans", "green"]}, {"sent": "those are green beans .", "words": ["those", "green beans", "are", "beans", "green"]}, {"sent": "the green beans .", "words": ["the", "green beans", "beans", "green"]}, {"sent": "green beans ?", "words": ["green beans", "beans", "green"]}, {"sent": "and some green beans or some sugar snap peas or something like that would be fun .", "words": ["be", "peas", "like", "green beans", "and", "beans", "green", "would", "some", "that"]}, {"sent": "not green beans jelly beans .", "words": ["jelly", "green beans", "beans", "green", "not"]}, {"sent": "Victor do you see any green beans ?", "words": ["do", "any", "green beans", "you", "beans", "green", "see"]}, {"sent": "I think those are green beans .", "words": ["think", "those", "green beans", "are", "beans", "green"]}, {"sent": "are those green beans ?", "words": ["those", "green beans", "are", "beans", "green"]}, {"sent": "look Macy green beans .", "words": ["green beans", "beans", "green", "look"]}, {"sent": "green beans ?", "words": ["green beans", "beans", "green"]}, {"sent": "green beans .", "words": ["green beans", "beans", "green"]}, {"sent": "do you want some frozen green beans to eat maybe ?", "words": ["do", "to", "green beans", "you", "eat", "beans", "green", "some"]}, {"sent": "are you enjoying those green beans ?", "words": ["those", "green beans", "you", "beans", "are", "green"]}, {"sent": "take up the the green beans and put them in a white bowl an then Ill be in there .", "words": ["up", "the", "put", "bowl", "be", "them", "green beans", "and", "there", "in", "beans", "an", "green", "then", "white", "take", "a"]}, {"sent": "are you gonna finish your green beans baby ?", "words": ["your", "finish", "green beans", "you", "are", "beans", "green"]}, {"sent": "you want some green beans ?", "words": ["green beans", "you", "beans", "green", "some"]}, {"sent": "uh huh well do um ice cream and cupcakes and green beans and hot dogs um we dont really and hamburgers .", "words": ["do", "ice cream", "we", "green beans", "and", "dont", "hot", "beans", "green", "ice"]}, {"sent": "yucky green beans ?", "words": ["green beans", "yucky", "beans", "green"]}, {"sent": "polar bears probably like these frozen green beans .", "words": ["these", "like", "green beans", "beans", "green"]}, {"sent": "green beans .", "words": ["green beans", "beans", "green"]}, {"sent": "and all your green beans ?", "words": ["all", "your", "green beans", "and", "beans", "green"]}, {"sent": "Daddys eating his green beans .", "words": ["his", "green beans", "beans", "green"]}, {"sent": "can you hand me the green beans ?", "words": ["can", "the", "hand", "green beans", "you", "beans", "green", "me"]}, {"sent": "ooh green beans and cheese thats a good idea .", "words": ["green beans", "and", "beans", "green", "good", "cheese", "a"]}, {"sent": "green beans are green .", "words": ["green beans", "beans", "are", "green"]}, {"sent": "did you eat all your green beans ?", "words": ["all", "your", "green beans", "you", "eat", "beans", "did", "green"]}, {"sent": "gree the green beans and the cheese bread and sometimes pizza not very often .", "words": ["the", "bread", "pizza", "and", "green beans", "beans", "green", "cheese", "not"]}, {"sent": "you gonna give the green beans to the girl ?", "words": ["the", "to", "green beans", "you", "beans", "green", "give"]}, {"sent": "oh and I found green beans .", "words": ["green beans", "beans", "green", "and"]}, {"sent": "whada you do when I say green beans ?", "words": ["do", "say", "green beans", "you", "beans", "green", "when"]}, {"sent": "dont you want your green beans ?", "words": ["your", "green beans", "you", "dont", "beans", "green"]}, {"sent": "and green beans .", "words": ["green beans", "beans", "green", "and"]}, {"sent": "yeah he loves green beans .", "words": ["green beans", "beans", "he", "green"]}, {"sent": "Kari eat some of your green beans too .", "words": ["your", "of", "green beans", "eat", "beans", "green", "too", "some"]}, {"sent": "thats an awful lot of green beans for that spoon .", "words": ["for", "of", "green beans", "an", "beans", "spoon", "green", "that"]}, {"sent": "we tried green beans but they werent soft enough .", "words": ["soft", "we", "green beans", "they", "beans", "green", "but"]}, {"sent": "green beans and carrots .", "words": ["green beans", "and", "beans", "carrots", "green"]}, {"sent": "and you like green beans and you like pasta and you like rice .", "words": ["like", "green beans", "you", "and", "beans", "green"]}, {"sent": "is the peach going on top of the green beans ?", "words": ["the", "is", "of", "green beans", "beans", "green", "on"]}, {"sent": "green beans .", "words": ["green beans", "beans", "green"]}, {"sent": "those are green beans yes they are .", "words": ["those", "green beans", "they", "are", "beans", "green"]}, {"sent": "do you see green beans ?", "words": ["do", "green beans", "you", "beans", "green", "see"]}, {"sent": "green beans or peas .", "words": ["peas", "green beans", "beans", "green"]}, {"sent": "why dont you make green beans this time .", "words": ["green beans", "you", "dont", "this", "beans", "green", "make", "why"]}, {"sent": "oh well youknow the green beans are a little funky too youknow theyre good conversation .", "words": ["the", "green beans", "little", "beans", "are", "green", "too", "good", "a"]}, {"sent": "do you think theyll grow green beans ?", "words": ["do", "think", "green beans", "you", "beans", "green"]}, {"sent": "Ill give you some green beans .", "words": ["green beans", "you", "beans", "green", "some", "give"]}, {"sent": "we dont have the beets or the green beans .", "words": ["the", "we", "green beans", "have", "dont", "beans", "green"]}, {"sent": "green beans ?", "words": ["green beans", "beans", "green"]}, {"sent": "yummy green beans .", "words": ["green beans", "beans", "green"]}, {"sent": "I think I brought green beans for your lunch too .", "words": ["think", "for", "your", "green beans", "beans", "green", "too"]}, {"sent": "you probably dont need that to cut green beans but .", "words": ["need", "cut", "to", "green beans", "you", "dont", "beans", "green", "that", "but"]}, {"sent": "do you see green beans ?", "words": ["do", "green beans", "you", "beans", "green", "see"]}, {"sent": "you eat green beans .", "words": ["green beans", "you", "eat", "beans", "green"]}, {"sent": "those are green beans .", "words": ["those", "green beans", "are", "beans", "green"]}, {"sent": "those are green beans .", "words": ["those", "green beans", "are", "beans", "green"]}, {"sent": "green beans .", "words": ["green beans", "beans", "green"]}, {"sent": "thats why you call them green beans .", "words": ["them", "green beans", "you", "beans", "green", "why"]}, {"sent": "and she already had green beans and potatoes .", "words": ["green beans", "and", "beans", "green", "she"]}, {"sent": "and green beans .", "words": ["green beans", "beans", "green", "and"]}, {"sent": "no green beans arent white .", "words": ["green beans", "white", "beans", "green"]}, {"sent": "can I have some green beans please ?", "words": ["can", "green beans", "have", "beans", "green", "some"]}, {"sent": "she eats oh no she has seen peas and green beans .", "words": ["peas", "green beans", "and", "beans", "green", "she"]}, {"sent": "sorry Stephanie that was only for seasoning the green beans with .", "words": ["the", "for", "was", "with", "green beans", "beans", "green", "that"]}, {"sent": "hey look at the green beans .", "words": ["the", "look", "green beans", "beans", "green", "at"]}, {"sent": "some green beans ?", "words": ["some", "green beans", "beans", "green"]}, {"sent": "green beans .", "words": ["green beans", "beans", "green"]}, {"sent": "are they peas broccoli or green beans ?", "words": ["peas", "green beans", "they", "are", "beans", "green"]}, {"sent": "and these green beans .", "words": ["these", "green beans", "and", "beans", "green"]}, {"sent": "no green beans .", "words": ["green beans", "beans", "green"]}, {"sent": "green beans and kiwi and .", "words": ["green beans", "beans", "green", "and"]}, {"sent": "green beans do you mean ?", "words": ["do", "green beans", "you", "beans", "green"]}, {"sent": "woah all these green beans .", "words": ["all", "these", "green beans", "beans", "green"]}, {"sent": "Abby who ate green beans last night ?", "words": ["last", "green beans", "beans", "green", "who"]}, {"sent": "green beans .", "words": ["green beans", "beans", "green"]}, {"sent": "Ive got some green beans .", "words": ["some", "green beans", "beans", "green"]}, {"sent": "where are your green beans ?", "words": ["where", "your", "green beans", "are", "beans", "green"]}, {"sent": "I mean the green beans .", "words": ["the", "green beans", "beans", "green"]}, {"sent": "want some yogurt with berries ?", "words": ["some", "yogurt", "with"]}, {"sent": "do you like that yogurt ?", "words": ["do", "yogurt", "like", "you", "that"]}, {"sent": "you can have some frozen blueberries do you want some yogurt also ?", "words": ["can", "do", "yogurt", "you", "have", "some"]}, {"sent": "yogurt and granola for dinner ?", "words": ["yogurt", "for", "and"]}, {"sent": "more yogurt perhaps ?", "words": ["more", "yogurt"]}, {"sent": "that bear really likes yogurt .", "words": ["yogurt", "that", "bear"]}, {"sent": "mm want some yogurt and cookies and raisins Daddy left this out oh its finished we hafta put it in the garbage .", "words": ["the", "put", "we", "yogurt", "and", "this", "in", "it", "out", "some", "garbage"]}, {"sent": "yogurt .", "words": ["yogurt"]}, {"sent": "I mean you dont sit on your yogurt when you wanna eat your yogurt do ya ?", "words": ["do", "your", "yogurt", "you", "eat", "dont", "wanna", "on", "sit", "when"]}, {"sent": "Im getting you yogurt xxx .", "words": ["yogurt", "you"]}, {"sent": "oh you could have yogurt Ill get some for you .", "words": ["for", "yogurt", "you", "have", "get", "some", "could"]}, {"sent": "dada yeah yogurt ?", "words": ["yogurt"]}, {"sent": "they eat yogurt .", "words": ["eat", "they", "yogurt"]}, {"sent": "oh you want this bowl okay you can have the bunny rabbit bowl let me just put some yogurt in to make it cooler .", "words": ["bowl", "can", "the", "bunny", "put", "to", "yogurt", "you", "have", "this", "in", "it", "me", "make", "some"]}, {"sent": "you have yogurt and food all over your chin and neck here .", "words": ["all", "your", "yogurt", "chin", "and", "have", "you", "food", "here", "over"]}, {"sent": "and youre eating yogurt ?", "words": ["yogurt", "and"]}, {"sent": "would you like some yogurt ?", "words": ["yogurt", "like", "you", "would", "some"]}, {"sent": "its yogurt sweetie .", "words": ["yogurt"]}, {"sent": "you like yogurt .", "words": ["yogurt", "like", "you"]}, {"sent": "and yogurt ?", "words": ["yogurt", "and"]}, {"sent": "no more yogurt .", "words": ["more", "yogurt"]}, {"sent": "doggie yogurt .", "words": ["yogurt"]}, {"sent": "theres no yogurt in our house except for the baby yogurt thats the flavor that you dont like .", "words": ["the", "for", "yogurt", "like", "our", "you", "dont", "in", "house", "that"]}, {"sent": "theres yogurt right here .", "words": ["here", "yogurt"]}, {"sent": "do you want some yogurt xxx ?", "words": ["do", "yogurt", "some", "you"]}, {"sent": "you wanna take a baby yogurt out of the container .", "words": ["the", "of", "yogurt", "you", "wanna", "out", "take", "a"]}, {"sent": "okay peach yogurt ?", "words": ["yogurt"]}, {"sent": "youre pretending to mix yogurt for Mommy ?", "words": ["to", "yogurt", "for"]}, {"sent": "wheres the yogurt ?", "words": ["the", "yogurt"]}, {"sent": "it does taste like yogurt its a little bit sour huh .", "words": ["does", "yogurt", "taste", "like", "little", "it", "a"]}, {"sent": "you want me to get you some some how about some yogurt you want some yogurt ?", "words": ["how", "to", "yogurt", "you", "get", "about", "me", "some"]}, {"sent": "Id like some yogurt too .", "words": ["some", "yogurt", "like", "too"]}, {"sent": "you want Mom to make more yogurt .", "words": ["to", "yogurt", "you", "more", "make"]}, {"sent": "you want some soy yogurt ?", "words": ["some", "yogurt", "you"]}, {"sent": "you eat the yogurt when youre finished with the yogurt then ya clean up .", "words": ["the", "clean", "up", "yogurt", "with", "you", "eat", "then", "when"]}, {"sent": "does the bear want more of the plain kind of yogurt ?", "words": ["the", "does", "of", "yogurt", "bear", "more"]}, {"sent": "you have yogurt there ?", "words": ["have", "there", "yogurt", "you"]}, {"sent": "do you want some yogurt in your some granola in your yogurt ?", "words": ["do", "your", "yogurt", "you", "in", "some"]}, {"sent": "right there just leave it right there Im not gonna eat your yogurt .", "words": ["your", "yogurt", "eat", "there", "it", "not"]}, {"sent": "you wanna sit and eat yogurt over there ?", "words": ["yogurt", "and", "eat", "you", "there", "wanna", "sit", "over"]}, {"sent": "yeah thats mostly plain theres just a little bit of yogurt there .", "words": ["of", "yogurt", "little", "there", "a"]}, {"sent": "she wants a yogurt maker .", "words": ["yogurt", "she", "a"]}, {"sent": "heres the yogurt .", "words": ["the", "yogurt"]}, {"sent": "youre having yogurt .", "words": ["yogurt"]}, {"sent": "how is the yogurt ?", "words": ["the", "how", "yogurt", "is"]}, {"sent": "it looks like your yogurt .", "words": ["yogurt", "it", "your", "like"]}, {"sent": "lets have some yogurt and ricotta .", "words": ["have", "some", "yogurt", "and"]}, {"sent": "oh yesterday you had a strawberry yogurt drink .", "words": ["drink", "strawberry", "yogurt", "you", "a"]}, {"sent": "more yogurt .", "words": ["more", "yogurt"]}, {"sent": "one yogurt for Grizzly .", "words": ["yogurt", "for"]}, {"sent": "and yogurt you can have both okay ?", "words": ["can", "yogurt", "you", "have", "and"]}, {"sent": "and the spoon and the yogurt has been a big deal recently as shes getting better with the spoon shes kinda again yeah so I think thats why shes been on that spoon .", "words": ["the", "better", "think", "so", "yogurt", "with", "and", "that", "spoon", "on", "big", "why", "a"]}, {"sent": "ya want some yogurt ?", "words": ["some", "yogurt"]}, {"sent": "want some yogurt ?", "words": ["some", "yogurt"]}, {"sent": "Mommy spilled the yogurt .", "words": ["the", "yogurt"]}, {"sent": "the thing is theres no yogurt left for her .", "words": ["the", "for", "is", "yogurt", "her"]}, {"sent": "well youre not quite finished with your yogurt yet Nomi .", "words": ["your", "yogurt", "with", "not"]}, {"sent": "do you want to have some yogurt ?", "words": ["do", "to", "yogurt", "you", "have", "want to", "some"]}, {"sent": "yogurty yogurt .", "words": ["yogurt"]}, {"sent": "Im gonna move this yogurt so no one gets yogurt on them .", "words": ["so", "yogurt", "them", "this", "on"]}, {"sent": "youve got strawberries in your yogurt .", "words": ["in", "yogurt", "your"]}, {"sent": "alright more yogurt coming up .", "words": ["up", "more", "yogurt"]}, {"sent": "you want yogurt drink or milk ?", "words": ["milk", "yogurt", "drink", "you"]}, {"sent": "have you got strawberry yogurt ?", "words": ["have", "yogurt", "strawberry", "you"]}, {"sent": "get the yogurt cup .", "words": ["get", "the", "yogurt", "cup"]}, {"sent": "would you like more yogurt ?", "words": ["yogurt", "like", "you", "more", "would"]}, {"sent": "is your yogurt allgone ?", "words": ["yogurt", "your", "is"]}, {"sent": "and you have yogurt .", "words": ["have", "yogurt", "you", "and"]}, {"sent": "you could have some plain yogurt if you want .", "words": ["if", "yogurt", "you", "have", "some", "could"]}, {"sent": "mm mm you got yogurt on your face .", "words": ["your", "face", "yogurt", "you", "on"]}, {"sent": "do you want smaller bowl for your yogurt ?", "words": ["bowl", "do", "for", "your", "yogurt", "you"]}, {"sent": "hm um yogurt ?", "words": ["yogurt"]}, {"sent": "yogurt .", "words": ["yogurt"]}, {"sent": "okay Ill just put some in your yogurt .", "words": ["put", "your", "yogurt", "in", "some"]}, {"sent": "what kind of yogurt is that ?", "words": ["what", "is", "of", "yogurt", "that"]}, {"sent": "which would you like to do first peach yogurt or books ?", "words": ["do", "to", "first", "yogurt", "like", "you", "would", "which"]}, {"sent": "well Beanice isnt too good at eating yogurt honey .", "words": ["yogurt", "at", "good", "too"]}, {"sent": "yogurty yogurt thats what youre gonna have all right .", "words": ["have", "what", "yogurt", "all"]}, {"sent": "this is the last baby yogurt .", "words": ["the", "is", "last", "yogurt", "this"]}, {"sent": "come have your yogurt stick .", "words": ["have", "stick", "yogurt", "your"]}, {"sent": "heres plain yogurt here here here .", "words": ["here", "yogurt"]}, {"sent": "does Georgie like yogurt ?", "words": ["does", "yogurt", "like"]}, {"sent": "would you like more yogurt in that yogurty yogurt ?", "words": ["yogurt", "like", "you", "more", "in", "would", "that"]}, {"sent": "you like yogurty yogurt the best of all dont you ?", "words": ["the", "all", "of", "yogurt", "like", "you", "dont"]}, {"sent": "Abe had listen to this Dad Abe had yogurt with honey cinnamon peanut butter and green food coloring .", "words": ["listen", "to", "yogurt", "with", "and", "this", "food", "butter", "green", "peanut butter"]}, {"sent": "hm we dont have any yogurt .", "words": ["we", "yogurt", "any", "have", "dont"]}, {"sent": "yes the yogurt caps too .", "words": ["the", "yogurt", "too"]}, {"sent": "do Baby and Georgie wanna eat yogurt ?", "words": ["do", "yogurt", "and", "eat", "wanna"]}, {"sent": "nursie again do you wanna eat some yogurt or some fromage ?", "words": ["do", "yogurt", "you", "eat", "wanna", "some"]}, {"sent": "you can dip it in the yogurt if you want .", "words": ["can", "the", "if", "yogurt", "you", "in", "it"]}, {"sent": "oh wait this is the low fat yogurt we need to get the we need to get the full fat yogurt for you .", "words": ["the", "wait", "for", "need", "full", "is", "we", "to", "need to", "yogurt", "you", "get", "this"]}, {"sent": "how about some yogurt ?", "words": ["some", "how", "yogurt", "about"]}, {"sent": "Im putting away the yogurt so you can have some later on .", "words": ["the", "can", "so", "yogurt", "you", "have", "away", "on", "some"]}, {"sent": "pieces of yogurt .", "words": ["of", "yogurt"]}, {"sent": "yogurt .", "words": ["yogurt"]}, {"sent": "yogurt .", "words": ["yogurt"]}, {"sent": "okay Justins got lotsof yogurt in there .", "words": ["there", "in", "yogurt"]}, {"sent": "is that sweeping or is that swinging like the bat ?", "words": ["the", "is", "like", "that", "bat"]}, {"sent": "mm a baseball and a bat .", "words": ["bat", "and", "a"]}, {"sent": "a bat and a ?", "words": ["bat", "and", "a"]}, {"sent": "thats right in the story pj funny bunnys cousin broke the bat .", "words": ["the", "in", "bat", "story"]}, {"sent": "do you find your bat ?", "words": ["do", "your", "you", "find", "bat"]}, {"sent": "squishy makes bat ?", "words": ["bat"]}, {"sent": "I have a bat and you have a bat at tennis .", "words": ["you", "have", "and", "at", "bat", "a"]}, {"sent": "did we see a bat ?", "words": ["we", "did", "see", "bat", "a"]}, {"sent": "sandbox a swing a ball a bat .", "words": ["sandbox", "swing", "ball", "bat", "a"]}, {"sent": "the bat .", "words": ["the", "bat"]}, {"sent": "bat .", "words": ["bat"]}, {"sent": "takes the bat .", "words": ["the", "bat"]}, {"sent": "you hit them with the bat ?", "words": ["the", "them", "with", "you", "hit", "bat"]}, {"sent": "and a bat .", "words": ["bat", "a", "and"]}, {"sent": "youre gonna be a bat ?", "words": ["be", "bat", "a"]}, {"sent": "and a bat .", "words": ["bat", "a", "and"]}, {"sent": "its a bat .", "words": ["bat", "a"]}, {"sent": "is that how bat where your batteries go then ?", "words": ["where", "how", "your", "is", "go", "that", "then", "bat"]}, {"sent": "an old bat .", "words": ["old", "an", "bat"]}, {"sent": "shall we take a bat and ball to play with on the beach ?", "words": ["the", "we", "to", "play", "with", "and", "ball", "beach", "on", "take", "bat", "a"]}, {"sent": "a bat .", "words": ["bat", "a"]}, {"sent": "thats a baseball bat .", "words": ["bat", "a"]}, {"sent": "a bat ?", "words": ["bat", "a"]}, {"sent": "a special PlayDoh bat .", "words": ["bat", "a"]}, {"sent": "you xxx ding bat .", "words": ["bat", "you"]}, {"sent": "baseball bat ?", "words": ["bat"]}, {"sent": "hes a bat .", "words": ["bat", "a"]}, {"sent": "when did you get that baseball bat Abe ?", "words": ["bat", "when", "you", "get", "did", "that"]}, {"sent": "and a bat ?", "words": ["bat", "a", "and"]}, {"sent": "you can hit it with your hand or with a bat right ?", "words": ["can", "your", "hand", "with", "you", "it", "hit", "bat", "a"]}, {"sent": "you could bat a baby when the clown goes off .", "words": ["the", "off", "when", "you", "could", "bat", "a"]}, {"sent": "a bat .", "words": ["bat", "a"]}, {"sent": "bat .", "words": ["bat"]}, {"sent": "you want your bat ?", "words": ["bat", "your", "you"]}, {"sent": "xxx make a big bat .", "words": ["bat", "big", "make", "a"]}, {"sent": "yours is broke too everybodys bat raw xxx is broke .", "words": ["too", "bat", "is"]}, {"sent": "thats a bat .", "words": ["bat", "a"]}, {"sent": "put your bat down .", "words": ["bat", "put", "down", "your"]}, {"sent": "put your bat down now .", "words": ["bat", "put", "down", "your"]}, {"sent": "no I dont think we did see a bat this halloween did we ?", "words": ["think", "we", "dont", "this", "did", "see", "bat", "a"]}, {"sent": "peeking out the cricket behind the cricket bat .", "words": ["the", "bat", "behind", "out"]}, {"sent": "apples squash beets and roast bat .", "words": ["bat", "and"]}, {"sent": "now Im an old bat .", "words": ["old", "an", "bat"]}, {"sent": "that says bat .", "words": ["bat", "that"]}, {"sent": "oh its a bat again .", "words": ["bat", "a"]}, {"sent": "mommy I got a small bat .", "words": ["bat", "a"]}, {"sent": "do you want the bat Ill get the bat .", "words": ["the", "do", "you", "get", "bat"]}, {"sent": "thats a bat yes .", "words": ["bat", "a"]}, {"sent": "wanna get out your bat ?", "words": ["your", "get", "out", "wanna", "bat"]}, {"sent": "blind as a bat .", "words": ["bat", "a"]}, {"sent": "its a bat .", "words": ["bat", "a"]}, {"sent": "is there a bat there ?", "words": ["there", "bat", "a", "is"]}, {"sent": "a bat .", "words": ["bat", "a"]}, {"sent": "how about bat ?", "words": ["how", "bat", "about"]}, {"sent": "bad bat should we put it up ?", "words": ["up", "put", "we", "bad", "it", "bat"]}, {"sent": "and which one of you hit JemimaPuddleduck with a cricket bat ?", "words": ["bat", "of", "with", "you", "and", "hit", "which", "a"]}, {"sent": "hold your hold the bat like this with two hands honey .", "words": ["the", "your", "hold", "with", "like", "this", "bat"]}, {"sent": "bat or .", "words": ["bat"]}, {"sent": "bat the ball huh ?", "words": ["the", "ball", "bat"]}, {"sent": "youre a bit dangerous with that bat .", "words": ["bat", "that", "with", "a"]}, {"sent": "bat .", "words": ["bat"]}, {"sent": "a bat .", "words": ["bat", "a"]}, {"sent": "your bat ?", "words": ["bat", "your"]}, {"sent": "give my bat .", "words": ["give", "bat", "my"]}, {"sent": "you found a bat that you made ?", "words": ["that", "bat", "a", "you"]}, {"sent": "dont trip over your bat .", "words": ["your", "dont", "bat", "over"]}, {"sent": "the bat ?", "words": ["the", "bat"]}, {"sent": "yeah but this bat is too xxx .", "words": ["is", "this", "too", "bat", "but"]}, {"sent": "its called bat be ball ?", "words": ["ball", "be", "bat"]}, {"sent": "you hit the ball with a bat .", "words": ["the", "with", "you", "ball", "hit", "bat", "a"]}, {"sent": "your bat ball .", "words": ["ball", "bat", "your"]}, {"sent": "bat very .", "words": ["bat"]}, {"sent": "its only the bat thats upside down .", "words": ["the", "down", "bat"]}, {"sent": "a bat .", "words": ["bat", "a"]}, {"sent": "bat .", "words": ["bat"]}, {"sent": "do you want this bat ?", "words": ["do", "bat", "this", "you"]}, {"sent": "bat .", "words": ["bat"]}, {"sent": "no dont hit me no Ill hit the ball to you Ill bat the ball to you this is a bat you ready ?", "words": ["the", "is", "to", "you", "ball", "dont", "this", "me", "hit", "bat", "a"]}, {"sent": "hold on I like to move mommy has to move the bat .", "words": ["the", "hold", "to", "like", "on", "bat"]}, {"sent": "a bat .", "words": ["bat", "a"]}, {"sent": "look thats a bat .", "words": ["bat", "a", "look"]}, {"sent": "its a bat .", "words": ["bat", "a"]}, {"sent": "ball bat ball bat bat ball .", "words": ["ball", "bat"]}, {"sent": "my bat .", "words": ["my", "bat"]}, {"sent": "and this is a bat .", "words": ["is", "and", "this", "bat", "a"]}, {"sent": "bat out of hell .", "words": ["of", "bat", "out"]}, {"sent": "bat .", "words": ["bat"]}, {"sent": "where are you bat ?", "words": ["where", "are", "bat", "you"]}, {"sent": "its coming its a big funny bat .", "words": ["bat", "big", "a"]}, {"sent": "no we cant hit the bat right now come on .", "words": ["the", "we", "on", "hit", "bat"]}, {"sent": "bring me the bat please .", "words": ["the", "me", "bat", "bring"]}, {"sent": "what what do they use the hat bat and helmet for ?", "words": ["the", "what", "do", "for", "and", "they", "hat", "bat"]}, {"sent": "you missed one bat .", "words": ["bat", "you"]}, {"sent": "a bat ?", "words": ["bat", "a"]}, {"sent": "yeah your real bat .", "words": ["bat", "your"]}, {"sent": "thats a vampire bat .", "words": ["bat", "a"]}, {"sent": "oh its a baseball bat wow you wanna show Amanda how you hit with a baseball bat ?", "words": ["how", "show", "with", "you", "wanna", "hit", "bat", "a"]}, {"sent": "you wont be able to to teach him how to hold the bat .", "words": ["the", "how", "hold", "to", "be", "you", "him", "bat"]}, {"sent": "and bat girl too ?", "words": ["too", "bat", "and"]}, {"sent": "can you find the sandbox ?", "words": ["can", "the", "sandbox", "you", "find"]}, {"sent": "do you think giraffe needs such a big sandbox ?", "words": ["sandbox", "do", "think", "giraffe", "you", "big", "a"]}, {"sent": "you know what I thought this that maybe this was a sandbox .", "words": ["sandbox", "what", "was", "you", "this", "that", "a"]}, {"sent": "thats like a turtle sandbox isnt it ?", "words": ["sandbox", "like", "it", "turtle", "a"]}, {"sent": "was there a little puddle on top of the sandbox ?", "words": ["the", "sandbox", "of", "was", "little", "there", "on", "a"]}, {"sent": "right do we play in the sandbox ?", "words": ["the", "do", "sandbox", "we", "play", "in"]}, {"sent": "the sandbox is chocolate .", "words": ["the", "sandbox", "chocolate", "is"]}, {"sent": "you useta like that sandbox .", "words": ["sandbox", "that", "like", "you"]}, {"sent": "is he playing in a sandbox ?", "words": ["sandbox", "is", "in", "he", "a"]}, {"sent": "playing in the sandbox .", "words": ["the", "in", "sandbox"]}, {"sent": "thats probably a sandbox .", "words": ["sandbox", "a"]}, {"sent": "looks like it may be a sandbox .", "words": ["sandbox", "be", "like", "it", "a"]}, {"sent": "did you tell Dad what was in the sandbox this morning ?", "words": ["the", "what", "sandbox", "was", "you", "this", "in", "did"]}, {"sent": "cause I think thats a sandbox .", "words": ["sandbox", "think", "a"]}, {"sent": "get more sandbox ?", "words": ["get", "more", "sandbox"]}, {"sent": "what did you make in the sandbox ?", "words": ["the", "what", "sandbox", "you", "in", "did", "make"]}, {"sent": "well do you see any toys on the table that might be good for using in the sandbox ?", "words": ["the", "do", "sandbox", "for", "table", "be", "any", "you", "in", "good", "on", "see", "that"]}, {"sent": "probably over in the sandbox .", "words": ["the", "in", "over", "sandbox"]}, {"sent": "what do you make in your sandbox ?", "words": ["sandbox", "what", "do", "your", "you", "in", "make"]}, {"sent": "hes in the sandbox wow neat .", "words": ["the", "in", "sandbox"]}, {"sent": "you know what I bet you could pretend this is a sandbox .", "words": ["sandbox", "what", "is", "you", "this", "pretend", "a", "could"]}, {"sent": "do you wanna take Carry over to the park to the big sandbox ?", "words": ["the", "do", "sandbox", "to", "you", "big", "park", "wanna", "take", "over"]}, {"sent": "a swimming pool or sandbox or something ?", "words": ["sandbox", "pool", "a"]}, {"sent": "is that a sandbox ?", "words": ["sandbox", "that", "a", "is"]}, {"sent": "the racoons playing in the sandbox .", "words": ["the", "in", "sandbox"]}, {"sent": "your dump truck goes back in the sandbox .", "words": ["the", "sandbox", "your", "back", "truck", "in", "dump"]}, {"sent": "is that a turtle sandbox ?", "words": ["sandbox", "is", "turtle", "that", "a"]}, {"sent": "do you like to play in the sandbox with Daddy ?", "words": ["the", "do", "sandbox", "to", "play", "with", "like", "you", "in"]}, {"sent": "it looks like maybe a sandbox where sand would go in it .", "words": ["sandbox", "where", "go", "like", "in", "it", "would", "a"]}, {"sent": "hey is there anything in the sandbox today ?", "words": ["the", "sandbox", "is", "there", "in"]}, {"sent": "pour the sand in the sandbox ?", "words": ["the", "in", "pour", "sandbox"]}, {"sent": "thiss gotta be a sandbox .", "words": ["sandbox", "be", "a"]}, {"sent": "a chair in the sandbox ?", "words": ["the", "sandbox", "chair", "in", "a"]}, {"sent": "a sandbox .", "words": ["sandbox", "a"]}, {"sent": "pretty soon well have a sandbox right down here .", "words": ["sandbox", "have", "here", "pretty", "down", "a"]}, {"sent": "yes hes playing in the sandbox .", "words": ["the", "in", "sandbox"]}, {"sent": "youre lucky to find that cause I found it buried in your sandbox and I took it and I washed it .", "words": ["sandbox", "your", "to", "and", "in", "it", "find", "that"]}, {"sent": "would you use this in a sandbox ?", "words": ["sandbox", "you", "this", "in", "would", "a"]}, {"sent": "is that a sandbox ?", "words": ["sandbox", "that", "a", "is"]}, {"sent": "would you want the dog in the sandbox ?", "words": ["the", "sandbox", "you", "in", "would", "dog"]}, {"sent": "toys yes how about a sandbox ?", "words": ["sandbox", "how", "about", "a"]}, {"sent": "what happened to your sandbox ?", "words": ["to", "what", "sandbox", "your"]}, {"sent": "in the sandbox .", "words": ["the", "in", "sandbox"]}, {"sent": "heres that little boy playing in the sandbox .", "words": ["the", "sandbox", "little", "in", "that"]}, {"sent": "yes in the sandbox .", "words": ["the", "in", "sandbox"]}, {"sent": "what do you do in the sandbox ?", "words": ["the", "what", "do", "sandbox", "you", "in"]}, {"sent": "so what did we do with the sandbox Matty ?", "words": ["the", "what", "do", "sandbox", "we", "so", "with", "did"]}, {"sent": "what was on top of the sandbox that you were playing with ?", "words": ["the", "what", "sandbox", "of", "was", "with", "you", "on", "that", "were"]}, {"sent": "do you think they could use those things in a sandbox ?", "words": ["sandbox", "do", "think", "those", "you", "they", "in", "could", "a"]}, {"sent": "yeah right well it makes shade but is it the pool or a sandbox ?", "words": ["the", "sandbox", "is", "pool", "it", "but", "a"]}, {"sent": "theres nobody in the sandbox .", "words": ["the", "in", "sandbox"]}, {"sent": "playing in the sandbox .", "words": ["the", "in", "sandbox"]}, {"sent": "I think thats for the sandbox .", "words": ["the", "think", "for", "sandbox"]}, {"sent": "I thought it looked like a sandbox .", "words": ["sandbox", "it", "like", "a"]}, {"sent": "he probably doesnt really wanna live in the sandbox anyway .", "words": ["the", "sandbox", "in", "he", "wanna"]}, {"sent": "no its a sandbox .", "words": ["sandbox", "a"]}, {"sent": "no not in your party dress if you wanna change your clothes we can play in the sandbox .", "words": ["can", "the", "sandbox", "your", "if", "we", "play", "party", "you", "in", "wanna", "dress", "not"]}, {"sent": "and then he played in the sandbox with his trucks .", "words": ["the", "sandbox", "with", "and", "in", "he", "his", "then"]}, {"sent": "I think when it gets warmer out daddys gonna havta put new sand in the sandbox because I bet the old sand is dirty .", "words": ["the", "sandbox", "think", "put", "old", "is", "in", "it", "dirty", "out", "new", "when", "because"]}, {"sent": "oh in the sandbox .", "words": ["the", "in", "sandbox"]}, {"sent": "playing in the mulch or the dirt or the sandbox .", "words": ["the", "in", "sandbox"]}, {"sent": "and you have a sandbox .", "words": ["sandbox", "you", "have", "and", "a"]}, {"sent": "good boring old sandbox .", "words": ["old", "sandbox", "good"]}, {"sent": "what did we play in the sandbox ?", "words": ["the", "what", "sandbox", "we", "play", "in", "did"]}, {"sent": "your dumptruck goes back in the sandbox said Ruby .", "words": ["the", "sandbox", "your", "back", "in"]}, {"sent": "do you know what happens when dogs and cats go in the sandbox ?", "words": ["the", "do", "what", "sandbox", "go", "and", "you", "in", "when"]}, {"sent": "its a sandbox .", "words": ["sandbox", "a"]}, {"sent": "get a new sandbox .", "words": ["get", "sandbox", "new", "a"]}, {"sent": "there is a huge sandbox at Maggies house .", "words": ["sandbox", "is", "there", "house", "at", "a"]}, {"sent": "look I think thats a sandbox .", "words": ["sandbox", "think", "a", "look"]}, {"sent": "we have pictures of you with that sandbox .", "words": ["sandbox", "we", "of", "with", "you", "have", "that"]}, {"sent": "you like the sandbox ?", "words": ["the", "sandbox", "like", "you"]}, {"sent": "you hurt yourself on the sandbox ?", "words": ["the", "sandbox", "yourself", "hurt", "you", "on"]}, {"sent": "you must have played in the sandbox today did you ?", "words": ["the", "sandbox", "you", "have", "in", "did"]}, {"sent": "is he playing in the sandbox ?", "words": ["the", "sandbox", "is", "in", "he"]}, {"sent": "this is a picture of a sandbox over here .", "words": ["sandbox", "is", "of", "picture", "here", "this", "over", "a"]}, {"sent": "a sandbox yeah .", "words": ["sandbox", "a"]}, {"sent": "Dad said why dont you go play in your sandbox .", "words": ["sandbox", "your", "go", "play", "you", "dont", "in", "why"]}, {"sent": "might you use a little rake in the sandbox ?", "words": ["the", "sandbox", "little", "you", "in", "a"]}, {"sent": "you get the swing the wagon and the sandbox and alls I get is an old park bench ?", "words": ["the", "sandbox", "old", "is", "swing", "and", "get", "you", "an", "park", "bench"]}, {"sent": "most people dont let their dogs and cats in their sandbox .", "words": ["sandbox", "their", "and", "dont", "in"]}, {"sent": "a sandbox .", "words": ["sandbox", "a"]}, {"sent": "another sandbox .", "words": ["another", "sandbox"]}, {"sent": "what do you make in the sandbox ?", "words": ["the", "what", "do", "sandbox", "you", "in", "make"]}, {"sent": "he is playing in a sandbox ?", "words": ["sandbox", "is", "in", "he", "a"]}, {"sent": "your dumptruck goes back in the sandbox .", "words": ["the", "sandbox", "your", "back", "in"]}, {"sent": "well I think I think that ones the sandbox .", "words": ["the", "think", "that", "sandbox"]}, {"sent": "a turtle sandbox thats right .", "words": ["sandbox", "turtle", "a"]}, {"sent": "theres a sandbox in the museum look .", "words": ["sandbox", "the", "in", "look", "a"]}, {"sent": "thats a um sandbox .", "words": ["sandbox", "a"]}, {"sent": "its the sandbox is yucky ?", "words": ["the", "sandbox", "yucky", "is"]}, {"sent": "sure is a big sandbox .", "words": ["sandbox", "big", "a", "is"]}, {"sent": "you hid in the sandbox didnt you ?", "words": ["the", "in", "sandbox", "you"]}, {"sent": "I think this is sposta be a sandbox .", "words": ["sandbox", "think", "is", "be", "this", "a"]}, {"sent": "what happened to your sandbox ?", "words": ["to", "what", "sandbox", "your"]}, {"sent": "is there a sandbox we can play ?", "words": ["sandbox", "can", "we", "is", "play", "there", "a"]}, {"sent": "ah what are you doing in the sandbox ?", "words": ["the", "what", "sandbox", "you", "in", "are"]}, {"sent": "look his sandbox has a little awning .", "words": ["sandbox", "little", "look", "his", "a"]}, {"sent": "theres not what theres no toys for the sandbox ?", "words": ["the", "what", "sandbox", "for", "not"]}, {"sent": "which sandbox ?", "words": ["sandbox", "which"]}, {"sent": "I think its a bucket a pail .", "words": ["bucket", "think", "a"]}, {"sent": "a nice bucket and some flowerpots .", "words": ["bucket", "nice", "and", "some", "a"]}, {"sent": "thats the rest of that bucket there int it ?", "words": ["the", "bucket", "of", "there", "it", "that"]}, {"sent": "xxx bucket and spade .", "words": ["and", "bucket"]}, {"sent": "put the blocks in the bucket to the beep beep beep .", "words": ["the", "put", "bucket", "to", "in"]}, {"sent": "and theres the bucket .", "words": ["the", "bucket", "and"]}, {"sent": "what colors my bucket ?", "words": ["what", "my", "bucket"]}, {"sent": "er the bucket again .", "words": ["the", "bucket"]}, {"sent": "oh in the bucket its heavy isnt it ?", "words": ["the", "bucket", "in", "it", "heavy"]}, {"sent": "in the water bucket .", "words": ["the", "bucket", "in", "water"]}, {"sent": "ah youve got a bucket and spade .", "words": ["bucket", "and", "a"]}, {"sent": "my bucket with the lid .", "words": ["the", "my", "with", "bucket"]}, {"sent": "the bucket .", "words": ["the", "bucket"]}, {"sent": "and a bucket .", "words": ["bucket", "a", "and"]}, {"sent": "no no dont take the lid off the bucket Thomas because it has got dirty clothes in it .", "words": ["the", "off", "bucket", "dont", "in", "it", "dirty", "take", "because"]}, {"sent": "meow bang bang bang on the bucket .", "words": ["on", "the", "bucket"]}, {"sent": "yes it has a big bucket .", "words": ["bucket", "it", "big", "a"]}, {"sent": "youre a sauce bucket you .", "words": ["bucket", "you", "sauce", "a"]}, {"sent": "bucket ?", "words": ["bucket"]}, {"sent": "in the bucket .", "words": ["the", "in", "bucket"]}, {"sent": "theyre going to the beach so theyll need their bucket and spade .", "words": ["the", "need", "their", "bucket", "to", "so", "and", "beach"]}, {"sent": "is he hiding in the the bucket of the digger ?", "words": ["the", "bucket", "is", "of", "in", "he"]}, {"sent": "Spots got the bucket .", "words": ["the", "bucket"]}, {"sent": "squeezing it out over the bucket .", "words": ["the", "bucket", "it", "out", "over"]}, {"sent": "alright put that one back and get something else get another bucket .", "words": ["put", "back", "bucket", "and", "get", "another", "that"]}, {"sent": "can you put the bucket .", "words": ["can", "the", "put", "bucket", "you"]}, {"sent": "those are pails or buckets you can call them either a pail or a bucket .", "words": ["can", "those", "bucket", "them", "you", "are", "a"]}, {"sent": "it goes in the bucket .", "words": ["the", "in", "it", "bucket"]}, {"sent": "find the white bucket for me .", "words": ["the", "for", "bucket", "find", "me", "white"]}, {"sent": "wheres the bucket of water go ?", "words": ["the", "go", "of", "bucket", "water"]}, {"sent": "it is a bucket .", "words": ["bucket", "it", "a", "is"]}, {"sent": "at least Granddads bucket of water was outside when he made a mess .", "words": ["bucket", "of", "was", "outside", "at", "he", "water", "when", "a"]}, {"sent": "full bucket .", "words": ["full", "bucket"]}, {"sent": "oh wow whats in this bucket ?", "words": ["this", "in", "bucket"]}, {"sent": "in the bucket .", "words": ["the", "in", "bucket"]}, {"sent": "in the little bucket .", "words": ["the", "bucket", "in", "little"]}, {"sent": "put the bucket .", "words": ["the", "put", "bucket"]}, {"sent": "its a bucket of crackers .", "words": ["of", "bucket", "a"]}, {"sent": "carrying the bucket .", "words": ["the", "bucket"]}, {"sent": "is there a little bucket and shovel in there maybe ?", "words": ["bucket", "is", "little", "and", "there", "in", "shovel", "a"]}, {"sent": "whats that doing in the bucket ?", "words": ["the", "in", "that", "bucket"]}, {"sent": "we can put this nice bucket .", "words": ["can", "put", "we", "bucket", "nice", "this"]}, {"sent": "and lets use this blue bucket .", "words": ["this", "blue", "bucket", "and"]}, {"sent": "Joseph can you get the bucket ?", "words": ["can", "the", "bucket", "you", "get"]}, {"sent": "xxx Sleepy Bear what are you doing in the bucket ?", "words": ["the", "what", "bucket", "you", "in", "are"]}, {"sent": "you put the alligator on top of the bucket okay ?", "words": ["the", "put", "bucket", "of", "you", "alligator", "on"]}, {"sent": "bucket .", "words": ["bucket"]}, {"sent": "thats a silly place to put a bucket isnt it ?", "words": ["put", "bucket", "to", "it", "a"]}, {"sent": "why didnt you get it in a bucket ?", "words": ["bucket", "you", "get", "in", "it", "why", "a"]}, {"sent": "see if hell go in the bucket .", "words": ["the", "if", "go", "bucket", "in", "see"]}, {"sent": "I like the color of the red bucket .", "words": ["the", "red", "bucket", "of", "like"]}, {"sent": "the bucket ?", "words": ["the", "bucket"]}, {"sent": "Im gonna buy a bucket .", "words": ["bucket", "buy", "a"]}, {"sent": "Mummys red bucket is far too big to put in the bag .", "words": ["red", "the", "put", "is", "bucket", "to", "in", "too", "big"]}, {"sent": "bang bang bang on the bucket .", "words": ["on", "the", "bucket"]}, {"sent": "what we gonna do with the bucket ?", "words": ["the", "what", "do", "we", "bucket", "with"]}, {"sent": "and into the bucket .", "words": ["the", "bucket", "into", "and"]}, {"sent": "a bucket is right .", "words": ["bucket", "is", "a"]}, {"sent": "and theres the bucket for the horses .", "words": ["the", "bucket", "for", "and"]}, {"sent": "put the blocks in the bucket to the beep beep beep .", "words": ["the", "put", "bucket", "to", "in"]}, {"sent": "piggie has the bucket ?", "words": ["the", "bucket"]}, {"sent": "do you wanna put all the blocks in the bucket ?", "words": ["the", "all", "do", "put", "bucket", "you", "in", "wanna"]}, {"sent": "have we got a bucket aswell ?", "words": ["have", "bucket", "a", "we"]}, {"sent": "thats a bucket .", "words": ["bucket", "a"]}, {"sent": "a bucket on her head a metal bucket on her head .", "words": ["bucket", "head", "on", "her", "a"]}, {"sent": "right put it on top of the bucket yeah ?", "words": ["the", "put", "bucket", "of", "it", "on"]}, {"sent": "like cement in a bucket .", "words": ["bucket", "in", "like", "a"]}, {"sent": "put them in the bucket and then it could be lowered down .", "words": ["the", "put", "bucket", "be", "them", "and", "in", "it", "then", "down", "could"]}, {"sent": "put the blocks in the bucket to the beep beep beep .", "words": ["the", "put", "bucket", "to", "in"]}, {"sent": "put the blocks in the bucket to the beep beep beep .", "words": ["the", "put", "bucket", "to", "in"]}, {"sent": "get all the pieces put them in your bucket .", "words": ["the", "all", "put", "your", "bucket", "them", "get", "in"]}, {"sent": "dyou wanna give the gigis a drink from the bucket ?", "words": ["the", "drink", "bucket", "wanna", "give", "a"]}, {"sent": "now why dont you put all of the animals on top of the bucket ?", "words": ["the", "all", "put", "bucket", "of", "you", "dont", "on", "why"]}, {"sent": "bucket .", "words": ["bucket"]}, {"sent": "oh look hes carrying a bucket .", "words": ["bucket", "a", "look"]}, {"sent": "put it in the bucket .", "words": ["the", "put", "bucket", "in", "it"]}, {"sent": "the bucket splits open .", "words": ["the", "open", "bucket"]}, {"sent": "bucket is gonna fall .", "words": ["fall", "is", "bucket"]}, {"sent": "bucket down hey .", "words": ["down", "bucket"]}, {"sent": "a bucket ?", "words": ["bucket", "a"]}, {"sent": "ah the drink of water had a turn first in the bucket .", "words": ["the", "drink", "bucket", "of", "first", "in", "water", "a"]}, {"sent": "put some sand in the bucket .", "words": ["the", "put", "bucket", "in", "some"]}, {"sent": "thats everything alright lets get a new bucket .", "words": ["get", "bucket", "new", "a"]}, {"sent": "youve got the bucket ?", "words": ["the", "bucket"]}, {"sent": "have you been collecting them in your bucket ?", "words": ["your", "bucket", "them", "you", "have", "in"]}, {"sent": "thats a light and a bucket .", "words": ["light", "bucket", "and", "a"]}, {"sent": "you have a nice big bucket .", "words": ["bucket", "nice", "you", "have", "big", "a"]}, {"sent": "whos behind the bucket ?", "words": ["the", "behind", "bucket"]}, {"sent": "would you like bucket and spade today ?", "words": ["bucket", "like", "and", "you", "would"]}, {"sent": "this bucket on the digger .", "words": ["on", "this", "the", "bucket"]}, {"sent": "I think that is the bucket for the apples .", "words": ["the", "think", "for", "bucket", "is", "that"]}, {"sent": "do you remember whatd we collect in the bucket ?", "words": ["the", "do", "we", "bucket", "you", "in"]}, {"sent": "ahhah right what about the bucket ?", "words": ["the", "what", "about", "bucket"]}, {"sent": "bring the bucket down and well put all the blocks in the bucket .", "words": ["the", "all", "put", "bucket", "bring", "and", "in", "down"]}, {"sent": "lets get your bucket .", "words": ["get", "your", "bucket"]}, {"sent": "when the bucket of the diggers upside down .", "words": ["the", "bucket", "of", "down", "when"]}, {"sent": "theres your bucket .", "words": ["your", "bucket"]}, {"sent": "they put all the milk in a bucket .", "words": ["the", "all", "put", "bucket", "they", "in", "milk", "a"]}, {"sent": "half half a cup of salt .", "words": ["cup", "of", "salt", "a"]}, {"sent": "I happen to be lenient with my salt so uh .", "words": ["my", "to", "be", "so", "with", "salt"]}, {"sent": "xxx one teaspoon of salt into a pot with a lid .", "words": ["into", "of", "with", "salt", "a"]}, {"sent": "are you cleaning off all the salt .", "words": ["off", "all", "the", "you", "salt", "are"]}, {"sent": "needs some more salt .", "words": ["salt", "some", "more"]}, {"sent": "pepper and salt ?", "words": ["salt", "and"]}, {"sent": "can I have salt and pepper ?", "words": ["can", "have", "salt", "and"]}, {"sent": "okay go and get the salt on the table then .", "words": ["the", "table", "go", "and", "get", "salt", "on", "then"]}, {"sent": "you need salt and pepper ?", "words": ["salt", "need", "and", "you"]}, {"sent": "well theres obviously enough salt in there .", "words": ["in", "salt", "there"]}, {"sent": "you needta test the food before you go putting salt in it .", "words": ["the", "go", "you", "food", "salt", "in", "it"]}, {"sent": "salt please .", "words": ["salt"]}, {"sent": "hey gimme the salt he nice middle class men I the family thats for sure .", "words": ["the", "for", "nice", "salt", "he"]}, {"sent": "salt herb extract and the color is patrek tsk paprika .", "words": ["the", "salt", "is", "and"]}, {"sent": "you want salt ?", "words": ["salt", "you"]}, {"sent": "give her the salt .", "words": ["the", "give", "her", "salt"]}, {"sent": "salt I need the salt .", "words": ["the", "salt", "need"]}, {"sent": "salt ?", "words": ["salt"]}, {"sent": "oh youre putting alotof salt .", "words": ["salt"]}, {"sent": "my husbands like that he will salt many things almost everything before he tastes it .", "words": ["my", "like", "salt", "it", "he", "will", "that"]}, {"sent": "you want salt ?", "words": ["salt", "you"]}, {"sent": "those are salt and vinegar crisps .", "words": ["salt", "are", "those", "and"]}, {"sent": "pausing here for salt but it had several faucets which sprinkled blue goo which somehow would sprinkle these Zukes as I flew and gum up those upside down butter xxx .", "words": ["up", "these", "for", "those", "blue", "here", "and", "salt", "it", "butter", "gum", "would", "down", "which", "but"]}, {"sent": "she said you mustnt use salt .", "words": ["salt", "she", "you"]}, {"sent": "have you found the salt ?", "words": ["have", "the", "salt", "you"]}, {"sent": "salt and vinegar hula hoops .", "words": ["salt", "and"]}, {"sent": "you do want salt ?", "words": ["do", "salt", "you"]}, {"sent": "xxx told me you didnt tell me to put no salt in the water .", "words": ["the", "put", "to", "you", "salt", "in", "water", "me"]}, {"sent": "like salt .", "words": ["salt", "like"]}, {"sent": "I should have left the machine plugged in because the reason I know it needs salt is because when it needs salt this little red light comes on .", "words": ["the", "red", "is", "light", "little", "have", "this", "in", "it", "salt", "on", "when", "because"]}, {"sent": "where is salt ?", "words": ["where", "salt", "is"]}, {"sent": "Laura I gave you salt .", "words": ["salt", "you"]}, {"sent": "alotof salt .", "words": ["salt"]}, {"sent": "and the salt is inside on the table when youre through okay ?", "words": ["the", "table", "is", "inside", "and", "salt", "on", "when"]}, {"sent": "more salt ?", "words": ["salt", "more"]}, {"sent": "I know you want it but youll get to do it when we add the salt but for now xxx .", "words": ["the", "do", "for", "we", "to", "you", "get", "salt", "it", "when", "but"]}, {"sent": "salt and pepper .", "words": ["salt", "and"]}, {"sent": "salt and pepper pots are usually quite small arent they ?", "words": ["they", "salt", "are", "and"]}, {"sent": "its a special salt .", "words": ["salt", "a"]}, {"sent": "salt and pepper ?", "words": ["salt", "and"]}, {"sent": "mhm I mean I salt it while its cooking .", "words": ["salt", "it"]}, {"sent": "Ill put some salt on them I usually put salt on my beans too .", "words": ["put", "my", "them", "salt", "beans", "too", "on", "some"]}, {"sent": "that airplane is going to salt lake .", "words": ["is", "airplane", "to", "salt", "that"]}, {"sent": "its not like the salt that we have in salt and pepper Thomas .", "words": ["the", "we", "like", "and", "have", "salt", "in", "that", "not"]}, {"sent": "xxx bit of salt ?", "words": ["of", "salt"]}, {"sent": "so what these guys went to salt lake ?", "words": ["what", "these", "to", "so", "salt"]}, {"sent": "salt and pepper chipper go get the salt and pepper please sothat Carl can .", "words": ["the", "can", "go", "and", "get", "salt"]}, {"sent": "um it actually has salt in it .", "words": ["in", "salt", "it"]}, {"sent": "we would rinse this out and I was going to try and take the top off and pour this salt into this container .", "words": ["the", "off", "into", "we", "was", "to", "pour", "and", "this", "salt", "out", "would", "take", "try"]}, {"sent": "this looks like salt .", "words": ["this", "salt", "like"]}, {"sent": "too much salt on there .", "words": ["there", "salt", "too", "on", "much"]}, {"sent": "oh you want salt and cheese on your macaroni .", "words": ["your", "and", "you", "salt", "on", "cheese"]}, {"sent": "very poor hand at putting salt and pepper in .", "words": ["hand", "and", "poor", "salt", "in", "at"]}, {"sent": "salt and pepper and a bit of milk .", "words": ["of", "and", "salt", "milk", "a"]}, {"sent": "does you have salt ?", "words": ["have", "does", "salt", "you"]}, {"sent": "xxx Wendoline salt pot .", "words": ["salt"]}, {"sent": "when we go shopping and I get some more salt we will make some more playdough .", "words": ["go", "we", "and", "get", "more", "salt", "make", "will", "some", "when"]}, {"sent": "is that the salt ?", "words": ["the", "salt", "that", "is"]}, {"sent": "I think it needs salt .", "words": ["salt", "think", "it"]}, {"sent": "put some salt in .", "words": ["salt", "some", "put", "in"]}, {"sent": "he does say salt .", "words": ["does", "salt", "he", "say"]}, {"sent": "the salt and pepper .", "words": ["the", "salt", "and"]}, {"sent": "are you shaking the salt out on the table ?", "words": ["the", "table", "you", "salt", "are", "out", "on"]}, {"sent": "Dwwws got salt .", "words": ["salt"]}, {"sent": "Anthony whyd you put all that salt over your um vegetables ?", "words": ["all", "put", "your", "you", "salt", "over", "that"]}, {"sent": "green need the greens need salt ?", "words": ["the", "salt", "need", "green"]}, {"sent": "because when you say blue crisps you mean salt and vinegar crisps .", "words": ["say", "blue", "you", "and", "salt", "when", "because"]}, {"sent": "going to salt lake City ?", "words": ["to", "salt"]}, {"sent": "xxx the salt and pepper .", "words": ["the", "salt", "and"]}, {"sent": "dont put salt on your eyes .", "words": ["put", "your", "dont", "salt", "on"]}, {"sent": "if you salt that yourself therell be too much salt on it and you wont eat it .", "words": ["yourself", "if", "be", "much", "you", "and", "eat", "salt", "it", "too", "on", "that"]}, {"sent": "too much salt .", "words": ["salt", "much", "too"]}, {"sent": "you wanna add any salt and pepper ?", "words": ["any", "and", "you", "salt", "wanna"]}, {"sent": "Ill have some salt on my tomatos .", "words": ["my", "have", "salt", "on", "some"]}, {"sent": "put some salt on it .", "words": ["put", "salt", "it", "on", "some"]}, {"sent": "I gave you quite a lot of salt .", "words": ["of", "a lot", "you", "salt", "a"]}, {"sent": "and that means xxx salt in it .", "words": ["and", "in", "salt", "it", "that"]}, {"sent": "you like that salt box ?", "words": ["box", "like", "you", "salt", "that"]}, {"sent": "you better not put salt all over me .", "words": ["better", "put", "all", "you", "salt", "me", "over", "not"]}, {"sent": "well lets see whats in here sugar citric acid salt imitation strawberry .", "words": ["strawberry", "here", "in", "salt", "see"]}, {"sent": "so it just takes a little bit of salt and pepper .", "words": ["of", "so", "little", "and", "salt", "it", "a"]}, {"sent": "but I dont put much salt in food because Im a hypertension they can put the salt in .", "words": ["can", "the", "put", "because", "dont", "salt", "in", "food", "they", "much", "but", "a"]}, {"sent": "I put more salt on it xxx .", "words": ["put", "salt", "it", "more", "on"]}, {"sent": "salt .", "words": ["salt"]}, {"sent": "salt ?", "words": ["salt"]}, {"sent": "salt .", "words": ["salt"]}, {"sent": "thats salt .", "words": ["salt"]}, {"sent": "I havent had salt .", "words": ["salt"]}, {"sent": "heres salt .", "words": ["salt"]}, {"sent": "rice flour soya flour salt yeast .", "words": ["salt"]}, {"sent": "I dont like salt .", "words": ["dont", "salt", "like"]}, {"sent": "it could use a little salt is what it could use .", "words": ["what", "is", "little", "salt", "it", "could", "a"]}, {"sent": "salt .", "words": ["salt"]}, {"sent": "cup of salt .", "words": ["cup", "of", "salt"]}, {"sent": "like salt or something .", "words": ["salt", "like"]}, {"sent": "sure is a good smile .", "words": ["smile", "good", "a", "is"]}, {"sent": "look through there and then you say smile and you press the pink button .", "words": ["the", "say", "you", "and", "there", "look", "then", "button", "smile"]}, {"sent": "daddys big smile .", "words": ["big", "smile"]}, {"sent": "lets see if I can make you smile .", "words": ["can", "if", "you", "see", "make", "smile"]}, {"sent": "put a smile on twenty eight .", "words": ["on", "put", "smile", "a"]}, {"sent": "trying the tea kettle smile .", "words": ["the", "smile"]}, {"sent": "oh lovely smile .", "words": ["smile"]}, {"sent": "its funny that the icing when it doesnt go to the bottom of the bun and it leaves a sort of semi circle at the bottom it looks like a smile doesnt it ?", "words": ["the", "bottom", "when", "go", "of", "to", "like", "and", "it", "at", "that", "smile", "a"]}, {"sent": "you hafta smile .", "words": ["smile", "you"]}, {"sent": "what a pretty smile .", "words": ["what", "pretty", "smile", "a"]}, {"sent": "because you smile so nice after the first flash .", "words": ["the", "so", "nice", "first", "because", "you", "smile"]}, {"sent": "we will miss your bright eyes and sweet smile .", "words": ["your", "we", "and", "will", "smile"]}, {"sent": "ah look at Warrens little smile .", "words": ["smile", "at", "little", "look"]}, {"sent": "thats how you smile when you eat alotof rice you smile like that .", "words": ["how", "like", "you", "eat", "that", "when", "smile"]}, {"sent": "so maybe if you give my plants a big smile itll work .", "words": ["my", "work", "if", "so", "you", "give", "big", "smile", "a"]}, {"sent": "a beautiful smile huh ?", "words": ["smile", "a"]}, {"sent": "big smile .", "words": ["big", "smile"]}, {"sent": "say smile Neville .", "words": ["say", "smile"]}, {"sent": "nice bright sunshine and a nice smile from PostmanPat .", "words": ["smile", "nice", "a", "and"]}, {"sent": "xxx that smile is .", "words": ["that", "smile", "is"]}, {"sent": "you smile for Daddy then .", "words": ["then", "for", "smile", "you"]}, {"sent": "smile .", "words": ["smile"]}, {"sent": "Betty smile .", "words": ["smile"]}, {"sent": "how about just a smile ?", "words": ["smile", "how", "about", "a"]}, {"sent": "smile pretty .", "words": ["pretty", "smile"]}, {"sent": "thats not a smile .", "words": ["not", "smile", "a"]}, {"sent": "what color smile should he have ?", "words": ["have", "what", "he", "smile"]}, {"sent": "because I do sometimes smile dont I ?", "words": ["smile", "do", "dont", "because"]}, {"sent": "could you smile Eleanor ?", "words": ["could", "smile", "you"]}, {"sent": "oh look at that big smile .", "words": ["look", "that", "at", "big", "smile"]}, {"sent": "did Jwww smile ?", "words": ["did", "smile"]}, {"sent": "he has a smile on his face .", "words": ["face", "his", "he", "on", "smile", "a"]}, {"sent": "smile .", "words": ["smile"]}, {"sent": "was that a small smile from my little man ?", "words": ["my", "was", "little", "that", "smile", "a"]}, {"sent": "they smile .", "words": ["they", "smile"]}, {"sent": "look shes xxx a lovely smile .", "words": ["smile", "a", "look"]}, {"sent": "smile .", "words": ["smile"]}, {"sent": "two eyes and a little bit of chocolate there that looks like a nose and can you see it looks like a smile at the bottom ?", "words": ["can", "the", "bottom", "of", "nose", "chocolate", "like", "little", "and", "there", "you", "it", "at", "see", "that", "smile", "a"]}, {"sent": "tell her to smile .", "words": ["to", "her", "smile"]}, {"sent": "hes got a smile on his face .", "words": ["face", "his", "on", "smile", "a"]}, {"sent": "will you give your doctor and dentist a great big smile the next time you see them ?", "words": ["the", "your", "them", "and", "you", "will", "see", "give", "big", "smile", "a"]}, {"sent": "its nice when they smile .", "words": ["they", "nice", "when", "smile"]}, {"sent": "and did you smile ?", "words": ["smile", "did", "you", "and"]}, {"sent": "does he smile ?", "words": ["does", "he", "smile"]}, {"sent": "smile .", "words": ["smile"]}, {"sent": "is that word smile ?", "words": ["that", "smile", "is"]}, {"sent": "smile .", "words": ["smile"]}, {"sent": "does she have a nice pretty smile ?", "words": ["does", "nice", "have", "pretty", "she", "smile", "a"]}, {"sent": "old million dollar smile .", "words": ["old", "smile"]}, {"sent": "dont smile .", "words": ["dont", "smile"]}, {"sent": "smile .", "words": ["smile"]}, {"sent": "yeah thats a smile .", "words": ["smile", "a"]}, {"sent": "he would just stare at the arm and smile .", "words": ["the", "would", "and", "he", "arm", "at", "smile"]}, {"sent": "thats a silly smile .", "words": ["smile", "a"]}, {"sent": "Betty are you going to smile for Daddy ?", "words": ["for", "to", "you", "are", "smile"]}, {"sent": "she brought a little smile to my face thats what happened .", "words": ["what", "my", "face", "to", "little", "she", "smile", "a"]}, {"sent": "right well you smile for Mummy .", "words": ["for", "smile", "you"]}, {"sent": "how about a smile ?", "words": ["smile", "how", "about", "a"]}, {"sent": "did you smile for her ?", "words": ["for", "you", "did", "her", "smile"]}, {"sent": "its got a smile on its face .", "words": ["on", "face", "smile", "a"]}, {"sent": "and theres the snails smile .", "words": ["the", "smile", "and"]}, {"sent": "oh hes gotta blue smile .", "words": ["smile", "blue"]}, {"sent": "the racing car driver has got a nice smile hasnt he ?", "words": ["the", "car", "nice", "he", "smile", "a"]}, {"sent": "smile .", "words": ["smile"]}, {"sent": "and the photographer came round and said smile .", "words": ["the", "smile", "and"]}, {"sent": "smile Eleanor .", "words": ["smile"]}, {"sent": "did you put a smile on the cats face ?", "words": ["the", "put", "face", "you", "did", "on", "smile", "a"]}, {"sent": "the yellow one has a happy smile on it .", "words": ["the", "happy", "it", "on", "yellow", "smile", "a"]}, {"sent": "wheres my smile ?", "words": ["my", "smile"]}, {"sent": "you like that smile ?", "words": ["that", "like", "smile", "you"]}, {"sent": "that always gets a smile huh .", "words": ["that", "smile", "a"]}, {"sent": "dont smile .", "words": ["dont", "smile"]}, {"sent": "smile .", "words": ["smile"]}, {"sent": "look at that funny smile .", "words": ["at", "that", "smile", "look"]}, {"sent": "you gimme a big smile ?", "words": ["smile", "big", "a", "you"]}, {"sent": "two eyes a big nose and a great big smile .", "words": ["a", "and", "big", "smile", "nose"]}, {"sent": "smile .", "words": ["smile"]}, {"sent": "smile .", "words": ["smile"]}, {"sent": "big smile .", "words": ["big", "smile"]}, {"sent": "can I see you smile honey happy face ?", "words": ["can", "face", "happy", "you", "see", "smile"]}, {"sent": "that was a lovely smile Tom .", "words": ["was", "that", "smile", "a"]}, {"sent": "please smile one smile .", "words": ["smile"]}, {"sent": "smile .", "words": ["smile"]}, {"sent": "smile .", "words": ["smile"]}, {"sent": "Lisa Forguarda the smile .", "words": ["the", "smile"]}, {"sent": "oh thats a big smile isnt it ?", "words": ["it", "big", "smile", "a"]}, {"sent": "is that ah smile ?", "words": ["that", "smile", "is"]}, {"sent": "wheres her smile ?", "words": ["her", "smile"]}, {"sent": "smile ?", "words": ["smile"]}, {"sent": "smile .", "words": ["smile"]}, {"sent": "Smile you say smile Mama .", "words": ["say", "smile", "you"]}, {"sent": "hes got a smile though .", "words": ["smile", "a"]}, {"sent": "how about if you put the smile on it .", "words": ["the", "how", "put", "if", "you", "it", "about", "on", "smile"]}, {"sent": "smile please .", "words": ["smile"]}, {"sent": "Ill never smile again .", "words": ["smile"]}, {"sent": "smile .", "words": ["smile"]}, {"sent": "Pingu put on his best camera smile and tried to look as handsome as possible .", "words": ["put", "camera", "to", "and", "his", "look", "on", "smile"]}, {"sent": "a smile .", "words": ["smile", "a"]}, {"sent": "who said smile at the camera ?", "words": ["the", "camera", "at", "who", "smile"]}, {"sent": "the poor deer must have stepped on a .", "words": ["the", "deer", "poor", "have", "on", "a"]}, {"sent": "oh and a little deer .", "words": ["deer", "little", "a", "and"]}, {"sent": "its a deer is it ?", "words": ["deer", "it", "is", "a"]}, {"sent": "you found a deer ?", "words": ["deer", "a", "you"]}, {"sent": "thats a deer on it .", "words": ["on", "deer", "it", "a"]}, {"sent": "what does a deer have on its head ?", "words": ["what", "does", "deer", "head", "have", "on", "a"]}, {"sent": "will you put deer together now .", "words": ["will", "deer", "put", "you"]}, {"sent": "what is it if its not a deer ?", "words": ["what", "if", "is", "deer", "it", "not", "a"]}, {"sent": "is that a deer ?", "words": ["deer", "that", "a", "is"]}, {"sent": "hes calling me deer thats thats so sad .", "words": ["me", "sad", "so", "deer"]}, {"sent": "its a baby deer .", "words": ["deer", "a"]}, {"sent": "or a deer .", "words": ["deer", "a"]}, {"sent": "where is the deer gonna play ?", "words": ["where", "the", "is", "deer", "play"]}, {"sent": "what does the deer do ?", "words": ["the", "what", "do", "does", "deer"]}, {"sent": "it looks a bit like a deer but its a goat .", "words": ["deer", "like", "it", "but", "a"]}, {"sent": "uh deer .", "words": ["deer"]}, {"sent": "what did we do with the deer ?", "words": ["the", "what", "do", "we", "deer", "with", "did"]}, {"sent": "deer weve not really come across deer yet uhn have we ?", "words": ["have", "deer", "not", "we"]}, {"sent": "no Im a giraffe not a deer .", "words": ["deer", "giraffe", "not", "a"]}, {"sent": "see the deer in your puzzle see him hes a deer .", "words": ["the", "your", "deer", "in", "him", "puzzle", "see", "a"]}, {"sent": "deer ?", "words": ["deer"]}, {"sent": "almost a deer like Bambi .", "words": ["deer", "like", "a"]}, {"sent": "you wanna use the deer okay .", "words": ["the", "deer", "wanna", "you"]}, {"sent": "the deer licked Snow Whites hand and ran into the forest .", "words": ["the", "into", "hand", "deer", "and"]}, {"sent": "then its a deer .", "words": ["deer", "then", "a"]}, {"sent": "deer .", "words": ["deer"]}, {"sent": "Nathan say deer .", "words": ["deer", "say"]}, {"sent": "and deer .", "words": ["deer", "and"]}, {"sent": "heres a deer .", "words": ["deer", "a"]}, {"sent": "the deer .", "words": ["the", "deer"]}, {"sent": "a baby what deer ?", "words": ["what", "deer", "a"]}, {"sent": "well a baby deer is called a fawn .", "words": ["deer", "is", "a"]}, {"sent": "like an otter and a deer .", "words": ["deer", "like", "and", "an", "a"]}, {"sent": "did the big deer keep chasing him away ?", "words": ["the", "deer", "him", "away", "did", "big"]}, {"sent": "baby deer .", "words": ["deer"]}, {"sent": "Bambi is no longer a little deer anymore .", "words": ["deer", "little", "a", "is"]}, {"sent": "thats a deer .", "words": ["deer", "a"]}, {"sent": "deer .", "words": ["deer"]}, {"sent": "where is the deer ?", "words": ["where", "the", "deer", "is"]}, {"sent": "um yeah he picked up like the deer .", "words": ["up", "the", "deer", "like", "he"]}, {"sent": "aw theres a deer like in our backyard .", "words": ["backyard", "deer", "like", "our", "in", "a"]}, {"sent": "the deer walks .", "words": ["the", "deer"]}, {"sent": "aw deer .", "words": ["deer"]}, {"sent": "thats not a deer .", "words": ["deer", "not", "a"]}, {"sent": "it is a deer .", "words": ["deer", "it", "a", "is"]}, {"sent": "maybe its a deer .", "words": ["deer", "a"]}, {"sent": "is that a deer ?", "words": ["deer", "that", "a", "is"]}, {"sent": "Ive never seen a deer eat peas .", "words": ["eat", "deer", "peas", "a"]}, {"sent": "its a deer .", "words": ["deer", "a"]}, {"sent": "theres a mommy deer oh here just a second .", "words": ["here", "deer", "a"]}, {"sent": "theyre counting deer that theyve bashed over the head with their big clubs there .", "words": ["the", "their", "deer", "head", "with", "there", "that", "big", "over"]}, {"sent": "do you remember deer like reindeer ?", "words": ["do", "deer", "like", "you"]}, {"sent": "thats a deer .", "words": ["deer", "a"]}, {"sent": "what is the rabbit giving to the deer ?", "words": ["the", "what", "is", "to", "deer"]}, {"sent": "oh youre going to use the deer to cook with .", "words": ["the", "to", "deer", "with", "cook"]}, {"sent": "here is a deer Bambi .", "words": ["here", "deer", "a", "is"]}, {"sent": "I see a deer .", "words": ["see", "deer", "a"]}, {"sent": "and heres a deer .", "words": ["deer", "a", "and"]}, {"sent": "thats just a regular deer .", "words": ["deer", "a"]}, {"sent": "okay he met a wonderful girl deer .", "words": ["deer", "he", "a"]}, {"sent": "whats whats the deer behind ?", "words": ["the", "deer", "behind"]}, {"sent": "deer .", "words": ["deer"]}, {"sent": "deer .", "words": ["deer"]}, {"sent": "where is dadas deer ?", "words": ["where", "deer", "is"]}, {"sent": "deer .", "words": ["deer"]}, {"sent": "looks a bit like a deer .", "words": ["deer", "like", "a"]}, {"sent": "deer oh okay .", "words": ["deer"]}, {"sent": "its a deer .", "words": ["deer", "a"]}, {"sent": "thats Bambi the deer .", "words": ["the", "deer"]}, {"sent": "its definitely a deer is it ?", "words": ["deer", "it", "is", "a"]}, {"sent": "where is dadas deer ?", "words": ["where", "deer", "is"]}, {"sent": "we dont needta we can feed the deer .", "words": ["can", "the", "we", "deer", "dont", "feed"]}, {"sent": "there was a baby deer looking at me ?", "words": ["was", "deer", "there", "at", "me", "a"]}, {"sent": "what does that deer have on his head ?", "words": ["what", "does", "deer", "head", "have", "his", "on", "that"]}, {"sent": "a deer .", "words": ["deer", "a"]}, {"sent": "because um like she picked up the deer and a couple other things .", "words": ["up", "the", "deer", "like", "and", "other", "she", "because", "a"]}, {"sent": "that deer looks awfully sad .", "words": ["deer", "sad", "that"]}, {"sent": "theres the deer .", "words": ["the", "deer"]}, {"sent": "she would definitely have known the deer .", "words": ["the", "deer", "have", "would", "she"]}, {"sent": "you have the deer .", "words": ["have", "the", "deer", "you"]}, {"sent": "and deer .", "words": ["deer", "and"]}, {"sent": "find the deer .", "words": ["the", "deer", "find"]}, {"sent": "well it looks like a horsie but its a deer .", "words": ["deer", "like", "it", "but", "a"]}, {"sent": "deer .", "words": ["deer"]}, {"sent": "does Daddy see some deer tracks ?", "words": ["see", "does", "deer", "some"]}, {"sent": "thats not a deer .", "words": ["deer", "not", "a"]}, {"sent": "he must be cold said Snow White covering the deer with her long cape .", "words": ["the", "be", "deer", "with", "long", "he", "her", "cold"]}, {"sent": "the bumblebee and the deer .", "words": ["the", "deer", "and"]}, {"sent": "besides what sound does a deer make ?", "words": ["what", "does", "deer", "make", "a"]}, {"sent": "xxx deer .", "words": ["deer"]}, {"sent": "thats really a deer Laura .", "words": ["deer", "a"]}, {"sent": "a deer ?", "words": ["deer", "a"]}, {"sent": "is that a deer ?", "words": ["deer", "that", "a", "is"]}, {"sent": "say deer .", "words": ["deer", "say"]}, {"sent": "a deer ?", "words": ["deer", "a"]}, {"sent": "theres a Mommy deer and what else ?", "words": ["deer", "what", "and", "a"]}, {"sent": "Id like to see the deer that Mrs Wood gave you .", "words": ["the", "to", "deer", "like", "you", "see", "that"]}, {"sent": "oh a baby deer .", "words": ["deer", "a"]}, {"sent": "uhhuh ask Mommy if we have any popcorn .", "words": ["if", "we", "any", "popcorn", "have"]}, {"sent": "we need a popcorn seller .", "words": ["need", "popcorn", "a", "we"]}, {"sent": "she wanted she had popcorn and she wanted seconds on popcorn and xxx .", "words": ["on", "she", "and", "popcorn"]}, {"sent": "popcorn what happens with popcorn ?", "words": ["what", "with", "popcorn"]}, {"sent": "Jwww likes popcorn and Mummy likes popcorn .", "words": ["popcorn", "and"]}, {"sent": "heres some popcorn .", "words": ["some", "popcorn"]}, {"sent": "you wanted some popcorn ?", "words": ["some", "popcorn", "you"]}, {"sent": "yeah well make popcorn Abe .", "words": ["make", "popcorn"]}, {"sent": "I xxx put some popcorn in a bowl for you .", "words": ["bowl", "put", "for", "popcorn", "you", "in", "some", "a"]}, {"sent": "did you tell Edna that were probably going to come by her house and expect alotof candy alotof candy and a candy apple and a popcorn ball ?", "words": ["to", "her", "popcorn", "you", "by", "and", "ball", "house", "did", "apple", "candy", "that", "were", "a"]}, {"sent": "that is the popcorn popper .", "words": ["the", "that", "popcorn", "is"]}, {"sent": "then get a bag of popcorn .", "words": ["of", "popcorn", "get", "then", "a"]}, {"sent": "well you can eat popcorn instead .", "words": ["can", "popcorn", "eat", "you"]}, {"sent": "yeah so you put the cover on when youre making popcorn .", "words": ["the", "put", "so", "popcorn", "you", "cover", "on", "when"]}, {"sent": "oh I dont see any popcorn .", "words": ["see", "dont", "any", "popcorn"]}, {"sent": "popcorn for breakfast ?", "words": ["for", "popcorn"]}, {"sent": "shes eating popcorn .", "words": ["popcorn"]}, {"sent": "get your popcorn okay .", "words": ["get", "your", "popcorn"]}, {"sent": "Phoebe likes popcorn .", "words": ["popcorn"]}, {"sent": "you done with the popcorn ?", "words": ["the", "with", "popcorn", "you"]}, {"sent": "popcorn is really neat fun to make .", "words": ["to", "make", "popcorn", "is"]}, {"sent": "did you have popcorn ?", "words": ["have", "popcorn", "did", "you"]}, {"sent": "xxx xxx xxx pop popcorn .", "words": ["pop", "popcorn"]}, {"sent": "you know like if Ive made some popcorn shell put the popcorn in their and play with it like shes cooking stirring it up .", "words": ["the", "up", "put", "if", "their", "play", "with", "popcorn", "like", "you", "and", "in", "it", "some"]}, {"sent": "thats popcorn .", "words": ["popcorn"]}, {"sent": "uh did you get popcorn Mia ?", "words": ["get", "popcorn", "did", "you"]}, {"sent": "pop poppy pop popcorn .", "words": ["pop", "popcorn"]}, {"sent": "Daddy didnt have any popcorn either did he ?", "words": ["any", "popcorn", "have", "he", "did"]}, {"sent": "youre too little for popcorn .", "words": ["for", "popcorn", "little", "too"]}, {"sent": "hes making popcorn .", "words": ["popcorn"]}, {"sent": "I will make some popcorn for you .", "words": ["for", "popcorn", "you", "will", "some", "make"]}, {"sent": "it does look like popcorn in the back of the train .", "words": ["the", "does", "back", "train", "of", "like", "popcorn", "in", "it", "look"]}, {"sent": "like popcorn ?", "words": ["like", "popcorn"]}, {"sent": "oh I uh I thought that we would have popcorn when we watched the movie .", "words": ["the", "when", "we", "popcorn", "have", "would", "that"]}, {"sent": "ohmygoodness its popcorn .", "words": ["popcorn"]}, {"sent": "what did I tell you two boys popcorn or peanuts ?", "words": ["what", "popcorn", "did", "you"]}, {"sent": "cooked some popcorn last year .", "words": ["some", "last", "popcorn"]}, {"sent": "did you have popcorn ?", "words": ["have", "popcorn", "did", "you"]}, {"sent": "you want me to make some popcorn for you ?", "words": ["for", "to", "popcorn", "you", "me", "make", "some"]}, {"sent": "no popcorn .", "words": ["popcorn"]}, {"sent": "its that sort of smell that you get with popcorn .", "words": ["of", "with", "popcorn", "you", "get", "that"]}, {"sent": "just like the popcorn with the Christmas tree .", "words": ["the", "tree", "with", "like", "popcorn"]}, {"sent": "in fact I dont think Ive every bought you popcorn .", "words": ["think", "popcorn", "you", "dont", "in", "every"]}, {"sent": "you did have popcorn .", "words": ["have", "popcorn", "did", "you"]}, {"sent": "you want your popcorn maker ?", "words": ["your", "popcorn", "you"]}, {"sent": "when you give him the popcorn .", "words": ["the", "popcorn", "you", "him", "give", "when"]}, {"sent": "and you eat popcorn .", "words": ["eat", "popcorn", "you", "and"]}, {"sent": "we dont have any popcorn balls Abe .", "words": ["we", "any", "popcorn", "have", "dont"]}, {"sent": "is that hard to say when you have your mouth full of popcorn ?", "words": ["your", "when", "full", "is", "of", "to", "mouth", "say", "popcorn", "you", "have", "hard", "that"]}, {"sent": "did you have sweet popcorn ?", "words": ["have", "popcorn", "did", "you"]}, {"sent": "I didnt get popcorn Taiyo .", "words": ["get", "popcorn"]}, {"sent": "can I have some popcorn ?", "words": ["can", "some", "have", "popcorn"]}, {"sent": "Phoebe eats popcorn .", "words": ["popcorn"]}, {"sent": "you didnt like the popcorn did you ?", "words": ["the", "popcorn", "like", "you", "did"]}, {"sent": "very very light like popcorn .", "words": ["light", "like", "popcorn"]}, {"sent": "and he get incase he runs outof popcorn .", "words": ["get", "he", "popcorn", "and"]}, {"sent": "Abe do you want me to make popcorn ?", "words": ["do", "to", "popcorn", "you", "me", "make"]}, {"sent": "still more popcorn to eat .", "words": ["eat", "to", "more", "popcorn"]}, {"sent": "youre the popcorn that goes with the milk what are you Dan ?", "words": ["the", "what", "with", "popcorn", "you", "are", "milk", "that"]}, {"sent": "you were eating a lot of popcorn .", "words": ["of", "a lot", "popcorn", "you", "were", "a"]}, {"sent": "Friday I thought it was just the popcorn hed threw up .", "words": ["the", "up", "was", "popcorn", "it"]}, {"sent": "popcorn is really neat its fun to make and fun to eat .", "words": ["is", "to", "popcorn", "and", "eat", "make"]}, {"sent": "because it was too early for Daddy to eat popcorn .", "words": ["for", "was", "to", "popcorn", "eat", "it", "too", "because"]}, {"sent": "youre getting more popcorn I see .", "words": ["see", "more", "popcorn"]}, {"sent": "what does popcorn do ?", "words": ["what", "do", "does", "popcorn"]}, {"sent": "the other day Abe found a nickel and told me that he wanted to buy a popcorn ball .", "words": ["the", "to", "popcorn", "and", "other", "ball", "he", "me", "that", "buy", "a"]}, {"sent": "well its unpopped popcorn .", "words": ["popcorn"]}, {"sent": "eating popcorn .", "words": ["popcorn"]}, {"sent": "wheres all your popcorn xxx jigsaw ?", "words": ["all", "your", "popcorn"]}, {"sent": "we hafta make popcorn .", "words": ["make", "popcorn", "we"]}, {"sent": "popcorn beard and he said .", "words": ["he", "popcorn", "and"]}, {"sent": "and then well make popcorn .", "words": ["then", "make", "popcorn", "and"]}, {"sent": "and ah here is popcorn .", "words": ["here", "popcorn", "is", "and"]}, {"sent": "oh my gosh thats a lotta popcorn in the mouth .", "words": ["the", "my", "mouth", "popcorn", "in", "a"]}, {"sent": "your popcorn fell .", "words": ["your", "popcorn"]}, {"sent": "Cookie I brought you some popcorn .", "words": ["some", "popcorn", "you"]}, {"sent": "wanna see a little eensie piece of popcorn ?", "words": ["of", "popcorn", "little", "wanna", "see", "a"]}, {"sent": "you want the popcorn man to wear the cowboy hat ?", "words": ["the", "to", "popcorn", "you", "hat"]}, {"sent": "alright Im gonna do your popcorn .", "words": ["do", "your", "popcorn"]}, {"sent": "dont you like soggy popcorn ?", "words": ["dont", "popcorn", "like", "you"]}, {"sent": "it wasnt chickennuggets it was popcorn chicken .", "words": ["was", "it", "popcorn", "chicken"]}, {"sent": "do we have enough popcorn to make when Mommy comes home ?", "words": ["do", "when", "we", "to", "popcorn", "have", "make", "home"]}, {"sent": "well I think that well take some peanuts and some popcorn .", "words": ["think", "popcorn", "and", "some", "take", "that"]}, {"sent": "everybody have popcorn ?", "words": ["have", "popcorn"]}, {"sent": "hell notice his popcorn string is missing .", "words": ["his", "popcorn", "is"]}, {"sent": "popcorn you can pop xxx xxx xxx xxx xxx .", "words": ["can", "pop", "popcorn", "you"]}, {"sent": "fill it with popcorn ?", "words": ["it", "with", "popcorn"]}, {"sent": "well thats good you know how to make popcorn .", "words": ["how", "to", "popcorn", "you", "good", "make"]}, {"sent": "popcorn who likes popcorn ?", "words": ["popcorn", "who"]}, {"sent": "but you said you didnt like popcorn .", "words": ["popcorn", "like", "but", "you"]}, {"sent": "thats your popcorn maker .", "words": ["your", "popcorn"]}, {"sent": "wheres the popcorn at ?", "words": ["the", "popcorn", "at"]}, {"sent": "here comes the popcorn .", "words": ["here", "the", "popcorn"]}, {"sent": "weve not had popcorn for a long time have we ?", "words": ["for", "we", "popcorn", "long", "have", "not", "a"]}, {"sent": "Alicia are gonna serve the popcorn right ?", "words": ["the", "are", "popcorn"]}, {"sent": "it was what popcorn ?", "words": ["was", "what", "it", "popcorn"]}, {"sent": "youre right I think a hamburger did .", "words": ["hamburger", "think", "did", "a"]}, {"sent": "ya may be babyd like a hamburger .", "words": ["hamburger", "be", "like", "a"]}, {"sent": "meat yeah it looks like hamburger doesnt it ?", "words": ["it", "like", "meat", "hamburger"]}, {"sent": "can mommy have the hamburger ?", "words": ["can", "the", "have", "hamburger"]}, {"sent": "you got a French fry team and a hamburger team .", "words": ["hamburger", "and", "a", "you"]}, {"sent": "hamburger .", "words": ["hamburger"]}, {"sent": "icecream like you wanna hotdog no you wanna hamburger no .", "words": ["hamburger", "like", "wanna", "you"]}, {"sent": "do ya wanna hamburger ?", "words": ["do", "wanna", "hamburger"]}, {"sent": "Id like a hamburger and some potato chip .", "words": ["like", "hamburger", "and", "some", "potato chip", "potato", "a"]}, {"sent": "hows your hamburger ?", "words": ["your", "hamburger"]}, {"sent": "and hamburger .", "words": ["hamburger", "and"]}, {"sent": "and she was looking at the cookie and the hamburger .", "words": ["the", "cookie", "was", "and", "hamburger", "at", "she"]}, {"sent": "doesnt taste like a hamburger huh ?", "words": ["hamburger", "taste", "like", "a"]}, {"sent": "ohp hamburger .", "words": ["hamburger"]}, {"sent": "one penny and you bought a hamburger ?", "words": ["hamburger", "you", "penny", "and", "a"]}, {"sent": "eat the hamburger .", "words": ["eat", "the", "hamburger"]}, {"sent": "do you want pickles on you hamburger ?", "words": ["on", "do", "hamburger", "you"]}, {"sent": "want a hamburger ?", "words": ["hamburger", "a"]}, {"sent": "are you a hamburger ?", "words": ["hamburger", "are", "a", "you"]}, {"sent": "wheres the hamburger ?", "words": ["the", "hamburger"]}, {"sent": "hamburger mm mm mm .", "words": ["hamburger"]}, {"sent": "oh thats hamburger .", "words": ["hamburger"]}, {"sent": "did you eat hamburger ?", "words": ["eat", "hamburger", "did", "you"]}, {"sent": "hamburger head ?", "words": ["head", "hamburger"]}, {"sent": "is that good hamburger ?", "words": ["hamburger", "that", "good", "is"]}, {"sent": "okay wheres your hamburger ?", "words": ["your", "hamburger"]}, {"sent": "theres the hamburger .", "words": ["the", "hamburger"]}, {"sent": "ah hamburger hamburger .", "words": ["hamburger"]}, {"sent": "hamburger .", "words": ["hamburger"]}, {"sent": "it goes on a hamburger .", "words": ["on", "it", "hamburger", "a"]}, {"sent": "hamburger .", "words": ["hamburger"]}, {"sent": "I did I called it hamburger and shes like .", "words": ["like", "and", "hamburger", "it", "did"]}, {"sent": "what else should go on the hamburger ?", "words": ["the", "what", "go", "hamburger", "on"]}, {"sent": "hamburger yes .", "words": ["hamburger"]}, {"sent": "oh no I think it is a hamburger .", "words": ["think", "is", "hamburger", "it", "a"]}, {"sent": "this is a good hamburger .", "words": ["is", "hamburger", "this", "good", "a"]}, {"sent": "whats that is that a hamburger ?", "words": ["hamburger", "that", "a", "is"]}, {"sent": "hamburger .", "words": ["hamburger"]}, {"sent": "hamburger ?", "words": ["hamburger"]}, {"sent": "did you find a hamburger ?", "words": ["hamburger", "you", "did", "find", "a"]}, {"sent": "is that you saying hamburger ?", "words": ["hamburger", "that", "you", "is"]}, {"sent": "wheres the hamburger ?", "words": ["the", "hamburger"]}, {"sent": "everyone likes the hamburger dont they ?", "words": ["the", "dont", "they", "hamburger"]}, {"sent": "can mommy have some hamburger now ?", "words": ["can", "some", "have", "hamburger"]}, {"sent": "Id like a hamburger .", "words": ["hamburger", "like", "a"]}, {"sent": "go back to the hamburger .", "words": ["the", "back", "go", "to", "hamburger"]}, {"sent": "this looks like a hamburger patty .", "words": ["hamburger", "this", "like", "a"]}, {"sent": "you loved that hamburger last time .", "words": ["hamburger", "last", "that", "you"]}, {"sent": "another hamburger ?", "words": ["another", "hamburger"]}, {"sent": "thats right Mommy likes peas on her hamburger .", "words": ["on", "peas", "her", "hamburger"]}, {"sent": "thats not a hamburger ?", "words": ["hamburger", "not", "a"]}, {"sent": "mm thats the best hamburger I ever had .", "words": ["the", "hamburger"]}, {"sent": "hamburger ?", "words": ["hamburger"]}, {"sent": "oh its hamburger and steak .", "words": ["and", "hamburger"]}, {"sent": "and youve got a hamburger .", "words": ["hamburger", "a", "and"]}, {"sent": "tomato pickle cheese hamburger and bread .", "words": ["bread", "hamburger", "and", "pickle", "cheese"]}, {"sent": "hamburger .", "words": ["hamburger"]}, {"sent": "you like the hamburger .", "words": ["the", "hamburger", "like", "you"]}, {"sent": "part of a hamburger ?", "words": ["of", "hamburger", "a"]}, {"sent": "hamburger ?", "words": ["hamburger"]}, {"sent": "you dont eat hamburger for breakfast .", "words": ["for", "hamburger", "eat", "dont", "you"]}, {"sent": "its a hamburger ?", "words": ["hamburger", "a"]}, {"sent": "hamburger ?", "words": ["hamburger"]}, {"sent": "maybe a hamburger if were willing to have it on bread .", "words": ["bread", "if", "to", "hamburger", "have", "it", "on", "were", "a"]}, {"sent": "get the hamburger .", "words": ["get", "the", "hamburger"]}, {"sent": "I would not like would not any hamburger but may I have some fish ?", "words": ["any", "like", "hamburger", "have", "would", "some", "not", "but", "fish"]}, {"sent": "peas and mashed potatoes and a baby hamburger .", "words": ["hamburger", "peas", "a", "and"]}, {"sent": "uh the ice cream and the hamburger were exciting .", "words": ["the", "ice cream", "hamburger", "and", "ice", "were"]}, {"sent": "its hamburger .", "words": ["hamburger"]}, {"sent": "thats a hamburger bun .", "words": ["hamburger", "a"]}, {"sent": "a hamburger .", "words": ["hamburger", "a"]}, {"sent": "look heres a hamburger .", "words": ["hamburger", "a", "look"]}, {"sent": "is there pickles on this hamburger ?", "words": ["is", "hamburger", "there", "this", "on"]}, {"sent": "well actually Matty you could get hamburger too or cheeseburger or chicken .", "words": ["hamburger", "get", "you", "too", "chicken", "could"]}, {"sent": "I would like some hamburger with lettuce and tomato .", "words": ["with", "like", "hamburger", "and", "would", "some"]}, {"sent": "oh ya gonna pour something on your hamburger ?", "words": ["on", "pour", "your", "hamburger"]}, {"sent": "what meal do you eat a hamburger at ?", "words": ["what", "do", "you", "eat", "hamburger", "at", "a"]}, {"sent": "a pretty sad hamburger .", "words": ["hamburger", "sad", "pretty", "a"]}, {"sent": "thats a lot of stuff to climb over to get to that hamburger .", "words": ["of", "to", "a lot", "hamburger", "get", "that", "climb", "over", "a"]}, {"sent": "please give me hamburger .", "words": ["me", "give", "hamburger"]}, {"sent": "well put some cheese and have coffee and a hamburger huh ?", "words": ["put", "and", "have", "hamburger", "coffee", "some", "cheese", "a"]}, {"sent": "we could make a hamburger .", "words": ["we", "hamburger", "could", "make", "a"]}, {"sent": "wanna taste a hamburger ?", "words": ["hamburger", "taste", "wanna", "a"]}, {"sent": "a hamburger or hotdog .", "words": ["hamburger", "a"]}, {"sent": "ut and a hamburger bun wow .", "words": ["hamburger", "a", "and"]}, {"sent": "its a hamburger ?", "words": ["hamburger", "a"]}, {"sent": "and so he ran over and sat down to eat the hamburger .", "words": ["the", "so", "to", "and", "eat", "hamburger", "he", "down", "over"]}, {"sent": "hamburger .", "words": ["hamburger"]}, {"sent": "wanna hamburger ?", "words": ["wanna", "hamburger"]}, {"sent": "hamburger .", "words": ["hamburger"]}, {"sent": "uhoh the hamburger patty thingy .", "words": ["the", "hamburger"]}, {"sent": "you want Jennys hamburger ?", "words": ["hamburger", "you"]}, {"sent": "well its a dollar nineteen a pound which is cheaper than lean hamburger .", "words": ["hamburger", "which", "is", "a"]}, {"sent": "hamburger I mean hes had a lot of the lunch meat and stuff .", "words": ["the", "of", "a lot", "meat", "hamburger", "and", "a"]}, {"sent": "hamburger and pizza .", "words": ["pizza", "and", "hamburger"]}, {"sent": "hamburger ?", "words": ["hamburger"]}, {"sent": "put the hamburger on there .", "words": ["the", "put", "hamburger", "there", "on"]}, {"sent": "oh theres a hamburger bun .", "words": ["hamburger", "a"]}, {"sent": "you wanna give me that raisin too ?", "words": ["you", "raisin", "wanna", "too", "me", "give", "that"]}, {"sent": "theres some raisin and sultanas .", "words": ["some", "raisin", "and"]}, {"sent": "she can have some raisin with cereal and .", "words": ["can", "with", "and", "have", "some", "raisin", "cereal", "she"]}, {"sent": "and raisin ?", "words": ["raisin", "and"]}, {"sent": "Panacka wants a raisin .", "words": ["raisin", "a"]}, {"sent": "this is a raisin .", "words": ["this", "raisin", "a", "is"]}, {"sent": "or is the raisin packet on the back of the lorry or something ?", "words": ["the", "back", "is", "of", "on", "raisin"]}, {"sent": "did you spit out the raisin ?", "words": ["the", "you", "did", "out", "raisin"]}, {"sent": "heres a man raisin man .", "words": ["raisin", "a"]}, {"sent": "oh he looks like a raisin alright .", "words": ["raisin", "he", "like", "a"]}, {"sent": "well is the raisin packet hitting the bridge on its own ?", "words": ["the", "raisin", "on", "is"]}, {"sent": "yes theres a raisin there wouldjou like it ?", "words": ["like", "there", "it", "raisin", "a"]}, {"sent": "a raisin .", "words": ["raisin", "a"]}, {"sent": "would you like a raisin ?", "words": ["like", "you", "would", "raisin", "a"]}, {"sent": "its a raisin .", "words": ["raisin", "a"]}, {"sent": "theres raisin here on the spoon .", "words": ["the", "here", "spoon", "on", "raisin"]}, {"sent": "where was I raisin oh I guess it does sound like that was I raisin .", "words": ["where", "does", "was", "like", "it", "raisin", "that"]}, {"sent": "theres the raisin .", "words": ["the", "raisin"]}, {"sent": "you still have raisin in your mouth ?", "words": ["your", "mouth", "you", "have", "in", "raisin"]}, {"sent": "and you have another little raisin no thats the microphone .", "words": ["the", "little", "have", "another", "you", "and", "raisin"]}, {"sent": "raisin .", "words": ["raisin"]}, {"sent": "oh is someone gonna eat raisin biscuits at Dolce ?", "words": ["eat", "raisin", "at", "is"]}, {"sent": "and a raisin too yummy yummy yummy .", "words": ["too", "raisin", "a", "and"]}, {"sent": "theres just the raisin one left and you didnt like that one .", "words": ["the", "like", "you", "and", "raisin", "that"]}, {"sent": "that its a raisin .", "words": ["raisin", "that", "a"]}, {"sent": "raisin toast and a xxx .", "words": ["raisin", "a", "toast", "and"]}, {"sent": "this is the raisin shop .", "words": ["the", "this", "raisin", "is"]}, {"sent": "its just you gave me a raisin so I thought you might like a piece of my cheese .", "words": ["my", "of", "so", "like", "you", "me", "raisin", "cheese", "a"]}, {"sent": "a pussy cat walking round the raisin shop ?", "words": ["the", "cat", "raisin", "a"]}, {"sent": "if you need a raisin Ill go get you a raisin .", "words": ["need", "if", "go", "you", "get", "raisin", "a"]}, {"sent": "would you like a raisin ?", "words": ["like", "you", "would", "raisin", "a"]}, {"sent": "nutritious lunch of pretzels and a little cheese a little ham a little oatmeal raisin cookie .", "words": ["cookie", "of", "little", "and", "raisin", "cheese", "a"]}, {"sent": "oh raisin rice pie right ?", "words": ["raisin"]}, {"sent": "your raisin container says have a nice day in sweetie shop does it ?", "words": ["does", "your", "nice", "have", "in", "it", "raisin", "a"]}, {"sent": "raisin yeah this is called raisin bran .", "words": ["this", "raisin", "is"]}, {"sent": "you can put a raisin on Tegamas plate .", "words": ["can", "put", "plate", "you", "on", "raisin", "a"]}, {"sent": "a muffled thankyou with a raisin in her mouth .", "words": ["mouth", "with", "in", "raisin", "her", "a"]}, {"sent": "this ones got a raisin on .", "words": ["on", "this", "raisin", "a"]}, {"sent": "I mean the raisin shop man .", "words": ["the", "raisin"]}, {"sent": "in that book but this is a raisin .", "words": ["is", "this", "in", "book", "raisin", "that", "but", "a"]}, {"sent": "raisin .", "words": ["raisin"]}, {"sent": "nothing is in that raisin container .", "words": ["raisin", "in", "that", "is"]}, {"sent": "raisin .", "words": ["raisin"]}, {"sent": "raisin .", "words": ["raisin"]}, {"sent": "cinnamon raisin bread pudding .", "words": ["bread", "raisin", "pudding"]}, {"sent": "can I have a raisin ?", "words": ["can", "raisin", "have", "a"]}, {"sent": "raisin bran cereal .", "words": ["raisin", "cereal"]}, {"sent": "you like raisin .", "words": ["raisin", "like", "you"]}, {"sent": "raisin .", "words": ["raisin"]}, {"sent": "raisin .", "words": ["raisin"]}, {"sent": "well to make it eight oclock you need one more raisin in the pot .", "words": ["the", "need", "to", "you", "more", "it", "in", "raisin", "make"]}, {"sent": "you have a raisin in your mouth .", "words": ["your", "mouth", "you", "have", "in", "raisin", "a"]}, {"sent": "you little raisin boy .", "words": ["raisin", "little", "you"]}, {"sent": "give Eeyore a raisin ?", "words": ["raisin", "give", "a"]}, {"sent": "dyou want smore cereal with raisin ?", "words": ["raisin", "cereal", "with"]}, {"sent": "hey you didnt want your raisin ?", "words": ["raisin", "your", "you"]}, {"sent": "I gave you a raisin .", "words": ["raisin", "a", "you"]}, {"sent": "raisin .", "words": ["raisin"]}, {"sent": "I mean get the raisin with the fork .", "words": ["the", "with", "get", "fork", "raisin"]}, {"sent": "are you the man from the raisin shop ?", "words": ["the", "raisin", "are", "you"]}, {"sent": "youve left your tissue in the raisin shop ?", "words": ["the", "your", "tissue", "in", "raisin"]}, {"sent": "a raisin .", "words": ["raisin", "a"]}, {"sent": "the raisin is arriving yeah .", "words": ["the", "raisin", "is"]}, {"sent": "you clever clever clever raisin seller man .", "words": ["raisin", "you"]}, {"sent": "thank you so much mister raisin seller .", "words": ["so", "raisin", "much", "you"]}, {"sent": "you sposta be combing the raisin doll .", "words": ["the", "be", "doll", "you", "raisin"]}, {"sent": "from now on up look for a raisin one .", "words": ["up", "for", "look", "on", "raisin", "a"]}, {"sent": "was I a raisin ?", "words": ["was", "raisin", "a"]}, {"sent": "bye bye raisin shop person .", "words": ["raisin"]}, {"sent": "its a raisin .", "words": ["raisin", "a"]}, {"sent": "raisin in the cake huh ?", "words": ["the", "in", "raisin", "cake"]}, {"sent": "what does the California raisin do ?", "words": ["the", "what", "does", "do", "raisin"]}, {"sent": "and raisin .", "words": ["raisin", "and"]}, {"sent": "and the raisin could be the bad guy .", "words": ["the", "be", "bad", "and", "raisin", "could"]}, {"sent": "I think I see a raisin raisins in here .", "words": ["think", "here", "in", "see", "raisin", "a"]}, {"sent": "you have a raisin on your face .", "words": ["your", "face", "you", "have", "on", "raisin", "a"]}, {"sent": "it had a raisin in it .", "words": ["in", "raisin", "it", "a"]}, {"sent": "raisin shop customer thats what I was thinking of .", "words": ["was", "what", "raisin", "of"]}, {"sent": "heres some raisin .", "words": ["some", "raisin"]}, {"sent": "one raisin .", "words": ["raisin"]}, {"sent": "I wont mind giving him a c o o key some of it because its oatmeal on raisin .", "words": ["of", "him", "it", "on", "some", "raisin", "because", "a"]}, {"sent": "Megans showing you what the California raisin does .", "words": ["the", "what", "does", "you", "raisin"]}, {"sent": "didjou want another raisin ?", "words": ["another", "raisin"]}, {"sent": "is it a raisin Bax ?", "words": ["raisin", "it", "a", "is"]}, {"sent": "thats the SunMaid raisin woman .", "words": ["the", "raisin"]}, {"sent": "wouldja like to try some raisin bran ?", "words": ["to", "like", "some", "raisin", "try"]}, {"sent": "those are oatmeal raisin .", "words": ["raisin", "are", "those"]}, {"sent": "its a raisin stuck to the book .", "words": ["the", "to", "stuck", "book", "raisin", "a"]}, {"sent": "no a raisin .", "words": ["raisin", "a"]}, {"sent": "Ill put some raisin on your cereal .", "words": ["put", "your", "on", "some", "raisin", "cereal"]}, {"sent": "well the raisin .", "words": ["the", "raisin"]}, {"sent": "oh uh cinnamon and raisin and bread pudding .", "words": ["bread", "raisin", "pudding", "and"]}, {"sent": "theres raisin on the spoon .", "words": ["on", "the", "raisin", "spoon"]}, {"sent": "didjou want another raisin ?", "words": ["another", "raisin"]}, {"sent": "is that just a nut and raisin xxx .", "words": ["is", "and", "raisin", "that", "a"]}, {"sent": "can you give Panacka a raisin ?", "words": ["can", "give", "you", "raisin", "a"]}, {"sent": "the raisin toast ?", "words": ["the", "raisin", "toast"]}, {"sent": "more raisin .", "words": ["raisin", "more"]}, {"sent": "dump it okay hey .", "words": ["it", "dump"]}, {"sent": "your dump truck goes .", "words": ["truck", "dump", "your"]}, {"sent": "and we dump it in a pile .", "words": ["we", "and", "in", "it", "dump", "a"]}, {"sent": "and were gonna dump and spread all around .", "words": ["all", "and", "around", "dump", "were"]}, {"sent": "dont dump it all over the place .", "words": ["the", "all", "dont", "it", "dump", "over"]}, {"sent": "dump them out and put them back .", "words": ["put", "back", "them", "and", "out", "dump"]}, {"sent": "youve dropped your dump truck .", "words": ["truck", "dump", "your"]}, {"sent": "are you gonna dump the people out ?", "words": ["the", "you", "are", "out", "dump"]}, {"sent": "dump it out .", "words": ["it", "dump", "out"]}, {"sent": "and one dump truck .", "words": ["dump", "truck", "and"]}, {"sent": "lets dump it on Tim .", "words": ["on", "it", "dump"]}, {"sent": "would you like to dump it into the tray ?", "words": ["the", "into", "to", "like", "you", "it", "would", "dump", "tray"]}, {"sent": "okay put the jeep and the dump truck next to eachother and well see how big it hasta be .", "words": ["the", "put", "how", "to", "be", "truck", "and", "it", "see", "dump", "big"]}, {"sent": "the dump truck can go .", "words": ["the", "can", "go", "truck", "dump"]}, {"sent": "does it dump them ?", "words": ["does", "it", "them", "dump"]}, {"sent": "lets dump this out wow you dump it out xxx .", "words": ["you", "this", "it", "out", "dump"]}, {"sent": "go ahead dump it over .", "words": ["it", "dump", "over", "go"]}, {"sent": "you dump it out ?", "words": ["it", "dump", "out", "you"]}, {"sent": "should we dump it in the washing machine and turn on spin dry or wash and wear or something ?", "words": ["the", "we", "dry", "and", "washing machine", "in", "it", "on", "dump", "wash"]}, {"sent": "and look theres a present in the back of the dump truck .", "words": ["the", "present", "back", "of", "truck", "and", "in", "look", "dump", "a"]}, {"sent": "youre sitting in a dump ?", "words": ["in", "dump", "a"]}, {"sent": "well where is the dump truck ?", "words": ["where", "the", "is", "truck", "dump"]}, {"sent": "dump trucks .", "words": ["dump"]}, {"sent": "dump it ?", "words": ["it", "dump"]}, {"sent": "about time to dump it again .", "words": ["to", "dump", "about", "it"]}, {"sent": "dump it dump it in .", "words": ["it", "in", "dump"]}, {"sent": "its called a dump truck .", "words": ["dump", "truck", "a"]}, {"sent": "put dump truck in blocks .", "words": ["in", "put", "dump", "truck"]}, {"sent": "lets dump it .", "words": ["it", "dump"]}, {"sent": "it means dump you out ?", "words": ["out", "it", "you", "dump"]}, {"sent": "what else shall we dump with a truck ?", "words": ["what", "we", "with", "truck", "dump", "a"]}, {"sent": "dump them all .", "words": ["all", "dump", "them"]}, {"sent": "dump it oh oh my gosh you are dumping it .", "words": ["my", "you", "it", "are", "dump"]}, {"sent": "you know if you cant get your fingers sometimes if you dump it out .", "words": ["your", "if", "you", "get", "it", "out", "dump"]}, {"sent": "thats a dump truck .", "words": ["dump", "truck", "a"]}, {"sent": "wheres the dump truck ?", "words": ["the", "dump", "truck"]}, {"sent": "to dump a load and the huge arm can go in any direction .", "words": ["the", "can", "go", "to", "any", "and", "in", "arm", "dump", "a"]}, {"sent": "yes the dump truck fell down .", "words": ["the", "down", "dump", "truck"]}, {"sent": "dont dump it .", "words": ["dont", "dump", "it"]}, {"sent": "dump truck .", "words": ["dump", "truck"]}, {"sent": "youre going to the dump ?", "words": ["the", "to", "dump"]}, {"sent": "dump .", "words": ["dump"]}, {"sent": "and he made that dumptruck dump out its load .", "words": ["and", "he", "out", "dump", "that"]}, {"sent": "I think we better just make it for the dump truck .", "words": ["the", "better", "think", "for", "we", "truck", "it", "dump", "make"]}, {"sent": "a dump truck ?", "words": ["dump", "truck", "a"]}, {"sent": "mhm the dump train .", "words": ["the", "dump", "train"]}, {"sent": "I dump it out .", "words": ["it", "dump", "out"]}, {"sent": "then he pushed the button on his dump truck and all the sand fell out .", "words": ["the", "all", "truck", "and", "his", "he", "out", "on", "then", "dump", "button"]}, {"sent": "go put it in your dump truck .", "words": ["put", "your", "go", "truck", "in", "it", "dump"]}, {"sent": "tables still a little bit dump isnt it ?", "words": ["it", "dump", "little", "a"]}, {"sent": "no dont wanna dump it out .", "words": ["dont", "it", "out", "wanna", "dump"]}, {"sent": "dump them all out .", "words": ["all", "dump", "them", "out"]}, {"sent": "heres the other one for the dump truck .", "words": ["the", "for", "truck", "other", "dump"]}, {"sent": "because hes a giant dump truck .", "words": ["dump", "truck", "because", "a"]}, {"sent": "dump truck ?", "words": ["dump", "truck"]}, {"sent": "dump that out .", "words": ["dump", "that", "out"]}, {"sent": "no here you wanna dump em ?", "words": ["here", "dump", "wanna", "you"]}, {"sent": "can we see a dump truck ?", "words": ["can", "we", "truck", "see", "dump", "a"]}, {"sent": "I think that bottle there goes in that dump .", "words": ["think", "there", "in", "bottle", "dump", "that"]}, {"sent": "the yellow is the part of the truck that dump things that dumps things .", "words": ["the", "is", "of", "truck", "yellow", "dump", "that"]}, {"sent": "thank you a little more dump it back in okay thats good .", "words": ["back", "little", "you", "more", "it", "in", "good", "dump", "a"]}, {"sent": "are you putting the sand in the dump truck ?", "words": ["the", "truck", "you", "in", "are", "dump"]}, {"sent": "youre gonna dump all the eggs out ?", "words": ["the", "all", "dump", "out"]}, {"sent": "dump train going down the road dump train haul .", "words": ["the", "down", "dump", "train"]}, {"sent": "its not a dump truck Nomi a tractor .", "words": ["truck", "tractor", "dump", "not", "a"]}, {"sent": "dump em out .", "words": ["dump", "out"]}, {"sent": "lets dump them all out and see .", "words": ["all", "them", "and", "out", "see", "dump"]}, {"sent": "the dump trucks not finished its work yet .", "words": ["the", "dump", "work", "not"]}, {"sent": "you like to dump huh ?", "words": ["to", "dump", "like", "you"]}, {"sent": "the drink dump ?", "words": ["the", "dump", "drink"]}, {"sent": "who has the dump truck ?", "words": ["the", "dump", "truck", "who"]}, {"sent": "you see him dump some xxx .", "words": ["you", "him", "see", "dump", "some"]}, {"sent": "is that sposta go in the dump ?", "words": ["the", "go", "is", "in", "dump", "that"]}, {"sent": "yes a dump truck .", "words": ["dump", "truck", "a"]}, {"sent": "theres a dump truck in there ?", "words": ["truck", "there", "in", "dump", "a"]}, {"sent": "youre just gonna dump arent you ?", "words": ["dump", "you"]}, {"sent": "you gonna dump em ?", "words": ["dump", "you"]}, {"sent": "should we dump him ?", "words": ["him", "dump", "we"]}, {"sent": "you gonna dump Mickeymouse all over the place ?", "words": ["the", "all", "you", "dump", "over"]}, {"sent": "they scoop up the sand and dump it over here .", "words": ["up", "the", "and", "here", "they", "it", "dump", "over"]}, {"sent": "dump dump .", "words": ["dump"]}, {"sent": "Im gonna dump em all out .", "words": ["all", "dump", "out"]}, {"sent": "now can he dump it into there ?", "words": ["can", "into", "there", "it", "he", "dump"]}, {"sent": "bump a dump a dump a dump a doo .", "words": ["bump", "dump", "a"]}, {"sent": "heres your dump truck .", "words": ["truck", "dump", "your"]}, {"sent": "dump that one in please .", "words": ["in", "dump", "that"]}, {"sent": "push push push dump push dump push .", "words": ["dump", "push"]}, {"sent": "theres your dump truck .", "words": ["truck", "dump", "your"]}, {"sent": "two in the dump truck .", "words": ["the", "in", "dump", "truck"]}, {"sent": "oh dont dump it .", "words": ["dont", "dump", "it"]}, {"sent": "you wanna dump it in from Cranky ?", "words": ["you", "in", "it", "wanna", "dump"]}, {"sent": "you dump them all ?", "words": ["all", "dump", "them", "you"]}, {"sent": "then he pushed the up and over button on his brand new power city rocker crushed dump truck .", "words": ["the", "up", "button", "dump", "truck", "and", "his", "he", "on", "then", "new", "over"]}, {"sent": "we dont need quite that much why dont you dump back in ?", "words": ["need", "back", "we", "much", "you", "dont", "in", "dump", "that", "why"]}, {"sent": "did you dump did you dump it ?", "words": ["it", "dump", "did", "you"]}, {"sent": "please dont dump all that out .", "words": ["all", "dont", "out", "dump", "that"]}, {"sent": "well dump it out .", "words": ["it", "dump", "out"]}, {"sent": "to a field and flies it and sees a squirrel collecting nuts .", "words": ["to", "and", "it", "squirrel", "a"]}, {"sent": "are you a squirrel ?", "words": ["are", "squirrel", "a", "you"]}, {"sent": "thats a squirrel and what is the squirrel eating ?", "words": ["the", "what", "is", "and", "squirrel", "a"]}, {"sent": "then he sees a squirrel collecting nuts .", "words": ["then", "he", "squirrel", "a"]}, {"sent": "squirrel .", "words": ["squirrel"]}, {"sent": "hes sposta be gray like gray squirrel bushy tail .", "words": ["be", "squirrel", "like"]}, {"sent": "squirrel .", "words": ["squirrel"]}, {"sent": "a squirrel do you remember what the squirrels name is ?", "words": ["the", "do", "what", "is", "you", "squirrel", "a"]}, {"sent": "ah going to see the the little squirrel ?", "words": ["the", "to", "little", "see", "squirrel"]}, {"sent": "no hes not eating a hand the squirrel is eating a nut .", "words": ["the", "hand", "is", "not", "squirrel", "a"]}, {"sent": "where has the squirrel gone ?", "words": ["where", "the", "squirrel"]}, {"sent": "heres s for squirrel .", "words": ["for", "squirrel"]}, {"sent": "a squirrel .", "words": ["squirrel", "a"]}, {"sent": "squirrel .", "words": ["squirrel"]}, {"sent": "wheres the squirrel ?", "words": ["the", "squirrel"]}, {"sent": "thats a squirrel .", "words": ["squirrel", "a"]}, {"sent": "oh now heres the gray squirrel song it says gray squirrel gray squirrel swish your bushy tail .", "words": ["the", "it", "your", "squirrel"]}, {"sent": "a squirrel there goes a squirrel .", "words": ["there", "squirrel", "a"]}, {"sent": "squirrel ?", "words": ["squirrel"]}, {"sent": "the squirrel xxx .", "words": ["the", "squirrel"]}, {"sent": "ah what a naughty squirrel .", "words": ["what", "squirrel", "naughty", "a"]}, {"sent": "its a squirrel .", "words": ["squirrel", "a"]}, {"sent": "look at the squirrel .", "words": ["the", "at", "squirrel", "look"]}, {"sent": "isnt there another squirrel somewhere ?", "words": ["another", "there", "squirrel"]}, {"sent": "I guess its a squirrel .", "words": ["squirrel", "a"]}, {"sent": "hello squirrel .", "words": ["squirrel"]}, {"sent": "squirrel .", "words": ["squirrel"]}, {"sent": "whos got the squirrel ?", "words": ["the", "squirrel"]}, {"sent": "a squirrel eating a whole nut .", "words": ["squirrel", "a"]}, {"sent": "I thought you said that was a squirrel .", "words": ["was", "you", "that", "squirrel", "a"]}, {"sent": "Naima is a squirrel .", "words": ["squirrel", "a", "is"]}, {"sent": "I saw a little squirrel a picking up acorns I saw a little squirrel she ran up a tree .", "words": ["up", "tree", "squirrel", "little", "she", "a"]}, {"sent": "no I didnt forget your squirrel .", "words": ["your", "squirrel"]}, {"sent": "well get some nuts for the squirrel .", "words": ["the", "for", "get", "some", "squirrel"]}, {"sent": "mister squirrel hes hoping theyll bring him some nuts for him .", "words": ["for", "bring", "him", "some", "squirrel"]}, {"sent": "color the squirrel brown .", "words": ["the", "squirrel", "brown"]}, {"sent": "the squirrel ?", "words": ["the", "squirrel"]}, {"sent": "a squirrel eats Ethans lunch .", "words": ["squirrel", "a"]}, {"sent": "a squirrel in the sandpit ?", "words": ["the", "in", "squirrel", "a"]}, {"sent": "with the squirrel .", "words": ["the", "with", "squirrel"]}, {"sent": "gray squirrel a bushy tail yes that was the song you sang .", "words": ["the", "was", "you", "that", "squirrel", "a"]}, {"sent": "a naughty squirrel .", "words": ["squirrel", "naughty", "a"]}, {"sent": "no its not a frog squirrel .", "words": ["squirrel", "frog", "not", "a"]}, {"sent": "whats the squirrel doing ?", "words": ["the", "squirrel"]}, {"sent": "its a squirrel trying to get to the nuts that are being put up by Brian for the pigeons or .", "words": ["the", "up", "put", "for", "to", "by", "get", "are", "that", "squirrel", "a"]}, {"sent": "when we were in the park with Grandma and Granddad the other day we saw a squirrel running up a tree didnt we ?", "words": ["the", "up", "tree", "we", "with", "and", "other", "in", "park", "when", "squirrel", "were", "a"]}, {"sent": "yep a squirrel eats the seeds .", "words": ["the", "squirrel", "a"]}, {"sent": "I just saw a squirrel outside .", "words": ["outside", "squirrel", "a"]}, {"sent": "heres a squirrel .", "words": ["squirrel", "a"]}, {"sent": "you saw a squirrel last night didnt you ?", "words": ["last", "squirrel", "a", "you"]}, {"sent": "a squirrel .", "words": ["squirrel", "a"]}, {"sent": "a squirrel ?", "words": ["squirrel", "a"]}, {"sent": "this little squirrel .", "words": ["this", "squirrel", "little"]}, {"sent": "squirrel .", "words": ["squirrel"]}, {"sent": "oh yeah back to the squirrel .", "words": ["the", "to", "squirrel", "back"]}, {"sent": "Tico the squirrel .", "words": ["the", "squirrel"]}, {"sent": "the squirrel food ?", "words": ["the", "food", "squirrel"]}, {"sent": "I think it is a squirrel actually .", "words": ["think", "is", "it", "squirrel", "a"]}, {"sent": "theres the squirrel in the pocket the kite the sugar cubes and all a the xxx .", "words": ["the", "all", "and", "in", "squirrel", "a"]}, {"sent": "whether hes a squirrel .", "words": ["squirrel", "a"]}, {"sent": "color your squirrel while I get your breakfast .", "words": ["get", "your", "squirrel"]}, {"sent": "oh I can hear that squirrel I can still hear it .", "words": ["can", "hear", "it", "that", "squirrel"]}, {"sent": "squirrel .", "words": ["squirrel"]}, {"sent": "its a squirrel .", "words": ["squirrel", "a"]}, {"sent": "thats his tail that squirrel has got a great big tail bushy tail .", "words": ["that", "his", "big", "squirrel", "a"]}, {"sent": "I dont think its a squirrel but I dont know what kind of animal it is .", "words": ["what", "think", "is", "of", "dont", "animal", "it", "squirrel", "but", "a"]}, {"sent": "toes gray squirrel gray squirrel swish your bushy tail .", "words": ["your", "squirrel"]}, {"sent": "thats a bird but thats a squirrel .", "words": ["bird", "squirrel", "but", "a"]}, {"sent": "thats a squirrel isnt it ?", "words": ["it", "squirrel", "a"]}, {"sent": "squirrel .", "words": ["squirrel"]}, {"sent": "a squirrel doesnt stink a skunk does .", "words": ["does", "squirrel", "a"]}, {"sent": "theyve had squirrel ?", "words": ["squirrel"]}, {"sent": "look at that squirrel .", "words": ["at", "that", "squirrel", "look"]}, {"sent": "squirrel ?", "words": ["squirrel"]}, {"sent": "a squirrel guy .", "words": ["squirrel", "a"]}, {"sent": "do you see a squirrel ?", "words": ["do", "you", "see", "squirrel", "a"]}, {"sent": "ahhah and theres another squirrel .", "words": ["another", "squirrel", "and"]}, {"sent": "wheres the squirrel ?", "words": ["the", "squirrel"]}, {"sent": "see hes right there see that little squirrel ?", "words": ["little", "there", "see", "that", "squirrel"]}, {"sent": "well get some nuts for the squirrel .", "words": ["the", "for", "get", "some", "squirrel"]}, {"sent": "square sounds like squirrel .", "words": ["squirrel", "like"]}, {"sent": "a squirrel are you ?", "words": ["are", "squirrel", "you", "a"]}, {"sent": "its a squirrel .", "words": ["squirrel", "a"]}, {"sent": "the squirrel is yours and the mouse the mouse is Jwwws .", "words": ["the", "is", "and", "squirrel", "mouse"]}, {"sent": "is that the squirrel ?", "words": ["the", "that", "squirrel", "is"]}, {"sent": "what a birthday well it really wasnt too bad for I still had the red silk from a kite and the squirrel the bad little fellow who came to my room that night and the jacket .", "words": ["the", "what", "red", "for", "my", "to", "bad", "and", "little", "it", "room", "jacket", "too", "who", "that", "squirrel", "a"]}, {"sent": "this frog and his squirrel .", "words": ["and", "frog", "this", "his", "squirrel"]}, {"sent": "that might be a squirrel .", "words": ["be", "that", "squirrel", "a"]}, {"sent": "I can see the squirrel .", "words": ["can", "see", "squirrel", "the"]}, {"sent": "squirrel very good .", "words": ["squirrel", "good"]}, {"sent": "is it a squirrel ?", "words": ["it", "squirrel", "a", "is"]}, {"sent": "a squirrel saw a green .", "words": ["squirrel", "green", "a"]}, {"sent": "can you say squirrel ?", "words": ["can", "say", "squirrel", "you"]}, {"sent": "squirrel .", "words": ["squirrel"]}, {"sent": "this the squirrel ?", "words": ["the", "this", "squirrel"]}, {"sent": "whats the squirrel doing ?", "words": ["the", "squirrel"]}, {"sent": "see the squirrel ?", "words": ["the", "see", "squirrel"]}, {"sent": "Huckel and Lolly worm were watching Huckel said you may borrow my tricycle to chase after him if you want to .", "words": ["my", "if", "to", "you", "and", "him", "want to", "tricycle", "chase", "were"]}, {"sent": "Ive not heard you call it a tricycle before .", "words": ["you", "it", "tricycle", "not", "a"]}, {"sent": "you didnt tell Ursula that Mommy hadta take you to the hospital and have a stitch put in your finger when you hurt it on the tricycle .", "words": ["the", "put", "your", "when", "hurt", "to", "you", "and", "have", "that", "in", "finger", "it", "tricycle", "on", "take", "a"]}, {"sent": "hes got a tricycle doesnt he ?", "words": ["he", "tricycle", "a"]}, {"sent": "its a woman riding a tricycle .", "words": ["tricycle", "a"]}, {"sent": "that was the little boy that was riding on the tricycle Helen .", "words": ["the", "was", "little", "tricycle", "on", "that"]}, {"sent": "tricycle yah .", "words": ["tricycle"]}, {"sent": "and hes telling another little boy on a tricycle to stop .", "words": ["to", "stop", "little", "another", "and", "tricycle", "on", "a"]}, {"sent": "a tricycle has a back yes .", "words": ["back", "tricycle", "a"]}, {"sent": "now the tricycle is fixed .", "words": ["the", "tricycle", "is"]}, {"sent": "its not a tricycle stick .", "words": ["stick", "not", "tricycle", "a"]}, {"sent": "remember Courtneys tricycle ?", "words": ["tricycle"]}, {"sent": "cause he looks short just like that little boy on the tricycle .", "words": ["the", "like", "little", "he", "tricycle", "on", "that"]}, {"sent": "Maxs tricycle is blue .", "words": ["blue", "tricycle", "is"]}, {"sent": "its on your tricycle ?", "words": ["on", "your", "tricycle"]}, {"sent": "I said Ive never heard you call your tricycle a tricycle before .", "words": ["a", "your", "tricycle", "you"]}, {"sent": "thats not a bicycle thats a tricycle .", "words": ["tricycle", "not", "bicycle", "a"]}, {"sent": "your tricycle is there .", "words": ["there", "your", "tricycle", "is"]}, {"sent": "why dont you go outside and ride the tricycle ?", "words": ["the", "go", "outside", "you", "and", "dont", "tricycle", "ride", "why"]}, {"sent": "riding his tricycle .", "words": ["his", "tricycle"]}, {"sent": "why dont you park the tricycle and finish the puzzle ?", "words": ["the", "finish", "you", "and", "dont", "park", "puzzle", "tricycle", "why"]}, {"sent": "you see him on his little tricycle ?", "words": ["you", "his", "little", "him", "tricycle", "on", "see"]}, {"sent": "a tricycle .", "words": ["tricycle", "a"]}, {"sent": "youre going to fall with that tricycle .", "words": ["to", "with", "fall", "tricycle", "that"]}, {"sent": "its a funny tricycle isnt it Nomi ?", "words": ["it", "tricycle", "a"]}, {"sent": "you have a tricycle .", "words": ["have", "tricycle", "a", "you"]}, {"sent": "tricycle .", "words": ["tricycle"]}, {"sent": "cowboy boots on his tricycle .", "words": ["on", "boots", "tricycle", "his"]}, {"sent": "tricycle .", "words": ["tricycle"]}, {"sent": "here comes the tricycle .", "words": ["here", "the", "tricycle"]}, {"sent": "is that your tricycle over there ?", "words": ["your", "is", "there", "that", "tricycle", "over"]}, {"sent": "well because a tricycle isnt a very good thing to ride on a beach .", "words": ["to", "beach", "tricycle", "good", "on", "ride", "because", "a"]}, {"sent": "you saw his tricycle .", "words": ["his", "tricycle", "you"]}, {"sent": "well thats it when you get to five you dont really need a little tricycle like that .", "words": ["need", "to", "like", "you", "get", "dont", "little", "it", "that", "tricycle", "when", "a"]}, {"sent": "first he tried a tricycle and that was too small .", "words": ["was", "first", "and", "he", "tricycle", "too", "that", "a"]}, {"sent": "thats a new tricycle isnt ?", "words": ["new", "tricycle", "a"]}, {"sent": "and somebody else with cowboy boots and a tricycle ?", "words": ["with", "and", "boots", "tricycle", "a"]}, {"sent": "how about if you take a quick tricycle ride ?", "words": ["how", "if", "ride", "you", "about", "tricycle", "take", "a"]}, {"sent": "that says Maisy likes riding her tricycle .", "words": ["her", "that", "tricycle"]}, {"sent": "Adam what are those papers under your tricycle ?", "words": ["what", "your", "those", "under", "are", "tricycle"]}, {"sent": "and theres a little boy here on a tricycle .", "words": ["little", "here", "and", "tricycle", "on", "a"]}, {"sent": "Lina do you have a tricycle ?", "words": ["do", "you", "have", "tricycle", "a"]}, {"sent": "his tricycle .", "words": ["his", "tricycle"]}, {"sent": "oh youre going to put it on your tricycle ?", "words": ["put", "your", "to", "it", "tricycle", "on"]}, {"sent": "remember the boy who was riding the tricycle .", "words": ["the", "was", "tricycle", "who"]}, {"sent": "he rode with him on the tricycle and Corky hadta run behind .", "words": ["the", "with", "behind", "and", "run", "him", "he", "tricycle", "on"]}, {"sent": "hes riding the tricycle .", "words": ["the", "tricycle"]}, {"sent": "yah he got a tricycle .", "words": ["he", "tricycle", "a"]}, {"sent": "you hafta have a pavement to ride a tricycle on .", "words": ["to", "you", "have", "tricycle", "on", "ride", "a"]}, {"sent": "I want a hat like that I Grover the trick rider will perform on my trusty tricycle .", "words": ["the", "my", "will", "like", "on", "tricycle", "hat", "that", "a"]}, {"sent": "yes that is a tricycle .", "words": ["tricycle", "that", "a", "is"]}, {"sent": "whatre you doing with your tricycle ?", "words": ["your", "with", "tricycle", "you"]}, {"sent": "tricycle .", "words": ["tricycle"]}, {"sent": "Maisy rides her tricycle rattle squeak rattle squeak .", "words": ["her", "tricycle"]}, {"sent": "tricycle okay .", "words": ["tricycle"]}, {"sent": "on the tricycle ?", "words": ["on", "the", "tricycle"]}, {"sent": "and things like Dobbin and the tricycle I would like to to think that somebody was using them nicely .", "words": ["the", "think", "to", "was", "them", "like", "and", "tricycle", "would", "that"]}, {"sent": "did you perhaps fall off a tricycle or a bike or a tractor or something ?", "words": ["off", "fall", "you", "tractor", "did", "tricycle", "a"]}, {"sent": "bicycle tricycle .", "words": ["tricycle", "bicycle"]}, {"sent": "just like your tricycle .", "words": ["your", "like", "tricycle"]}, {"sent": "yeah is the monkey going on the tricycle or on the train ?", "words": ["the", "is", "train", "monkey", "tricycle", "on"]}, {"sent": "you skipped the tricycle stage I take it ?", "words": ["the", "you", "it", "tricycle", "take"]}, {"sent": "you show me how to put the rope round the tricycle and pull it out of the mud .", "words": ["the", "how", "put", "show", "to", "of", "you", "and", "pull", "it", "tricycle", "out", "me"]}, {"sent": "Dwww brang his tricycle but we couldnt fit it in the car at all .", "words": ["the", "all", "we", "car", "fit", "in", "it", "tricycle", "at", "his", "but"]}, {"sent": "I like your Mums new tricycle .", "words": ["new", "your", "like", "tricycle"]}, {"sent": "Maisy likes riding her tricycle .", "words": ["her", "tricycle"]}, {"sent": "Barry stay on that tricycle Barry .", "words": ["on", "stay", "that", "tricycle"]}, {"sent": "a small tricycle ?", "words": ["tricycle", "a"]}, {"sent": "yeah what about riding a tricycle ?", "words": ["what", "tricycle", "about", "a"]}, {"sent": "is that all you can think of to say about your tricycle ?", "words": ["can", "all", "think", "your", "is", "of", "to", "say", "you", "about", "tricycle", "that"]}, {"sent": "they raced through a crowded streets dont you ever ride your tricycle in the street .", "words": ["the", "your", "you", "dont", "they", "in", "tricycle", "street", "ride", "a"]}, {"sent": "or a tricycle .", "words": ["tricycle", "a"]}, {"sent": "its a tricycle .", "words": ["tricycle", "a"]}, {"sent": "did he buy a tricycle ?", "words": ["he", "did", "tricycle", "buy", "a"]}, {"sent": "the tricycle ?", "words": ["the", "tricycle"]}, {"sent": "thats a tricycle .", "words": ["tricycle", "a"]}, {"sent": "tricycle .", "words": ["tricycle"]}, {"sent": "and heres another kid on a tricycle .", "words": ["and", "another", "tricycle", "on", "a"]}, {"sent": "yeah its a tricycle .", "words": ["tricycle", "a"]}, {"sent": "oh there hes getting a tricycle .", "words": ["there", "tricycle", "a"]}, {"sent": "a three wheeler bike its like your tricycle isnt it ?", "words": ["your", "like", "it", "tricycle", "a"]}, {"sent": "tricycle .", "words": ["tricycle"]}, {"sent": "is that a tricycle ?", "words": ["tricycle", "that", "a", "is"]}, {"sent": "did we ride the tricycle ?", "words": ["the", "we", "did", "tricycle", "ride"]}, {"sent": "he likes riding his tricycle dont he ?", "words": ["his", "dont", "he", "tricycle"]}, {"sent": "another tricycle .", "words": ["another", "tricycle"]}, {"sent": "thats a tricycle .", "words": ["tricycle", "a"]}, {"sent": "hes playing on a tricycle .", "words": ["on", "tricycle", "a"]}, {"sent": "tricycle .", "words": ["tricycle"]}, {"sent": "you wanna play on your tricycle for a while ?", "words": ["for", "your", "play", "you", "wanna", "tricycle", "on", "a"]}, {"sent": "its not your job to ride my tricycle .", "words": ["my", "your", "to", "tricycle", "ride", "not"]}, {"sent": "that is a tricycle .", "words": ["tricycle", "that", "a", "is"]}, {"sent": "a tricycle thats right .", "words": ["tricycle", "a"]}, {"sent": "what do you want to hear .", "words": ["what", "do", "to", "hear", "you", "want to"]}, {"sent": "but I want to do it .", "words": ["do", "to", "it", "want to", "but"]}, {"sent": "well I mean if theres other cars going you can always come home with whoever you want to .", "words": ["can", "if", "to", "with", "you", "other", "want to", "home"]}, {"sent": "you want to get up here and go ninight ?", "words": ["up", "go", "to", "you", "get", "here", "and", "want to"]}, {"sent": "sweetie pie should we try this and see if we want to bring it to Debbies with us ?", "words": ["if", "we", "to", "with", "bring", "and", "this", "us", "it", "want to", "see", "try"]}, {"sent": "or dyou want to try something different .", "words": ["to", "try", "want to"]}, {"sent": "like youd actually want to eat them but .", "words": ["to", "them", "like", "eat", "want to", "but"]}, {"sent": "xxx you want to blow your nose ?", "words": ["your", "to", "you", "want to", "blow", "nose"]}, {"sent": "do you want to sit in this chair ?", "words": ["do", "to", "chair", "you", "this", "in", "want to", "sit"]}, {"sent": "oh you want to see .", "words": ["see", "to", "want to", "you"]}, {"sent": "you want to tell story ?", "words": ["to", "story", "want to", "you"]}, {"sent": "want to see the .", "words": ["see", "to", "want to", "the"]}, {"sent": "you think she does I think he does want to help Ruby .", "words": ["does", "think", "help", "to", "you", "he", "want to", "she"]}, {"sent": "you want to see the light .", "words": ["the", "light", "to", "you", "want to", "see"]}, {"sent": "you want to sit down ?", "words": ["to", "you", "want to", "sit", "down"]}, {"sent": "dyou want to drink your water no okay .", "words": ["your", "drink", "to", "want to", "water"]}, {"sent": "yay do you want to get xxx your clip clip the microphone you can give your clip to Amanda put it on your sleeve ?", "words": ["the", "do", "can", "put", "your", "to", "you", "get", "it", "want to", "on", "give"]}, {"sent": "do you want to see some more ?", "words": ["do", "to", "you", "more", "want to", "see", "some"]}, {"sent": "uh do you want to listen to your record ?", "words": ["do", "your", "listen", "to", "you", "want to"]}, {"sent": "you could doinkaboink it with Oatios or wheat if you want to .", "words": ["if", "to", "with", "you", "it", "want to", "could"]}, {"sent": "and you want to learn how to write some words you said ?", "words": ["how", "to", "write", "you", "and", "want to", "some"]}, {"sent": "oh you want to .", "words": ["to", "want to", "you"]}, {"sent": "you want to pull the top off .", "words": ["the", "off", "to", "pull", "you", "want to"]}, {"sent": "would you want to nap in a monkeys lap ?", "words": ["to", "you", "in", "want to", "would", "a"]}, {"sent": "how big and strong do you want to be ?", "words": ["do", "how", "to", "be", "you", "and", "want to", "big"]}, {"sent": "well if you want to talk to Dimitra a moment you had better go and see her hadnt you ?", "words": ["better", "if", "go", "to", "you", "and", "want to", "talk", "see", "her", "a"]}, {"sent": "do you want to do it or do you want Mommy to do it ?", "words": ["do", "to", "you", "it", "want to"]}, {"sent": "now if you do not want to eat what Im giving you .", "words": ["do", "what", "if", "to", "you", "eat", "want to", "not"]}, {"sent": "I want to walk home okay ?", "words": ["to", "walk", "want to", "home"]}, {"sent": "do you want to get down ?", "words": ["do", "to", "you", "get", "want to", "down"]}, {"sent": "do you want to go to Grammys ?", "words": ["do", "go", "to", "you", "want to"]}, {"sent": "do you want to put a green ?", "words": ["do", "put", "to", "you", "want to", "green", "a"]}, {"sent": "what do you want to do .", "words": ["what", "do", "to", "you", "want to"]}, {"sent": "oh you want to color ?", "words": ["to", "want to", "you"]}, {"sent": "you want to go get a drink of water ?", "words": ["drink", "go", "of", "to", "you", "get", "want to", "water", "a"]}, {"sent": "you want to go now ?", "words": ["go", "to", "want to", "you"]}, {"sent": "want to show Mommy ?", "words": ["show", "to", "want to"]}, {"sent": "I think I want to give Alex a hug .", "words": ["think", "to", "want to", "hug", "give", "a"]}, {"sent": "hey wasza want to draw to get an apple ?", "words": ["to", "draw", "get", "an", "want to", "apple"]}, {"sent": "Mommy I want to see xxx .", "words": ["see", "to", "want to"]}, {"sent": "want to jump horsie ?", "words": ["to", "jump", "want to"]}, {"sent": "you want to see one more stone ?", "words": ["to", "you", "more", "want to", "stone", "see"]}, {"sent": "okay do I think they want to see me .", "words": ["do", "think", "to", "me", "they", "want to", "see"]}, {"sent": "you know you could also eat some hummus or some spicy beans if you want to if youre still hungry .", "words": ["if", "to", "hungry", "you", "eat", "beans", "want to", "some", "could"]}, {"sent": "thats important because I want to keep my underpants clean and dry .", "words": ["clean", "my", "to", "dry", "and", "underpants", "want to", "because"]}, {"sent": "if youre happy and ya know it and you really want to show it if youre happy and ya know it beep yer nose .", "words": ["if", "happy", "show", "to", "you", "and", "it", "want to", "nose"]}, {"sent": "you want to come play with Daddy .", "words": ["to", "play", "with", "you", "want to"]}, {"sent": "do you want to put somebody in here do you want to put this lady ?", "words": ["do", "put", "to", "you", "here", "this", "in", "want to"]}, {"sent": "you want to pick you want me to pick you up ?", "words": ["up", "pick", "to", "you", "want to", "me"]}, {"sent": "so what do you want to do .", "words": ["what", "do", "to", "so", "you", "want to"]}, {"sent": "mm you pick what you want to read .", "words": ["what", "pick", "to", "you", "read", "want to"]}, {"sent": "yes maybe later you want to show Jenny your squishy pillow ?", "words": ["your", "show", "to", "you", "want to", "pillow"]}, {"sent": "a C for cat well you might want to turn upside .", "words": ["for", "to", "you", "cat", "want to", "a"]}, {"sent": "what do you want to do Little Bird ?", "words": ["what", "do", "to", "you", "want to"]}, {"sent": "do you want to get up on Daddys chest ?", "words": ["up", "do", "to", "you", "get", "want to", "on"]}, {"sent": "want to like a big boy ?", "words": ["to", "like", "want to", "big", "a"]}, {"sent": "we want to see you on camera .", "words": ["camera", "we", "to", "you", "want to", "on", "see"]}, {"sent": "I mean if you want to call him and say that the power .", "words": ["the", "if", "to", "say", "and", "you", "him", "want to", "that"]}, {"sent": "do I want to not now but remember we used to when were vacation last year .", "words": ["do", "were", "we", "to", "last", "want to", "when", "not", "but"]}, {"sent": "if youre happy and you know it and you really want to show it .", "words": ["if", "happy", "show", "to", "you", "and", "it", "want to"]}, {"sent": "alright you want to sit up on my knee .", "words": ["up", "my", "knee", "to", "you", "want to", "on", "sit"]}, {"sent": "I want you to sit there cause I want to talk to you .", "words": ["to", "you", "there", "want to", "talk", "sit"]}, {"sent": "do you want to read either one of these ?", "words": ["do", "these", "of", "to", "you", "read", "want to"]}, {"sent": "want to play with the guitar ?", "words": ["the", "to", "play", "with", "want to"]}, {"sent": "want to play horsie is dead ?", "words": ["to", "play", "want to", "is"]}, {"sent": "you hafta sit and listen then if you want to read it .", "words": ["if", "listen", "to", "then", "and", "you", "it", "read", "want to", "sit"]}, {"sent": "do you want to keep this one ?", "words": ["do", "to", "you", "this", "want to"]}, {"sent": "you dont want to go on a sail boat ?", "words": ["go", "to", "you", "dont", "want to", "on", "boat", "a"]}, {"sent": "do you want to see him ?", "words": ["do", "to", "you", "him", "want to", "see"]}, {"sent": "what do you want to hear .", "words": ["what", "do", "to", "hear", "you", "want to"]}, {"sent": "Phoebe was telling a story to auntie and auntie couldnt hear what she said and Phoebe didnt want to tell the story anymore so auntie said what did you say Phoebe and Phoebe said dede .", "words": ["the", "what", "to", "was", "so", "say", "and", "hear", "you", "did", "want to", "story", "she", "a"]}, {"sent": "you want to go to school ?", "words": ["go", "to", "school", "you", "want to"]}, {"sent": "do you want to read one of your books ?", "words": ["do", "your", "of", "to", "you", "read", "want to"]}, {"sent": "you want to hold on .", "words": ["hold", "to", "you", "want to", "on"]}, {"sent": "no you want to go up the stairs ?", "words": ["the", "up", "go", "to", "you", "want to", "stairs"]}, {"sent": "do you want to have a tail ?", "words": ["do", "to", "you", "have", "want to", "a"]}, {"sent": "didjou want to give mommy a kiss ?", "words": ["to", "want to", "give", "kiss", "a"]}, {"sent": "no I dont want to no no I dont want to whine whine whine whine .", "words": ["to", "dont", "want to"]}, {"sent": "you want to get down ?", "words": ["to", "you", "get", "want to", "down"]}, {"sent": "do you want to .", "words": ["to", "do", "want to", "you"]}, {"sent": "once hes nice and cozy hell want to listen to .", "words": ["listen", "to", "nice", "and", "want to"]}, {"sent": "obviously I what I want to do is have have him come in and and and have the .", "words": ["the", "what", "do", "is", "to", "and", "have", "him", "in", "want to"]}, {"sent": "you want to do a tune ?", "words": ["do", "to", "you", "want to", "a"]}, {"sent": "you dont want to walk to the farm ?", "words": ["the", "to", "you", "dont", "want to", "walk"]}, {"sent": "want to take them out first ?", "words": ["to", "first", "them", "want to", "out", "take"]}, {"sent": "you want to leave him out ?", "words": ["to", "you", "him", "want to", "out"]}, {"sent": "I just want to be an astronaut .", "words": ["to", "an", "be", "want to"]}, {"sent": "no no you dont want to put your socks in the bag .", "words": ["the", "put", "your", "to", "you", "dont", "in", "want to"]}, {"sent": "if if you want to .", "words": ["to", "want to", "if", "you"]}, {"sent": "you want to pittypat ?", "words": ["to", "want to", "you"]}, {"sent": "you want to pick up ?", "words": ["up", "pick", "to", "you", "want to"]}, {"sent": "you want you want to pull them down ?", "words": ["to", "them", "pull", "you", "want to", "down"]}, {"sent": "Im gonna move it to the side because we dont want to get it wet .", "words": ["the", "wet", "we", "to", "get", "dont", "it", "want to", "because"]}, {"sent": "Daddy Daddy does not want to swing you right this second .", "words": ["does", "swing", "to", "you", "this", "want to", "not"]}, {"sent": "you want to go ninight some more ?", "words": ["go", "to", "you", "more", "want to", "some"]}, {"sent": "that you want to read me ?", "words": ["to", "you", "read", "want to", "me", "that"]}, {"sent": "so hell want to make ornaments .", "words": ["to", "so", "make", "want to"]}, {"sent": "you want to hear the bird again ?", "words": ["the", "to", "bird", "hear", "you", "want to"]}, {"sent": "and well go and watch them making rock .", "words": ["watch", "go", "them", "and", "rock"]}, {"sent": "Ill rock you like you rock your babies .", "words": ["rock", "your", "like", "you"]}, {"sent": "early one morning he and Mufasa climbed to the top of Pride rock .", "words": ["the", "of", "to", "and", "he", "rock"]}, {"sent": "you can eat your fishies and rock .", "words": ["can", "your", "and", "eat", "you", "rock"]}, {"sent": "rock rock .", "words": ["rock"]}, {"sent": "do you rock on the horsie ?", "words": ["the", "do", "you", "on", "rock"]}, {"sent": "every morning as the sun peaks over the horizon a giant rock formation catches the first rays of light .", "words": ["the", "of", "light", "sun", "first", "every", "rock", "over", "a"]}, {"sent": "on your rock .", "words": ["on", "rock", "your"]}, {"sent": "youre moving a rock .", "words": ["rock", "a"]}, {"sent": "why dont you rock ?", "words": ["dont", "rock", "why", "you"]}, {"sent": "rock like this .", "words": ["this", "rock", "like"]}, {"sent": "better rock her up and down then .", "words": ["up", "better", "then", "and", "rock", "down", "her"]}, {"sent": "are you making a rockingchair rock ?", "words": ["rock", "are", "a", "you"]}, {"sent": "his little rock a tot .", "words": ["his", "rock", "a", "little"]}, {"sent": "I scared you you rock head ?", "words": ["rock", "head", "scared", "you"]}, {"sent": "and we were throwing it and hit the rock and explode .", "words": ["the", "we", "and", "it", "rock", "hit", "were"]}, {"sent": "sitting on a rock .", "words": ["on", "rock", "a"]}, {"sent": "rock .", "words": ["rock"]}, {"sent": "Im on my rock .", "words": ["on", "my", "rock"]}, {"sent": "there is a big rock over there .", "words": ["is", "there", "over", "rock", "big", "a"]}, {"sent": "why didnt you come back to Pride rock she asked you are the king .", "words": ["the", "back", "to", "you", "are", "rock", "she", "why"]}, {"sent": "maybe it bumped into a rock .", "words": ["rock", "it", "into", "a"]}, {"sent": "a rock ride ?", "words": ["rock", "ride", "a"]}, {"sent": "I didnt say it had a rock in it .", "words": ["say", "in", "it", "rock", "a"]}, {"sent": "thats right on the rock hes sitting on a rock see the big rock ?", "words": ["the", "on", "see", "rock", "big", "a"]}, {"sent": "do you rock on that chair ?", "words": ["do", "chair", "you", "on", "rock", "that"]}, {"sent": "rock a rock .", "words": ["rock", "a"]}, {"sent": "Mommy threw the rock .", "words": ["the", "rock"]}, {"sent": "bahroom oh your rock .", "words": ["rock", "your"]}, {"sent": "oh youre going to rock yourself are you ?", "words": ["yourself", "to", "you", "are", "rock"]}, {"sent": "theres bubbles theres a paper airplane a balloon feathers leaves powder puff dandelion puff and here is a big rock .", "words": ["is", "airplane", "bubbles", "paper", "and", "here", "balloon", "rock", "big", "a"]}, {"sent": "we put babies in rock a tots .", "words": ["put", "we", "in", "rock", "a"]}, {"sent": "rock her from side to side and sing to her .", "words": ["to", "and", "sing", "rock", "her"]}, {"sent": "youll be able to rock her on there .", "words": ["to", "be", "there", "on", "rock", "her"]}, {"sent": "small bear swam and swam until he reached the shore where he lay on a rock all shivering .", "words": ["the", "where", "all", "and", "bear", "he", "on", "rock", "a"]}, {"sent": "the white rock ?", "words": ["the", "rock", "white"]}, {"sent": "Scar returned to Pride rock to take over the throne .", "words": ["the", "to", "take", "rock", "over"]}, {"sent": "rock music ?", "words": ["rock"]}, {"sent": "Lilys in the japanese rock garden .", "words": ["the", "garden", "in", "rock"]}, {"sent": "are you having a rock ?", "words": ["rock", "are", "a", "you"]}, {"sent": "rock is rock that you get at the seaside .", "words": ["the", "is", "you", "get", "at", "rock", "that"]}, {"sent": "wait for me big rock .", "words": ["wait", "for", "me", "rock", "big"]}, {"sent": "and she started to rock and rock .", "words": ["to", "rock", "she", "and"]}, {"sent": "rock hop are they dancing ?", "words": ["they", "rock", "are"]}, {"sent": "a rock polishing kit .", "words": ["rock", "a"]}, {"sent": "rock a my soul in the bosom of Abraham .", "words": ["the", "my", "of", "in", "rock", "a"]}, {"sent": "rock a bye baby .", "words": ["rock", "a"]}, {"sent": "all the way through the rock it says Blackpool Blackpool Scarborough Scarborough Scarborough .", "words": ["the", "all", "rock", "it"]}, {"sent": "thats a fragile rock .", "words": ["rock", "a"]}, {"sent": "lets rock a bye baby .", "words": ["rock", "a"]}, {"sent": "rock music .", "words": ["rock"]}, {"sent": "because thats what you do with a rock gardens .", "words": ["what", "do", "with", "you", "rock", "because", "a"]}, {"sent": "oh you could put the rock in the garden look .", "words": ["the", "put", "you", "in", "garden", "look", "could", "rock"]}, {"sent": "make your clown rock make your clown rock fall down fall down fall down fall down .", "words": ["your", "fall", "rock", "down", "make"]}, {"sent": "so you tripped over a rock and hurt yourself .", "words": ["yourself", "hurt", "so", "you", "and", "rock", "over", "a"]}, {"sent": "oh where did Chris find that big rock ?", "words": ["where", "that", "did", "find", "rock", "big"]}, {"sent": "I think that you could probably find a better rock .", "words": ["better", "think", "you", "rock", "find", "could", "that", "a"]}, {"sent": "rock ?", "words": ["rock"]}, {"sent": "rock a rock a rock a rock .", "words": ["rock", "a"]}, {"sent": "this big rock is not light its very very heavy .", "words": ["heavy", "is", "light", "this", "rock", "big", "not"]}, {"sent": "not far from the ceremony in a cave at the backside of Pride rock a scraggly lion with a dark mane grumbled lifes not fair I shall never be king .", "words": ["the", "of", "be", "dark", "with", "in", "at", "rock", "lion", "not", "a"]}, {"sent": "okay well leave the rock here .", "words": ["the", "rock", "here"]}, {"sent": "rock a bye baby .", "words": ["rock", "a"]}, {"sent": "rock .", "words": ["rock"]}, {"sent": "blows the cradle will rock .", "words": ["the", "rock", "will"]}, {"sent": "you can rock them .", "words": ["can", "rock", "them", "you"]}, {"sent": "youre sposta rock in it .", "words": ["in", "rock", "it"]}, {"sent": "oh this one doesnt rock sweetie pie .", "words": ["this", "rock"]}, {"sent": "rock .", "words": ["rock"]}, {"sent": "I Ill just Ill give you a rock .", "words": ["give", "rock", "a", "you"]}, {"sent": "Ill rock her .", "words": ["rock", "her"]}, {"sent": "are you going to sit down and rock them ?", "words": ["to", "them", "you", "and", "are", "sit", "down", "rock"]}, {"sent": "little bears still asleep by the rock but whos coming down the hill ?", "words": ["the", "by", "little", "asleep", "rock", "down", "but"]}, {"sent": "do you remember what this rock is called ?", "words": ["do", "what", "is", "you", "this", "rock"]}, {"sent": "before long Pride rock flourished again .", "words": ["rock", "long"]}, {"sent": "before long Pride rock flourished again Nala remained by Simbas side and soon they had they had their own newborn cub .", "words": ["their", "long", "and", "by", "they", "rock"]}, {"sent": "uhuh it its a rock I think .", "words": ["think", "rock", "it", "a"]}, {"sent": "rock a rock a bye .", "words": ["rock", "a"]}, {"sent": "careful youre stepping on a rock .", "words": ["on", "rock", "a", "careful"]}, {"sent": "I might make some rock .", "words": ["some", "rock", "make"]}, {"sent": "do you wanna do rock a bye baby ?", "words": ["do", "you", "wanna", "rock", "a"]}, {"sent": "help Daddy rock in the chair .", "words": ["the", "help", "chair", "in", "rock"]}, {"sent": "yeah hes sucking on the rock .", "words": ["on", "the", "rock"]}, {"sent": "rock hop .", "words": ["rock"]}, {"sent": "rock a bye baby on the tree top .", "words": ["the", "tree", "on", "rock", "a"]}, {"sent": "ah he looks really upset when hes in the water but he swims and he manages to reach that huge rock .", "words": ["the", "to", "and", "in", "that", "he", "water", "rock", "when", "but"]}, {"sent": "but little birds mommy lifts that baby in her arms alright she rocks that baby back and forth rock rock rock rock .", "words": ["back", "her", "little", "and", "in", "rock", "that", "she", "but"]}, {"sent": "theres a cute little rock .", "words": ["cute", "rock", "little", "a"]}, {"sent": "they got up on the rock and this mommy is hugging her babies .", "words": ["up", "the", "is", "and", "they", "this", "on", "rock", "her"]}, {"sent": "rock rock .", "words": ["rock"]}, {"sent": "do you know the mint rock like the pink rock that you buy at the seaside ?", "words": ["the", "do", "like", "you", "at", "rock", "that", "buy"]}, {"sent": "rock you ?", "words": ["rock", "you"]}, {"sent": "and hay and a big rock .", "words": ["rock", "big", "a", "and"]}, {"sent": "the rock .", "words": ["the", "rock"]}, {"sent": "who want rock ?", "words": ["rock", "who"]}, {"sent": "still a rock .", "words": ["rock", "a"]}, {"sent": "thats a rock .", "words": ["rock", "a"]}, {"sent": "heres a rock .", "words": ["rock", "a"]}, {"sent": "but rot that is different from rock .", "words": ["rock", "that", "but", "is"]}, {"sent": "its a rock ?", "words": ["rock", "a"]}, {"sent": "what did you just tear ?", "words": ["what", "tear", "did", "you"]}, {"sent": "hold the glass firmly on the red paper and tear around the edge to make a ladybirds body .", "words": ["the", "red", "hold", "to", "paper", "tear", "and", "glass", "around", "on", "make", "a"]}, {"sent": "lets try and tear it .", "words": ["tear", "it", "try", "and"]}, {"sent": "oh okay I wont tear it .", "words": ["it", "tear"]}, {"sent": "tear it apart tear it apart .", "words": ["it", "tear"]}, {"sent": "tear it up .", "words": ["up", "it", "tear"]}, {"sent": "oh xxx good tear away .", "words": ["away", "tear", "good"]}, {"sent": "you dont wanna tear those off .", "words": ["off", "those", "tear", "you", "dont", "wanna"]}, {"sent": "and shell tear you in two .", "words": ["in", "tear", "you", "and"]}, {"sent": "you dont tear them .", "words": ["dont", "them", "tear", "you"]}, {"sent": "well they are so jealous of Cinderella that they tear her outfit to pieces .", "words": ["of", "so", "to", "tear", "her", "they", "are", "that"]}, {"sent": "dont tear it .", "words": ["dont", "it", "tear"]}, {"sent": "we dont want it to tear .", "words": ["we", "to", "tear", "dont", "it"]}, {"sent": "now tear out a leaf shape from the green paper .", "words": ["the", "paper", "tear", "green", "out", "a"]}, {"sent": "okay lets tear the paper here .", "words": ["the", "paper", "here", "tear"]}, {"sent": "then what will you do if you tear your cowboy book up ?", "words": ["up", "what", "do", "your", "if", "tear", "you", "will", "book", "then"]}, {"sent": "dont tear that .", "words": ["dont", "that", "tear"]}, {"sent": "dont tear it dont tear it dont tear it .", "words": ["dont", "it", "tear"]}, {"sent": "did you tear it ?", "words": ["it", "tear", "did", "you"]}, {"sent": "rattles and tops and books that wont tear old pots and pans and a fuzzy brown bear .", "words": ["old", "brown", "tear", "and", "bear", "that", "a"]}, {"sent": "no dont tear it .", "words": ["dont", "it", "tear"]}, {"sent": "tear out a leaf .", "words": ["tear", "out", "a"]}, {"sent": "dont tear it .", "words": ["dont", "it", "tear"]}, {"sent": "tear drops you mean .", "words": ["tear", "you"]}, {"sent": "did you tear him all up ?", "words": ["up", "all", "tear", "you", "him", "did"]}, {"sent": "you want something to tear honey ?", "words": ["to", "tear", "you"]}, {"sent": "like Ill just give her a magazine to tear up .", "words": ["up", "to", "tear", "like", "give", "her", "a"]}, {"sent": "xxx tear it because you can stencil .", "words": ["can", "tear", "you", "it", "because"]}, {"sent": "did you tear it in half ?", "words": ["tear", "you", "in", "it", "did"]}, {"sent": "dont tear it .", "words": ["dont", "it", "tear"]}, {"sent": "hm dont tear the book .", "words": ["the", "dont", "book", "tear"]}, {"sent": "wanna get over there and tear up the blinds ?", "words": ["up", "the", "tear", "and", "get", "there", "wanna", "over"]}, {"sent": "youre not supposed to tear the box .", "words": ["the", "box", "to", "tear", "not"]}, {"sent": "you dont dont tear .", "words": ["dont", "tear", "you"]}, {"sent": "tear what ?", "words": ["what", "tear"]}, {"sent": "ooh we dont wanna tear that sweetie pie .", "words": ["we", "tear", "dont", "wanna", "that"]}, {"sent": "here okay you tear it off .", "words": ["off", "tear", "you", "here", "it"]}, {"sent": "did I tear your toys up ?", "words": ["your", "up", "tear", "did"]}, {"sent": "did you tear that ?", "words": ["that", "tear", "did", "you"]}, {"sent": "that thing is going to tear in a Moment .", "words": ["is", "to", "tear", "in", "that", "a"]}, {"sent": "tear up .", "words": ["up", "tear"]}, {"sent": "dont tear the babies .", "words": ["the", "dont", "tear"]}, {"sent": "you forgot to tear that towel down off the rack .", "words": ["off", "the", "to", "tear", "you", "towel", "down", "that"]}, {"sent": "dont tear it .", "words": ["dont", "it", "tear"]}, {"sent": "mustnt tear your sock .", "words": ["your", "tear", "sock"]}, {"sent": "shall I tear it for you ?", "words": ["it", "tear", "for", "you"]}, {"sent": "dont tear the book .", "words": ["the", "dont", "book", "tear"]}, {"sent": "oh I didnt mean for you to tear off her paper .", "words": ["off", "for", "to", "paper", "tear", "you", "her"]}, {"sent": "aw honey youre gonna tear them .", "words": ["tear", "them"]}, {"sent": "did you ever tear anything or rip anything ?", "words": ["rip", "tear", "did", "you"]}, {"sent": "tear at the end .", "words": ["the", "tear", "at"]}, {"sent": "no you cant tear my flowers out .", "words": ["my", "tear", "out", "you"]}, {"sent": "why did you tear that ?", "words": ["tear", "you", "did", "that", "why"]}, {"sent": "hell tear if you do that Becky .", "words": ["do", "if", "tear", "you", "that"]}, {"sent": "the the sad face with the tear .", "words": ["the", "face", "tear", "with", "sad"]}, {"sent": "you working your way around the counter trying to find something to grab and tear up .", "words": ["the", "up", "your", "to", "tear", "find", "you", "and", "around"]}, {"sent": "lemme just tear off that .", "words": ["off", "that", "tear"]}, {"sent": "something to just tear up .", "words": ["up", "to", "tear"]}, {"sent": "oh oh oh dont tear it .", "words": ["dont", "it", "tear"]}, {"sent": "you dont think itll tear ?", "words": ["dont", "think", "tear", "you"]}, {"sent": "dont tear them down .", "words": ["dont", "down", "tear", "them"]}, {"sent": "youre going to tear it .", "words": ["to", "it", "tear"]}, {"sent": "shall we just have it like that because I dont wanna tear it .", "words": ["we", "tear", "like", "have", "dont", "it", "wanna", "that", "because"]}, {"sent": "you want me to tear it for you ?", "words": ["for", "to", "tear", "you", "it", "me"]}, {"sent": "rattles and tops and books that wont tear old pots and pans and a fuzzy brown bear .", "words": ["old", "brown", "tear", "and", "bear", "that", "a"]}, {"sent": "dont tear it .", "words": ["dont", "it", "tear"]}, {"sent": "you shouldnt tear it .", "words": ["it", "tear", "you"]}, {"sent": "dont tear it sweetheart .", "words": ["dont", "it", "tear"]}, {"sent": "dont tear them more love .", "words": ["tear", "them", "dont", "more", "love"]}, {"sent": "hitty bitty tear song .", "words": ["tear"]}, {"sent": "you want me to tear it out ?", "words": ["to", "tear", "you", "it", "out", "me"]}, {"sent": "you gonna tear one ?", "words": ["tear", "you"]}, {"sent": "Mama Mama can tear it .", "words": ["can", "it", "tear"]}, {"sent": "give it Lois and let her tear it for you .", "words": ["for", "tear", "and", "you", "it", "give", "her"]}, {"sent": "oh and you have a tear .", "words": ["tear", "you", "have", "and", "a"]}, {"sent": "tear it ?", "words": ["it", "tear"]}, {"sent": "hey dont tear that apart .", "words": ["dont", "that", "tear"]}, {"sent": "youll tear that .", "words": ["that", "tear"]}, {"sent": "for them to tear apart .", "words": ["tear", "to", "for", "them"]}, {"sent": "tear the paper ?", "words": ["the", "paper", "tear"]}, {"sent": "no dont tear it you stick it there .", "words": ["stick", "tear", "you", "dont", "there", "it"]}, {"sent": "wait dont tear them up .", "words": ["up", "wait", "tear", "them", "dont"]}, {"sent": "remember Mama said you dont tear things .", "words": ["dont", "tear", "you"]}, {"sent": "now dont dont dont tear it off .", "words": ["off", "dont", "it", "tear"]}, {"sent": "tear some of the paper off so we can get to the pencil .", "words": ["the", "off", "can", "we", "of", "so", "paper", "to", "tear", "pencil", "get", "some"]}, {"sent": "let me tear it .", "words": ["me", "it", "tear"]}, {"sent": "its not a tear .", "words": ["tear", "not", "a"]}, {"sent": "tear pages or ri youknow or eat it so .", "words": ["eat", "so", "it", "tear"]}, {"sent": "we wont get to look at the kites if you tear them up .", "words": ["the", "up", "if", "we", "to", "tear", "them", "you", "get", "at", "look"]}, {"sent": "a couple of minutes ago you told me there was a face that was scaring you to come tear it out .", "words": ["face", "of", "was", "to", "tear", "you", "there", "it", "out", "me", "that", "a"]}, {"sent": "its a little tear coming from her eye .", "words": ["tear", "eye", "little", "her", "a"]}, {"sent": "you dont tear them off when you find them .", "words": ["off", "them", "tear", "you", "dont", "find", "when"]}, {"sent": "you have a real tear on your face .", "words": ["your", "face", "tear", "you", "have", "on", "a"]}, {"sent": "dont tear the paper Adam .", "words": ["the", "dont", "paper", "tear"]}, {"sent": "shes got a little tear .", "words": ["tear", "little", "a"]}, {"sent": "dont tear the pages .", "words": ["the", "dont", "tear"]}, {"sent": "I tear all of it .", "words": ["of", "all", "it", "tear"]}, {"sent": "and a penguin .", "words": ["penguin", "a", "and"]}, {"sent": "penguin .", "words": ["penguin"]}, {"sent": "did you teach him how did to be a penguin ?", "words": ["how", "to", "be", "penguin", "you", "him", "did", "a"]}, {"sent": "whats this penguin doing ?", "words": ["this", "penguin"]}, {"sent": "because youve got a penguin Pok\u00e9mon .", "words": ["penguin", "because", "a"]}, {"sent": "shall I put penguin on for you ?", "words": ["put", "for", "penguin", "you", "on"]}, {"sent": "its penguin .", "words": ["penguin"]}, {"sent": "or do you think hed like to eat the odd penguin or something ?", "words": ["the", "do", "think", "to", "penguin", "like", "you", "eat"]}, {"sent": "penguin .", "words": ["penguin"]}, {"sent": "whats the penguin doing ?", "words": ["the", "penguin"]}, {"sent": "is penguin gonna drive the train ?", "words": ["the", "train", "is", "penguin", "drive"]}, {"sent": "a penguin .", "words": ["penguin", "a"]}, {"sent": "and we have a penguin .", "words": ["we", "penguin", "and", "have", "a"]}, {"sent": "shall the penguin come through ?", "words": ["the", "penguin"]}, {"sent": "theres your penguin .", "words": ["penguin", "your"]}, {"sent": "a penguin ?", "words": ["penguin", "a"]}, {"sent": "penguin .", "words": ["penguin"]}, {"sent": "penguin and panda .", "words": ["penguin", "and"]}, {"sent": "take the monkey out and put the penguin in .", "words": ["the", "put", "monkey", "penguin", "and", "in", "out", "take"]}, {"sent": "but its a penguin .", "words": ["penguin", "but", "a"]}, {"sent": "penguin ?", "words": ["penguin"]}, {"sent": "penguin go under the bridge .", "words": ["the", "under", "penguin", "go"]}, {"sent": "thats a penguin .", "words": ["penguin", "a"]}, {"sent": "its a penguin .", "words": ["penguin", "a"]}, {"sent": "penguin doesnt wanna stand up does he ?", "words": ["up", "does", "stand", "penguin", "he", "wanna"]}, {"sent": "see if penguin will go .", "words": ["if", "go", "penguin", "will", "see"]}, {"sent": "oh one good penguin left .", "words": ["penguin", "good"]}, {"sent": "theres Ruths penguin isnt it ?", "words": ["it", "penguin"]}, {"sent": "and the penguin .", "words": ["the", "penguin", "and"]}, {"sent": "do you think the penguin wants a sausage ?", "words": ["the", "do", "think", "penguin", "you", "a"]}, {"sent": "tell the penguin to eat the dolphin .", "words": ["the", "to", "penguin", "eat"]}, {"sent": "whered the penguin go ?", "words": ["the", "penguin", "go"]}, {"sent": "youre sitting on the penguin .", "words": ["on", "the", "penguin"]}, {"sent": "its like your little penguin isnt it ?", "words": ["your", "penguin", "like", "little", "it"]}, {"sent": "if Mummys a penguin that means youre a penguin too .", "words": ["if", "penguin", "too", "that", "a"]}, {"sent": "its just aswell weve got the penguin isnt it ?", "words": ["the", "it", "penguin"]}, {"sent": "does that penguin want some ?", "words": ["does", "penguin", "that", "some"]}, {"sent": "theres your other penguin .", "words": ["penguin", "your", "other"]}, {"sent": "thats a penguin .", "words": ["penguin", "a"]}, {"sent": "I might give the penguin this fish .", "words": ["the", "penguin", "this", "fish", "give"]}, {"sent": "the penguin the penguin would love you to throw him some fish .", "words": ["the", "throw", "to", "penguin", "you", "him", "love", "would", "some", "fish"]}, {"sent": "penguin .", "words": ["penguin"]}, {"sent": "does Mummy look like a penguin ?", "words": ["does", "penguin", "like", "look", "a"]}, {"sent": "is there a baby penguin ?", "words": ["there", "penguin", "a", "is"]}, {"sent": "yeah thats where our penguin goes .", "words": ["where", "penguin", "our"]}, {"sent": "penguin stay .", "words": ["stay", "penguin"]}, {"sent": "penguin .", "words": ["penguin"]}, {"sent": "penguin go in there .", "words": ["there", "in", "penguin", "go"]}, {"sent": "penguin .", "words": ["penguin"]}, {"sent": "Joseph can make a penguin .", "words": ["can", "penguin", "make", "a"]}, {"sent": "penguin .", "words": ["penguin"]}, {"sent": "youre gonna rip the poor penguin apart huh .", "words": ["the", "rip", "penguin", "poor"]}, {"sent": "not behind the penguin .", "words": ["the", "penguin", "behind", "not"]}, {"sent": "they thats a penguin isnt it ?", "words": ["they", "penguin", "it", "a"]}, {"sent": "penguin .", "words": ["penguin"]}, {"sent": "so the penguin she loves .", "words": ["the", "so", "penguin", "she"]}, {"sent": "theres a penguin sitting on top .", "words": ["on", "penguin", "a"]}, {"sent": "is that a penguin ?", "words": ["penguin", "that", "a", "is"]}, {"sent": "the penguin likes it in the water .", "words": ["the", "penguin", "in", "it", "water"]}, {"sent": "thats a snake on top of the penguin isnt it ?", "words": ["the", "of", "penguin", "it", "on", "a"]}, {"sent": "one penguin is jumping over the ice .", "words": ["the", "is", "penguin", "ice", "over"]}, {"sent": "oh is that a penguin ?", "words": ["penguin", "that", "a", "is"]}, {"sent": "theres a penguin .", "words": ["penguin", "a"]}, {"sent": "can you show us how a penguin walks ?", "words": ["can", "how", "show", "penguin", "you", "us", "a"]}, {"sent": "the penguin .", "words": ["the", "penguin"]}, {"sent": "a penguin .", "words": ["penguin", "a"]}, {"sent": "the daddy penguin keeps the egg warm .", "words": ["the", "penguin", "egg"]}, {"sent": "its a penguin .", "words": ["penguin", "a"]}, {"sent": "are you going to put that penguin on it .", "words": ["put", "to", "penguin", "you", "are", "it", "on", "that"]}, {"sent": "does he wanna talk to the penguin ?", "words": ["the", "does", "to", "penguin", "he", "talk", "wanna"]}, {"sent": "wheres your penguin ?", "words": ["penguin", "your"]}, {"sent": "wheres penguin ?", "words": ["penguin"]}, {"sent": "thats a baby penguin .", "words": ["penguin", "a"]}, {"sent": "another penguin .", "words": ["another", "penguin"]}, {"sent": "that penguin useta go there didnt it ?", "words": ["go", "penguin", "there", "it", "that"]}, {"sent": "this isnt a penguin .", "words": ["this", "penguin", "a"]}, {"sent": "penguin .", "words": ["penguin"]}, {"sent": "what kind of drink are you giving your penguin ?", "words": ["what", "your", "drink", "of", "penguin", "you", "are"]}, {"sent": "thats a penguin .", "words": ["penguin", "a"]}, {"sent": "whats penguin doing to Paddington ?", "words": ["to", "penguin"]}, {"sent": "a penguin bone .", "words": ["penguin", "a"]}, {"sent": "whos that penguin gonna play with at nursery ?", "words": ["penguin", "with", "play", "at", "that"]}, {"sent": "you can put a penguin in there .", "words": ["can", "put", "penguin", "you", "there", "in", "a"]}, {"sent": "shall we put penguin through ?", "words": ["put", "penguin", "we"]}, {"sent": "I can see the penguin .", "words": ["can", "see", "penguin", "the"]}, {"sent": "is that the Mummy penguin ?", "words": ["the", "penguin", "that", "is"]}, {"sent": "can you find the penguin here ?", "words": ["can", "the", "penguin", "you", "here", "find"]}, {"sent": "whats hes doing that penguin ?", "words": ["penguin", "that"]}, {"sent": "xxx put the penguin .", "words": ["the", "put", "penguin"]}, {"sent": "thats a big penguin .", "words": ["penguin", "big", "a"]}, {"sent": "she knows she she would think thats a penguin is my guess .", "words": ["think", "my", "is", "penguin", "would", "she", "a"]}, {"sent": "its a penguin .", "words": ["penguin", "a"]}, {"sent": "penguin puzzle .", "words": ["penguin", "puzzle"]}, {"sent": "wheres the penguin ?", "words": ["the", "penguin"]}, {"sent": "the penguin having some grape ?", "words": ["the", "some", "penguin"]}, {"sent": "penguin .", "words": ["penguin"]}, {"sent": "thats a black penguin .", "words": ["black", "penguin", "a"]}, {"sent": "and where is the penguin ?", "words": ["where", "the", "is", "penguin", "and"]}, {"sent": "comb comb comb .", "words": ["comb"]}, {"sent": "no comb in there .", "words": ["there", "in", "comb"]}, {"sent": "and a comb and a brush and a bowl full of mush .", "words": ["bowl", "full", "brush", "of", "comb", "and", "a"]}, {"sent": "you dont comb mommys hair with this .", "words": ["comb", "with", "you", "dont", "this", "hair"]}, {"sent": "good night comb and good night brush .", "words": ["brush", "comb", "good", "and"]}, {"sent": "we can comb the crumbs out .", "words": ["can", "the", "we", "comb", "out"]}, {"sent": "comb his hair .", "words": ["his", "hair", "comb"]}, {"sent": "let me comb your hair .", "words": ["your", "me", "hair", "comb"]}, {"sent": "or do you comb your hair with combs ?", "words": ["do", "your", "comb", "with", "you", "hair"]}, {"sent": "and thats a comb .", "words": ["comb", "a", "and"]}, {"sent": "yeah you have teeth and the comb has teeth .", "words": ["the", "comb", "and", "have", "you"]}, {"sent": "and a comb .", "words": ["comb", "a", "and"]}, {"sent": "this part of the rooster is called the comb .", "words": ["the", "is", "of", "comb", "this", "rooster"]}, {"sent": "mommys comb mommy uses a comb ?", "words": ["comb", "a"]}, {"sent": "look there we got all those out we got one more side to comb .", "words": ["all", "those", "we", "to", "comb", "there", "more", "out", "look"]}, {"sent": "I dont think you know a comb .", "words": ["think", "comb", "you", "dont", "a"]}, {"sent": "what do you do with a comb ?", "words": ["what", "do", "with", "comb", "you", "a"]}, {"sent": "thats different than the kind of comb you comb your hair with isnt it ?", "words": ["the", "your", "of", "comb", "with", "you", "it", "hair"]}, {"sent": "yeah you comb it but what is it ?", "words": ["what", "is", "comb", "you", "it", "but"]}, {"sent": "with a comb ?", "words": ["comb", "with", "a"]}, {"sent": "I want that comb to brush your hair .", "words": ["your", "brush", "to", "comb", "hair", "that"]}, {"sent": "can you comb mammys hair ?", "words": ["can", "hair", "comb", "you"]}, {"sent": "you wanna comb Sleepy Bears fur ?", "words": ["comb", "wanna", "you"]}, {"sent": "alright should we comb your hair ?", "words": ["your", "hair", "comb", "we"]}, {"sent": "would you like to brush your hair with your comb ?", "words": ["your", "brush", "to", "with", "like", "comb", "you", "would", "hair"]}, {"sent": "because if I take my comb out my hair will fall .", "words": ["my", "if", "comb", "fall", "out", "will", "hair", "take", "because"]}, {"sent": "thats a comb .", "words": ["comb", "a"]}, {"sent": "shall comb your hair ?", "words": ["your", "hair", "comb"]}, {"sent": "youre using your comb to bang arent you ?", "words": ["comb", "to", "your", "you"]}, {"sent": "no heres the comb .", "words": ["the", "comb"]}, {"sent": "goodnight comb and goodnight brush .", "words": ["brush", "comb", "and"]}, {"sent": "its your comb .", "words": ["comb", "your"]}, {"sent": "its a comb .", "words": ["comb", "a"]}, {"sent": "why dont you put the comb back in Patsys pocketbook ?", "words": ["the", "put", "back", "comb", "you", "dont", "in", "why"]}, {"sent": "comb .", "words": ["comb"]}, {"sent": "you do that you comb your hair .", "words": ["do", "your", "comb", "you", "hair", "that"]}, {"sent": "we need a little comb so we can comb his fur right ?", "words": ["can", "need", "we", "so", "comb", "little", "his", "a"]}, {"sent": "upstairs to comb my hair .", "words": ["to", "my", "hair", "comb"]}, {"sent": "goodnight comb goodnight brush .", "words": ["brush", "comb"]}, {"sent": "I dont know where the comb is today love .", "words": ["where", "the", "is", "comb", "dont", "love"]}, {"sent": "you have a comb dont you ?", "words": ["comb", "you", "have", "dont", "a"]}, {"sent": "comb .", "words": ["comb"]}, {"sent": "now you hafta comb my hair .", "words": ["my", "hair", "comb", "you"]}, {"sent": "you going to comb him ?", "words": ["to", "him", "comb", "you"]}, {"sent": "brush comb .", "words": ["brush", "comb"]}, {"sent": "comb .", "words": ["comb"]}, {"sent": "should we comb her hair ?", "words": ["her", "hair", "comb", "we"]}, {"sent": "comb .", "words": ["comb"]}, {"sent": "theres a comb .", "words": ["comb", "a"]}, {"sent": "Melissa dont lick that comb its dirty .", "words": ["comb", "dont", "dirty", "lick", "that"]}, {"sent": "shall I comb your fringe ?", "words": ["your", "comb"]}, {"sent": "but you cant comb her hair .", "words": ["comb", "you", "hair", "her", "but"]}, {"sent": "can you comb my hair ?", "words": ["can", "my", "comb", "you", "hair"]}, {"sent": "youre getting enough hair we can actually start to comb it .", "words": ["can", "we", "to", "comb", "it", "hair"]}, {"sent": "theyre comb teeth .", "words": ["comb"]}, {"sent": "see Megan wont fuss if we comb hers .", "words": ["if", "we", "comb", "hers", "see"]}, {"sent": "you comb your hair .", "words": ["your", "hair", "comb", "you"]}, {"sent": "a brush and a comb .", "words": ["brush", "comb", "and", "a"]}, {"sent": "if youre happy and your comb and your face will really show it if youre happy and you know it comb your hair .", "words": ["your", "if", "happy", "face", "show", "comb", "and", "you", "it", "will", "hair"]}, {"sent": "I my comb .", "words": ["my", "comb"]}, {"sent": "its a comb .", "words": ["comb", "a"]}, {"sent": "wheres your comb ?", "words": ["comb", "your"]}, {"sent": "tadah okay go get me your comb please .", "words": ["your", "go", "comb", "get", "me"]}, {"sent": "you comb your hair .", "words": ["your", "hair", "comb", "you"]}, {"sent": "comb your hair .", "words": ["your", "hair", "comb"]}, {"sent": "it looks like a comb to me .", "words": ["to", "comb", "like", "it", "me", "a"]}, {"sent": "comb their hair make their hair pretty .", "words": ["their", "comb", "pretty", "hair", "make"]}, {"sent": "little comb .", "words": ["comb", "little"]}, {"sent": "thankyou for the little comb .", "words": ["the", "for", "comb", "little"]}, {"sent": "hes looking for the comb .", "words": ["the", "for", "comb"]}, {"sent": "comb .", "words": ["comb"]}, {"sent": "comb your hair .", "words": ["your", "hair", "comb"]}, {"sent": "thats what color my comb is .", "words": ["what", "my", "comb", "is"]}, {"sent": "lemme just comb your hair .", "words": ["your", "hair", "comb"]}, {"sent": "how do you comb your hair how do you comb your hair how do you comb your hair ?", "words": ["do", "how", "your", "comb", "you", "hair"]}, {"sent": "we hafta comb your hair make it all pretty now .", "words": ["all", "your", "we", "comb", "it", "pretty", "hair", "make"]}, {"sent": "Naima can comb his fur .", "words": ["can", "his", "comb"]}, {"sent": "where is the where is the comb ?", "words": ["where", "the", "comb", "is"]}, {"sent": "you chewing the comb now ?", "words": ["the", "comb", "you"]}, {"sent": "you wanna comb your comb ?", "words": ["your", "comb", "wanna", "you"]}, {"sent": "this is a comb for the little ponies dont throw them all out .", "words": ["the", "all", "for", "throw", "is", "comb", "them", "little", "this", "dont", "out", "a"]}, {"sent": "a comb very good honey .", "words": ["comb", "good", "a"]}, {"sent": "oh and theres your comb .", "words": ["comb", "your", "and"]}, {"sent": "a brush and comb .", "words": ["brush", "comb", "and", "a"]}, {"sent": "wanna comb my hair ?", "words": ["my", "hair", "comb", "wanna"]}, {"sent": "did was it be a comb .", "words": ["was", "be", "comb", "it", "did", "a"]}, {"sent": "thats a comb .", "words": ["comb", "a"]}, {"sent": "whats the comb doing in there ?", "words": ["the", "there", "in", "comb"]}, {"sent": "Melissa brought it in where she could comb the ponys hair can you get me your hair bow ?", "words": ["where", "the", "can", "your", "comb", "you", "get", "in", "it", "me", "could", "hair", "she"]}, {"sent": "is panda going to comb my hair ?", "words": ["my", "is", "to", "comb", "hair"]}, {"sent": "because he dont know because he aint got a pick to comb his hair with .", "words": ["pick", "to", "comb", "with", "dont", "he", "his", "hair", "because", "a"]}, {"sent": "are you gonna comb your own hair ?", "words": ["your", "comb", "you", "are", "hair"]}, {"sent": "thats called the comb of the rooster .", "words": ["the", "of", "comb", "rooster"]}, {"sent": "can you give the comb for a minute .", "words": ["can", "the", "for", "comb", "you", "give", "a"]}, {"sent": "comb your hair with your comb I should say .", "words": ["your", "comb", "with", "say", "hair"]}, {"sent": "theres a comb to do her hair .", "words": ["do", "to", "comb", "hair", "her", "a"]}, {"sent": "you can make a necklace maybe .", "words": ["can", "necklace", "you", "make", "a"]}, {"sent": "a necklace .", "words": ["necklace", "a"]}, {"sent": "you can put the chain on it and wear it like a necklace .", "words": ["can", "the", "put", "necklace", "like", "you", "and", "it", "on", "a"]}, {"sent": "when she got there she found Laney and Fritz proudly displaying a beautiful Lily sized necklace .", "words": ["necklace", "and", "there", "when", "she", "a"]}, {"sent": "but that that necklace xxx .", "words": ["necklace", "that", "but"]}, {"sent": "do you know what its necklace is made from Thomas ?", "words": ["do", "what", "necklace", "is", "you"]}, {"sent": "I dont know if its the pink uh necklace thing .", "words": ["the", "dont", "necklace", "if"]}, {"sent": "yes take off the necklace .", "words": ["off", "the", "necklace", "take"]}, {"sent": "beautiful necklace why is she all dressed up ?", "words": ["up", "all", "necklace", "is", "she", "why"]}, {"sent": "do you wanna make a necklace ?", "words": ["do", "necklace", "you", "wanna", "make", "a"]}, {"sent": "oh part of a necklace ?", "words": ["of", "necklace", "a"]}, {"sent": "here shes wearing a different necklace .", "words": ["here", "necklace", "a"]}, {"sent": "well its not a necklace yet you have to put beads on it .", "words": ["put", "necklace", "to", "you", "have", "it", "beads", "on", "not", "a"]}, {"sent": "would you like a necklace ?", "words": ["necklace", "like", "you", "would", "a"]}, {"sent": "its a necklace .", "words": ["necklace", "a"]}, {"sent": "I can go somewhere really swish in this necklace cant I ?", "words": ["can", "necklace", "go", "this", "in"]}, {"sent": "a necklace .", "words": ["necklace", "a"]}, {"sent": "Ill show you how to draw a necklace .", "words": ["how", "necklace", "show", "to", "draw", "you", "a"]}, {"sent": "lets make a bead necklace .", "words": ["necklace", "make", "a"]}, {"sent": "to make a necklace ?", "words": ["to", "necklace", "make", "a"]}, {"sent": "are they a necklace ?", "words": ["they", "necklace", "are", "a"]}, {"sent": "a lovely long necklace .", "words": ["necklace", "long", "a"]}, {"sent": "what colors this necklace Nicole ?", "words": ["what", "necklace", "this"]}, {"sent": "its a necklace ?", "words": ["necklace", "a"]}, {"sent": "they all have necklaces well the goat is wearing a bell around her neck Glenmae is wearing a necklace around her neck .", "words": ["the", "all", "necklace", "is", "have", "they", "around", "her", "a"]}, {"sent": "you can hook the two ends together and make a necklace .", "words": ["can", "the", "necklace", "and", "you", "make", "a"]}, {"sent": "she chipped his necklace alright .", "words": ["his", "necklace", "she"]}, {"sent": "what are you doing with the necklace ?", "words": ["the", "what", "necklace", "with", "you", "are"]}, {"sent": "our necklace is here from last time .", "words": ["necklace", "is", "last", "our", "here"]}, {"sent": "a necklace .", "words": ["necklace", "a"]}, {"sent": "could make a necklace with those beads if you want .", "words": ["necklace", "those", "if", "with", "you", "beads", "could", "make", "a"]}, {"sent": "a blue necklace .", "words": ["necklace", "blue", "a"]}, {"sent": "yes like a necklace and a bracelet .", "words": ["necklace", "like", "and", "a"]}, {"sent": "thats mommys necklace .", "words": ["necklace"]}, {"sent": "this that your necklace ?", "words": ["your", "this", "necklace", "that"]}, {"sent": "I love oh did my necklace hurt you ?", "words": ["my", "necklace", "hurt", "you", "did", "love"]}, {"sent": "is it Gabbys necklace ?", "words": ["necklace", "it", "is"]}, {"sent": "my microphones caught in the necklace .", "words": ["the", "in", "my", "necklace"]}, {"sent": "is it candy necklace dolly beads ?", "words": ["necklace", "is", "it", "beads", "candy"]}, {"sent": "oh thats right thats the necklace that broke youre absolutely right .", "words": ["the", "necklace", "that"]}, {"sent": "I thought she wore a diamond necklace .", "words": ["necklace", "she", "a"]}, {"sent": "its a necklace .", "words": ["necklace", "a"]}, {"sent": "youve just been telling me that Ayisha broke Roses necklace uh sorry her crown and made her cry .", "words": ["necklace", "cry", "and", "that", "me", "her"]}, {"sent": "no its not a necklace ?", "words": ["necklace", "not", "a"]}, {"sent": "did AuntCarey make you a necklace last time ?", "words": ["necklace", "last", "you", "did", "make", "a"]}, {"sent": "kind of a necklace ?", "words": ["of", "necklace", "a"]}, {"sent": "we can give her her necklace back when she gets back .", "words": ["can", "necklace", "when", "back", "we", "give", "her", "she"]}, {"sent": "do you wanna make a necklace ?", "words": ["do", "necklace", "you", "wanna", "make", "a"]}, {"sent": "necklace ?", "words": ["necklace"]}, {"sent": "necklace .", "words": ["necklace"]}, {"sent": "its not a necklace darling .", "words": ["necklace", "not", "a"]}, {"sent": "you necklace comere sweetie pie .", "words": ["necklace", "you"]}, {"sent": "ooh a necklace .", "words": ["necklace", "a"]}, {"sent": "lets tie it to the end and then we can make a whole row of them like a necklace .", "words": ["the", "can", "necklace", "we", "of", "to", "them", "like", "and", "it", "then", "make", "a"]}, {"sent": "your necklace .", "words": ["necklace", "your"]}, {"sent": "you gonna to put the necklace on ?", "words": ["the", "put", "necklace", "to", "you", "on"]}, {"sent": "remember that necklace you had that broke ?", "words": ["necklace", "that", "you"]}, {"sent": "Scuttle snatched the necklace from Vanessa which fell to the ground and shattered and my voice flowed back to me .", "words": ["the", "my", "necklace", "back", "to", "and", "me", "which"]}, {"sent": "isnt that a pretty necklace ?", "words": ["pretty", "necklace", "that", "a"]}, {"sent": "Eleanor do you like my necklace ?", "words": ["do", "my", "necklace", "like", "you"]}, {"sent": "and do you like this little necklace or bracelet ?", "words": ["do", "necklace", "like", "you", "and", "this", "little"]}, {"sent": "necklace .", "words": ["necklace"]}, {"sent": "like a beautiful starry sparkly necklace .", "words": ["necklace", "like", "a"]}, {"sent": "what about your necklace you wanna put your necklace on ?", "words": ["what", "put", "necklace", "your", "you", "about", "wanna", "on"]}, {"sent": "yeah but you need a whole bunch to make a necklace .", "words": ["necklace", "need", "to", "you", "make", "but", "a"]}, {"sent": "oh what a beautiful necklace she said Ive been so selfish .", "words": ["what", "necklace", "so", "she", "a"]}, {"sent": "I like your necklace too .", "words": ["necklace", "your", "like", "too"]}, {"sent": "thats a necklace not a bracelet .", "words": ["necklace", "not", "a"]}, {"sent": "FiremanSams not wearing a necklace ?", "words": ["necklace", "not", "a"]}, {"sent": "a wee another necklace .", "words": ["another", "necklace", "a"]}, {"sent": "shed a wee necklace .", "words": ["necklace", "a"]}, {"sent": "you didnt wear your necklace today .", "words": ["necklace", "your", "you"]}, {"sent": "and a necklace yes .", "words": ["necklace", "a", "and"]}, {"sent": "Delta do you wanna make a necklace ?", "words": ["do", "necklace", "you", "wanna", "make", "a"]}, {"sent": "she sat down and the string on her necklace beads broke and the beads fell on the floor but I dont think she sat on mash potato .", "words": ["the", "think", "necklace", "and", "dont", "beads", "on", "down", "her", "she", "but", "potato"]}, {"sent": "are you giving the necklace to Sugar ?", "words": ["the", "necklace", "to", "you", "are"]}, {"sent": "what color is the necklace ?", "words": ["the", "what", "necklace", "is"]}, {"sent": "a necklace .", "words": ["necklace", "a"]}, {"sent": "a few of the women put jewelry on a necklace earrings but no bracelets .", "words": ["the", "put", "necklace", "of", "on", "but", "a"]}, {"sent": "yeah thats a pretty necklace .", "words": ["necklace", "pretty", "a"]}, {"sent": "a necklace ?", "words": ["necklace", "a"]}, {"sent": "necklace yeah .", "words": ["necklace"]}, {"sent": "a necklace .", "words": ["necklace", "a"]}, {"sent": "which part of the necklace do you think is the label ?", "words": ["the", "do", "think", "necklace", "is", "of", "you", "which"]}, {"sent": "fancy necklace .", "words": ["necklace"]}, {"sent": "your necklace ?", "words": ["necklace", "your"]}, {"sent": "and it still looks like a necklace doesnt it ?", "words": ["necklace", "like", "and", "it", "a"]}, {"sent": "make a necklace ?", "words": ["necklace", "make", "a"]}, {"sent": "shes wearing a necklace .", "words": ["necklace", "a"]}, {"sent": "okay can we have your necklace ?", "words": ["can", "necklace", "your", "we", "have"]}, {"sent": "at least you didnt break your wedding necklace huh ?", "words": ["necklace", "your", "you", "at", "break"]}, {"sent": "youve got a necklace on .", "words": ["on", "necklace", "a"]}, {"sent": "who made the necklace ?", "words": ["the", "necklace", "who"]}, {"sent": "another necklace ?", "words": ["another", "necklace"]}, {"sent": "this makes into a necklace .", "words": ["this", "necklace", "into", "a"]}, {"sent": "what about the necklace would you say is interesting ?", "words": ["the", "what", "necklace", "is", "say", "you", "about", "would"]}, {"sent": "necklace .", "words": ["necklace"]}, {"sent": "the owie make the the bandaid makes his owie feel all better .", "words": ["the", "all", "better", "owie", "his", "make"]}, {"sent": "owie owie .", "words": ["owie"]}, {"sent": "youve gotta nice big owie huh ?", "words": ["owie", "nice", "big"]}, {"sent": "it doesnt look like an owie .", "words": ["like", "owie", "an", "it", "look"]}, {"sent": "you have an owie huh ?", "words": ["have", "owie", "an", "you"]}, {"sent": "owie .", "words": ["owie"]}, {"sent": "owie owie ?", "words": ["owie"]}, {"sent": "you got another owie ?", "words": ["another", "owie", "you"]}, {"sent": "did Trevor get an owie ?", "words": ["get", "owie", "an", "did"]}, {"sent": "owie on your sock ?", "words": ["on", "owie", "your", "sock"]}, {"sent": "you gave you gave Ryan an owie .", "words": ["owie", "an", "you"]}, {"sent": "I know your owie hurts .", "words": ["owie", "your"]}, {"sent": "hey owie .", "words": ["owie"]}, {"sent": "oh youve got an owie on your finger dont you ?", "words": ["your", "you", "owie", "an", "finger", "dont", "on"]}, {"sent": "you got an owie let me kiss it .", "words": ["you", "owie", "an", "it", "me", "kiss"]}, {"sent": "oh poor elephant give the elephant a kiss he has an owie .", "words": ["the", "elephant", "poor", "owie", "an", "he", "give", "kiss", "a"]}, {"sent": "did she get an owie ?", "words": ["get", "owie", "an", "did", "she"]}, {"sent": "shes got an owie on her foot doesnt she ?", "words": ["owie", "an", "foot", "on", "her", "she"]}, {"sent": "you got an owie on your toe ?", "words": ["your", "toe", "you", "owie", "an", "on"]}, {"sent": "oh owie .", "words": ["owie"]}, {"sent": "oh owie .", "words": ["owie"]}, {"sent": "how did you know you had an owie Ross ?", "words": ["how", "you", "owie", "an", "did"]}, {"sent": "its alot like your owie .", "words": ["owie", "your", "like"]}, {"sent": "I have a little owie .", "words": ["have", "owie", "little", "a"]}, {"sent": "I know she looks like she has an owie .", "words": ["owie", "an", "she", "like"]}, {"sent": "owie .", "words": ["owie"]}, {"sent": "owie owie .", "words": ["owie"]}, {"sent": "owie owie .", "words": ["owie"]}, {"sent": "oh why do I have an owie .", "words": ["do", "have", "owie", "an", "why"]}, {"sent": "how did you get that owie ?", "words": ["how", "you", "get", "owie", "did", "that"]}, {"sent": "I was showing Dad my owie .", "words": ["was", "my", "owie"]}, {"sent": "does he have an owie ?", "words": ["does", "have", "owie", "an", "he"]}, {"sent": "okay its owie ?", "words": ["owie"]}, {"sent": "neerow owie .", "words": ["owie"]}, {"sent": "owie .", "words": ["owie"]}, {"sent": "did you get an owie ?", "words": ["you", "get", "owie", "an", "did"]}, {"sent": "wheres the owie .", "words": ["the", "owie"]}, {"sent": "owie .", "words": ["owie"]}, {"sent": "owie .", "words": ["owie"]}, {"sent": "oh its an owie .", "words": ["owie", "an"]}, {"sent": "owie you got my hair .", "words": ["owie", "my", "hair", "you"]}, {"sent": "wheres owie ?", "words": ["owie"]}, {"sent": "she has a owie .", "words": ["owie", "she", "a"]}, {"sent": "owie .", "words": ["owie"]}, {"sent": "uh owie .", "words": ["owie"]}, {"sent": "owie .", "words": ["owie"]}, {"sent": "owie kazowwie kadoodle .", "words": ["owie"]}, {"sent": "oh no owie owie stop stop crash boom ow .", "words": ["owie", "stop"]}, {"sent": "owie .", "words": ["owie"]}, {"sent": "owie did you get an owie on your toe ?", "words": ["your", "toe", "you", "get", "owie", "an", "did", "on"]}, {"sent": "why does she have a owie ?", "words": ["does", "have", "owie", "she", "why", "a"]}, {"sent": "you got an owie ?", "words": ["owie", "an", "you"]}, {"sent": "well howd she get the owie ?", "words": ["get", "the", "she", "owie"]}, {"sent": "owie owie dont no .", "words": ["dont", "owie"]}, {"sent": "I have an owie in my head .", "words": ["my", "head", "have", "owie", "an", "in"]}, {"sent": "would you like a kiss on your owie ?", "words": ["your", "like", "you", "owie", "would", "on", "kiss", "a"]}, {"sent": "your what owie owie ?", "words": ["what", "your", "owie"]}, {"sent": "do you see if baby has an owie ?", "words": ["do", "if", "you", "owie", "an", "see"]}, {"sent": "look the baby has an owie on its foot .", "words": ["the", "owie", "an", "foot", "look", "on"]}, {"sent": "owie .", "words": ["owie"]}, {"sent": "he got a little owie ?", "words": ["owie", "he", "little", "a"]}, {"sent": "oh owie .", "words": ["owie"]}, {"sent": "owie .", "words": ["owie"]}, {"sent": "wheres an owie ?", "words": ["owie", "an"]}, {"sent": "got an owie ?", "words": ["owie", "an"]}, {"sent": "millenium Falcon owie ?", "words": ["owie"]}, {"sent": "an owie .", "words": ["owie", "an"]}, {"sent": "owie .", "words": ["owie"]}, {"sent": "owie .", "words": ["owie"]}, {"sent": "shes got an owie .", "words": ["owie", "an"]}, {"sent": "he said owie .", "words": ["owie", "he"]}, {"sent": "you told me you had an owie last night .", "words": ["last", "you", "owie", "an", "me"]}, {"sent": "owie you bumped yourself .", "words": ["owie", "yourself", "you"]}, {"sent": "dont touch that owie .", "words": ["dont", "owie", "that", "touch"]}, {"sent": "owie .", "words": ["owie"]}, {"sent": "no hell get an owie .", "words": ["get", "owie", "an"]}, {"sent": "oh I was just telling her about Grandpas owie .", "words": ["was", "owie", "her", "about"]}, {"sent": "did she fall down and go owie ?", "words": ["go", "fall", "and", "owie", "did", "down", "she"]}, {"sent": "owie .", "words": ["owie"]}, {"sent": "owie you bited me .", "words": ["owie", "me", "you"]}, {"sent": "you didnt get an owie .", "words": ["get", "owie", "an", "you"]}, {"sent": "neerow owie .", "words": ["owie"]}, {"sent": "shes got an owie on her foot .", "words": ["owie", "an", "foot", "on", "her"]}, {"sent": "owie .", "words": ["owie"]}, {"sent": "an owie how ?", "words": ["owie", "an", "how"]}, {"sent": "owie owie huh ?", "words": ["owie"]}, {"sent": "know how I got this owie ?", "words": ["this", "how", "owie"]}, {"sent": "owie .", "words": ["owie"]}, {"sent": "did the boy get an owie ?", "words": ["the", "get", "owie", "an", "did"]}, {"sent": "and itll make your owie feel better .", "words": ["better", "your", "and", "owie", "make"]}, {"sent": "the beads .", "words": ["the", "beads"]}, {"sent": "Melissa heres some beads that .", "words": ["some", "that", "beads"]}, {"sent": "can we count the beads ?", "words": ["can", "the", "beads", "we"]}, {"sent": "beads green .", "words": ["green", "beads"]}, {"sent": "pick these beads up Parson .", "words": ["pick", "these", "up", "beads"]}, {"sent": "there are some orange beads and red beads on the floor to pick up .", "words": ["red", "the", "up", "pick", "to", "and", "there", "are", "orange", "beads", "on", "some"]}, {"sent": "beads .", "words": ["beads"]}, {"sent": "are these your beads ?", "words": ["these", "are", "your", "beads"]}, {"sent": "Zic move the beads ?", "words": ["the", "beads"]}, {"sent": "were beads and blocks and um .", "words": ["were", "beads", "and"]}, {"sent": "those are beads .", "words": ["are", "those", "beads"]}, {"sent": "beads .", "words": ["beads"]}, {"sent": "all the beads are strung onto some wire arent they ?", "words": ["the", "all", "they", "are", "beads", "some"]}, {"sent": "here is beads on this side .", "words": ["is", "here", "this", "beads", "on"]}, {"sent": "whos stringing beads .", "words": ["beads"]}, {"sent": "remember how to put the pop beads together ?", "words": ["the", "how", "pop", "put", "to", "beads"]}, {"sent": "are those the ones you repaired Grandmas beads with ?", "words": ["the", "those", "with", "you", "are", "beads"]}, {"sent": "where is the rest of the beads here ?", "words": ["where", "the", "is", "of", "here", "beads"]}, {"sent": "they are beads ?", "words": ["they", "are", "beads"]}, {"sent": "these are beads you wear around your neck .", "words": ["these", "your", "you", "are", "beads", "around"]}, {"sent": "one of those beads we one of those beads that go together like this .", "words": ["those", "we", "of", "go", "like", "this", "beads", "that"]}, {"sent": "theyre beads as well .", "words": ["beads"]}, {"sent": "in some societies men wear beads .", "words": ["some", "in", "beads"]}, {"sent": "mm we have some beads it could scoop up .", "words": ["up", "we", "have", "it", "beads", "some", "could"]}, {"sent": "beads .", "words": ["beads"]}, {"sent": "they are beads .", "words": ["they", "are", "beads"]}, {"sent": "because of all the beads in the back because its sposta be xxx and cement mixing around .", "words": ["the", "all", "back", "of", "be", "and", "in", "beads", "around", "because"]}, {"sent": "youre making beads ?", "words": ["beads"]}, {"sent": "fixing my beads .", "words": ["my", "beads"]}, {"sent": "flopflops covered in beads .", "words": ["in", "beads"]}, {"sent": "your counting beads .", "words": ["your", "beads"]}, {"sent": "these are beads beads .", "words": ["these", "are", "beads"]}, {"sent": "thankyou for those beads ?", "words": ["for", "those", "beads"]}, {"sent": "beads .", "words": ["beads"]}, {"sent": "look at all the beads .", "words": ["the", "all", "look", "beads", "at"]}, {"sent": "do you remember pushing all the beads ?", "words": ["the", "all", "do", "you", "beads"]}, {"sent": "do you like those beads ?", "words": ["do", "those", "like", "you", "beads"]}, {"sent": "there is one bead here there is two beads here .", "words": ["here", "there", "beads", "is"]}, {"sent": "oh no you like the beads .", "words": ["the", "like", "beads", "you"]}, {"sent": "little pop beads .", "words": ["pop", "beads", "little"]}, {"sent": "beads .", "words": ["beads"]}, {"sent": "these little beads are all on an elasticated thread arent they ?", "words": ["all", "these", "little", "they", "an", "are", "beads", "on"]}, {"sent": "alright lookee how many beads are on that hand ?", "words": ["how", "hand", "are", "beads", "on", "that"]}, {"sent": "beads on .", "words": ["on", "beads"]}, {"sent": "these are beads .", "words": ["these", "are", "beads"]}, {"sent": "and you hafta push the beads round on the wire frame dont you ?", "words": ["the", "push", "you", "and", "dont", "beads", "on"]}, {"sent": "beads says beads .", "words": ["beads"]}, {"sent": "with blue beads .", "words": ["with", "beads", "blue"]}, {"sent": "why dont you put all the beads back .", "words": ["the", "all", "put", "back", "you", "dont", "beads", "why"]}, {"sent": "I dont think you can close the hood with the beads in there .", "words": ["can", "the", "think", "with", "you", "dont", "in", "there", "beads", "close"]}, {"sent": "there is beads .", "words": ["there", "beads", "is"]}, {"sent": "pick up your beads .", "words": ["pick", "up", "your", "beads"]}, {"sent": "say here Lisa heres some more beads .", "words": ["say", "here", "more", "beads", "some"]}, {"sent": "when you were in the doctors surgery today you were playing with a toy that had lotsof beads on it do you remember ?", "words": ["the", "do", "with", "you", "in", "toy", "that", "it", "beads", "on", "when", "were", "a"]}, {"sent": "little beads .", "words": ["beads", "little"]}, {"sent": "would you like some more little beads ?", "words": ["like", "you", "little", "more", "beads", "would", "some"]}, {"sent": "you pulling those beads .", "words": ["those", "beads", "you"]}, {"sent": "you were playing with some wooden beads werent you ?", "words": ["with", "you", "beads", "some", "were"]}, {"sent": "the beads are for getting wet ?", "words": ["the", "wet", "for", "are", "beads"]}, {"sent": "how many beads do you have ?", "words": ["do", "how", "you", "have", "beads"]}, {"sent": "those are counting beads .", "words": ["are", "those", "beads"]}, {"sent": "with lots and lotsof brightly colored beads .", "words": ["with", "beads", "and"]}, {"sent": "are you taking the beads off because youre a policeman ?", "words": ["the", "off", "you", "are", "beads", "because", "a"]}, {"sent": "look bubble beads .", "words": ["beads", "look"]}, {"sent": "because they look like beads .", "words": ["like", "they", "beads", "look", "because"]}, {"sent": "play with the beads .", "words": ["the", "play", "with", "beads"]}, {"sent": "the little girls wearing a fancy hat with a feather in it and some fancy beads and shes carrying a handbag .", "words": ["the", "with", "little", "and", "in", "it", "beads", "hat", "some", "a"]}, {"sent": "arent your beads in here ?", "words": ["here", "in", "your", "beads"]}, {"sent": "dont eat the beads .", "words": ["eat", "dont", "beads", "the"]}, {"sent": "do you like those beads ?", "words": ["do", "those", "like", "you", "beads"]}, {"sent": "you gonna make some beads ?", "words": ["some", "make", "beads", "you"]}, {"sent": "the train the beads and the car are all cars are all on the same side now arent they ?", "words": ["the", "all", "train", "car", "and", "they", "are", "same", "beads", "on"]}, {"sent": "would you like to get some beads ?", "words": ["to", "like", "you", "get", "beads", "would", "some"]}, {"sent": "lookit these beads look .", "words": ["these", "beads", "look"]}, {"sent": "beads allgone .", "words": ["beads"]}, {"sent": "bring your the beads up here .", "words": ["the", "up", "your", "bring", "here", "beads"]}, {"sent": "beads .", "words": ["beads"]}, {"sent": "its clippable and it goes in your hair and then you have beads in your hair huh ?", "words": ["your", "and", "you", "have", "in", "it", "beads", "then", "hair"]}, {"sent": "theyre beads .", "words": ["beads"]}, {"sent": "is this a beads ?", "words": ["this", "beads", "a", "is"]}, {"sent": "its got a wooden frame and and a metal frame and lotsof different colored beads .", "words": ["beads", "and", "a"]}, {"sent": "theyre beads .", "words": ["beads"]}, {"sent": "are those little beads ?", "words": ["are", "those", "beads", "little"]}, {"sent": "orange beads ?", "words": ["orange", "beads"]}, {"sent": "you gotta put the beads beads back in here babe .", "words": ["the", "put", "back", "you", "here", "in", "beads"]}, {"sent": "theyre beads rather than buttons arent they ?", "words": ["they", "beads"]}, {"sent": "beads ?", "words": ["beads"]}, {"sent": "they are your beads .", "words": ["they", "are", "your", "beads"]}, {"sent": "Mummy puts these beads in here .", "words": ["here", "these", "in", "beads"]}, {"sent": "it looked like a plastic lemon but it had shaky beads inside didnt it ?", "words": ["inside", "like", "it", "beads", "but", "a"]}, {"sent": "down go the beads .", "words": ["the", "down", "beads", "go"]}, {"sent": "little string of beads ?", "words": ["of", "beads", "little"]}, {"sent": "these look like beads that ladys wear round their necks .", "words": ["these", "their", "like", "beads", "look", "that"]}, {"sent": "beads ?", "words": ["beads"]}, {"sent": "theyre beads huh ?", "words": ["beads"]}, {"sent": "your silver beads .", "words": ["your", "beads"]}, {"sent": "quite fond of your beads at the moment arent you ?", "words": ["the", "your", "of", "you", "beads", "at"]}, {"sent": "black black beads ?", "words": ["black", "beads"]}, {"sent": "beads .", "words": ["beads"]}, {"sent": "zipper .", "words": ["zipper"]}, {"sent": "oh your zipper thats always fun .", "words": ["zipper", "your"]}, {"sent": "its got a zipper .", "words": ["zipper", "a"]}, {"sent": "zipper .", "words": ["zipper"]}, {"sent": "zipper .", "words": ["zipper"]}, {"sent": "zipper and that is a teepee .", "words": ["is", "and", "zipper", "that", "a"]}, {"sent": "say zipper .", "words": ["zipper", "say"]}, {"sent": "take hold of the zipper .", "words": ["the", "hold", "of", "zipper", "take"]}, {"sent": "from inside that teepee with the zipper huh ?", "words": ["the", "inside", "with", "zipper", "that"]}, {"sent": "thats a zipper .", "words": ["zipper", "a"]}, {"sent": "zipper ?", "words": ["zipper"]}, {"sent": "look at the zipper .", "words": ["the", "zipper", "at", "look"]}, {"sent": "there is the zipper .", "words": ["the", "there", "zipper", "is"]}, {"sent": "say zipper .", "words": ["zipper", "say"]}, {"sent": "zipper .", "words": ["zipper"]}, {"sent": "show them how to zipper .", "words": ["how", "show", "to", "them", "zipper"]}, {"sent": "do you know how to zipper ?", "words": ["do", "how", "to", "you", "zipper"]}, {"sent": "look here is a zipper .", "words": ["is", "here", "zipper", "look", "a"]}, {"sent": "can you do the zipper one again ?", "words": ["can", "do", "the", "you", "zipper"]}, {"sent": "a zipper .", "words": ["zipper", "a"]}, {"sent": "pull it pull the zipper down .", "words": ["the", "pull", "zipper", "it", "down"]}, {"sent": "the zipper .", "words": ["the", "zipper"]}, {"sent": "can you pull the zipper ?", "words": ["can", "the", "pull", "you", "zipper"]}, {"sent": "zipper .", "words": ["zipper"]}, {"sent": "can you say zipper ?", "words": ["can", "zipper", "say", "you"]}, {"sent": "zipper it down zipper it up .", "words": ["up", "zipper", "down", "it"]}, {"sent": "can you do the zipper ?", "words": ["can", "do", "the", "you", "zipper"]}, {"sent": "look there is the zipper .", "words": ["the", "is", "there", "zipper", "look"]}, {"sent": "a zipper .", "words": ["zipper", "a"]}, {"sent": "pull the zipper .", "words": ["the", "zipper", "pull"]}, {"sent": "zipper .", "words": ["zipper"]}, {"sent": "thats right Sela it is a zipper .", "words": ["zipper", "it", "a", "is"]}, {"sent": "which zipper ?", "words": ["zipper", "which"]}, {"sent": "is that a zipper like on the front of your coat ?", "words": ["the", "your", "is", "of", "like", "zipper", "on", "that", "coat", "a"]}, {"sent": "theres a zipper .", "words": ["zipper", "a"]}, {"sent": "there is a zipper over there .", "words": ["is", "there", "zipper", "over", "a"]}, {"sent": "look at the zipper look at the zipper .", "words": ["the", "zipper", "at", "look"]}, {"sent": "xxx to do that xxx that zipper ?", "words": ["to", "do", "that", "zipper"]}, {"sent": "do you wanna do the zipper ?", "words": ["the", "do", "you", "zipper", "wanna"]}, {"sent": "my zipper .", "words": ["zipper", "my"]}, {"sent": "its a zipper .", "words": ["zipper", "a"]}, {"sent": "you dont wanna play with zipper either ?", "words": ["play", "with", "you", "dont", "zipper", "wanna"]}, {"sent": "here look you wanna see the zipper ?", "words": ["the", "you", "here", "zipper", "wanna", "look", "see"]}, {"sent": "zipper yes thats a zipper .", "words": ["zipper", "a"]}, {"sent": "it is a hard zipper .", "words": ["is", "zipper", "it", "hard", "a"]}, {"sent": "lemme see the zipper .", "words": ["the", "see", "zipper"]}, {"sent": "pull the zipper down .", "words": ["the", "zipper", "down", "pull"]}, {"sent": "because its got a zipper .", "words": ["zipper", "because", "a"]}, {"sent": "zipper .", "words": ["zipper"]}, {"sent": "and a zipper .", "words": ["zipper", "a", "and"]}, {"sent": "look Bro is opening the zipper .", "words": ["the", "zipper", "is", "look"]}, {"sent": "a zipper .", "words": ["zipper", "a"]}, {"sent": "say zipper say zip zip zip .", "words": ["zipper", "say"]}, {"sent": "is it a button or a zipper ?", "words": ["is", "zipper", "it", "button", "a"]}, {"sent": "pull with the zipper .", "words": ["the", "zipper", "with", "pull"]}, {"sent": "zipper .", "words": ["zipper"]}, {"sent": "Alex this is not a zipper .", "words": ["is", "this", "zipper", "not", "a"]}, {"sent": "my zipper .", "words": ["zipper", "my"]}, {"sent": "no that is a zipper huh ?", "words": ["zipper", "that", "a", "is"]}, {"sent": "wanna pull the zipper down ?", "words": ["the", "pull", "zipper", "wanna", "down"]}, {"sent": "ooh can you find the zipper ?", "words": ["can", "the", "you", "zipper", "find"]}, {"sent": "zipper it up .", "words": ["up", "zipper", "it"]}, {"sent": "with a zipper and a snap .", "words": ["zipper", "with", "and", "a"]}, {"sent": "zipper .", "words": ["zipper"]}, {"sent": "you can say zipper .", "words": ["can", "zipper", "say", "you"]}, {"sent": "hold the zipper .", "words": ["the", "zipper", "hold"]}, {"sent": "thats the zipper .", "words": ["the", "zipper"]}, {"sent": "zipper .", "words": ["zipper"]}, {"sent": "a zipper .", "words": ["zipper", "a"]}, {"sent": "can you zipper yourself up ?", "words": ["can", "up", "yourself", "you", "zipper"]}, {"sent": "you can put it in that thats a zipper .", "words": ["can", "put", "you", "zipper", "in", "it", "that", "a"]}, {"sent": "pull that zipper .", "words": ["zipper", "that", "pull"]}, {"sent": "zipper see .", "words": ["see", "zipper"]}, {"sent": "Mommy when did you get a zipper in your tummy ?", "words": ["your", "you", "get", "zipper", "in", "did", "tummy", "when", "a"]}, {"sent": "shes gotta learn how to put her boots on her leggings on zipper her coat button her coat xxx tie her shoes .", "words": ["how", "put", "button", "to", "boots", "zipper", "on", "her", "coat"]}, {"sent": "zipper pull the zipper .", "words": ["the", "zipper", "pull"]}, {"sent": "see the zipper ?", "words": ["the", "see", "zipper"]}, {"sent": "pull the zipper down .", "words": ["the", "zipper", "down", "pull"]}, {"sent": "a zipper .", "words": ["zipper", "a"]}, {"sent": "say zip zip zipper .", "words": ["zipper", "say"]}, {"sent": "we did the zipper .", "words": ["the", "zipper", "did", "we"]}, {"sent": "is that a zipper ?", "words": ["zipper", "that", "a", "is"]}, {"sent": "zipper .", "words": ["zipper"]}, {"sent": "has a zipper .", "words": ["zipper", "a"]}, {"sent": "lookit here zipper .", "words": ["here", "zipper"]}, {"sent": "yes that is a zipper .", "words": ["zipper", "that", "a", "is"]}, {"sent": "do the zipper .", "words": ["the", "do", "zipper"]}, {"sent": "oh look what is in the zipper .", "words": ["the", "what", "is", "zipper", "in", "look"]}, {"sent": "Nomi can zipper .", "words": ["can", "zipper"]}, {"sent": "does it have a zipper ?", "words": ["does", "have", "zipper", "it", "a"]}, {"sent": "wanna zip the zipper back up ?", "words": ["the", "up", "back", "zipper", "wanna"]}, {"sent": "in other words youre sposta be the zipper ?", "words": ["the", "be", "other", "zipper", "in"]}, {"sent": "let us see you zipper .", "words": ["see", "us", "zipper", "you"]}, {"sent": "zipper .", "words": ["zipper"]}, {"sent": "here is a zipper .", "words": ["here", "zipper", "a", "is"]}, {"sent": "do you know how to zipper ?", "words": ["do", "how", "to", "you", "zipper"]}, {"sent": "theres the zipper .", "words": ["the", "zipper"]}, {"sent": "can you pull the zipper down ?", "words": ["can", "the", "pull", "you", "zipper", "down"]}, {"sent": "we put a bib on her .", "words": ["put", "we", "bib", "on", "her", "a"]}, {"sent": "so but even at home when he has a when he notices his bib and whats on his bib he has a tendency to if its colorful .", "words": ["if", "a", "so", "to", "and", "bib", "on", "he", "at", "his", "when", "but", "home"]}, {"sent": "lets put a WinniethePooh bib on you .", "words": ["put", "you", "bib", "on", "a"]}, {"sent": "why dont we get a different bib then ?", "words": ["we", "get", "dont", "bib", "then", "why", "a"]}, {"sent": "you took your bib off .", "words": ["off", "bib", "your", "you"]}, {"sent": "I suppose it does look a bit like a bib .", "words": ["does", "like", "bib", "it", "look", "a"]}, {"sent": "the bib .", "words": ["the", "bib"]}, {"sent": "take do you wanna take her bib off ?", "words": ["off", "do", "you", "bib", "wanna", "take", "her"]}, {"sent": "he is putting on improvised bib .", "words": ["on", "bib", "he", "is"]}, {"sent": "you find your bib ?", "words": ["bib", "your", "find", "you"]}, {"sent": "a bib .", "words": ["bib", "a"]}, {"sent": "and what does this bib say ?", "words": ["what", "does", "say", "and", "bib", "this"]}, {"sent": "heres your bib .", "words": ["bib", "your"]}, {"sent": "heres a bib .", "words": ["bib", "a"]}, {"sent": "shes got a bib .", "words": ["bib", "a"]}, {"sent": "the blue bib .", "words": ["the", "bib", "blue"]}, {"sent": "do you want that bib off now ?", "words": ["off", "do", "you", "bib", "that"]}, {"sent": "I should have put a bib on you .", "words": ["put", "you", "have", "bib", "on", "a"]}, {"sent": "just looking at your bib .", "words": ["bib", "your", "at"]}, {"sent": "I believe that bib is getting a little small for you .", "words": ["for", "is", "little", "you", "bib", "that", "a"]}, {"sent": "I think we better put a bib on dont you ?", "words": ["better", "think", "put", "we", "you", "bib", "dont", "on", "a"]}, {"sent": "wheres your bib ?", "words": ["bib", "your"]}, {"sent": "shes got a bib on .", "words": ["on", "bib", "a"]}, {"sent": "you look like superman with your bib flying backward .", "words": ["your", "with", "like", "you", "bib", "look"]}, {"sent": "youve got to have a bib Thomas .", "words": ["have", "to", "bib", "a"]}, {"sent": "its a very colorful bib .", "words": ["bib", "a"]}, {"sent": "I think you managed to have your meal and not get anything on your bib but .", "words": ["think", "your", "to", "you", "have", "and", "get", "bib", "on", "not", "but"]}, {"sent": "do you want the baby to wear a bib ?", "words": ["the", "do", "to", "you", "bib", "a"]}, {"sent": "here put the bib on .", "words": ["the", "put", "here", "bib", "on"]}, {"sent": "its a its a good job Mummy Mummys wearing her bib isnt it ?", "words": ["bib", "it", "good", "her", "a"]}, {"sent": "oh babys bib .", "words": ["bib"]}, {"sent": "and theres a butterfly on your bib .", "words": ["butterfly", "your", "and", "bib", "on", "a"]}, {"sent": "thats your Sunday bib .", "words": ["bib", "your"]}, {"sent": "okay Im gonna take off your bib while youre thinking about that ?", "words": ["off", "your", "bib", "that", "about", "take"]}, {"sent": "lets take your bib off .", "words": ["your", "bib", "take", "off"]}, {"sent": "you want your bib off .", "words": ["off", "bib", "your", "you"]}, {"sent": "does the baby need a bib too ?", "words": ["the", "does", "need", "bib", "too", "a"]}, {"sent": "fell down the bib .", "words": ["the", "bib", "down"]}, {"sent": "okay lemme take off your bib .", "words": ["off", "bib", "take", "your"]}, {"sent": "well take this silly bib off of you .", "words": ["off", "of", "you", "this", "bib", "take"]}, {"sent": "you dont want your bib on ?", "words": ["your", "you", "dont", "bib", "on"]}, {"sent": "well fix the bib .", "words": ["the", "fix", "bib"]}, {"sent": "do we have a bib ?", "words": ["do", "we", "have", "bib", "a"]}, {"sent": "lets put your bib on .", "words": ["on", "bib", "put", "your"]}, {"sent": "sometimes Id like to glue your bib on .", "words": ["your", "to", "like", "bib", "on", "glue"]}, {"sent": "and how about even you could use a bib with this one I think .", "words": ["how", "think", "with", "you", "and", "bib", "this", "about", "could", "a"]}, {"sent": "no bib .", "words": ["bib"]}, {"sent": "dollys going to put her bib .", "words": ["to", "put", "her", "bib"]}, {"sent": "does she wear it like a bib ?", "words": ["does", "like", "bib", "it", "she", "a"]}, {"sent": "thats a bib .", "words": ["bib", "a"]}, {"sent": "heres your bib .", "words": ["bib", "your"]}, {"sent": "do it like a bib .", "words": ["do", "like", "bib", "it", "a"]}, {"sent": "her bib .", "words": ["bib", "her"]}, {"sent": "have you got toast on your bib ?", "words": ["your", "toast", "you", "have", "bib", "on"]}, {"sent": "he needs a bib doesnt he ?", "words": ["bib", "he", "a"]}, {"sent": "put a bib on .", "words": ["on", "bib", "put", "a"]}, {"sent": "wheres Mandys bib ?", "words": ["bib"]}, {"sent": "youve got your bib on ?", "words": ["on", "bib", "your"]}, {"sent": "get your highchair and bib .", "words": ["get", "bib", "your", "and"]}, {"sent": "uh huh get on your bib uh huh .", "words": ["get", "bib", "on", "your"]}, {"sent": "bib off ?", "words": ["off", "bib"]}, {"sent": "Mommys gonna adjust your bib a little bit okay ?", "words": ["bib", "your", "little", "a"]}, {"sent": "lets put on a bib would you like some apple blueberry sauce too ?", "words": ["put", "sauce", "apple", "like", "you", "bib", "would", "on", "some", "too", "a"]}, {"sent": "shes got her bib on .", "words": ["on", "bib", "her"]}, {"sent": "you put the moocow in the bib .", "words": ["the", "put", "you", "bib", "in"]}, {"sent": "green bib on .", "words": ["on", "bib", "green"]}, {"sent": "every bib you have that has a nice smily sun on it doesnt it ?", "words": ["sun", "nice", "you", "have", "bib", "it", "every", "on", "that", "a"]}, {"sent": "should we put the bib on the baby ?", "words": ["the", "put", "we", "bib", "on"]}, {"sent": "your bib ?", "words": ["bib", "your"]}, {"sent": "come on bib .", "words": ["on", "bib"]}, {"sent": "shall we pop a bib on ?", "words": ["pop", "we", "bib", "on", "a"]}, {"sent": "should Mummy be wearing a bib ?", "words": ["be", "bib", "a"]}, {"sent": "you dont want the bib on .", "words": ["the", "you", "dont", "bib", "on"]}, {"sent": "it was only when I was putting the bib on .", "words": ["the", "was", "bib", "it", "on", "when"]}, {"sent": "we gotta get a bib too .", "words": ["we", "get", "bib", "too", "a"]}, {"sent": "I forgot you had this bib .", "words": ["this", "bib", "you"]}, {"sent": "this bib was in your closet .", "words": ["your", "was", "closet", "this", "in", "bib"]}, {"sent": "I dont think the bib goes that way .", "words": ["the", "think", "dont", "bib", "that"]}, {"sent": "thats her bib .", "words": ["bib", "her"]}, {"sent": "comere mister superman with that bib on .", "words": ["on", "bib", "that", "with"]}, {"sent": "or do you mean that youd like Mummy to wear a bib ?", "words": ["do", "to", "like", "you", "bib", "that", "a"]}, {"sent": "I got wise and put a bib on her because she spits up so much lately .", "words": ["up", "put", "so", "much", "and", "bib", "on", "her", "she", "because", "a"]}, {"sent": "youre wearing your bib and it says flowers grow in my garden .", "words": ["my", "your", "and", "bib", "in", "it", "garden"]}, {"sent": "NinNin bib ?", "words": ["bib"]}, {"sent": "thats her bib .", "words": ["bib", "her"]}, {"sent": "lemme put a bib on you .", "words": ["put", "you", "bib", "on", "a"]}, {"sent": "wheres that bib ?", "words": ["bib", "that"]}, {"sent": "you go and get her bib .", "words": ["go", "and", "get", "you", "bib", "her"]}, {"sent": "what has the bib got on it ?", "words": ["the", "what", "bib", "it", "on"]}, {"sent": "xxx uh Dexter puts the towel on like its a bib .", "words": ["the", "like", "bib", "towel", "on", "a"]}, {"sent": "xxx skirt with a bib .", "words": ["bib", "with", "a"]}, {"sent": "lets put your bib on bib on .", "words": ["on", "bib", "put", "your"]}, {"sent": "theres a big spider on Amys bib .", "words": ["on", "bib", "big", "a"]}, {"sent": "thats a nice bib .", "words": ["bib", "nice", "a"]}, {"sent": "right lets put a nice bib on you .", "words": ["put", "nice", "you", "bib", "on", "a"]}, {"sent": "lets take lets take your bib off and then lets clean you up .", "words": ["off", "clean", "up", "your", "and", "you", "bib", "then", "take"]}, {"sent": "you can not walk while Im tying the bib .", "words": ["can", "the", "you", "bib", "walk", "not"]}, {"sent": "thats the picture on your bib isnt it ?", "words": ["the", "your", "picture", "bib", "it", "on"]}, {"sent": "lets put the bib on shall we shall I put a bib on ?", "words": ["the", "put", "we", "bib", "on", "a"]}, {"sent": "scarf ?", "words": ["scarf"]}, {"sent": "and what about your hat scarf and gloves ?", "words": ["what", "your", "scarf", "and", "gloves", "about", "hat"]}, {"sent": "Daddys scarf ?", "words": ["scarf"]}, {"sent": "who has the scarf yeah .", "words": ["the", "scarf", "who"]}, {"sent": "because they get cold if I dont have a scarf on .", "words": ["if", "scarf", "get", "they", "dont", "have", "on", "cold", "because", "a"]}, {"sent": "and a scarf I forgot .", "words": ["scarf", "a", "and"]}, {"sent": "xxx maybe thats like a scarf to keep him warm in the winter when its cold .", "words": ["the", "to", "scarf", "like", "him", "in", "when", "cold", "a"]}, {"sent": "xxx you do need the scarf .", "words": ["the", "do", "need", "scarf", "you"]}, {"sent": "make just a scarf .", "words": ["scarf", "make", "a"]}, {"sent": "s is for scarf she wears on her curls .", "words": ["for", "is", "scarf", "on", "her", "she"]}, {"sent": "oh and give her a different color scarf .", "words": ["scarf", "and", "give", "her", "a"]}, {"sent": "Liverpool scarf .", "words": ["scarf"]}, {"sent": "are you going to take off your hat and scarf and join us ?", "words": ["off", "your", "to", "scarf", "you", "and", "us", "are", "hat", "take"]}, {"sent": "hes got a scarf and gloves on .", "words": ["scarf", "and", "gloves", "on", "a"]}, {"sent": "xxx wheres your yellow scarf ?", "words": ["yellow", "scarf", "your"]}, {"sent": "a lil scarf .", "words": ["scarf", "a"]}, {"sent": "its a scarf .", "words": ["scarf", "a"]}, {"sent": "I had forgotten to take off my scarf but our classmates didnt notice .", "words": ["off", "my", "to", "scarf", "our", "take", "but"]}, {"sent": "one two three like a button and wed wrap a big scarf round the snowmans neck .", "words": ["the", "button", "scarf", "like", "and", "big", "a"]}, {"sent": "shall I put this scarf on your head ?", "words": ["put", "your", "head", "scarf", "this", "on"]}, {"sent": "your coat this ones your bag and this is your hat and this is your scarf .", "words": ["your", "is", "scarf", "and", "this", "hat", "coat"]}, {"sent": "can you say scarf ?", "words": ["can", "scarf", "say", "you"]}, {"sent": "see you did not get scarf Dexter .", "words": ["scarf", "you", "get", "did", "see", "not"]}, {"sent": "she put on her her woolly scarf .", "words": ["put", "scarf", "on", "her", "she"]}, {"sent": "and a red scarf aswell .", "words": ["red", "scarf", "a", "and"]}, {"sent": "its your scarf .", "words": ["scarf", "your"]}, {"sent": "xxx scarf .", "words": ["scarf"]}, {"sent": "who wears a scarf ?", "words": ["scarf", "a", "who"]}, {"sent": "put on your scarf .", "words": ["on", "put", "scarf", "your"]}, {"sent": "Pippo isnt there but Tom finds Pippos scarf .", "words": ["there", "scarf", "but"]}, {"sent": "you need a scarf says Thomas .", "words": ["scarf", "need", "a", "you"]}, {"sent": "you havent got a scarf .", "words": ["scarf", "a", "you"]}, {"sent": "maybe we could put it on lion like a head scarf .", "words": ["put", "we", "head", "scarf", "like", "it", "on", "could", "lion", "a"]}, {"sent": "I think her scarf flew off .", "words": ["off", "think", "scarf", "her"]}, {"sent": "and a scarf .", "words": ["scarf", "a", "and"]}, {"sent": "now I just need my scarf theres my scarf over there .", "words": ["my", "need", "scarf", "there", "over"]}, {"sent": "right a scarf .", "words": ["scarf", "a"]}, {"sent": "Ill put my scarf on first .", "words": ["put", "my", "scarf", "first", "on"]}, {"sent": "I think its a scarf .", "words": ["think", "scarf", "a"]}, {"sent": "wheres your scarf gone ?", "words": ["scarf", "your"]}, {"sent": "like a scarf .", "words": ["scarf", "like", "a"]}, {"sent": "and hes wearing an old scarf and hat and he has got two birds resting on his arms .", "words": ["old", "scarf", "and", "his", "an", "on", "he", "hat"]}, {"sent": "what color is his scarf ?", "words": ["his", "what", "scarf", "is"]}, {"sent": "and I think he might need a scarf mightnt he ?", "words": ["think", "need", "scarf", "and", "he", "a"]}, {"sent": "it reminds me of the scarf you got at Julias birthday party .", "words": ["the", "of", "scarf", "you", "party", "it", "at", "me"]}, {"sent": "dont wear a scarf like that over my head .", "words": ["my", "head", "scarf", "like", "dont", "over", "that", "a"]}, {"sent": "youll hafta put your hat and gloves and scarf on .", "words": ["put", "your", "scarf", "and", "on", "gloves", "hat"]}, {"sent": "have you got a green scarf ?", "words": ["scarf", "you", "have", "green", "a"]}, {"sent": "PBBear had a hat and a scarf in the story this morning .", "words": ["the", "scarf", "and", "this", "in", "hat", "story", "a"]}, {"sent": "xxx be sure you put your hat in the drawers and put your hat and scarf in the drawer or either over there in the bag .", "words": ["the", "put", "your", "be", "scarf", "you", "and", "there", "in", "hat", "drawer", "over"]}, {"sent": "can you see the scarf and the Robin anywhere together ?", "words": ["can", "the", "scarf", "you", "and", "see"]}, {"sent": "a scarf .", "words": ["scarf", "a"]}, {"sent": "thats a scarf .", "words": ["scarf", "a"]}, {"sent": "a scarf ?", "words": ["scarf", "a"]}, {"sent": "um so the snowman has a lovely warm red scarf on .", "words": ["the", "red", "so", "scarf", "snowman", "on", "a"]}, {"sent": "hes got a big scarf on .", "words": ["on", "scarf", "big", "a"]}, {"sent": "now what do you do with your scarf ?", "words": ["what", "do", "your", "scarf", "with", "you"]}, {"sent": "we can pretend thats his scarf cant we ?", "words": ["can", "we", "scarf", "his", "pretend"]}, {"sent": "scarf is that Patsys scarf ?", "words": ["scarf", "that", "is"]}, {"sent": "shall we have a red scarf ?", "words": ["red", "we", "scarf", "have", "a"]}, {"sent": "Mama buy you a nice scarf .", "words": ["scarf", "nice", "you", "buy", "a"]}, {"sent": "and look hes got a scarf on .", "words": ["scarf", "and", "look", "on", "a"]}, {"sent": "and when do you wear a scarf ?", "words": ["do", "scarf", "you", "and", "when", "a"]}, {"sent": "ah wow thats a nice scarf .", "words": ["scarf", "nice", "a"]}, {"sent": "we call that a scarf .", "words": ["scarf", "that", "a", "we"]}, {"sent": "ah I was gonna put on my scarf today but dat wouldnt be ready .", "words": ["put", "my", "was", "be", "scarf", "on", "but"]}, {"sent": "did you get your scarf ?", "words": ["your", "scarf", "you", "get", "did"]}, {"sent": "it looks like hes wearing a scarf .", "words": ["scarf", "it", "like", "a"]}, {"sent": "draw a scarf .", "words": ["scarf", "draw", "a"]}, {"sent": "scarf .", "words": ["scarf"]}, {"sent": "its a little boy with a hat and a scarf on .", "words": ["scarf", "with", "and", "little", "on", "hat", "a"]}, {"sent": "scarf .", "words": ["scarf"]}, {"sent": "scarf .", "words": ["scarf"]}, {"sent": "do you wear a scarf today ?", "words": ["do", "scarf", "a", "you"]}, {"sent": "put your scarf on ?", "words": ["on", "put", "scarf", "your"]}, {"sent": "does mommy wear a scarf when we go out ?", "words": ["does", "we", "go", "scarf", "out", "when", "a"]}, {"sent": "the nurse took off my scarf and put a white gown on me .", "words": ["the", "off", "my", "put", "scarf", "and", "on", "me", "white", "a"]}, {"sent": "find your coat hat and scarf .", "words": ["your", "scarf", "and", "find", "hat", "coat"]}, {"sent": "dont pretend its a scarf .", "words": ["dont", "pretend", "scarf", "a"]}, {"sent": "oh David was wearing that yellow scarf this morning .", "words": ["was", "scarf", "this", "yellow", "that"]}, {"sent": "purple scarf where ?", "words": ["where", "scarf"]}, {"sent": "who wears a scarf ?", "words": ["scarf", "a", "who"]}, {"sent": "I want a scarf I want a scarf I want a scarf .", "words": ["scarf", "a"]}, {"sent": "you need the scarf on .", "words": ["the", "need", "scarf", "you", "on"]}, {"sent": "little scarf .", "words": ["scarf", "little"]}, {"sent": "thats a scarf .", "words": ["scarf", "a"]}, {"sent": "you just wear the scarf then .", "words": ["the", "then", "scarf", "you"]}, {"sent": "thats my scarf .", "words": ["my", "scarf"]}, {"sent": "thats Patsys scarf .", "words": ["scarf"]}, {"sent": "you can wrap up nice and warm with a scarf .", "words": ["can", "up", "nice", "with", "scarf", "and", "you", "a"]}, {"sent": "xxx a red scarf .", "words": ["red", "scarf", "a"]}, {"sent": "the nurse took off my white scarf and put a white gown on me .", "words": ["the", "off", "my", "put", "scarf", "and", "on", "me", "white", "a"]}, {"sent": "got your scarf ?", "words": ["scarf", "your"]}, {"sent": "did you wear a scarf this morning ?", "words": ["scarf", "you", "this", "did", "a"]}, {"sent": "hes wearing a hat and a scarf .", "words": ["hat", "scarf", "and", "a"]}, {"sent": "and your scarf and your coat .", "words": ["scarf", "your", "coat", "and"]}, {"sent": "Im having a job getting my horrible gloves off .", "words": ["off", "my", "gloves", "a"]}, {"sent": "gloves ?", "words": ["gloves"]}, {"sent": "three gloves .", "words": ["gloves"]}, {"sent": "bites through I put on a pair of plastic gloves and his heavy Bobby socks .", "words": ["put", "of", "and", "his", "gloves", "on", "heavy", "a"]}, {"sent": "would you like to put these gloves on ?", "words": ["these", "put", "to", "like", "you", "gloves", "would", "on"]}, {"sent": "whered you lose the gloves ?", "words": ["the", "gloves", "you"]}, {"sent": "wheres your gloves ?", "words": ["gloves", "your"]}, {"sent": "blue gloves ?", "words": ["gloves", "blue"]}, {"sent": "he meant that the gloves were clothes .", "words": ["the", "gloves", "he", "that", "were"]}, {"sent": "you dont need your gloves on while youre eating that do you ?", "words": ["do", "your", "need", "you", "dont", "gloves", "on", "that"]}, {"sent": "some nice gloves to wear .", "words": ["to", "some", "nice", "gloves"]}, {"sent": "hes got a green hat and blue gloves and a big red nose .", "words": ["red", "nose", "blue", "and", "gloves", "green", "hat", "big", "a"]}, {"sent": "Ive used these oven gloves havent I ?", "words": ["these", "gloves", "oven"]}, {"sent": "you why you gonna take two pair a gloves to school Roland ?", "words": ["to", "school", "you", "gloves", "take", "why", "a"]}, {"sent": "no gloves .", "words": ["gloves"]}, {"sent": "if you were doing a lot of messy gardening youd hafta have some special little gardening gloves wouldnt you ?", "words": ["if", "of", "a lot", "you", "have", "little", "gloves", "some", "were", "a"]}, {"sent": "get your gloves .", "words": ["get", "gloves", "your"]}, {"sent": "where where are those sequined gloves anyway ?", "words": ["where", "are", "those", "gloves"]}, {"sent": "gloves on .", "words": ["on", "gloves"]}, {"sent": "gimme these gloves please thankyou .", "words": ["these", "gloves"]}, {"sent": "that man has gloves on because he is trying to keep his hands from being burned while he lights the fire .", "words": ["the", "is", "to", "his", "gloves", "he", "on", "that", "because"]}, {"sent": "hey do you know where his gloves are ?", "words": ["where", "do", "you", "gloves", "are", "his"]}, {"sent": "wherere your gloves Lara ?", "words": ["gloves", "your"]}, {"sent": "the men arent wearing gloves today are they ?", "words": ["the", "they", "gloves", "are"]}, {"sent": "wheres the oven gloves ?", "words": ["the", "gloves", "oven"]}, {"sent": "get your gloves .", "words": ["get", "gloves", "your"]}, {"sent": "always wear gloves .", "words": ["gloves"]}, {"sent": "these fantastic rubber gloves and a xxx isnt it ?", "words": ["these", "and", "gloves", "it", "a"]}, {"sent": "why did I put gloves on my hands ?", "words": ["put", "my", "gloves", "did", "on", "why"]}, {"sent": "what are those gloves ?", "words": ["what", "are", "those", "gloves"]}, {"sent": "you put your gloves on .", "words": ["put", "your", "you", "gloves", "on"]}, {"sent": "its my rubber gloves .", "words": ["my", "gloves"]}, {"sent": "youre wearing gloves ?", "words": ["gloves"]}, {"sent": "well they look like gloves dont they ?", "words": ["like", "dont", "they", "gloves", "look"]}, {"sent": "dyou see the gloves .", "words": ["the", "see", "gloves"]}, {"sent": "its cold outside so we hadta put wellies and hats and gloves on didnt we ?", "words": ["put", "we", "outside", "so", "and", "gloves", "on", "cold"]}, {"sent": "mommy wears gloves ?", "words": ["gloves"]}, {"sent": "wash those gloves of daddys .", "words": ["of", "gloves", "wash", "those"]}, {"sent": "tell him not to pop your gloves .", "words": ["pop", "your", "to", "him", "gloves", "not"]}, {"sent": "oh really what color are her gloves ?", "words": ["what", "are", "her", "gloves"]}, {"sent": "oh theyre gloves .", "words": ["gloves"]}, {"sent": "its quite good isnt it to have these things as oven g gloves .", "words": ["these", "to", "have", "it", "gloves", "good", "oven"]}, {"sent": "wheres your gloves ?", "words": ["gloves", "your"]}, {"sent": "look at these really nice outdoor gloves .", "words": ["these", "look", "nice", "gloves", "at"]}, {"sent": "weve got to wear oven gloves .", "words": ["to", "gloves", "oven"]}, {"sent": "special gloves .", "words": ["gloves"]}, {"sent": "the goose is wearing gloves .", "words": ["the", "goose", "gloves", "is"]}, {"sent": "I got new rubber gloves .", "words": ["new", "gloves"]}, {"sent": "Roman when did you get your gloves ?", "words": ["your", "you", "get", "gloves", "did", "when"]}, {"sent": "if you put your socks on your fingers then theyre very much like gloves .", "words": ["put", "your", "if", "like", "you", "gloves", "on", "then", "much"]}, {"sent": "and a scarf and some gloves .", "words": ["scarf", "and", "gloves", "some", "a"]}, {"sent": "xxx wheres are your gloves .", "words": ["are", "your", "gloves"]}, {"sent": "you have beautiful gloves on .", "words": ["have", "on", "gloves", "you"]}, {"sent": "and the gloves .", "words": ["the", "gloves", "and"]}, {"sent": "you dont hafta have gloves on .", "words": ["you", "have", "dont", "gloves", "on"]}, {"sent": "theres no and no any gloves in there ?", "words": ["any", "and", "there", "in", "gloves"]}, {"sent": "dont really iron gloves normally but .", "words": ["dont", "gloves", "but"]}, {"sent": "gloves keep your hands warm .", "words": ["gloves", "your"]}, {"sent": "theyre gloves .", "words": ["gloves"]}, {"sent": "does he have any gloves for his hands ?", "words": ["does", "for", "any", "have", "gloves", "he", "his"]}, {"sent": "here you have some gloves here .", "words": ["you", "here", "have", "gloves", "some"]}, {"sent": "are your gloves in the car ?", "words": ["the", "your", "car", "in", "are", "gloves"]}, {"sent": "your gloves have fallen down ?", "words": ["have", "down", "gloves", "your"]}, {"sent": "eventually but it wont come off your gloves .", "words": ["off", "your", "it", "gloves", "but"]}, {"sent": "FiremanSam said if you havent got gloves put your socks on your hands .", "words": ["put", "your", "if", "you", "gloves", "on"]}, {"sent": "but shes got gloves on hasnt she ?", "words": ["on", "gloves", "she", "but"]}, {"sent": "where are your gloves ?", "words": ["where", "are", "your", "gloves"]}, {"sent": "you dont wanna wear your gloves ?", "words": ["your", "you", "dont", "gloves", "wanna"]}, {"sent": "gloves are off .", "words": ["off", "gloves", "are"]}, {"sent": "yeah but its the second pair of gloves Ive lost this winter .", "words": ["the", "of", "this", "gloves", "but"]}, {"sent": "you put your gloves on .", "words": ["put", "your", "you", "gloves", "on"]}, {"sent": "a pair of gloves .", "words": ["of", "gloves", "a"]}, {"sent": "has he got gloves on ?", "words": ["on", "gloves", "he"]}, {"sent": "theyre called gloves ?", "words": ["gloves"]}, {"sent": "didnt wanna buy anymore gloves right away .", "words": ["away", "gloves", "buy", "wanna"]}, {"sent": "Mummy usually puts rubber gloves on when shes cleaning shoes .", "words": ["on", "gloves", "when"]}, {"sent": "your gloves .", "words": ["gloves", "your"]}, {"sent": "hat and gloves .", "words": ["hat", "gloves", "and"]}, {"sent": "youre putting gloves on .", "words": ["on", "gloves"]}, {"sent": "the gloves gloves gloves gloves .", "words": ["the", "gloves"]}, {"sent": "you have special gloves on do you ?", "words": ["do", "you", "have", "gloves", "on"]}, {"sent": "youve got two gloves because youve got one hand inside Mummys shoe and the other inside an empty box .", "words": ["the", "box", "hand", "inside", "shoe", "and", "other", "an", "gloves", "empty", "because"]}, {"sent": "and then I went and got some gloves and he says .", "words": ["and", "gloves", "he", "some", "then"]}, {"sent": "shes wearing your gloves as well Thomas .", "words": ["gloves", "your"]}, {"sent": "where is the gloves pinkie ?", "words": ["where", "the", "gloves", "is"]}, {"sent": "got your gloves ?", "words": ["gloves", "your"]}, {"sent": "he was also wearing gloves .", "words": ["was", "gloves", "he"]}, {"sent": "no you call those gloves .", "words": ["gloves", "those", "you"]}, {"sent": "right and now weve got to use these oven gloves because this trays very hot .", "words": ["these", "to", "and", "this", "hot", "gloves", "oven", "because"]}, {"sent": "put your gloves in your pocket and lets go .", "words": ["put", "your", "go", "and", "in", "gloves"]}, {"sent": "okay you changed your mind you want your gloves on .", "words": ["on", "gloves", "your", "you"]}, {"sent": "you have pretty gloves ?", "words": ["have", "gloves", "pretty", "you"]}, {"sent": "they dont mention gloves do they ?", "words": ["dont", "they", "do", "gloves"]}, {"sent": "I think gloves would be best .", "words": ["be", "think", "gloves", "would"]}, {"sent": "get your gloves .", "words": ["get", "gloves", "your"]}, {"sent": "the goose is wearing gloves .", "words": ["the", "goose", "gloves", "is"]}, {"sent": "that means you have to chase me .", "words": ["to", "you", "have", "chase", "me", "that"]}, {"sent": "what can you why can you chase things away Tony ?", "words": ["can", "what", "you", "away", "chase", "why"]}, {"sent": "its about the great the great cat chase .", "words": ["the", "cat", "chase", "about"]}, {"sent": "the giant woke up and started to chase after Jack .", "words": ["the", "up", "to", "and", "chase"]}, {"sent": "so they chase him in another taxi .", "words": ["so", "another", "they", "him", "in", "chase"]}, {"sent": "theyre gonna chase the cat again .", "words": ["chase", "the", "cat"]}, {"sent": "ah haha anything that is mechanical in nature and anything that he can kinda chase after .", "words": ["can", "is", "and", "in", "he", "chase", "that"]}, {"sent": "well but you always but you sa but you be cats so you can chase him .", "words": ["can", "be", "so", "you", "him", "chase", "but"]}, {"sent": "its about some people who chase a cat .", "words": ["cat", "about", "who", "chase", "some", "a"]}, {"sent": "in the summertime when the butterflys are fluttering in the garden have you ever seen Purdie chase one ?", "words": ["the", "you", "have", "garden", "in", "are", "chase", "when"]}, {"sent": "well chase him .", "words": ["chase", "him"]}, {"sent": "dont they chase eachother ?", "words": ["chase", "dont", "they"]}, {"sent": "oh we decided it was to chase burglars in high places .", "words": ["we", "to", "was", "in", "it", "high", "chase"]}, {"sent": "hm chase after the noonoo ?", "words": ["chase", "the"]}, {"sent": "right well have them chase us okay .", "words": ["have", "us", "chase", "them"]}, {"sent": "I love to chase after small fishes it keeps me from getting too bored .", "words": ["to", "it", "love", "too", "chase", "me"]}, {"sent": "come on come on Im not going to chase you .", "words": ["to", "chase", "you", "on", "not"]}, {"sent": "chase him away .", "words": ["chase", "him", "away"]}, {"sent": "okay and they go to chase the cat he says .", "words": ["the", "go", "to", "and", "they", "cat", "he", "chase"]}, {"sent": "Im going to chase you .", "words": ["chase", "to", "you"]}, {"sent": "cmon lets chase him yall .", "words": ["chase", "him"]}, {"sent": "Im not going to chase you .", "words": ["chase", "to", "not", "you"]}, {"sent": "so you took the croquet mallets to break them apart and chase them away and you put her in the car .", "words": ["the", "put", "so", "to", "car", "them", "her", "you", "and", "in", "away", "chase", "break"]}, {"sent": "I chase the baby ?", "words": ["chase", "the"]}, {"sent": "though I bet dogs chase squirrels .", "words": ["chase"]}, {"sent": "but where are you going to chase him ?", "words": ["where", "to", "you", "him", "are", "chase", "but"]}, {"sent": "is he going to chase you ?", "words": ["is", "to", "you", "he", "chase"]}, {"sent": "so they started to chase the cat .", "words": ["the", "to", "so", "they", "cat", "chase"]}, {"sent": "well thats why the shark cant really explain why he wants to chase Dori .", "words": ["the", "to", "he", "chase", "why"]}, {"sent": "back off or Ill chase you down .", "words": ["off", "back", "you", "chase", "down"]}, {"sent": "is he going to chase the Star trooper with his blaster ?", "words": ["the", "is", "to", "with", "his", "he", "chase"]}, {"sent": "to chase them outof that barn ?", "words": ["chase", "to", "that", "them"]}, {"sent": "okay now what do I do I hafta chase you .", "words": ["chase", "what", "do", "you"]}, {"sent": "did you chase him away ?", "words": ["you", "him", "away", "did", "chase"]}, {"sent": "hes coming to chase you Thomas .", "words": ["chase", "to", "you"]}, {"sent": "hes really giving them a big chase oh okay and look at him up the tree .", "words": ["up", "the", "tree", "them", "and", "him", "at", "look", "chase", "big", "a"]}, {"sent": "shall we make the big dog chase the little dog .", "words": ["the", "we", "little", "make", "chase", "big", "dog"]}, {"sent": "sometimes it was true he did chase mice but maybe because it was Christmas or maybe because he was remembering the time when he had played with the children by the farmhouse Christmas tree he said the mice could borrow his toy .", "words": ["the", "tree", "was", "with", "because", "by", "his", "toy", "it", "he", "did", "chase", "could", "when", "but"]}, {"sent": "they chase him throught the pipe .", "words": ["chase", "they", "him", "the"]}, {"sent": "we must chase away the fox .", "words": ["chase", "the", "away", "we"]}, {"sent": "when she gets out of the front door sometimes she tries to chase Toby doesnt she ?", "words": ["the", "of", "to", "out", "chase", "when", "she", "door"]}, {"sent": "do chase us cried Molly mouse .", "words": ["chase", "do", "mouse", "us"]}, {"sent": "did he chase him with his walking stick ?", "words": ["stick", "with", "his", "him", "he", "did", "chase"]}, {"sent": "and Chickys gonna chase them .", "words": ["chase", "them", "and"]}, {"sent": "no dont chase him Diandra he was playing with that .", "words": ["was", "with", "dont", "him", "he", "chase", "that"]}, {"sent": "what do you do when those geese chase you ?", "words": ["what", "do", "those", "you", "chase", "when"]}, {"sent": "chase Charlotte ?", "words": ["chase"]}, {"sent": "now we will chase all of you .", "words": ["all", "we", "of", "chase", "you", "will"]}, {"sent": "now usually the magpie chase mice but maybe because it was Christmas or maybe because she too had been admiring the Christmas tree in the window she let go of the other end of the ribbon and Molly took it away .", "words": ["the", "tree", "go", "of", "was", "other", "and", "in", "it", "away", "but", "too", "chase", "window", "she", "because"]}, {"sent": "I think you should chase this monster away .", "words": ["think", "you", "this", "away", "chase"]}, {"sent": "hell chase you if you go into his field .", "words": ["if", "into", "go", "you", "his", "chase"]}, {"sent": "oh the meat eater could chase us it could happen .", "words": ["the", "meat", "us", "it", "chase", "could"]}, {"sent": "so I guess the chase well Ill chase her down the hall .", "words": ["the", "so", "chase", "down", "her"]}, {"sent": "Peter lost something in the chase .", "words": ["the", "in", "chase"]}, {"sent": "did Daddy chase the spider ?", "words": ["chase", "the", "did"]}, {"sent": "no its usually the cats that chase the fish uhn isnt it ?", "words": ["the", "it", "fish", "chase", "that"]}, {"sent": "think youd like me to chase that dont you ?", "words": ["think", "to", "like", "you", "dont", "chase", "me", "that"]}, {"sent": "dont chase him off Thomas .", "words": ["chase", "dont", "him", "off"]}, {"sent": "I chase the butterflies and the butterflies chase me .", "words": ["chase", "the", "me", "and"]}, {"sent": "she does like um she has a couple of balls she does like to to chase those around and watch them roll .", "words": ["does", "watch", "those", "of", "to", "them", "like", "and", "around", "chase", "she", "a"]}, {"sent": "she says chase my chase my child and .", "words": ["chase", "my", "she", "and"]}, {"sent": "the animals all chase the rabbit ?", "words": ["the", "all", "chase"]}, {"sent": "shell chase it round the toybox .", "words": ["chase", "the", "it"]}, {"sent": "oh Ive got to make him chase the tortoise .", "words": ["the", "to", "him", "chase", "make"]}, {"sent": "itll hurt if I hafta chase you .", "words": ["hurt", "chase", "if", "you"]}, {"sent": "I cant chase you anymore .", "words": ["chase", "you"]}, {"sent": "she plays pattycake and will play um like Ill chase her or shell youknow like Ill hide and well creep up on each other .", "words": ["up", "hide", "each", "chase", "play", "like", "and", "other", "on", "will", "her", "she"]}, {"sent": "oh bad wolf and the big bad wolf is going to chase the wee pig ?", "words": ["the", "wolf", "is", "to", "bad", "and", "pig", "chase", "big"]}, {"sent": "all right Im going to chase after you .", "words": ["to", "all", "chase", "you"]}, {"sent": "are you trying to get him to chase you ?", "words": ["to", "you", "get", "him", "are", "chase"]}, {"sent": "ch chase .", "words": ["chase"]}, {"sent": "hafta be ever so careful so he doesnt get out and chase us .", "words": ["be", "so", "and", "get", "us", "he", "out", "careful", "chase"]}, {"sent": "did you chase the chickens ?", "words": ["chase", "the", "did", "you"]}, {"sent": "tiny tiger turns to chase her own tail round in circles she goes .", "words": ["tiny", "to", "in", "tiger", "chase", "her", "she"]}, {"sent": "did it chase you ?", "words": ["chase", "it", "did", "you"]}, {"sent": "Im not going chase you .", "words": ["chase", "not", "you"]}, {"sent": "does Chantilly chase your grandpas cat ?", "words": ["chase", "does", "your", "cat"]}, {"sent": "follow and chase that rabbit .", "words": ["chase", "that", "and"]}, {"sent": "wanna chase a cat ?", "words": ["chase", "cat", "wanna", "a"]}, {"sent": "but where are you going to chase him Thomas ?", "words": ["where", "to", "you", "him", "are", "chase", "but"]}, {"sent": "see she went out to chase the ball okay now we can go off in front .", "words": ["the", "can", "off", "we", "go", "to", "ball", "in", "out", "chase", "see", "she"]}, {"sent": "Im gonna let you get down and chase this ball all around .", "words": ["all", "and", "get", "you", "this", "ball", "around", "chase", "down"]}, {"sent": "and Im going to chase the crocodile .", "words": ["chase", "to", "the", "and"]}, {"sent": "chase the little dog .", "words": ["chase", "the", "dog", "little"]}, {"sent": "wanna chase a cat ?", "words": ["chase", "cat", "wanna", "a"]}, {"sent": "xxx we hafta chase this car .", "words": ["chase", "this", "car", "we"]}, {"sent": "daddys going to chase the lions .", "words": ["chase", "to", "the"]}, {"sent": "it is a chase game isnt it ?", "words": ["game", "is", "it", "chase", "a"]}, {"sent": "say dont chase my cat .", "words": ["my", "say", "dont", "cat", "chase"]}, {"sent": "hes just learning how to drive it very fast so he can chase people .", "words": ["can", "how", "fast", "to", "so", "drive", "it", "he", "chase"]}, {"sent": "is he going to chase you ?", "words": ["is", "to", "you", "he", "chase"]}, {"sent": "we chase them all away .", "words": ["all", "we", "them", "away", "chase"]}, {"sent": "Im gonna chase you .", "words": ["chase", "you"]}, {"sent": "Im going to chase you .", "words": ["chase", "to", "you"]}, {"sent": "you gonna put him in a play pen .", "words": ["put", "play", "play pen", "you", "in", "him", "pen", "a"]}, {"sent": "this is a play pen .", "words": ["is", "play", "play pen", "this", "pen", "a"]}, {"sent": "put her in the play pen .", "words": ["the", "put", "play", "play pen", "in", "pen", "her"]}, {"sent": "by the librarians desk there is a play pen with a baby in it the baby turns the page in a cloth book but the book is upside down .", "words": ["the", "is", "play", "with", "play pen", "by", "there", "in", "it", "pen", "book", "down", "but", "a"]}, {"sent": "lets put him in this play pen .", "words": ["put", "play", "play pen", "this", "in", "him", "pen"]}, {"sent": "and sometimes she plays in her uh play pen at home .", "words": ["play", "play pen", "and", "in", "at", "pen", "her", "she", "home"]}, {"sent": "yours went under the play pen .", "words": ["the", "play", "play pen", "under", "pen"]}, {"sent": "its in the play pen .", "words": ["the", "play", "play pen", "in", "pen"]}, {"sent": "look at the baby beds and the play pen and the Mommy .", "words": ["the", "look", "pen", "play", "play pen", "and", "at"]}, {"sent": "hm a play pen .", "words": ["pen", "play", "play pen", "a"]}, {"sent": "did you drop the pretzel ?", "words": ["the", "pretzel", "you", "drop", "did"]}, {"sent": "or a pretzel ?", "words": ["pretzel", "a"]}, {"sent": "theyre in pieces thats not a whole pretzel they fell apart .", "words": ["pretzel", "they", "in", "not", "a"]}, {"sent": "cooking my pretzel ?", "words": ["my", "pretzel"]}, {"sent": "a pretzel cigar huh .", "words": ["pretzel", "a"]}, {"sent": "a little of pretzel .", "words": ["of", "pretzel", "little", "a"]}, {"sent": "it there any one that I did not give a pretzel to ?", "words": ["pretzel", "to", "any", "there", "it", "did", "give", "that", "not", "a"]}, {"sent": "anyone needs a pretzel xxx ?", "words": ["pretzel", "a"]}, {"sent": "Ill have a pretzel .", "words": ["have", "pretzel", "a"]}, {"sent": "maybe the alligator wants to eat some of your pretzel .", "words": ["the", "your", "pretzel", "of", "to", "eat", "alligator", "some"]}, {"sent": "mhm pretzel .", "words": ["pretzel"]}, {"sent": "Peter youve got pretzel .", "words": ["pretzel"]}, {"sent": "would you like a pretzel ?", "words": ["pretzel", "like", "you", "would", "a"]}, {"sent": "no pretzel ?", "words": ["pretzel"]}, {"sent": "she finished the pretzel she had .", "words": ["the", "she", "pretzel"]}, {"sent": "look I made you a pretzel .", "words": ["a", "pretzel", "you", "look"]}, {"sent": "\u8fd8 \u6709 \u4e00 \u4e2a pretzel \u5462 ?", "words": ["pretzel"]}, {"sent": "what are you doing to that pretzel ?", "words": ["what", "pretzel", "to", "you", "are", "that"]}, {"sent": "I get a pretzel every .", "words": ["get", "pretzel", "every", "a"]}, {"sent": "you want a pretzel ?", "words": ["pretzel", "a", "you"]}, {"sent": "pretzel sticks ?", "words": ["pretzel"]}, {"sent": "feed your doggie a pretzel .", "words": ["pretzel", "your", "a", "feed"]}, {"sent": "can you feed that dog a pretzel ?", "words": ["can", "pretzel", "you", "feed", "that", "dog", "a"]}, {"sent": "a pretzel .", "words": ["pretzel", "a"]}, {"sent": "look at that pretzel .", "words": ["pretzel", "at", "that", "look"]}, {"sent": "another pretzel ?", "words": ["another", "pretzel"]}, {"sent": "and pretzel .", "words": ["pretzel", "and"]}, {"sent": "pretzel thats right .", "words": ["pretzel"]}, {"sent": "pretzel and cheerio .", "words": ["and", "pretzel"]}, {"sent": "would you like a pretzel ?", "words": ["pretzel", "like", "you", "would", "a"]}, {"sent": "I have pretzels do you want a pretzel ?", "words": ["do", "pretzel", "you", "have", "a"]}, {"sent": "heres a pretzel want this ?", "words": ["this", "pretzel", "a"]}, {"sent": "pretzel .", "words": ["pretzel"]}, {"sent": "pretzel .", "words": ["pretzel"]}, {"sent": "you have a pretzel here and I want you to have more sandwich .", "words": ["pretzel", "to", "you", "have", "here", "and", "more", "sandwich", "a"]}, {"sent": "theres a pretzel there .", "words": ["there", "pretzel", "a"]}, {"sent": "wouldja like another pretzel ?", "words": ["another", "like", "pretzel"]}, {"sent": "you can eat the pretzel .", "words": ["can", "the", "pretzel", "you", "eat"]}, {"sent": "pretzel ?", "words": ["pretzel"]}, {"sent": "did you give him a pretzel ?", "words": ["pretzel", "you", "him", "did", "give", "a"]}, {"sent": "pretzel .", "words": ["pretzel"]}, {"sent": "the pretzel ?", "words": ["the", "pretzel"]}, {"sent": "pretzel ?", "words": ["pretzel"]}, {"sent": "look theres a pretzel .", "words": ["pretzel", "a", "look"]}, {"sent": "Gabriella look a pretzel truck ?", "words": ["truck", "pretzel", "a", "look"]}, {"sent": "a pretzel ?", "words": ["pretzel", "a"]}, {"sent": "youre not a pretzel .", "words": ["pretzel", "not", "a"]}, {"sent": "how come we only get a little bit of pretzel ?", "words": ["how", "we", "of", "pretzel", "little", "get", "a"]}, {"sent": "Im a Spiderman because Im my pretzel is tall .", "words": ["my", "pretzel", "is", "because", "a"]}, {"sent": "Im talking to your pretzel .", "words": ["to", "your", "pretzel"]}, {"sent": "Im the tallest pretzel in the whole wide world .", "words": ["the", "in", "pretzel"]}, {"sent": "are you having pretzel and gum at the same time ?", "words": ["the", "pretzel", "and", "you", "are", "same", "gum", "at"]}, {"sent": "Peter you left a pretzel on the floor .", "words": ["the", "pretzel", "you", "on", "a"]}, {"sent": "oh would you like a pretzel ?", "words": ["pretzel", "like", "you", "would", "a"]}, {"sent": "why dont you ask Patsy and Loi uh Patsy and Lyn if theyd like a pretzel .", "words": ["if", "pretzel", "like", "and", "you", "dont", "why", "a"]}, {"sent": "pretzel .", "words": ["pretzel"]}, {"sent": "its a pretzel .", "words": ["pretzel", "a"]}, {"sent": "maybe the alligator wants to eat your pretzel not you eating the gorilla .", "words": ["the", "your", "pretzel", "to", "you", "eat", "alligator", "not"]}, {"sent": "no dont put that on your mouth its not as good as the pretzel its not as salty is your pretzel salty ?", "words": ["the", "put", "your", "pretzel", "is", "mouth", "dont", "good", "on", "that", "not"]}, {"sent": "you want a pretzel ?", "words": ["pretzel", "a", "you"]}, {"sent": "okay Ill give you one pretzel .", "words": ["give", "pretzel", "you"]}, {"sent": "you want a pretzel ?", "words": ["pretzel", "a", "you"]}, {"sent": "the pretzel .", "words": ["the", "pretzel"]}, {"sent": "would you like a tissue to um wipe your hands that you just sneezed pretzel upon ?", "words": ["your", "tissue", "pretzel", "to", "like", "you", "would", "wipe", "that", "a"]}, {"sent": "oh you want a a pretzel ?", "words": ["pretzel", "a", "you"]}, {"sent": "its a pretzel .", "words": ["pretzel", "a"]}, {"sent": "okay its a pretty good pretzel .", "words": ["pretzel", "pretty", "good", "a"]}, {"sent": "you want a pretzel ?", "words": ["pretzel", "a", "you"]}, {"sent": "its a pretzel .", "words": ["pretzel", "a"]}, {"sent": "say pretzel .", "words": ["say", "pretzel"]}, {"sent": "a pretzel .", "words": ["pretzel", "a"]}, {"sent": "\u4e70 \u4e2a pretzel \u597d\u4e0d\u597d ?", "words": ["pretzel"]}, {"sent": "\u90a3\u4e2a pretzel \u54ea\u513f \u53bb \u4e86 ?", "words": ["pretzel"]}, {"sent": "pretzel \u5462 ?", "words": ["pretzel"]}, {"sent": "\u5bf9 \u8fd9 \u662f pretzel .", "words": ["pretzel"]}, {"sent": "\u90a3 \u5c31 \u518d \u4e70 \u70b9\u513f pretzel \u5427 .", "words": ["pretzel"]}, {"sent": "\u8fd8 \u6709 \u8fd9\u4e2a pretzel \u8981 \u653e \u597d .", "words": ["pretzel"]}, {"sent": "its a pretzel .", "words": ["pretzel", "a"]}, {"sent": "you want a pretzel ?", "words": ["pretzel", "a", "you"]}, {"sent": "you want a whole huge pretzel ?", "words": ["pretzel", "a", "you"]}, {"sent": "what are you doing to that pretzel ?", "words": ["what", "pretzel", "to", "you", "are", "that"]}, {"sent": "but um why didnt you want the pretzel Grandma bought you ?", "words": ["the", "pretzel", "you", "but", "why"]}, {"sent": "well get some cheese and a pretzel ?", "words": ["pretzel", "and", "get", "some", "cheese", "a"]}, {"sent": "thats the only thing I forgot to tell them today is not to pick up his clothes because I gotta to put my clothes in the dryer .", "words": ["the", "up", "put", "my", "is", "pick", "to", "them", "in", "his", "not", "because", "dryer"]}, {"sent": "finally the dryer stopped whirling and the man gathered up the clothes .", "words": ["the", "up", "and", "dryer"]}, {"sent": "there are no clothes in the dryer .", "words": ["the", "there", "in", "are", "dryer"]}, {"sent": "can you hear the dryer ?", "words": ["can", "the", "hear", "you", "dryer"]}, {"sent": "soap cost a dollar the washer cost a dollar and the dryer cost another dollar .", "words": ["the", "soap", "and", "another", "a", "dryer"]}, {"sent": "I think I hear that the washing machine stopped and that means I need to go and put the laundry thats in the washing machine into the dryer .", "words": ["the", "think", "put", "need", "into", "go", "need to", "to", "and", "hear", "washing machine", "in", "that", "dryer"]}, {"sent": "that was what we put in the dryer to make the things smell good .", "words": ["the", "what", "put", "we", "was", "to", "in", "make", "good", "that", "dryer"]}, {"sent": "is that the blow dryer ?", "words": ["the", "is", "that", "blow", "dryer"]}, {"sent": "you got a washing machine and dryer didnt you ?", "words": ["washing machine", "and", "you", "a", "dryer"]}, {"sent": "from when you climbed onto the dryer .", "words": ["the", "dryer", "when", "you"]}, {"sent": "thats the hair dryer bonnet .", "words": ["the", "hair", "dryer"]}, {"sent": "why did you hafta use the hair dryer ?", "words": ["the", "you", "did", "hair", "why", "dryer"]}, {"sent": "theyll shrink in the dryer .", "words": ["the", "in", "dryer"]}, {"sent": "theres the dryer .", "words": ["the", "dryer"]}, {"sent": "the fucking dryer was just fixed yesterday .", "words": ["the", "was", "dryer"]}, {"sent": "you help me put the clothes in the dryer .", "words": ["the", "help", "put", "you", "in", "me", "dryer"]}, {"sent": "let us go see if our clothes are dry in the dryer yet .", "words": ["the", "if", "go", "dry", "our", "us", "in", "are", "see", "dryer"]}, {"sent": "xxx shall I dry it with a dryer now ?", "words": ["dry", "with", "it", "a", "dryer"]}, {"sent": "not the dryer .", "words": ["the", "not", "dryer"]}, {"sent": "get them out of the dryer .", "words": ["the", "of", "them", "get", "out", "dryer"]}, {"sent": "into the dryer .", "words": ["the", "into", "dryer"]}, {"sent": "I can hear the dryer .", "words": ["can", "the", "dryer", "hear"]}, {"sent": "wash clothing in warm or hot water okay mm check pockets for crayons before placing in dryer mm .", "words": ["for", "hot", "in", "water", "wash", "dryer"]}, {"sent": "lets put the diapers in the dryer .", "words": ["the", "in", "put", "dryer"]}, {"sent": "boy that dryer is very loud .", "words": ["that", "loud", "is", "dryer"]}, {"sent": "Im putting the wet clothes into the dryer .", "words": ["the", "wet", "into", "dryer"]}, {"sent": "we dont have any covers I think all the covers are in the dryer so were gonna hafta use another paper diaper .", "words": ["the", "all", "think", "we", "diaper", "so", "paper", "any", "have", "dont", "in", "are", "another", "were", "dryer"]}, {"sent": "will you come with me so I can put these things in the dryer ?", "words": ["can", "these", "put", "the", "so", "with", "you", "in", "will", "me", "dryer"]}, {"sent": "out of the dryer .", "words": ["of", "the", "out", "dryer"]}, {"sent": "we do need to tug up more weeds well maybe who knows maybe today it wont rain very much and maybe it will get a little dryer and maybe we can do things outside .", "words": ["need to", "more", "can", "it", "who", "much", "up", "to", "and", "little", "will", "dryer", "a", "do", "need", "rain", "we", "outside", "get"]}, {"sent": "xxx turn on the dryer ?", "words": ["on", "the", "dryer"]}, {"sent": "here Im unna help you turn on the dryer .", "words": ["the", "help", "you", "here", "on", "dryer"]}, {"sent": "there were two different loads from the dryer in there .", "words": ["the", "there", "in", "were", "dryer"]}, {"sent": "it was two loads of xxx of the dryer .", "words": ["the", "of", "was", "it", "dryer"]}, {"sent": "uh I just think I just put it in the dryer .", "words": ["the", "think", "put", "in", "it", "dryer"]}, {"sent": "put your shoes in the dryer ?", "words": ["the", "put", "your", "in", "dryer"]}, {"sent": "why do they needta go in the dryer ?", "words": ["the", "do", "go", "they", "in", "why", "dryer"]}, {"sent": "Mummys dryer .", "words": ["dryer"]}, {"sent": "it honked at the hair dryer .", "words": ["the", "it", "at", "hair", "dryer"]}, {"sent": "no thats a lady with a hair dryer on her head .", "words": ["head", "with", "on", "hair", "her", "a", "dryer"]}, {"sent": "did you use the hair dryer last night ?", "words": ["the", "last", "you", "did", "hair", "dryer"]}, {"sent": "and later on remember to put them in the dryer .", "words": ["the", "put", "to", "them", "and", "in", "on", "dryer"]}, {"sent": "theres the dryer .", "words": ["the", "dryer"]}, {"sent": "lets finish putting everything in the dryer .", "words": ["the", "in", "finish", "dryer"]}, {"sent": "put all the clothes in the dryer .", "words": ["the", "all", "put", "in", "dryer"]}, {"sent": "do you think theyll all fit in the dryer ?", "words": ["the", "all", "do", "think", "fit", "you", "in", "dryer"]}, {"sent": "its a dryer .", "words": ["dryer", "a"]}, {"sent": "the dryer ?", "words": ["the", "dryer"]}, {"sent": "I mean xxx dryer .", "words": ["dryer"]}, {"sent": "do you remember when you had your hair dried with a hair dryer ?", "words": ["do", "dryer", "your", "with", "you", "hair", "when", "a"]}, {"sent": "because its not for squirting on the spin dryer .", "words": ["the", "for", "on", "not", "because", "dryer"]}, {"sent": "yeah but before it was fixed you werent even washing them before before it was fixed I washed them all except for the white wash and thats when the dryer br .", "words": ["the", "all", "for", "when", "was", "them", "you", "and", "it", "white", "wash", "but", "dryer"]}, {"sent": "and I fixed the uh dryer .", "words": ["the", "dryer", "and"]}, {"sent": "yeah xxx the ah dryer .", "words": ["the", "dryer"]}, {"sent": "I was so involved in the thing that I forget to take them off from the hair dryer .", "words": ["the", "off", "was", "so", "to", "them", "in", "take", "hair", "that", "dryer"]}, {"sent": "I hafta put the wash in the dryer .", "words": ["the", "put", "in", "wash", "dryer"]}, {"sent": "oh you just leave this dryer alone .", "words": ["this", "dryer", "you"]}, {"sent": "put them in the dryer .", "words": ["the", "put", "them", "in", "dryer"]}, {"sent": "put the clothes in the dryer ?", "words": ["the", "in", "put", "dryer"]}, {"sent": "in the dryer .", "words": ["the", "in", "dryer"]}, {"sent": "he gonna put two articles in the dryer .", "words": ["the", "put", "in", "he", "dryer"]}, {"sent": "stop using this dryer so much .", "words": ["so", "stop", "this", "much", "dryer"]}, {"sent": "dryer is going .", "words": ["is", "dryer"]}, {"sent": "thats Daddys hair dryer .", "words": ["hair", "dryer"]}, {"sent": "he blows the baby dry with the hair dryer .", "words": ["the", "dry", "with", "he", "hair", "dryer"]}, {"sent": "I think Im gonna hafta put the washing in the dryer before it gets too late .", "words": ["the", "think", "put", "in", "it", "too", "dryer"]}, {"sent": "Im gonna hafta put the washing in the dryer before it gets too late .", "words": ["the", "put", "in", "it", "too", "dryer"]}, {"sent": "but if it doesnt go in the dryer now there wont wont be time .", "words": ["the", "if", "go", "be", "there", "in", "it", "but", "dryer"]}, {"sent": "you can be official dryer ?", "words": ["can", "be", "dryer", "you"]}, {"sent": "going to put this stuff in the dryer .", "words": ["the", "put", "to", "this", "in", "dryer"]}, {"sent": "she said shes got to go to the dryer .", "words": ["the", "go", "to", "she", "dryer"]}, {"sent": "its the clothes dryer .", "words": ["the", "dryer"]}, {"sent": "weve got a clothes hanger dryer havent we ?", "words": ["dryer", "we", "a"]}, {"sent": "well well wait till its a little bit dryer than shall we ?", "words": ["wait", "we", "little", "a", "dryer"]}, {"sent": "or it could be somebodys tumble dryer and they hafta have a pipe outside to let the .", "words": ["the", "be", "outside", "to", "and", "have", "they", "it", "could", "a", "dryer"]}, {"sent": "and I think the most exciting thing for you was washing your hands in the cloakroom using the dryer .", "words": ["the", "think", "for", "your", "was", "you", "and", "in", "dryer"]}, {"sent": "well hes fixing the dryer Jerry .", "words": ["the", "dryer"]}, {"sent": "oh that sounds just the its just the dryer working .", "words": ["the", "that", "dryer"]}, {"sent": "hey what was Bailey doing on the dryer ?", "words": ["the", "what", "was", "on", "dryer"]}, {"sent": "is that your dryer ?", "words": ["your", "dryer", "that", "is"]}, {"sent": "Id better wash and xxx take those clothes down in the dryer xxx .", "words": ["the", "better", "those", "wash", "and", "in", "down", "take", "dryer"]}, {"sent": "dryer .", "words": ["dryer"]}, {"sent": "just put those little underpants in .", "words": ["put", "those", "little", "in", "underpants"]}, {"sent": "youre such a big girl and you do such a good job making weewee that we can get you some real underpants without any rubber on them .", "words": ["can", "do", "we", "any", "them", "you", "and", "get", "that", "underpants", "good", "on", "some", "big", "a"]}, {"sent": "gonna put all these underpants .", "words": ["all", "put", "underpants", "these"]}, {"sent": "underpants down .", "words": ["down", "underpants"]}, {"sent": "and are you gonna wear big girl underpants one day ?", "words": ["you", "and", "underpants", "are", "big"]}, {"sent": "remember that Phoebe was wearing her big girl underpants with Hello Kitty on them ?", "words": ["was", "with", "them", "that", "underpants", "big", "on", "her"]}, {"sent": "he needs some underpants .", "words": ["some", "underpants", "he"]}, {"sent": "and she wears big girl underpants .", "words": ["underpants", "big", "she", "and"]}, {"sent": "and shes like what colors your underpants ?", "words": ["what", "your", "like", "and", "underpants"]}, {"sent": "whats Thomass underpants doing in here ?", "words": ["here", "in", "underpants"]}, {"sent": "I meant to change your underpants uhn didnt I ?", "words": ["to", "underpants", "your"]}, {"sent": "it looks like underpants .", "words": ["underpants", "it", "like"]}, {"sent": "underpants pantyhose stockings petticoats slips and bras .", "words": ["underpants", "and"]}, {"sent": "oh look theres Fraser in his underpants .", "words": ["his", "in", "underpants", "look"]}, {"sent": "what does she do when she takes her underpants off ?", "words": ["off", "what", "do", "does", "her", "underpants", "when", "she"]}, {"sent": "why has Jake been on the television in his underpants Thomas ?", "words": ["the", "his", "in", "underpants", "on", "why"]}, {"sent": "let me go and just get some underpants stay there .", "words": ["stay", "go", "and", "get", "there", "underpants", "me", "some"]}, {"sent": "maybe they do they think they wanna wear princess underpants those goblins ?", "words": ["do", "think", "those", "they", "underpants", "wanna"]}, {"sent": "youre putting your underpants on my foot now are you ?", "words": ["my", "your", "you", "foot", "underpants", "are", "on"]}, {"sent": "how can I pull up underpants if theyre not on ?", "words": ["can", "up", "how", "if", "pull", "underpants", "on", "not"]}, {"sent": "youve got your underpants on the beach uhn havent you ?", "words": ["the", "your", "you", "beach", "underpants", "on"]}, {"sent": "and lets have a look at your new underpants as well .", "words": ["your", "and", "have", "at", "underpants", "look", "new", "a"]}, {"sent": "shes putting her underpants on .", "words": ["on", "underpants", "her"]}, {"sent": "underpants .", "words": ["underpants"]}, {"sent": "youve only got your underpants on havent you ?", "words": ["on", "underpants", "your", "you"]}, {"sent": "theyre called underpants .", "words": ["underpants"]}, {"sent": "are those the dolls underpants ?", "words": ["the", "underpants", "are", "those"]}, {"sent": "yeah lets put the blue one on the guy with the red underpants and the red one on the guy with the blue underpants .", "words": ["the", "red", "put", "with", "blue", "and", "underpants", "on"]}, {"sent": "thats her underpants .", "words": ["underpants", "her"]}, {"sent": "huh whats the matter with your underpants ?", "words": ["the", "underpants", "with", "your"]}, {"sent": "hes look at his underpants whats on his underpants ?", "words": ["look", "on", "underpants", "at", "his"]}, {"sent": "Sweetie are you gonna wear big girl underpants one day ?", "words": ["underpants", "are", "big", "you"]}, {"sent": "do orange underpants ?", "words": ["do", "underpants", "orange"]}, {"sent": "thats your socks and your underpants .", "words": ["underpants", "your", "and"]}, {"sent": "mm hm keep on your underpants .", "words": ["on", "underpants", "your"]}, {"sent": "why dont you put it back in your underpants .", "words": ["put", "your", "back", "you", "dont", "in", "it", "underpants", "why"]}, {"sent": "underpants wow .", "words": ["underpants"]}, {"sent": "and howcome there are so many pairs of underpants in the wash ?", "words": ["the", "of", "so", "and", "there", "in", "are", "underpants", "wash"]}, {"sent": "underpants .", "words": ["underpants"]}, {"sent": "take your shirt off but keep underpants on .", "words": ["off", "your", "shirt", "underpants", "on", "take", "but"]}, {"sent": "go and get some underpants .", "words": ["go", "and", "get", "underpants", "some"]}, {"sent": "underpants .", "words": ["underpants"]}, {"sent": "you left your underpants on .", "words": ["on", "underpants", "your", "you"]}, {"sent": "oh theres her underpants .", "words": ["underpants", "her"]}, {"sent": "shes wearing underpants does that mean she sits on the potty ?", "words": ["the", "does", "underpants", "potty", "on", "that", "she"]}, {"sent": "do you know your underpants is inside out sweetheart ?", "words": ["do", "your", "is", "inside", "you", "underpants", "out"]}, {"sent": "a pair of Daddys underpants on your head .", "words": ["your", "of", "head", "underpants", "on", "a"]}, {"sent": "theyre Daddys underpants arent they ?", "words": ["they", "underpants"]}, {"sent": "shes got underpants hm ?", "words": ["underpants"]}, {"sent": "well you need underpants yeah .", "words": ["underpants", "need", "you"]}, {"sent": "Catherine please put on your underpants .", "words": ["on", "put", "underpants", "your"]}, {"sent": "whove we got on the underpants ?", "words": ["on", "the", "underpants", "we"]}, {"sent": "do big girls use underpants and use the potty ?", "words": ["the", "do", "and", "underpants", "potty", "big"]}, {"sent": "looks like Isabelles wearing her underpants too .", "words": ["underpants", "her", "like", "too"]}, {"sent": "big boy underpants wanna try the puzzle ?", "words": ["the", "big", "underpants", "puzzle", "wanna", "try"]}, {"sent": "hes in his little underpants .", "words": ["his", "in", "underpants", "little"]}, {"sent": "theyre all underpants that I use .", "words": ["all", "underpants", "that"]}, {"sent": "and shes wearing underpants for a little while .", "words": ["for", "little", "and", "underpants", "a"]}, {"sent": "underpants .", "words": ["underpants"]}, {"sent": "I dont know I dont know how to say underpants in spanish .", "words": ["how", "to", "say", "dont", "in", "underpants"]}, {"sent": "big boy underpants .", "words": ["underpants", "big"]}, {"sent": "theyre my underpants and theyre nothing wrong one of mine you cheeky monkey .", "words": ["my", "of", "monkey", "and", "you", "mine", "underpants"]}, {"sent": "youve got a thing about Daddys underpants .", "words": ["underpants", "about", "a"]}, {"sent": "but the last time we got some underpants they had footballs on and aeroplanes and buss and things .", "words": ["the", "we", "last", "and", "they", "underpants", "on", "some", "but"]}, {"sent": "are you wearing underpants for awhile ?", "words": ["underpants", "are", "for", "you"]}, {"sent": "underpants on .", "words": ["on", "underpants"]}, {"sent": "Superman wears his underpants on the outside of his trousers .", "words": ["the", "of", "outside", "on", "underpants", "his"]}, {"sent": "are you wearing big boy underpants right now ?", "words": ["underpants", "are", "big", "you"]}, {"sent": "put your underpants on before your trousers .", "words": ["on", "put", "underpants", "your"]}, {"sent": "ah looks like hes wearing his underpants .", "words": ["his", "underpants", "like"]}, {"sent": "she takes her underpants off .", "words": ["off", "underpants", "her", "she"]}, {"sent": "underpants go in the too big pile .", "words": ["the", "go", "in", "underpants", "too", "big"]}, {"sent": "underpants .", "words": ["underpants"]}, {"sent": "and look at his underpants Ethan .", "words": ["and", "underpants", "at", "look", "his"]}, {"sent": "new kinda diapers theyre like underpants .", "words": ["underpants", "new", "like"]}, {"sent": "do you wear your underpants under your pajamas ?", "words": ["do", "your", "you", "under", "underpants", "pajamas"]}, {"sent": "babas underpants .", "words": ["underpants"]}, {"sent": "is SleepingBeauty wearing underpants ?", "words": ["underpants", "is"]}, {"sent": "I dont think MrsWilson xxx you going round with underpants on your head do you ?", "words": ["do", "think", "your", "head", "with", "you", "dont", "underpants", "on"]}, {"sent": "with your underpants ?", "words": ["your", "underpants", "with"]}, {"sent": "I dont think you get free toys with underpants Thomas .", "words": ["think", "with", "you", "get", "dont", "underpants"]}, {"sent": "theres your underpants .", "words": ["underpants", "your"]}, {"sent": "thats a little dangerous my little boy who wears big boy underpants .", "words": ["my", "little", "underpants", "who", "big", "a"]}, {"sent": "wheres your gotta find your underpants first .", "words": ["underpants", "your", "first", "find"]}, {"sent": "okay lets find you some underpants .", "words": ["some", "underpants", "you", "find"]}, {"sent": "those are Daddys underpants .", "words": ["underpants", "are", "those"]}, {"sent": "Stella called Sam I cant find my underpants .", "words": ["my", "underpants", "find"]}, {"sent": "whats in your underpants ?", "words": ["in", "underpants", "your"]}, {"sent": "right and when youre a big girl youre going to wear underpants all the time right and not diapers the candle holders ?", "words": ["the", "all", "when", "to", "and", "underpants", "big", "not", "a"]}, {"sent": "they all need underpants .", "words": ["all", "they", "underpants", "need"]}, {"sent": "well Daddys underpants arent they ?", "words": ["they", "underpants"]}, {"sent": "she must be a big girl if shes wearing underpants right ?", "words": ["if", "be", "underpants", "big", "she", "a"]}, {"sent": "she has underpants too can I see ?", "words": ["can", "underpants", "too", "see", "she"]}, {"sent": "do we have any Wednesday underpants ?", "words": ["do", "we", "any", "have", "underpants"]}, {"sent": "lets put on some underpants .", "words": ["on", "some", "put", "underpants"]}, {"sent": "yes thats the cloud .", "words": ["the", "cloud"]}, {"sent": "do you know where a cloud goes ?", "words": ["where", "do", "you", "cloud", "a"]}, {"sent": "the cloud ?", "words": ["the", "cloud"]}, {"sent": "where is cloud ?", "words": ["where", "cloud", "is"]}, {"sent": "its starting to cloud ?", "words": ["to", "cloud"]}, {"sent": "cloud .", "words": ["cloud"]}, {"sent": "thats a cloud .", "words": ["cloud", "a"]}, {"sent": "the sun went behind a cloud .", "words": ["the", "sun", "behind", "cloud", "a"]}, {"sent": "with what color cloud ?", "words": ["what", "with", "cloud"]}, {"sent": "cloud City .", "words": ["cloud"]}, {"sent": "thats a cloud uhn isnt it in the sky ?", "words": ["the", "sky", "in", "it", "cloud", "a"]}, {"sent": "the cloud mustve blown away .", "words": ["the", "away", "cloud"]}, {"sent": "oh a cloud .", "words": ["cloud", "a"]}, {"sent": "a big big cloud ?", "words": ["big", "cloud", "a"]}, {"sent": "thats a cloud .", "words": ["cloud", "a"]}, {"sent": "your cloud is only a little cloud isnt he ?", "words": ["your", "is", "little", "he", "cloud", "a"]}, {"sent": "some of the bees fly round the cloud .", "words": ["of", "some", "cloud", "the"]}, {"sent": "lets see if it comes out of that cloud in a minute .", "words": ["if", "of", "in", "it", "out", "cloud", "see", "that", "a"]}, {"sent": "cloud .", "words": ["cloud"]}, {"sent": "so shes on cloud nine .", "words": ["on", "so", "cloud"]}, {"sent": "do you think its a cloud ?", "words": ["do", "think", "you", "cloud", "a"]}, {"sent": "youre drawing the cloud ?", "words": ["the", "cloud"]}, {"sent": "teletubbies followed the cloud indoors it was time for tubby .", "words": ["the", "for", "was", "it", "cloud"]}, {"sent": "I got my cloud .", "words": ["my", "cloud"]}, {"sent": "and a cloud .", "words": ["cloud", "a", "and"]}, {"sent": "the end of a cloud ?", "words": ["the", "of", "cloud", "a"]}, {"sent": "is it a cloud storm ?", "words": ["it", "cloud", "a", "is"]}, {"sent": "another cloud there .", "words": ["another", "there", "cloud"]}, {"sent": "whos behind the cloud ?", "words": ["the", "behind", "cloud"]}, {"sent": "cloud .", "words": ["cloud"]}, {"sent": "and the cloud can you find the cloud ?", "words": ["the", "can", "you", "and", "cloud", "find"]}, {"sent": "a cloud .", "words": ["cloud", "a"]}, {"sent": "the cloud the cloud became a slow turning disk and the disks center shrank and became the Sun .", "words": ["the", "slow", "and", "cloud", "a"]}, {"sent": "almost its a cloud .", "words": ["cloud", "a"]}, {"sent": "and thats a cloud .", "words": ["cloud", "a", "and"]}, {"sent": "and whats behind the cloud Fraser ?", "words": ["the", "behind", "cloud", "and"]}, {"sent": "heres a cloud in the sky .", "words": ["the", "sky", "in", "cloud", "a"]}, {"sent": "um its a cloud .", "words": ["cloud", "a"]}, {"sent": "look I have a cloud .", "words": ["have", "cloud", "a", "look"]}, {"sent": "thats for wining the balloon race said the cloud .", "words": ["the", "for", "balloon", "cloud"]}, {"sent": "I think the sun went behind a cloud didnt it ?", "words": ["the", "think", "sun", "behind", "it", "cloud", "a"]}, {"sent": "its cloud shaped in the sky .", "words": ["the", "in", "cloud", "sky"]}, {"sent": "but the teletubbies found it a little inconvenient having a cloud indoors .", "words": ["the", "little", "it", "cloud", "but", "a"]}, {"sent": "look at that big grey cloud over there Fraser .", "words": ["look", "there", "that", "cloud", "at", "over", "big"]}, {"sent": "oh but the storm cloud made the ground all wet and boots is stuck in the icky sticky sand .", "words": ["the", "all", "wet", "is", "sticky", "and", "boots", "in", "stuck", "cloud", "but"]}, {"sent": "thats a big black cloud in the sky .", "words": ["the", "sky", "black", "in", "cloud", "big", "a"]}, {"sent": "or cloud sheep .", "words": ["sheep", "cloud"]}, {"sent": "the solar system was once a cloud of dust and gas that stretched for more than a trillion miles .", "words": ["the", "for", "of", "was", "and", "more", "cloud", "that", "a"]}, {"sent": "youve got that one the cloud .", "words": ["the", "that", "cloud"]}, {"sent": "its a purple cloud isnt it ?", "words": ["it", "cloud", "a"]}, {"sent": "but theres a storm cloud and its going to rain .", "words": ["rain", "to", "and", "cloud", "but", "a"]}, {"sent": "on a cloud .", "words": ["on", "cloud", "a"]}, {"sent": "other particles from the cloud stuck together and formed planets which travel in roughly circular orbits around the Sun .", "words": ["the", "and", "other", "in", "stuck", "cloud", "around", "which"]}, {"sent": "wheres the cloud gone ?", "words": ["the", "cloud"]}, {"sent": "two sausages and a silver cloud .", "words": ["cloud", "a", "and"]}, {"sent": "cloud .", "words": ["cloud"]}, {"sent": "cloud dont rain on me .", "words": ["rain", "dont", "cloud", "on", "me"]}, {"sent": "no the sun went behind a cloud .", "words": ["the", "sun", "behind", "cloud", "a"]}, {"sent": "are we going up to in the sky to have our tea on a fluffy cloud ?", "words": ["up", "the", "we", "sky", "to", "our", "have", "in", "are", "cloud", "on", "a"]}, {"sent": "whos behind this cloud ?", "words": ["this", "behind", "cloud"]}, {"sent": "theres a cloud in the sky isnt there ?", "words": ["the", "sky", "there", "in", "cloud", "a"]}, {"sent": "that cloud too ?", "words": ["that", "cloud", "too"]}, {"sent": "that cloud looks like .", "words": ["that", "like", "cloud"]}, {"sent": "a wooly black faced cloud sheep .", "words": ["black", "sheep", "cloud", "a"]}, {"sent": "wh wh where is the where is the cloud ?", "words": ["where", "the", "cloud", "is"]}, {"sent": "the cloud gave a little puff .", "words": ["the", "little", "cloud", "a"]}, {"sent": "cloud .", "words": ["cloud"]}, {"sent": "a little white puffy cloud .", "words": ["white", "cloud", "little", "a"]}, {"sent": "Catherine did you ever try doing these shapes like the cloud or the tree or anything ?", "words": ["the", "these", "tree", "like", "you", "did", "cloud", "try"]}, {"sent": "and you can see the sun but its very much covered by cloud .", "words": ["can", "the", "sun", "you", "and", "by", "cloud", "see", "much", "but"]}, {"sent": "oh Dimitra look at this cloud .", "words": ["this", "at", "cloud", "look"]}, {"sent": "it looks like a cloud now .", "words": ["it", "like", "cloud", "a"]}, {"sent": "dont be afraid said hawk all that noise is just cloud giants playing drums in the sky .", "words": ["the", "all", "is", "sky", "be", "dont", "in", "cloud", "that"]}, {"sent": "cloud .", "words": ["cloud"]}, {"sent": "under that cloud ?", "words": ["under", "that", "cloud"]}, {"sent": "oh she said in that case a large helping for me please I love to eat cloud .", "words": ["for", "to", "eat", "in", "love", "cloud", "me", "that", "she", "a"]}, {"sent": "a white cloud .", "words": ["white", "cloud", "a"]}, {"sent": "byebye special pink tubbiecustard cloud .", "words": ["cloud"]}, {"sent": "you want a cloud ?", "words": ["cloud", "a", "you"]}, {"sent": "thats a cloud .", "words": ["cloud", "a"]}, {"sent": "cloud you know uhn dont you Nathan ?", "words": ["dont", "cloud", "you"]}, {"sent": "a big big cloud ?", "words": ["big", "cloud", "a"]}, {"sent": "do a cloud like this .", "words": ["do", "like", "this", "cloud", "a"]}, {"sent": "that look like a cloud ?", "words": ["like", "cloud", "look", "that", "a"]}, {"sent": "a cloud ?", "words": ["cloud", "a"]}, {"sent": "okay you make a cloud .", "words": ["make", "cloud", "a", "you"]}, {"sent": "youre making a a cloud ?", "words": ["cloud", "a"]}, {"sent": "a nice big blue cloud .", "words": ["nice", "blue", "cloud", "big", "a"]}, {"sent": "did you ever see a cloud sheep ?", "words": ["you", "sheep", "did", "cloud", "see", "a"]}, {"sent": "cloud .", "words": ["cloud"]}, {"sent": "is it a big big cloud ?", "words": ["is", "it", "cloud", "big", "a"]}, {"sent": "who is behind the cloud ?", "words": ["the", "is", "behind", "cloud", "who"]}, {"sent": "all of a sudden the suns come out of the cloud hasnt it ?", "words": ["the", "all", "of", "it", "out", "cloud", "a"]}, {"sent": "cloud .", "words": ["cloud"]}, {"sent": "that cloud opens .", "words": ["that", "cloud"]}, {"sent": "thats a cloud .", "words": ["cloud", "a"]}, {"sent": "the stroller right .", "words": ["the", "stroller"]}, {"sent": "gee were gonna hafta get a different stroller I think .", "words": ["think", "get", "stroller", "were", "a"]}, {"sent": "oh Sugars goin in the stroller ?", "words": ["the", "in", "stroller"]}, {"sent": "you wanna pick your stroller up ?", "words": ["up", "your", "pick", "you", "stroller", "wanna"]}, {"sent": "you wanna push your stroller out to the kitty ?", "words": ["the", "your", "push", "to", "you", "kitty", "stroller", "wanna", "out"]}, {"sent": "I think a stroller .", "words": ["think", "stroller", "a"]}, {"sent": "I dont see the stroller ?", "words": ["see", "dont", "stroller", "the"]}, {"sent": "its a stroller and who sits in the stroller usually ?", "words": ["the", "and", "in", "stroller", "who", "a"]}, {"sent": "baby stroller .", "words": ["stroller"]}, {"sent": "Floppys in the stroller .", "words": ["the", "in", "stroller"]}, {"sent": "did you push somebody in a stroller ?", "words": ["push", "you", "in", "stroller", "did", "a"]}, {"sent": "hes in his stroller now .", "words": ["his", "in", "stroller"]}, {"sent": "now the dolls in the stroller .", "words": ["the", "in", "stroller"]}, {"sent": "and hes in his stroller right ?", "words": ["his", "in", "stroller", "and"]}, {"sent": "here well go down this side and around back to our stroller huh ?", "words": ["back", "go", "to", "and", "here", "this", "our", "stroller", "around", "down"]}, {"sent": "stroller yep .", "words": ["stroller"]}, {"sent": "and some that goes in a stroller right ?", "words": ["and", "in", "stroller", "some", "that", "a"]}, {"sent": "we walked and Paul was in the stroller .", "words": ["the", "we", "was", "and", "in", "stroller"]}, {"sent": "wheres the stroller ?", "words": ["the", "stroller"]}, {"sent": "of your stroller .", "words": ["of", "your", "stroller"]}, {"sent": "she took you in Kirstens stroller ?", "words": ["in", "stroller", "she", "you"]}, {"sent": "may be we can walk today with the stroller .", "words": ["can", "the", "we", "be", "with", "stroller", "walk"]}, {"sent": "but what happened to the stroller did it fall down ?", "words": ["the", "what", "to", "fall", "it", "stroller", "did", "down", "but"]}, {"sent": "thats a stroller ?", "words": ["stroller", "a"]}, {"sent": "what are you doing with your stroller Boneen ?", "words": ["what", "your", "with", "you", "are", "stroller"]}, {"sent": "and whos pushing the stroller ?", "words": ["the", "stroller", "and"]}, {"sent": "yeah you were riding in stroller thats right .", "words": ["in", "were", "stroller", "you"]}, {"sent": "wheres your little err stroller today .", "words": ["your", "stroller", "little"]}, {"sent": "but why have you go a stroller if hes a year and a half and were going on a a were going to the park ?", "words": ["the", "if", "go", "to", "you", "have", "and", "park", "stroller", "but", "on", "were", "why", "a"]}, {"sent": "youre gonna throw what in the stroller ?", "words": ["the", "what", "throw", "in", "stroller"]}, {"sent": "mkay you wanna push the stroller along ?", "words": ["the", "push", "you", "stroller", "wanna"]}, {"sent": "one in the stroller .", "words": ["the", "in", "stroller"]}, {"sent": "Arthur is taking his baby brother for a ride in the stroller .", "words": ["the", "for", "is", "in", "stroller", "his", "ride", "a"]}, {"sent": "stroller ?", "words": ["stroller"]}, {"sent": "do you wanna sit in the stroller ?", "words": ["the", "do", "you", "in", "stroller", "wanna", "sit"]}, {"sent": "you want the daddy in the stroller ?", "words": ["the", "in", "stroller", "you"]}, {"sent": "flying stroller ?", "words": ["stroller"]}, {"sent": "push the stroller .", "words": ["the", "push", "stroller"]}, {"sent": "yeah shes in her stroller .", "words": ["in", "her", "stroller"]}, {"sent": "oh Mommys drivin the car and youre gonna push the stroller ?", "words": ["the", "push", "car", "and", "stroller"]}, {"sent": "wheres the stroller ?", "words": ["the", "stroller"]}, {"sent": "and Floppy in the stroller again .", "words": ["the", "in", "stroller", "and"]}, {"sent": "she she will walk her baby dolls everywhere um with the stroller um .", "words": ["the", "with", "walk", "stroller", "will", "her", "she"]}, {"sent": "but you went in a stroller last year .", "words": ["last", "you", "in", "stroller", "but", "a"]}, {"sent": "gonna go in the stroller and get some milk at the store .", "words": ["the", "go", "store", "and", "get", "in", "stroller", "at", "milk", "some"]}, {"sent": "maybe well take the stroller with us .", "words": ["the", "with", "us", "stroller", "take"]}, {"sent": "were walkin the stroller walkin the stroller .", "words": ["the", "were", "stroller"]}, {"sent": "lets just go in the stroller .", "words": ["the", "in", "stroller", "go"]}, {"sent": "in the stroller .", "words": ["the", "in", "stroller"]}, {"sent": "and we ride in the stroller .", "words": ["the", "we", "and", "in", "stroller", "ride"]}, {"sent": "and our stroller outside .", "words": ["our", "outside", "stroller", "and"]}, {"sent": "now those are kind of all over because we have like the stroller and we have the shopping cart and we do have those kinda things so .", "words": ["the", "all", "do", "those", "we", "of", "so", "like", "and", "have", "are", "stroller", "over", "because"]}, {"sent": "a stroller carriage .", "words": ["stroller", "a"]}, {"sent": "you called up daddy on the phone did ya go in your stroller for a ride ?", "words": ["up", "the", "for", "your", "go", "did ya", "you", "in", "stroller", "did", "on", "ride", "a"]}, {"sent": "huh Floppys in the stroller .", "words": ["the", "in", "stroller"]}, {"sent": "and you helped Mommy push the stroller didnt you ?", "words": ["the", "push", "you", "and", "stroller"]}, {"sent": "the wheels on the car and the stroller go round and round .", "words": ["the", "go", "car", "and", "stroller", "on"]}, {"sent": "she does she likes to feed them and push them around in the stroller .", "words": ["the", "does", "around", "push", "to", "them", "and", "in", "stroller", "feed", "she"]}, {"sent": "a stroller .", "words": ["stroller", "a"]}, {"sent": "but first we are gonna get the stroller .", "words": ["the", "we", "first", "get", "are", "stroller", "but"]}, {"sent": "in the stroller .", "words": ["the", "in", "stroller"]}, {"sent": "thats Katies stroller .", "words": ["stroller"]}, {"sent": "shes like yeah I mean Abby shes like every single day I say to the kids okay were gonna go to the stroller and they crawl to the top .", "words": ["the", "go", "to", "say", "like", "and", "they", "stroller", "every", "were"]}, {"sent": "and shell put the baby doll in the stroller .", "words": ["the", "put", "doll", "and", "in", "stroller"]}, {"sent": "so I talk about those while were when hes in the stroller and were walking around .", "words": ["the", "around", "those", "when", "so", "and", "in", "stroller", "about", "talk", "were"]}, {"sent": "she likes to push them in a little stroller .", "words": ["push", "to", "them", "little", "in", "stroller", "she", "a"]}, {"sent": "so they hadta put the little boy into the stroller and wheel him home .", "words": ["the", "put", "into", "so", "little", "and", "they", "him", "stroller", "home"]}, {"sent": "thats your stroller .", "words": ["your", "stroller"]}, {"sent": "can you say stroller ?", "words": ["can", "say", "stroller", "you"]}, {"sent": "are you gonna put your ball in the stroller ?", "words": ["the", "put", "your", "you", "ball", "in", "are", "stroller"]}, {"sent": "and she put him in the stroller and she take him for a ride with the dolly .", "words": ["the", "put", "for", "with", "ride", "and", "in", "him", "stroller", "take", "she", "a"]}, {"sent": "and the stroller .", "words": ["the", "stroller", "and"]}, {"sent": "yeah she just for her birthday uh my mom just got her a babydoll and so thats been a big um thats been a favorite too and a little stroller and she really likes she really likes that .", "words": ["my", "for", "so", "and", "little", "big", "that", "stroller", "too", "her", "she", "a"]}, {"sent": "oh remember what you did yesterday when we went out to the sandbox and Jennifer was sitting in the stroller and I gave her a cookie ?", "words": ["the", "what", "sandbox", "cookie", "we", "to", "was", "her", "you", "and", "in", "stroller", "did", "out", "when", "a"]}, {"sent": "well go do our business at the bank and then we will go in the stroller from the bank and well walk to xxx .", "words": ["the", "do", "go", "we", "to", "our", "and", "in", "stroller", "at", "will", "then", "walk"]}, {"sent": "and then we will go in the stroller .", "words": ["the", "go", "we", "and", "in", "stroller", "will", "then"]}, {"sent": "youre water bottles in the stroller .", "words": ["the", "in", "stroller", "water"]}, {"sent": "come on would you like to hold my hand or would you like to go in the stroller today ?", "words": ["the", "my", "hold", "hand", "go", "to", "like", "you", "in", "stroller", "would", "on"]}, {"sent": "where do you think hes gonna go in his stroller ?", "words": ["where", "do", "think", "go", "you", "in", "stroller", "his"]}, {"sent": "so but she does like to push her stroller .", "words": ["does", "push", "so", "to", "like", "stroller", "her", "she", "but"]}, {"sent": "Floppy looks great in the stroller you wanna take him for a walk ?", "words": ["the", "for", "you", "in", "him", "stroller", "walk", "wanna", "take", "a"]}, {"sent": "okay you can push your stroller .", "words": ["can", "push", "your", "you", "stroller"]}, {"sent": "the kitty jumps outof the the kitty jumps outof the bas of the the stroller .", "words": ["the", "of", "kitty", "stroller"]}, {"sent": "well take a stroller with us .", "words": ["with", "us", "stroller", "take", "a"]}, {"sent": "maybe one baby wants to push the other baby in the stroller .", "words": ["the", "push", "to", "other", "in", "stroller"]}, {"sent": "is she pushing the stroller ?", "words": ["the", "stroller", "she", "is"]}, {"sent": "and we have a little stroller that like pushes a baby doll .", "words": ["we", "like", "doll", "little", "have", "and", "stroller", "that", "a"]}, {"sent": "someone wants to climb outof the stroller .", "words": ["the", "to", "climb", "stroller"]}, {"sent": "they have diapers and bottles and even has a little stroller that she likes to play with .", "words": ["to", "play", "with", "little", "have", "they", "and", "stroller", "that", "she", "a"]}, {"sent": "oh Mister stroller .", "words": ["stroller"]}, {"sent": "and shes pushing a cat in a stroller .", "words": ["and", "cat", "in", "stroller", "a"]}, {"sent": "how about his stroller ?", "words": ["his", "how", "stroller", "about"]}, {"sent": "thats a pretty cool stroller .", "words": ["pretty", "stroller", "a"]}, {"sent": "lets get in the stroller .", "words": ["get", "the", "in", "stroller"]}, {"sent": "you take the baby for a walk in the stroller ?", "words": ["the", "for", "you", "walk", "in", "stroller", "take", "a"]}, {"sent": "and then she does other stuff with them like she likes to push them in a stroller or she has a little doll carrier to put them in .", "words": ["does", "put", "push", "to", "with", "them", "like", "other", "and", "little", "in", "doll", "stroller", "then", "she", "a"]}, {"sent": "its a stroller .", "words": ["stroller", "a"]}, {"sent": "your stroller do you wanna put Po in the stroller ?", "words": ["the", "do", "put", "your", "you", "in", "stroller", "wanna"]}, {"sent": "it is not a toothbrush .", "words": ["toothbrush", "is", "it", "not", "a"]}, {"sent": "can you say toothbrush .", "words": ["can", "toothbrush", "say", "you"]}, {"sent": "oh no Dot did one of your teeth get caught on that toothbrush ?", "words": ["toothbrush", "your", "of", "get", "did", "on", "that"]}, {"sent": "we dont use your toothbrush on Purdie do we ?", "words": ["do", "toothbrush", "your", "we", "dont", "on"]}, {"sent": "whatve you done with the toothbrush .", "words": ["the", "toothbrush", "with", "you"]}, {"sent": "bye toothbrush .", "words": ["toothbrush"]}, {"sent": "now Ive got to go back upstairs for your shoes and your toothbrush .", "words": ["toothbrush", "for", "your", "back", "go", "to", "and"]}, {"sent": "weve got to go and get the toothbrush .", "words": ["the", "toothbrush", "go", "to", "and", "get"]}, {"sent": "or youve got a green toothbrush ?", "words": ["toothbrush", "green", "a"]}, {"sent": "Jwww puts the toothpaste on your toothbrush for you in the morning doesnt he ?", "words": ["the", "toothbrush", "for", "your", "you", "in", "he", "on"]}, {"sent": "is your toothbrush .", "words": ["toothbrush", "your", "is"]}, {"sent": "come let us find your toothbrush .", "words": ["us", "toothbrush", "your", "find"]}, {"sent": "where is your toothbrush .", "words": ["where", "toothbrush", "your", "is"]}, {"sent": "toothpaste and toothbrush are upstairs Thomas .", "words": ["toothbrush", "are", "and"]}, {"sent": "wanna hold Daddys toothbrush and brush Daddys teeth too ?", "words": ["toothbrush", "hold", "brush", "and", "wanna", "too"]}, {"sent": "where is your toothbrush .", "words": ["where", "toothbrush", "your", "is"]}, {"sent": "a purple toothbrush ?", "words": ["toothbrush", "a"]}, {"sent": "let us go find your toothbrush .", "words": ["toothbrush", "your", "go", "us", "find"]}, {"sent": "Ill get your er toothbrush .", "words": ["get", "toothbrush", "your"]}, {"sent": "where is your toothbrush .", "words": ["where", "toothbrush", "your", "is"]}, {"sent": "Mommy fix your toothbrush .", "words": ["fix", "toothbrush", "your"]}, {"sent": "can you pick up your toothbrush Ill be right down .", "words": ["can", "up", "toothbrush", "your", "pick", "be", "you", "down"]}, {"sent": "shes the one who gave you the new toothbrush .", "words": ["the", "toothbrush", "you", "who", "new"]}, {"sent": "Lily look I have a toothbrush .", "words": ["have", "toothbrush", "a", "look"]}, {"sent": "now give Daddy your toothbrush .", "words": ["toothbrush", "give", "your"]}, {"sent": "well shes holding the toothbrush .", "words": ["the", "toothbrush"]}, {"sent": "toothbrush and your toothpaste .", "words": ["toothbrush", "your", "and"]}, {"sent": "bye toothbrush .", "words": ["toothbrush"]}, {"sent": "and Jwwws got an Actionman toothbrush .", "words": ["toothbrush", "an", "and"]}, {"sent": "a Noddy toothbrush holder .", "words": ["toothbrush", "a"]}, {"sent": "clean your toothbrush and then put it back in the pot .", "words": ["clean", "the", "toothbrush", "put", "your", "back", "and", "in", "it", "then"]}, {"sent": "youre gonna brush your teeth with the toothbrush .", "words": ["the", "toothbrush", "your", "brush", "with"]}, {"sent": "I think it might be a top to a toothbrush .", "words": ["think", "toothbrush", "to", "be", "it", "a"]}, {"sent": "thats the toothbrush .", "words": ["the", "toothbrush"]}, {"sent": "Mark bent his toothbrush with his own hands .", "words": ["his", "toothbrush", "with"]}, {"sent": "Naima what color is your toothbrush ?", "words": ["what", "toothbrush", "your", "is"]}, {"sent": "theres the toothbrush .", "words": ["the", "toothbrush"]}, {"sent": "you already have a toothbrush in your mouth .", "words": ["toothbrush", "your", "mouth", "you", "have", "in", "a"]}, {"sent": "did we get you a new toothbrush ?", "words": ["toothbrush", "we", "you", "get", "did", "new", "a"]}, {"sent": "a yellow toothbrush with bristles .", "words": ["yellow", "toothbrush", "with", "a"]}, {"sent": "you go upstairs darling and give Dada her his toothbrush .", "words": ["toothbrush", "go", "and", "you", "his", "give", "her"]}, {"sent": "Daddyll get toothbrush a little wet ?", "words": ["wet", "toothbrush", "little", "get", "a"]}, {"sent": "Lil you wanna go get your toothbrush ?", "words": ["toothbrush", "your", "go", "you", "get", "wanna"]}, {"sent": "you know at the moment how we put your toothbrush in a beaker with your toothpaste well thats a .", "words": ["the", "how", "put", "your", "toothbrush", "we", "with", "you", "in", "at", "a"]}, {"sent": "we do not stand with the toothbrush .", "words": ["the", "do", "toothbrush", "we", "stand", "with", "not"]}, {"sent": "she has a new toothbrush so do I .", "words": ["do", "toothbrush", "so", "new", "she", "a"]}, {"sent": "what is wrong with the toothbrush .", "words": ["the", "what", "toothbrush", "is", "with"]}, {"sent": "oh now that shes seen the toothbrush .", "words": ["the", "toothbrush", "that"]}, {"sent": "its a toothbrush .", "words": ["toothbrush", "a"]}, {"sent": "you got a WinniethePooh toothbrush .", "words": ["toothbrush", "a", "you"]}, {"sent": "your toothbrush ?", "words": ["toothbrush", "your"]}, {"sent": "not with your toothbrush .", "words": ["your", "toothbrush", "with", "not"]}, {"sent": "its Jwwws toothbrush .", "words": ["toothbrush"]}, {"sent": "heres a toothbrush for elephant .", "words": ["elephant", "toothbrush", "for", "a"]}, {"sent": "a Snoopy toothbrush ?", "words": ["toothbrush", "a"]}, {"sent": "xxx dangerous to walk around with your toothbrush .", "words": ["toothbrush", "your", "to", "with", "around", "walk"]}, {"sent": "put those toothbrush back there .", "words": ["toothbrush", "put", "those", "back", "there"]}, {"sent": "huh your Clifford toothbrush .", "words": ["toothbrush", "your"]}, {"sent": "what color is your toothbrush .", "words": ["what", "toothbrush", "your", "is"]}, {"sent": "okay lets rinse off your toothbrush .", "words": ["off", "toothbrush", "your"]}, {"sent": "there is your toothbrush .", "words": ["there", "toothbrush", "your", "is"]}, {"sent": "you hang on to your toothbrush .", "words": ["toothbrush", "your", "to", "you", "on"]}, {"sent": "where did your toothbrush go ?", "words": ["where", "toothbrush", "your", "go", "did"]}, {"sent": "did you take your toothbrush ?", "words": ["toothbrush", "your", "you", "did", "take"]}, {"sent": "where is your toothbrush .", "words": ["where", "toothbrush", "your", "is"]}, {"sent": "here is your toothbrush ?", "words": ["here", "toothbrush", "your", "is"]}, {"sent": "you want me get your toothbrush ?", "words": ["toothbrush", "your", "you", "get", "me"]}, {"sent": "about when you were trying to put the top on your toothbrush ?", "words": ["the", "put", "toothbrush", "your", "when", "to", "you", "about", "on", "were"]}, {"sent": "and some like your toothbrush ?", "words": ["toothbrush", "your", "like", "and", "some"]}, {"sent": "just let me do your www tune on the toothbrush xxx .", "words": ["the", "do", "toothbrush", "your", "on", "me"]}, {"sent": "hes like dont let her have my toothbrush .", "words": ["my", "toothbrush", "like", "have", "dont", "her"]}, {"sent": "I wish I had a pink toothbrush .", "words": ["toothbrush", "a", "wish"]}, {"sent": "your toothbrush is in my room .", "words": ["toothbrush", "my", "your", "is", "in", "room"]}, {"sent": "thats a purple toothbrush .", "words": ["toothbrush", "a"]}, {"sent": "honey you need to brush your teeth not suck on your toothbrush .", "words": ["toothbrush", "need", "your", "brush", "to", "need to", "you", "on", "not"]}, {"sent": "where does your toothbrush go Fraser ?", "words": ["where", "does", "toothbrush", "your", "go"]}, {"sent": "here is your toothbrush .", "words": ["here", "toothbrush", "your", "is"]}, {"sent": "there is the toothbrush .", "words": ["the", "there", "toothbrush", "is"]}, {"sent": "you hafta use your toothbrush .", "words": ["toothbrush", "your", "you"]}, {"sent": "wheres the toothbrush ?", "words": ["the", "toothbrush"]}, {"sent": "and Ive got hes got a toothbrush in his hand .", "words": ["toothbrush", "hand", "and", "in", "his", "a"]}, {"sent": "I dont know where you put your toothbrush .", "words": ["where", "put", "toothbrush", "your", "you", "dont"]}, {"sent": "you have a new toothbrush ?", "words": ["toothbrush", "you", "have", "new", "a"]}, {"sent": "oh that toothbrush .", "words": ["toothbrush", "that"]}, {"sent": "I dont want you walking around with your toothbrush .", "words": ["toothbrush", "your", "with", "you", "dont", "around"]}, {"sent": "wheres your toothbrush ?", "words": ["toothbrush", "your"]}, {"sent": "hes brushing his teeth with a toothbrush .", "words": ["his", "toothbrush", "with", "a"]}, {"sent": "looking at your toothbrush .", "words": ["toothbrush", "your", "at"]}, {"sent": "whats your toothbrush doing here ?", "words": ["here", "toothbrush", "your"]}, {"sent": "alright wheres your toothbrush ?", "words": ["toothbrush", "your"]}, {"sent": "theyre cleaning their teeth with a toothbrush .", "words": ["toothbrush", "with", "their", "a"]}, {"sent": "has he got a red toothbrush has the teddy got a red toothbrush in his hand ?", "words": ["red", "the", "toothbrush", "hand", "in", "he", "his", "a"]}, {"sent": "thankyou for putting your toothbrush back ?", "words": ["toothbrush", "for", "your", "back"]}, {"sent": "yeah thats my toothbrush from home .", "words": ["toothbrush", "my", "home"]}, {"sent": "where is your toothbrush .", "words": ["where", "toothbrush", "your", "is"]}, {"sent": "where is the toothbrush ?", "words": ["where", "the", "toothbrush", "is"]}, {"sent": "here is your toothbrush ?", "words": ["here", "toothbrush", "your", "is"]}, {"sent": "hes holding the toothbrush in his hands .", "words": ["the", "in", "toothbrush", "his"]}, {"sent": "I asked you what was on your toothbrush .", "words": ["what", "toothbrush", "your", "was", "you", "on"]}, {"sent": "thats not a teddybear .", "words": ["not", "teddybear", "a"]}, {"sent": "a teddybear ?", "words": ["teddybear", "a"]}, {"sent": "and you slept with your teddybear .", "words": ["your", "with", "teddybear", "you", "and"]}, {"sent": "it is a teddybear .", "words": ["it", "teddybear", "a", "is"]}, {"sent": "who who bought you a teddybear ?", "words": ["a", "teddybear", "you", "who"]}, {"sent": "that teddybear that we got yesterday at the shops .", "words": ["the", "we", "teddybear", "at", "that"]}, {"sent": "Im sorry but my teddybear doesnt like that medicine you gave him .", "words": ["my", "medicine", "teddybear", "like", "you", "him", "that", "but"]}, {"sent": "thats a teddybear .", "words": ["teddybear", "a"]}, {"sent": "that is the teddybear .", "words": ["the", "that", "teddybear", "is"]}, {"sent": "xxx next to your teddybear and .", "words": ["to", "your", "teddybear", "and"]}, {"sent": "Mummys drawn a teddybear for you .", "words": ["for", "teddybear", "you", "a"]}, {"sent": "which teddybear book ?", "words": ["book", "which", "teddybear"]}, {"sent": "squadgy teddybear .", "words": ["teddybear"]}, {"sent": "a white teddybear .", "words": ["white", "teddybear", "a"]}, {"sent": "youre giving her the teddybear ?", "words": ["the", "her", "teddybear"]}, {"sent": "heres your teddybear .", "words": ["your", "teddybear"]}, {"sent": "thats a green teddybear .", "words": ["teddybear", "green", "a"]}, {"sent": "Ive brought me teddybear to see you .", "words": ["see", "to", "teddybear", "you", "me"]}, {"sent": "well look at the teddybear .", "words": ["the", "at", "teddybear", "look"]}, {"sent": "are you okay teddybear ?", "words": ["are", "teddybear", "you"]}, {"sent": "lookee what is this teddybear doing ?", "words": ["what", "teddybear", "this", "is"]}, {"sent": "a teddybear .", "words": ["teddybear", "a"]}, {"sent": "theres a teddybear .", "words": ["teddybear", "a"]}, {"sent": "the teddybear and the baby are going to have a picnic .", "words": ["the", "to", "teddybear", "and", "have", "are", "a"]}, {"sent": "its a teddybear saying you go on holiday very cheaply .", "words": ["go", "teddybear", "you", "on", "a"]}, {"sent": "would you like to draw a teddybear .", "words": ["to", "draw", "like", "teddybear", "you", "would", "a"]}, {"sent": "poor teddybear .", "words": ["teddybear", "poor"]}, {"sent": "um probably its his teddybear thats also his security blanket its like attached .", "words": ["his", "blanket", "teddybear", "like"]}, {"sent": "thats your teddybear .", "words": ["your", "teddybear"]}, {"sent": "is there a teddybear house in that book ?", "words": ["is", "teddybear", "there", "in", "house", "book", "that", "a"]}, {"sent": "no what color is the teddybear ?", "words": ["the", "what", "teddybear", "is"]}, {"sent": "its a teddybear .", "words": ["teddybear", "a"]}, {"sent": "is that your teddybear ?", "words": ["your", "that", "teddybear", "is"]}, {"sent": "xxx on the teddybear ?", "words": ["on", "the", "teddybear"]}, {"sent": "I bet the teddybear wants to hear this story .", "words": ["the", "to", "teddybear", "hear", "this", "story"]}, {"sent": "and what did teddybear say ?", "words": ["what", "say", "teddybear", "and", "did"]}, {"sent": "uht oh theres the missing teddybear .", "words": ["the", "teddybear"]}, {"sent": "one little teddybear ?", "words": ["teddybear", "little"]}, {"sent": "teddybear would like to do it .", "words": ["do", "to", "teddybear", "like", "it", "would"]}, {"sent": "Mummys putting a cardigan on her teddybear .", "words": ["on", "her", "teddybear", "a"]}, {"sent": "a teddybear .", "words": ["teddybear", "a"]}, {"sent": "hi teddybear .", "words": ["teddybear"]}, {"sent": "teddybear there .", "words": ["there", "teddybear"]}, {"sent": "its a wafer biscuit in the shape of a teddybear .", "words": ["the", "of", "teddybear", "in", "a"]}, {"sent": "does your teddybear sing does he ?", "words": ["does", "your", "teddybear", "sing", "he"]}, {"sent": "dont take her teddybear off her .", "words": ["off", "teddybear", "dont", "take", "her"]}, {"sent": "yeah huh wheres the bear oh help Elmo find his missing teddybear .", "words": ["the", "help", "teddybear", "bear", "find", "his"]}, {"sent": "a bad teddybear .", "words": ["teddybear", "bad", "a"]}, {"sent": "Ill hafta go and talk to teddybear I think .", "words": ["think", "go", "to", "teddybear", "and", "talk"]}, {"sent": "press the teddybear .", "words": ["the", "teddybear"]}, {"sent": "heres your teddybear .", "words": ["your", "teddybear"]}, {"sent": "oh yeah there is a little teddybear there isnt there ?", "words": ["is", "teddybear", "little", "there", "a"]}, {"sent": "teddybear teddybear up the .", "words": ["up", "the", "teddybear"]}, {"sent": "and that teddybear has a red ribbon with white spots on it .", "words": ["red", "with", "teddybear", "and", "it", "on", "white", "that", "a"]}, {"sent": "you wanna do the teddybear ?", "words": ["the", "do", "teddybear", "you", "wanna"]}, {"sent": "have you pulled the teddybear apart ?", "words": ["have", "the", "teddybear", "you"]}, {"sent": "Mummy is putting the cardigan back onto teddybear .", "words": ["the", "teddybear", "back", "is"]}, {"sent": "youve got that teddybear look .", "words": ["that", "teddybear", "look"]}, {"sent": "its a teddybear yes it is .", "words": ["it", "teddybear", "is", "a"]}, {"sent": "orange teddybear .", "words": ["orange", "teddybear"]}, {"sent": "did the big bad teddybear get you ?", "words": ["the", "teddybear", "bad", "you", "get", "did", "big"]}, {"sent": "lucky its only a playdough teddybear .", "words": ["teddybear", "a"]}, {"sent": "do you know what this teddybear is doing ?", "words": ["do", "what", "is", "teddybear", "you", "this"]}, {"sent": "your teddybear ?", "words": ["your", "teddybear"]}, {"sent": "I see do you see that teddybear ?", "words": ["do", "teddybear", "you", "see", "that"]}, {"sent": "and Mummys teddybear .", "words": ["teddybear", "and"]}, {"sent": "I cant see the teddybear .", "words": ["the", "see", "teddybear"]}, {"sent": "thats the teddybear .", "words": ["the", "teddybear"]}, {"sent": "wherere you going with PoohBear and teddybear ?", "words": ["with", "teddybear", "and", "you"]}, {"sent": "I wanna see the teddybear on this page .", "words": ["the", "teddybear", "this", "wanna", "on", "see"]}, {"sent": "Im looking at your birthday cards and I can see a teddybear has a hat .", "words": ["can", "your", "teddybear", "and", "at", "hat", "see", "a"]}, {"sent": "to cut some teddybear shapes out of the bread .", "words": ["the", "bread", "cut", "of", "to", "teddybear", "out", "some"]}, {"sent": "whats that teddybear got ?", "words": ["that", "teddybear"]}, {"sent": "take the teddybear off .", "words": ["the", "take", "teddybear", "off"]}, {"sent": "thats your teddybear .", "words": ["your", "teddybear"]}, {"sent": "on teddybear ?", "words": ["on", "teddybear"]}, {"sent": "yes didnt you have a teddybear balloon ?", "words": ["teddybear", "you", "have", "balloon", "a"]}, {"sent": "its your teddybear .", "words": ["your", "teddybear"]}, {"sent": "thats a teddybear .", "words": ["teddybear", "a"]}, {"sent": "look at that teddybear .", "words": ["at", "that", "teddybear", "look"]}, {"sent": "thats beautiful teddybear .", "words": ["teddybear"]}, {"sent": "its Ninas teddybear .", "words": ["teddybear"]}, {"sent": "thats the baby teddybear .", "words": ["the", "teddybear"]}, {"sent": "the teddybear .", "words": ["the", "teddybear"]}, {"sent": "you like your teddybear .", "words": ["your", "like", "teddybear", "you"]}, {"sent": "heres another teddybear too .", "words": ["another", "teddybear", "too"]}, {"sent": "you wanna look at the teddybear ?", "words": ["the", "teddybear", "you", "at", "wanna", "look"]}, {"sent": "he has a bow like your teddybear at home does huh ?", "words": ["does", "your", "like", "teddybear", "he", "at", "home", "a"]}, {"sent": "wheres the teddybear gone ?", "words": ["the", "teddybear"]}, {"sent": "can you see paprikas dummy and her teddybear ?", "words": ["can", "teddybear", "and", "you", "see", "her"]}, {"sent": "shall we draw round that teddybear ?", "words": ["draw", "that", "teddybear", "we"]}, {"sent": "does the teddybear go there ?", "words": ["the", "does", "go", "teddybear", "there"]}, {"sent": "because Kippers asking him if theyve seen his teddybear .", "words": ["if", "teddybear", "him", "his", "because"]}, {"sent": "whos behind the teddybear ?", "words": ["the", "behind", "teddybear"]}, {"sent": "youve still got a car a van and a teddybear havent you ?", "words": ["car", "teddybear", "you", "and", "a"]}, {"sent": "lookee the teddybear .", "words": ["the", "teddybear"]}, {"sent": "do you know where Kippers teddybear is ?", "words": ["where", "do", "is", "teddybear", "you"]}, {"sent": "oh the teddybear .", "words": ["the", "teddybear"]}, {"sent": "a green teddybear .", "words": ["teddybear", "green", "a"]}, {"sent": "hes got a little teddybear there .", "words": ["there", "teddybear", "little", "a"]}, {"sent": "hur soda .", "words": ["soda"]}, {"sent": "Ill give you your soda .", "words": ["give", "your", "soda", "you"]}, {"sent": "dont have no soda .", "words": ["have", "dont", "soda"]}, {"sent": "my soda .", "words": ["my", "soda"]}, {"sent": "soda .", "words": ["soda"]}, {"sent": "yeah like a soda machine I was thinking of a gumball machine .", "words": ["soda", "of", "was", "like", "a"]}, {"sent": "soda .", "words": ["soda"]}, {"sent": "I want my soda .", "words": ["my", "soda"]}, {"sent": "pretend that I fainted because I make believed that I drank too much if the ice cream soda .", "words": ["the", "ice cream", "if", "soda", "much", "that", "too", "ice", "pretend", "make", "because"]}, {"sent": "you xxx eat before you drink that that j u i c e soda .", "words": ["i", "soda", "drink", "you", "eat", "that"]}, {"sent": "you been drinking all that soda .", "words": ["all", "that", "soda", "you"]}, {"sent": "Ill get that yellow that we didnt hafta put there that playdough in there and made that playdough ice cream soda .", "words": ["ice cream", "put", "soda", "we", "and", "get", "there", "in", "ice", "yellow", "that"]}, {"sent": "ice cream soda .", "words": ["ice", "ice cream", "soda"]}, {"sent": "soda .", "words": ["soda"]}, {"sent": "you want some soda ?", "words": ["some", "soda", "you"]}, {"sent": "if you want the soda youre gonna hafta drink it in the highchair .", "words": ["the", "soda", "if", "drink", "you", "in", "it"]}, {"sent": "wheres the soda ?", "words": ["the", "soda"]}, {"sent": "do you want soda ?", "words": ["do", "soda", "you"]}, {"sent": "its a soda bottle .", "words": ["bottle", "soda", "a"]}, {"sent": "so you said they put sugar baking powder baking soda green food coloring anything else go in the cookies ?", "words": ["the", "put", "soda", "go", "so", "you", "they", "food", "in", "green"]}, {"sent": "now you can have some soda .", "words": ["can", "soda", "you", "have", "some"]}, {"sent": "I want mah soda .", "words": ["soda"]}, {"sent": "thats soda ?", "words": ["soda"]}, {"sent": "soda pop .", "words": ["pop", "soda"]}, {"sent": "soda .", "words": ["soda"]}, {"sent": "soda .", "words": ["soda"]}, {"sent": "soda bottle .", "words": ["bottle", "soda"]}, {"sent": "Carl you care for a beer or a soda or some juice ?", "words": ["for", "soda", "you", "some", "juice", "a"]}, {"sent": "twenty eight is a soda ?", "words": ["soda", "a", "is"]}, {"sent": "yup weve got baking soda and salt .", "words": ["salt", "soda", "and"]}, {"sent": "soda pop .", "words": ["pop", "soda"]}, {"sent": "finish you soda please .", "words": ["soda", "finish", "you"]}, {"sent": "Ma can I have some more soda ?", "words": ["can", "soda", "have", "more", "some"]}, {"sent": "grape soda ?", "words": ["soda"]}, {"sent": "xxx my soda .", "words": ["my", "soda"]}, {"sent": "you aint gonna get no soda .", "words": ["get", "soda", "you"]}, {"sent": "uhhuh its baking soda .", "words": ["soda"]}, {"sent": "back my soda .", "words": ["my", "soda", "back"]}, {"sent": "oh in the drugstore at the soda fountain ?", "words": ["the", "in", "soda", "at"]}, {"sent": "is that from the soda ?", "words": ["the", "that", "soda", "is"]}, {"sent": "alright so I get a soda .", "words": ["get", "so", "soda", "a"]}, {"sent": "it could be another soda xxx I cant drink it .", "words": ["soda", "drink", "be", "another", "it", "could"]}, {"sent": "can I have some soda ?", "words": ["can", "some", "have", "soda"]}, {"sent": "soda bottle .", "words": ["bottle", "soda"]}, {"sent": "and a soda .", "words": ["soda", "a", "and"]}, {"sent": "maybe a little baking soda .", "words": ["soda", "little", "a"]}, {"sent": "Carl what a you wanna drink water soda tea ?", "words": ["what", "soda", "drink", "you", "wanna", "water", "a"]}, {"sent": "you gonna buy him a soda ?", "words": ["soda", "you", "him", "buy", "a"]}, {"sent": "do you want some soda ?", "words": ["do", "soda", "some", "you"]}, {"sent": "I want cream soda .", "words": ["soda"]}, {"sent": "Bobby Bobby wont drink soda .", "words": ["soda", "drink"]}, {"sent": "soda .", "words": ["soda"]}, {"sent": "wanna drink of soda ?", "words": ["of", "drink", "soda", "wanna"]}, {"sent": "ice cream soda .", "words": ["ice", "ice cream", "soda"]}, {"sent": "and dont you touch my soda .", "words": ["my", "soda", "touch", "you", "and", "dont"]}, {"sent": "baking soda .", "words": ["soda"]}, {"sent": "is that Daddys soda ?", "words": ["that", "soda", "is"]}, {"sent": "I want my soda .", "words": ["my", "soda"]}, {"sent": "you dont need special time to pick up that soda .", "words": ["up", "need", "soda", "pick", "to", "you", "dont", "that"]}, {"sent": "Philip dont open the soda I told you theres one in the freezer .", "words": ["the", "soda", "open", "you", "dont", "in"]}, {"sent": "soda .", "words": ["soda"]}, {"sent": "dont want no soda .", "words": ["dont", "soda"]}, {"sent": "soda straw .", "words": ["soda"]}, {"sent": "I got some soda .", "words": ["some", "soda"]}, {"sent": "I want my soda .", "words": ["my", "soda"]}, {"sent": "Ill make xxx soda and Ill drink that .", "words": ["soda", "drink", "and", "make", "that"]}, {"sent": "its not cream soda .", "words": ["soda", "not"]}, {"sent": "I dont really want water can I have soda please ?", "words": ["can", "soda", "have", "dont", "water"]}, {"sent": "and soda soda ohmygoodness .", "words": ["soda", "and"]}, {"sent": "soda bottle .", "words": ["bottle", "soda"]}, {"sent": "and I got a soda .", "words": ["soda", "a", "and"]}, {"sent": "Ill give you your soda .", "words": ["give", "your", "soda", "you"]}, {"sent": "I want my soda .", "words": ["my", "soda"]}, {"sent": "Im my soda .", "words": ["my", "soda"]}, {"sent": "come get a soda .", "words": ["get", "soda", "a"]}, {"sent": "whats this soda ?", "words": ["this", "soda"]}, {"sent": "hes not gonna have soda until hes finished .", "words": ["have", "soda", "not"]}, {"sent": "want some milk or some soda ?", "words": ["milk", "some", "soda"]}, {"sent": "oh I forgot I I put that soda in the freezer .", "words": ["the", "put", "soda", "in", "that"]}, {"sent": "baking soda .", "words": ["soda"]}, {"sent": "soda water .", "words": ["soda", "water"]}, {"sent": "eat and leda soda .", "words": ["eat", "soda", "and"]}, {"sent": "you want some soda ?", "words": ["some", "soda", "you"]}, {"sent": "soda .", "words": ["soda"]}, {"sent": "eat something first and then you get some soda .", "words": ["soda", "first", "you", "eat", "and", "get", "some", "then"]}, {"sent": "cloves cinnamon ginger baking soda .", "words": ["soda"]}, {"sent": "and he does like to play with um like real uh soda cans and things like that or water bottles .", "words": ["does", "soda", "to", "play", "with", "like", "and", "he", "water", "that"]}, {"sent": "um soda .", "words": ["soda"]}, {"sent": "what kind of soda ?", "words": ["of", "what", "soda"]}, {"sent": "I want my soda .", "words": ["my", "soda"]}, {"sent": "soda .", "words": ["soda"]}, {"sent": "youre gonna have your orange juice and then later on I always give you a little bit of soda .", "words": ["give", "your", "soda", "of", "and", "have", "you", "little", "orange", "on", "then", "juice", "a"]}, {"sent": "because she look and think Im gawn give her soda .", "words": ["think", "soda", "and", "look", "give", "her", "she", "because"]}, {"sent": "soda bottle .", "words": ["bottle", "soda"]}, {"sent": "you spilled the soda ?", "words": ["the", "soda", "you"]}, {"sent": "ice cream soda .", "words": ["ice", "ice cream", "soda"]}, {"sent": "soda .", "words": ["soda"]}, {"sent": "that doggys chewing on a sneaker .", "words": ["on", "sneaker", "that", "a"]}, {"sent": "yeah how about a sneaker or a slipper ?", "words": ["how", "sneaker", "about", "slipper", "a"]}, {"sent": "my own sneaker .", "words": ["my", "sneaker"]}, {"sent": "and I shot him wid a wid a bullet Joe Joe hit the broom over his back Bobby threw a sneaker at him and Danny kicked him .", "words": ["the", "back", "and", "broom", "him", "sneaker", "at", "his", "hit", "over", "a"]}, {"sent": "do you want me to tie your sneaker ?", "words": ["do", "your", "to", "you", "sneaker", "me"]}, {"sent": "whats a sneaker car ?", "words": ["car", "sneaker", "a"]}, {"sent": "he ate Mookys stinky sneaker for breakfast .", "words": ["sneaker", "for", "he"]}, {"sent": "its sneaker time .", "words": ["sneaker"]}, {"sent": "would you like me to tie your sneaker ?", "words": ["your", "to", "like", "you", "sneaker", "would", "me"]}, {"sent": "his other sneaker ?", "words": ["his", "sneaker", "other"]}, {"sent": "put the sneaker on .", "words": ["the", "put", "sneaker", "on"]}, {"sent": "my look at this other sneaker .", "words": ["my", "look", "other", "this", "sneaker", "at"]}, {"sent": "Sarah wheres your sneaker .", "words": ["sneaker", "your"]}, {"sent": "Ill initiate your sneaker .", "words": ["sneaker", "your"]}, {"sent": "the old sneaker ?", "words": ["the", "sneaker", "old"]}, {"sent": "yes theyre big theyre going to be very big on you probably that type of a sneaker .", "words": ["of", "to", "be", "you", "that", "sneaker", "on", "big", "a"]}, {"sent": "thats a type of sneaker aint it ?", "words": ["of", "it", "sneaker", "a"]}, {"sent": "lets put on your sneaker here .", "words": ["put", "your", "here", "sneaker", "on"]}, {"sent": "sneaker .", "words": ["sneaker"]}, {"sent": "your sneaker cars ?", "words": ["sneaker", "your"]}, {"sent": "sneaker ?", "words": ["sneaker"]}, {"sent": "he was drawing a picture of his sneaker .", "words": ["of", "was", "picture", "he", "sneaker", "his", "a"]}, {"sent": "Im gonna go up and go potty on your potty okay ?", "words": ["up", "your", "go", "and", "potty", "on"]}, {"sent": "little potty or big potty ?", "words": ["big", "potty", "little"]}, {"sent": "well that can come off because shes potty trained now .", "words": ["can", "off", "potty", "that", "because"]}, {"sent": "do you wanna use your potty now Thomas ?", "words": ["do", "your", "you", "potty", "wanna"]}, {"sent": "it was a potty .", "words": ["was", "it", "potty", "a"]}, {"sent": "Laura said her need for poos in the potty today .", "words": ["the", "for", "need", "in", "potty", "her"]}, {"sent": "I think you must be looking for the pink potty that belongs to the dolly because when you were playing with her and Daddy the other day she was sitting on the potty wasnt she ?", "words": ["the", "think", "for", "were", "when", "be", "to", "was", "with", "her", "you", "and", "other", "potty", "on", "that", "she", "because"]}, {"sent": "oh but hes gone now he went potty .", "words": ["he", "potty", "but"]}, {"sent": "but like Santa brought her a potty training book that has like a little button that youknow you hear the toilet flushing or um the Thomas tank engine books that youknow if you see the picture you push and you get a noise .", "words": ["the", "push", "if", "button", "picture", "like", "little", "you", "hear", "that", "and", "get", "potty", "book", "see", "her", "but", "a"]}, {"sent": "you wanna sit on this potty ?", "words": ["you", "this", "potty", "wanna", "on", "sit"]}, {"sent": "dont let her fall in the potty .", "words": ["the", "fall", "dont", "in", "potty", "her"]}, {"sent": "then I want you to go to the potty .", "words": ["the", "go", "to", "you", "potty", "then"]}, {"sent": "yes Im going to my potty potty .", "words": ["to", "my", "potty"]}, {"sent": "did you have a weewees on your potty ?", "words": ["your", "you", "have", "did", "potty", "on", "a"]}, {"sent": "what did you do on your potty ?", "words": ["what", "do", "your", "you", "did", "potty", "on"]}, {"sent": "potty potty .", "words": ["potty"]}, {"sent": "dyou need to use the potty ?", "words": ["the", "need", "need to", "to", "potty"]}, {"sent": "you sit on the potty .", "words": ["the", "you", "potty", "on", "sit"]}, {"sent": "wanna try the potty for a minute ?", "words": ["the", "for", "potty", "wanna", "try", "a"]}, {"sent": "this book says here is a bunnys brand new purple potty .", "words": ["is", "here", "this", "potty", "book", "new", "a"]}, {"sent": "do you wanna sit on the potty to poop ?", "words": ["the", "do", "to", "you", "potty", "wanna", "on", "sit"]}, {"sent": "you need the potty ?", "words": ["the", "need", "potty", "you"]}, {"sent": "that sounds like potty to me .", "words": ["to", "like", "potty", "me", "that"]}, {"sent": "and read the book the potty book whoops .", "words": ["the", "and", "read", "potty", "book"]}, {"sent": "come and sit here on the potty for a few more moments .", "words": ["the", "for", "and", "here", "more", "potty", "on", "sit", "a"]}, {"sent": "going to the potty .", "words": ["the", "to", "potty"]}, {"sent": "you have a second potty now you have two potties .", "words": ["have", "potty", "a", "you"]}, {"sent": "if you wanna use the potty though lets use the potty because .", "words": ["the", "if", "you", "potty", "wanna", "because"]}, {"sent": "what color do you want for your potty chart ?", "words": ["what", "do", "for", "your", "you", "potty"]}, {"sent": "oh do you wanna sit on your potty ?", "words": ["do", "your", "you", "potty", "wanna", "on", "sit"]}, {"sent": "are you talking about Tom and Pippin and Sonia and Isabel and a potty ?", "words": ["and", "you", "are", "potty", "about", "a"]}, {"sent": "yes Im going to my potty potty .", "words": ["to", "my", "potty"]}, {"sent": "youve been very good with your potty though havent you ?", "words": ["your", "with", "you", "potty", "good"]}, {"sent": "I think well hafta persevere with this potty training Thomas .", "words": ["this", "think", "with", "potty"]}, {"sent": "well you can when you use the potty .", "words": ["can", "the", "you", "potty", "when"]}, {"sent": "she sat on her new potty and sat and sat and sat and sat but nothing came out .", "words": ["and", "potty", "out", "on", "new", "her", "she", "but"]}, {"sent": "well animals dont usually do that although cats and some animals do clean themselves up after they poop or pee in the potty .", "words": ["clean", "do", "up", "the", "and", "dont", "they", "in", "potty", "some", "that"]}, {"sent": "youre sposta be telling Mummy when you wanna use your potty arent you ?", "words": ["your", "be", "you", "potty", "wanna", "when"]}, {"sent": "if you use your potty you get a sweetie .", "words": ["your", "if", "you", "get", "potty", "a"]}, {"sent": "so hand Mummy the potty .", "words": ["the", "so", "potty", "hand"]}, {"sent": "do you hafta go potty ?", "words": ["go", "do", "potty", "you"]}, {"sent": "its a small potty this is like your little yellow potty which is a bit small for you .", "words": ["for", "your", "is", "like", "little", "you", "this", "potty", "yellow", "which", "a"]}, {"sent": "then lets go potty .", "words": ["then", "potty", "go"]}, {"sent": "xxx say potty is not it .", "words": ["is", "say", "it", "potty", "not"]}, {"sent": "dya think he wants to sit on the potty ?", "words": ["the", "think", "to", "he", "potty", "on", "sit"]}, {"sent": "shall we sit you on your potty ?", "words": ["your", "we", "you", "potty", "on", "sit"]}, {"sent": "yeah the potty book says sit sit sit .", "words": ["the", "book", "sit", "potty"]}, {"sent": "you did you did put pee in the potty you can have potty sticker .", "words": ["the", "can", "put", "you", "have", "in", "did", "potty"]}, {"sent": "next time you go to the potty .", "words": ["the", "go", "to", "you", "potty"]}, {"sent": "go and sit on your potty .", "words": ["your", "go", "and", "potty", "on", "sit"]}, {"sent": "lets go to the potty .", "words": ["the", "to", "potty", "go"]}, {"sent": "you like that better than the big potty ?", "words": ["the", "better", "like", "you", "big", "potty", "that"]}, {"sent": "show me where the potty .", "words": ["where", "the", "show", "potty", "me"]}, {"sent": "which potty do you think we should take with us on the journey should we take this potty seat and leave these two at home ?", "words": ["the", "do", "think", "these", "we", "with", "you", "and", "us", "this", "potty", "at", "on", "take", "which", "home"]}, {"sent": "we useta sit on the potty .", "words": ["the", "we", "potty", "on", "sit"]}, {"sent": "now thats a squeaky potty .", "words": ["potty", "a"]}, {"sent": "Sues sitting on her potty .", "words": ["on", "her", "potty"]}, {"sent": "can you find the potty ?", "words": ["can", "the", "you", "potty", "find"]}, {"sent": "because then we dont hafta empty the potty .", "words": ["the", "we", "empty", "dont", "potty", "then", "because"]}, {"sent": "do you hafta go potty ?", "words": ["go", "do", "potty", "you"]}, {"sent": "if youve finished sitting on the potty do you think its time that we got you dressed ?", "words": ["the", "do", "think", "if", "we", "you", "potty", "on", "that"]}, {"sent": "can you see what there is on your potty ?", "words": ["can", "what", "your", "is", "you", "there", "potty", "on", "see"]}, {"sent": "doing potty well what do you hafta do when you do potty ?", "words": ["what", "do", "you", "potty", "when"]}, {"sent": "babies do weewee on the potty .", "words": ["on", "do", "potty", "the"]}, {"sent": "Im gonna put the little potty next to the big potty .", "words": ["the", "put", "to", "little", "potty", "big"]}, {"sent": "yeah doesnt he wash his hands after he goes potty ?", "words": ["his", "he", "potty", "wash"]}, {"sent": "okay and then he goes on the potty .", "words": ["the", "and", "he", "potty", "on", "then"]}, {"sent": "were not no you can go potty at home no you can go potty at home okay ?", "words": ["can", "go", "you", "potty", "at", "were", "not", "home"]}, {"sent": "a talking potty .", "words": ["potty", "a"]}, {"sent": "cause you wanna try the potty ?", "words": ["the", "you", "potty", "wanna", "try"]}, {"sent": "is she um are you teaching her to potty train or just like xxx .", "words": ["is", "train", "to", "like", "you", "are", "potty", "her", "she"]}, {"sent": "then I fill the potty with my very own peepee .", "words": ["the", "my", "with", "potty", "then"]}, {"sent": "going to the potty ?", "words": ["the", "to", "potty"]}, {"sent": "yeah I moved the potty by accident .", "words": ["the", "potty", "by"]}, {"sent": "I got a potty gotta figure it out .", "words": ["it", "potty", "out", "a"]}, {"sent": "the dog potty .", "words": ["the", "potty", "dog"]}, {"sent": "Naima do you need to sit on potty ?", "words": ["do", "need", "need to", "to", "you", "potty", "on", "sit"]}, {"sent": "weve just read the blue book and it said wheres Maxs potty .", "words": ["the", "and", "blue", "it", "read", "potty", "book"]}, {"sent": "I would like you to tell me if you need to use the potty .", "words": ["the", "need", "if", "need to", "to", "like", "you", "potty", "would", "me"]}, {"sent": "is this the potty ?", "words": ["the", "this", "potty", "is"]}, {"sent": "its not a potty uhn is it ?", "words": ["is", "it", "potty", "not", "a"]}, {"sent": "do you hafta go potty Emily ?", "words": ["go", "do", "potty", "you"]}, {"sent": "lets try the potty .", "words": ["the", "try", "potty"]}, {"sent": "probably after drinking all that juice you have to go potty .", "words": ["all", "go", "to", "you", "have", "potty", "juice", "that"]}, {"sent": "hell hafta go potty .", "words": ["potty", "go"]}, {"sent": "its time to go potty .", "words": ["to", "potty", "go"]}, {"sent": "I know you wanna go potty .", "words": ["go", "potty", "wanna", "you"]}, {"sent": "tripod potty .", "words": ["potty"]}, {"sent": "where is the potty .", "words": ["where", "the", "potty", "is"]}, {"sent": "the potty .", "words": ["the", "potty"]}, {"sent": "and Max laughed at Lulu and said why have you got your toys in the potty ?", "words": ["the", "your", "you", "have", "and", "in", "potty", "at", "why"]}, {"sent": "the potty training has been coming along very nicely hasnt it ?", "words": ["the", "it", "potty"]}, {"sent": "so are you going to tell me about this potty ?", "words": ["to", "so", "you", "this", "are", "potty", "about", "me"]}, {"sent": "because Thomas forgot to ask for his potty .", "words": ["for", "to", "potty", "his", "because"]}, {"sent": "youve been a good boy using that potty .", "words": ["that", "potty", "good", "a"]}, {"sent": "like your potty ?", "words": ["your", "like", "potty"]}, {"sent": "you wanna to sit on a potty and poop .", "words": ["to", "you", "and", "potty", "wanna", "on", "sit", "a"]}, {"sent": "shes using the potty .", "words": ["the", "potty"]}, {"sent": "what do you do with a potty ?", "words": ["what", "do", "with", "you", "potty", "a"]}, {"sent": "the lamp wouldnt light so papa bear put a new bulb in it .", "words": ["the", "put", "light", "so", "bear", "in", "it", "lamp", "new", "a"]}, {"sent": "and like right now Im trying to outof this boat dat was a sposta be a lamp .", "words": ["to", "was", "be", "like", "and", "this", "lamp", "boat", "a"]}, {"sent": "theres a picture of a table lamp or a bedside lamp at the side of the bed .", "words": ["the", "table", "of", "picture", "bed", "lamp", "at", "a"]}, {"sent": "just put the table lamp on .", "words": ["the", "put", "table", "lamp", "on"]}, {"sent": "right weve rescued the man fishing and one of the lamp posts .", "words": ["the", "of", "lamp", "and"]}, {"sent": "Catherine was saying well I hafta get a good lamp up here I hafta get a good lamp up here .", "words": ["up", "was", "get", "here", "lamp", "good", "a"]}, {"sent": "I w I dont have a lamp like that but I have a lamp .", "words": ["like", "have", "dont", "lamp", "that", "but", "a"]}, {"sent": "put the lamp on .", "words": ["the", "put", "lamp", "on"]}, {"sent": "how come you put it right on top of the table and the lamp ?", "words": ["the", "how", "put", "table", "of", "you", "and", "it", "lamp", "on"]}, {"sent": "what do you do with that lamp ?", "words": ["what", "do", "with", "you", "lamp", "that"]}, {"sent": "help me find a very special lamp .", "words": ["help", "lamp", "find", "me", "a"]}, {"sent": "did you make a pumpkin lamp at nursery ?", "words": ["you", "lamp", "did", "at", "pumpkin", "make", "a"]}, {"sent": "why do you sleep with your lamp on ?", "words": ["do", "your", "sleep", "with", "you", "lamp", "on", "why"]}, {"sent": "it is the lamp on Daddys drafting table .", "words": ["the", "table", "is", "it", "lamp", "on"]}, {"sent": "a lamp .", "words": ["lamp", "a"]}, {"sent": "whos on your lamp ?", "words": ["on", "your", "lamp"]}, {"sent": "thats the lamp .", "words": ["the", "lamp"]}, {"sent": "oh the lamp .", "words": ["the", "lamp"]}, {"sent": "Ill put the lamp in there .", "words": ["the", "put", "there", "in", "lamp"]}, {"sent": "oh I was gonna make a lamp .", "words": ["was", "make", "lamp", "a"]}, {"sent": "its not a street lamp Thomas .", "words": ["street", "lamp", "not", "a"]}, {"sent": "Ill buy a new one for my room and Ill give you that nice lamp with the leaves on it that I have okay ?", "words": ["the", "my", "for", "nice", "with", "and", "you", "have", "room", "lamp", "it", "on", "give", "new", "that", "buy", "a"]}, {"sent": "wheres the other lamp gone ?", "words": ["the", "lamp", "other"]}, {"sent": "theres a lamp .", "words": ["lamp", "a"]}, {"sent": "now its a lamp .", "words": ["lamp", "a"]}, {"sent": "thats a oil lamp .", "words": ["lamp", "a"]}, {"sent": "its a lamp .", "words": ["lamp", "a"]}, {"sent": "in the bedroom in the tall lamp .", "words": ["the", "in", "lamp", "bedroom"]}, {"sent": "see the lamp up there ?", "words": ["the", "up", "there", "lamp", "see"]}, {"sent": "and he falls down knocking over a lamp .", "words": ["and", "he", "lamp", "down", "over", "a"]}, {"sent": "pictures and a lamp .", "words": ["lamp", "a", "and"]}, {"sent": "dont mess with this lamp Xavier .", "words": ["dont", "with", "lamp", "this"]}, {"sent": "I dont know what you mean by lamp post come off .", "words": ["off", "what", "by", "you", "dont", "lamp"]}, {"sent": "got a new lamp ?", "words": ["new", "lamp", "a"]}, {"sent": "somebody had hit the lamp post and they had hadta turn the electricity off to mend it .", "words": ["the", "off", "to", "and", "they", "it", "lamp", "hit"]}, {"sent": "yeah it looks like a lamp .", "words": ["it", "lamp", "like", "a"]}, {"sent": "its over there beside the lamp .", "words": ["the", "beside", "there", "lamp", "over"]}, {"sent": "um I know I have a plug in here somewhere because I have a lamp plugged in over there .", "words": ["have", "here", "in", "there", "lamp", "over", "because", "a"]}, {"sent": "its not a street lamp that .", "words": ["lamp", "street", "that", "not", "a"]}, {"sent": "whats so special about this old lamp ?", "words": ["old", "so", "this", "lamp", "about"]}, {"sent": "dont mess with the lamp Fraser .", "words": ["the", "dont", "with", "lamp"]}, {"sent": "does the lamp come off ?", "words": ["the", "does", "lamp", "off"]}, {"sent": "shes got a lamp with dolphins .", "words": ["lamp", "with", "a"]}, {"sent": "youre building a street lamp ?", "words": ["street", "lamp", "a"]}, {"sent": "its over there next to the lamp .", "words": ["the", "to", "there", "lamp", "over"]}, {"sent": "you dont normally find jigsaw puzzles in a lamp do you ?", "words": ["do", "you", "dont", "in", "lamp", "find", "a"]}, {"sent": "good I spy a lamp .", "words": ["lamp", "good", "a"]}, {"sent": "theres a lamp in this room so we have got a lamp in here or in these rooms here .", "words": ["these", "we", "so", "have", "this", "in", "room", "lamp", "here", "a"]}, {"sent": "swinging from the lamp ?", "words": ["the", "lamp"]}, {"sent": "put the lamp right there .", "words": ["the", "there", "put", "lamp"]}, {"sent": "a little lamp .", "words": ["lamp", "little", "a"]}, {"sent": "and turn the lamp on .", "words": ["the", "on", "lamp", "and"]}, {"sent": "I dont sleep with my lamp on .", "words": ["my", "sleep", "with", "dont", "lamp", "on"]}, {"sent": "its a thin strip of cheese and when you stand it up like that it does look like a lamp post .", "words": ["up", "does", "when", "of", "stand", "like", "and", "you", "that", "it", "lamp", "look", "cheese", "a"]}, {"sent": "ah a little lamp like mummys .", "words": ["lamp", "like", "little", "a"]}, {"sent": "and a lamp .", "words": ["lamp", "a", "and"]}, {"sent": "whats on the pavement there right by the lamp post well by the bus stop ?", "words": ["the", "stop", "by", "there", "lamp", "on", "bus"]}, {"sent": "lamp .", "words": ["lamp"]}, {"sent": "lamp ?", "words": ["lamp"]}, {"sent": "yeah theres a lamp right there .", "words": ["there", "lamp", "a"]}, {"sent": "a lamp uhn isnt it ?", "words": ["it", "lamp", "a"]}, {"sent": "lamp .", "words": ["lamp"]}, {"sent": "not on mummies lamp .", "words": ["on", "lamp", "not"]}, {"sent": "turn the lamp off .", "words": ["the", "lamp", "off"]}, {"sent": "well done and else another one another lamp lady ?", "words": ["another", "lamp", "and"]}, {"sent": "no thats the lamp .", "words": ["the", "lamp"]}, {"sent": "hot lamp .", "words": ["hot", "lamp"]}, {"sent": "lamp shades ?", "words": ["lamp"]}, {"sent": "youre taking a lamp out .", "words": ["lamp", "out", "a"]}, {"sent": "near the end so theyve got room for a table and lamp .", "words": ["the", "for", "table", "so", "and", "room", "lamp", "a"]}, {"sent": "its only a tiny lamp .", "words": ["tiny", "lamp", "a"]}, {"sent": "its only a play lamp .", "words": ["play", "lamp", "a"]}, {"sent": "well just have a lamp there because we can have a lamp for later .", "words": ["can", "for", "we", "have", "there", "lamp", "because", "a"]}, {"sent": "bang into a lamp .", "words": ["lamp", "into", "a"]}, {"sent": "is it your street lamp ?", "words": ["your", "is", "it", "lamp", "street"]}, {"sent": "dont touch the lamp .", "words": ["the", "dont", "lamp", "touch"]}, {"sent": "thats the lamp .", "words": ["the", "lamp"]}, {"sent": "theres a lamp .", "words": ["lamp", "a"]}, {"sent": "well wheres that lamp go ?", "words": ["that", "lamp", "go"]}, {"sent": "no you dont want a lamp on the roof .", "words": ["the", "you", "dont", "lamp", "roof", "on", "a"]}, {"sent": "Thomas Im a talking lamp post .", "words": ["lamp", "a"]}, {"sent": "and the lamp ?", "words": ["the", "lamp", "and"]}, {"sent": "a lamp .", "words": ["lamp", "a"]}, {"sent": "but the globe hit the lamp and knocked Buzz outof the window .", "words": ["the", "and", "lamp", "window", "hit", "but"]}, {"sent": "is it hot the lamp ?", "words": ["the", "is", "hot", "it", "lamp"]}, {"sent": "you trying to turn the lamp down round ?", "words": ["the", "to", "you", "lamp", "down"]}, {"sent": "oh look theres a lamp post there .", "words": ["there", "lamp", "a", "look"]}, {"sent": "hes got his lamp .", "words": ["his", "lamp"]}, {"sent": "by your lamp .", "words": ["your", "lamp", "by"]}, {"sent": "they might bang their heads on the lamp .", "words": ["the", "their", "they", "lamp", "on"]}, {"sent": "do you wanna put the lamp on the table ?", "words": ["the", "do", "put", "table", "you", "lamp", "wanna", "on"]}, {"sent": "the street lamp .", "words": ["the", "street", "lamp"]}, {"sent": "the lamp post come off dig dig dig .", "words": ["the", "lamp", "off"]}, {"sent": "its hiding behind that lamp isnt it ?", "words": ["that", "it", "behind", "lamp"]}, {"sent": "somebody had hit the lamp post hadnt they ?", "words": ["the", "they", "hit", "lamp"]}, {"sent": "a red lamp .", "words": ["red", "lamp", "a"]}, {"sent": "round and round street lamp ?", "words": ["street", "lamp", "and"]}, {"sent": "Aladin returned to the palace with the lamp .", "words": ["the", "to", "with", "lamp"]}, {"sent": "youre checking the street lamp .", "words": ["the", "street", "lamp"]}, {"sent": "but I know Im gonna get your box but I need to unzip your wait wait wait wait .", "words": ["wait", "need", "box", "your", "need to", "to", "get", "but"]}, {"sent": "I need to take that garbage out .", "words": ["need", "need to", "to", "that", "out", "take", "garbage"]}, {"sent": "I dont need to say it again if I dont wanna .", "words": ["need", "if", "need to", "to", "say", "dont", "it", "wanna"]}, {"sent": "okay I need to type something .", "words": ["need to", "to", "need"]}, {"sent": "no all done you need to have more .", "words": ["all", "need", "need to", "to", "you", "have", "more"]}, {"sent": "I just need to put on my jacket up in here .", "words": ["up", "put", "my", "need", "need to", "to", "here", "in", "on", "jacket"]}, {"sent": "no honey you need to hold your microphone .", "words": ["your", "need", "hold", "need to", "to", "you"]}, {"sent": "first we need to find Boots the monkey .", "words": ["the", "need", "we", "need to", "to", "monkey", "first", "find"]}, {"sent": "so it says we need to find .", "words": ["need", "we", "need to", "to", "so", "it", "find"]}, {"sent": "now these we need to find some tissue for you .", "words": ["these", "for", "need", "we", "tissue", "to", "need to", "you", "find", "some"]}, {"sent": "oh you need to go potty .", "words": ["need", "go", "need to", "to", "you", "potty"]}, {"sent": "no you need to go to bed .", "words": ["need", "go", "need to", "to", "bed", "you"]}, {"sent": "we need to find Owls door .", "words": ["need", "we", "need to", "to", "find", "door"]}, {"sent": "thats good they need to have breaks .", "words": ["need", "need to", "to", "have", "they", "good"]}, {"sent": "Naima do you not need to poop ?", "words": ["do", "need", "need to", "to", "you", "not"]}, {"sent": "were gonna have lunch soon you dont need to each your button .", "words": ["need", "your", "each", "button", "need to", "to", "you", "have", "dont", "were"]}, {"sent": "thats a big piece youll need to chew it well .", "words": ["need", "need to", "to", "it", "big", "a"]}, {"sent": "youre gonna need to look here .", "words": ["need", "need to", "to", "here", "look"]}, {"sent": "thats what we need to be looking for ah .", "words": ["what", "for", "need", "we", "need to", "to", "be"]}, {"sent": "yep we need to put some clothes on you and change your diaper .", "words": ["put", "need", "your", "we", "diaper", "to", "need to", "you", "and", "on", "some"]}, {"sent": "what time do you need to be picked up ?", "words": ["up", "what", "do", "need", "need to", "to", "be", "you"]}, {"sent": "maybe you need to sing it a good night xxx a good night song .", "words": ["need", "need to", "to", "you", "sing", "it", "good", "a"]}, {"sent": "or do you need to make a bridge over that river ?", "words": ["do", "need", "need to", "to", "you", "that", "over", "make", "a"]}, {"sent": "whatever goes here would need to have a straight edge .", "words": ["need", "need to", "to", "here", "have", "would", "a"]}, {"sent": "now we need to take that boat across turtle river before we get into the boat what should we wear to be safe ?", "words": ["the", "what", "need", "into", "we", "need to", "to", "boat", "be", "get", "that", "turtle", "take"]}, {"sent": "maybe we need to get some fingerpaints huh William ?", "words": ["need", "we", "need to", "to", "get", "some"]}, {"sent": "we need to clean up a little bit if were gonna read that .", "words": ["clean", "up", "need", "if", "we", "need to", "to", "little", "that", "read", "were", "a"]}, {"sent": "okay lets see what you need to do next .", "words": ["what", "do", "need", "need to", "to", "you", "see"]}, {"sent": "and I dont really need to get .", "words": ["need", "need to", "to", "and", "get", "dont"]}, {"sent": "getting all the things you need to go ?", "words": ["the", "all", "need", "go", "need to", "to", "you"]}, {"sent": "we dont need to have the timer up that high .", "words": ["the", "up", "need", "we", "need to", "to", "have", "dont", "high", "that"]}, {"sent": "we need to go for a walk huh ?", "words": ["for", "need", "go", "we", "to", "need to", "walk", "a"]}, {"sent": "that tells you you need to turn it around and open it on that side .", "words": ["need", "open", "to", "need to", "you", "and", "it", "around", "on", "that"]}, {"sent": "we need to get ready to go to the postoffice .", "words": ["the", "need", "go", "we", "to", "need to", "get"]}, {"sent": "you need to get ones with two sides .", "words": ["need", "need to", "to", "with", "you", "get"]}, {"sent": "oh do you need to call the ambulance ?", "words": ["the", "do", "need", "need to", "to", "you"]}, {"sent": "yeah I know but you need to find another orange .", "words": ["need", "need to", "to", "you", "another", "orange", "find", "but"]}, {"sent": "well I think you said that you wanted to get a yoga card which means what you need to do is brush teeth now .", "words": ["what", "think", "do", "need", "is", "brush", "to", "need to", "which", "you", "get", "that", "a"]}, {"sent": "this is why I need to clip it on and have you not touch it because it keeps falling apart the more you play with it okay ?", "words": ["the", "need", "is", "need to", "to", "play", "with", "because", "touch", "and", "have", "this", "you", "it", "more", "on", "not", "why"]}, {"sent": "huh you wanna draw what do you need to draw ?", "words": ["what", "do", "need", "need to", "to", "draw", "you", "wanna"]}, {"sent": "well I think you need to get down to do the chair because I see some on the side that maybe you didnt see .", "words": ["the", "do", "think", "need", "need to", "to", "chair", "you", "get", "on", "see", "down", "that", "because", "some"]}, {"sent": "you need to put the brush in the water first .", "words": ["the", "put", "need", "brush", "to", "need to", "first", "you", "in", "water"]}, {"sent": "you need to stay seated .", "words": ["need", "need to", "to", "you", "stay"]}, {"sent": "if you need to .", "words": ["need", "if", "need to", "to", "you"]}, {"sent": "if you wanna if you wanna hear Ethan on the camera you need to wear the backpack .", "words": ["the", "camera", "if", "need", "need to", "to", "hear", "you", "wanna", "on"]}, {"sent": "well you need to make sure you have lunch before you have icecream .", "words": ["need", "need to", "to", "you", "have", "make"]}, {"sent": "we need to wash him up .", "words": ["up", "need", "we", "need to", "to", "him", "wash"]}, {"sent": "you need to eat a few more okay ?", "words": ["need", "need to", "to", "you", "eat", "more", "a"]}, {"sent": "now we need to find .", "words": ["need", "we", "need to", "to", "find"]}, {"sent": "then you need to behave .", "words": ["need", "need to", "to", "you", "then"]}, {"sent": "need to get up and play ?", "words": ["up", "need", "need to", "to", "play", "and", "get"]}, {"sent": "and I need to find Pal he began calling .", "words": ["need", "need to", "to", "and", "he", "find"]}, {"sent": "well need to keep looking .", "words": ["need to", "to", "need"]}, {"sent": "we need to get Bob to fix it dont we ?", "words": ["need", "we", "need to", "to", "fix", "get", "dont", "it"]}, {"sent": "we need to get ready to go soon .", "words": ["need", "go", "we", "to", "need to", "get"]}, {"sent": "you dont really need to be worried about the fly flies dont really bite .", "words": ["the", "need", "need to", "to", "be", "you", "dont", "about", "bite"]}, {"sent": "does it need to be fixed ?", "words": ["does", "need", "need to", "to", "be", "it"]}, {"sent": "does the fox need to have a den ?", "words": ["the", "does", "need", "need to", "to", "have", "a"]}, {"sent": "xxx people really need to do that .", "words": ["do", "need", "need to", "to", "that"]}, {"sent": "do we need to fix it ?", "words": ["do", "need", "we", "need to", "to", "fix", "it"]}, {"sent": "so we dont need to keep .", "words": ["need", "we", "need to", "so", "to", "dont"]}, {"sent": "I need a garbage pile is what I need to do .", "words": ["what", "do", "need", "is", "need to", "to", "garbage", "a"]}, {"sent": "but shes very clean but you still need to wash your hands before you touch food .", "words": ["clean", "need", "your", "need to", "to", "touch", "you", "food", "wash", "but"]}, {"sent": "does your diaper need to be changed ?", "words": ["does", "your", "need", "diaper", "to", "be", "need to"]}, {"sent": "you need a hole you need to put this in a hole do you see a hole ?", "words": ["do", "put", "need", "need to", "to", "you", "this", "in", "see", "a"]}, {"sent": "these bubbles might need to rest .", "words": ["these", "need", "need to", "to", "bubbles"]}, {"sent": "you have anything else you need to look at when we go to the mall ?", "words": ["the", "need", "we", "go", "to", "need to", "you", "have", "at", "look", "when"]}, {"sent": "we need to clean this up .", "words": ["clean", "up", "need", "we", "need to", "to", "this"]}, {"sent": "do I need to open it ?", "words": ["do", "need", "open", "to", "need to", "it"]}, {"sent": "well that wouldnt need to have a straight .", "words": ["need", "need to", "to", "have", "that", "a"]}, {"sent": "you dont need to take them all .", "words": ["all", "need", "need to", "to", "them", "you", "dont", "take"]}, {"sent": "do you need to go into timeout ?", "words": ["do", "need", "into", "go", "need to", "to", "you"]}, {"sent": "hm we need to fix this book .", "words": ["need", "we", "need to", "to", "fix", "this", "book"]}, {"sent": "I might pack some other lunch things for you because if we have lunch with Daddy then well need to have more stuff .", "words": ["for", "need", "if", "we", "need to", "to", "with", "you", "other", "have", "more", "some", "then", "because"]}, {"sent": "you can get down now you need to wash your hands .", "words": ["can", "need", "your", "need to", "to", "you", "get", "down", "wash"]}, {"sent": "his ribbons coming untied we need to tie it .", "words": ["need", "we", "need to", "to", "it", "his"]}, {"sent": "well Im putting them on your table first but I really need to remind you to not touch them until theyre dry when theyre on your little black table in the living room .", "words": ["the", "table", "need to", "dry", "first", "when", "not", "touch", "on", "to", "living room", "you", "little", "your", "need", "them", "black", "in", "room", "but"]}, {"sent": "youre gonna give me that I need to put your socks on here .", "words": ["put", "need", "your", "need to", "to", "here", "on", "me", "give", "that"]}, {"sent": "yeah do you need to use the potty ?", "words": ["the", "do", "need", "need to", "to", "you", "potty"]}, {"sent": "Naima I hafta go check the laundry I xxx xxx need to stop playing for xxx .", "words": ["the", "for", "need", "go", "need to", "to", "stop"]}, {"sent": "we need to snap these .", "words": ["these", "need", "we", "need to", "to"]}, {"sent": "yes said his Mommy you need to be brave .", "words": ["need", "need to", "to", "be", "you", "his"]}, {"sent": "and a lot of times shell say something to me and I dont know what shes saying and I just need to like not even put it in context and just keep repeating the word ?", "words": ["the", "what", "put", "need", "of", "to", "a lot", "need to", "say", "like", "and", "dont", "in", "it", "me", "not", "a"]}, {"sent": "what will you tell me if you need to go ?", "words": ["what", "need", "if", "go", "need to", "to", "you", "will", "me"]}, {"sent": "okay so we need to put some of this is for the ice cream .", "words": ["the", "ice cream", "put", "for", "need", "we", "of", "to", "so", "is", "need to", "this", "ice", "some"]}, {"sent": "uh maybe we need to push it in a little more and rub it .", "words": ["need", "push", "we", "need to", "to", "little", "and", "in", "it", "more", "a"]}, {"sent": "and actually I have something I need to do in .", "words": ["do", "need", "need to", "to", "and", "have", "in"]}, {"sent": "you need to poop now ?", "words": ["need to", "to", "need", "you"]}, {"sent": "maybe we need to draw SpiderMan some eyes .", "words": ["need", "we", "need to", "to", "draw", "some"]}, {"sent": "now you need to pick up all that money .", "words": ["up", "all", "money", "need", "pick", "to", "need to", "you", "that"]}, {"sent": "you dont need to twist it honey .", "words": ["need", "need to", "to", "you", "dont", "it"]}, {"sent": "and I need to talk to my broker .", "words": ["my", "need", "need to", "to", "and", "talk"]}, {"sent": "well I need to measure your .", "words": ["your", "to", "need", "need to"]}, {"sent": "oh no were not gonna do glue you wanna do magnets ?", "words": ["do", "you", "wanna", "glue", "were", "not"]}, {"sent": "put some glue on it .", "words": ["put", "it", "on", "some", "glue"]}, {"sent": "looks like you have hardly any room to glue anything else .", "words": ["to", "any", "like", "you", "have", "room", "glue"]}, {"sent": "it doesnt have glue on it .", "words": ["have", "glue", "on", "it"]}, {"sent": "you dont hafta put glue all over the thing .", "words": ["the", "all", "put", "you", "dont", "glue", "over"]}, {"sent": "look heres your other glue and this glue is a good one .", "words": ["your", "is", "other", "and", "this", "good", "look", "glue", "a"]}, {"sent": "can we glue these pieces of paper on here ?", "words": ["can", "these", "we", "of", "paper", "here", "on", "glue"]}, {"sent": "the glue .", "words": ["the", "glue"]}, {"sent": "youve got glue under there .", "words": ["glue", "under", "there"]}, {"sent": "wheres the lid for the glue ?", "words": ["the", "glue", "for"]}, {"sent": "Kell wanna get something outof that bag over there and maybe well cut and glue ?", "words": ["cut", "and", "get", "there", "that", "wanna", "glue", "over"]}, {"sent": "may be shes gonna glue hearts on the notes like you did when you made valentines .", "words": ["the", "be", "like", "you", "did", "on", "glue", "when"]}, {"sent": "I put glue on him so he wouldnt fall off .", "words": ["off", "put", "so", "fall", "him", "he", "on", "glue"]}, {"sent": "and theyre gonna glue the wallpaper to the wall .", "words": ["the", "glue", "to", "and"]}, {"sent": "I hadta glue it on didnt I ?", "words": ["on", "glue", "it"]}, {"sent": "glue ?", "words": ["glue"]}, {"sent": "alright thats enough glue on your hands .", "words": ["on", "glue", "your"]}, {"sent": "dried glue on your chin .", "words": ["on", "glue", "your", "chin"]}, {"sent": "Daddy keeps saying hell glue it .", "words": ["glue", "it"]}, {"sent": "where is your glue one ?", "words": ["where", "glue", "your", "is"]}, {"sent": "you still gonna glue some things on it ?", "words": ["you", "it", "on", "glue", "some"]}, {"sent": "Pete gobbled the glue stick with gusto .", "words": ["the", "glue", "stick", "with"]}, {"sent": "and you sprinkled glue on and then onto the glue you stuck lotsof sparkly glitter .", "words": ["the", "you", "and", "stuck", "on", "glue", "then"]}, {"sent": "I think we oughta put a blob of glue on this and stick it on permanently .", "words": ["think", "put", "stick", "we", "of", "and", "this", "it", "on", "glue", "a"]}, {"sent": "just pop the glue up on the up on there for the moment .", "words": ["the", "up", "pop", "for", "there", "on", "glue"]}, {"sent": "see you put the glue .", "words": ["the", "put", "you", "see", "glue"]}, {"sent": "but its not baby glue glue otherwise youll get too sticky .", "words": ["sticky", "get", "too", "glue", "not", "but"]}, {"sent": "without some glue .", "words": ["some", "glue"]}, {"sent": "uhhuh if you break it you hafta glue it again right ?", "words": ["if", "you", "it", "glue", "break"]}, {"sent": "glue ?", "words": ["glue"]}, {"sent": "glue ?", "words": ["glue"]}, {"sent": "get some glue and glue those red ones together .", "words": ["red", "those", "and", "get", "some", "glue"]}, {"sent": "Ill get you some glue from Belly yeah ?", "words": ["get", "some", "glue", "you"]}, {"sent": "where did that glue ?", "words": ["where", "glue", "that", "did"]}, {"sent": "and give the glue back to Mama .", "words": ["the", "back", "to", "and", "glue", "give"]}, {"sent": "and the glue .", "words": ["the", "glue", "and"]}, {"sent": "its had glue on it Becky .", "words": ["on", "glue", "it"]}, {"sent": "its alright we can glue them back on .", "words": ["can", "back", "we", "them", "on", "glue"]}, {"sent": "glue ?", "words": ["glue"]}, {"sent": "it was like some of the glue from the envelope had stuck on the card which was why I couldnt open it .", "words": ["the", "of", "was", "open", "like", "it", "stuck", "on", "some", "which", "why", "glue"]}, {"sent": "well we could glue things and we could also use stickers .", "words": ["glue", "could", "and", "we"]}, {"sent": "hafta we need the glue .", "words": ["the", "glue", "need", "we"]}, {"sent": "in a minute we could get a plain piece of paper and some glue and you could stick the little .", "words": ["the", "stick", "we", "of", "paper", "and", "get", "you", "in", "little", "glue", "some", "could", "a"]}, {"sent": "oh youd like to glue them ?", "words": ["to", "them", "like", "glue"]}, {"sent": "have you got very good glue very strong glue yeah ?", "words": ["have", "glue", "good", "you"]}, {"sent": "go upstairs and get the paper and the glue .", "words": ["the", "go", "paper", "and", "get", "glue"]}, {"sent": "paste or glue ?", "words": ["glue"]}, {"sent": "mhm I dont know if we can I think we need some glue I think its broken .", "words": ["can", "think", "need", "if", "we", "broken", "dont", "some", "glue"]}, {"sent": "got some glue havent we ?", "words": ["some", "we", "glue"]}, {"sent": "Ill get some glue for your head then ?", "words": ["for", "your", "head", "get", "some", "then", "glue"]}, {"sent": "glue on the back .", "words": ["on", "glue", "back", "the"]}, {"sent": "the pussy cat with the glue .", "words": ["the", "cat", "with", "glue"]}, {"sent": "that glue .", "words": ["glue", "that"]}, {"sent": "put some glue on it .", "words": ["put", "it", "on", "some", "glue"]}, {"sent": "you dont eat glue .", "words": ["eat", "dont", "glue", "you"]}, {"sent": "I can cut and glue .", "words": ["can", "glue", "cut", "and"]}, {"sent": "put some glue on it then .", "words": ["put", "it", "on", "some", "then", "glue"]}, {"sent": "pop some glue on there .", "words": ["pop", "there", "on", "some", "glue"]}, {"sent": "I cant stand this glue up .", "words": ["up", "stand", "this", "glue"]}, {"sent": "some glue in there .", "words": ["some", "in", "there", "glue"]}, {"sent": "the nozzle of the glue tube snapped didnt it ?", "words": ["the", "of", "it", "glue"]}, {"sent": "let the glue dry and it wont come off .", "words": ["the", "off", "dry", "and", "it", "glue"]}, {"sent": "you shouldnt move them after you glue them .", "words": ["glue", "them", "you"]}, {"sent": "have we got any glue gun and well just stick that down ?", "words": ["stick", "we", "any", "and", "have", "glue", "down", "that"]}, {"sent": "no use the glue darling .", "words": ["the", "glue"]}, {"sent": "well hafta get some glue and then Ill ask daddy to glue it on .", "words": ["to", "and", "get", "it", "on", "some", "then", "glue"]}, {"sent": "I hadta throw the other glue away .", "words": ["the", "throw", "other", "away", "glue"]}, {"sent": "I can put it together but it wont stay without glue .", "words": ["can", "put", "it", "stay", "but", "glue"]}, {"sent": "we must have glue we have stickers .", "words": ["have", "glue", "we"]}, {"sent": "our glue stick is a little .", "words": ["stick", "is", "our", "little", "glue", "a"]}, {"sent": "its glue .", "words": ["glue"]}, {"sent": "there you spread them around and stick them into the glue .", "words": ["the", "stick", "into", "them", "and", "you", "there", "around", "glue"]}, {"sent": "now wheres that glue .", "words": ["glue", "that"]}, {"sent": "this glue isnt so great .", "words": ["this", "so", "glue"]}, {"sent": "yeah sticky glue .", "words": ["sticky", "glue"]}, {"sent": "you dont put glue when youve spilt water .", "words": ["put", "you", "dont", "water", "glue", "when"]}, {"sent": "yeah well you can glue it right down to the paper and then we can glue a stem on the paper .", "words": ["can", "the", "we", "to", "paper", "you", "and", "it", "on", "glue", "then", "down", "a"]}, {"sent": "no youre not having the glue because its Wills and its nearly allgone .", "words": ["the", "and", "glue", "not", "because"]}, {"sent": "I dont have any glue .", "words": ["have", "dont", "any", "glue"]}, {"sent": "well hafta glue it on wont we or tape it on at least .", "words": ["we", "tape", "it", "at", "on", "glue"]}, {"sent": "with glue ?", "words": ["glue", "with"]}, {"sent": "the tube of glue is almost empty before .", "words": ["the", "is", "of", "empty", "glue"]}, {"sent": "oh is that one of the boys with sticky glue behind him ?", "words": ["the", "is", "of", "sticky", "with", "behind", "him", "glue", "that"]}, {"sent": "if I glue it you cant play with it for a while so the glue can dry .", "words": ["the", "can", "for", "if", "so", "dry", "play", "with", "you", "it", "glue", "a"]}, {"sent": "what do you think shes going to do with the glue ?", "words": ["the", "what", "do", "think", "to", "with", "you", "glue"]}, {"sent": "you can decorate them and then you can also glue them onto things later .", "words": ["can", "them", "and", "you", "glue", "then"]}, {"sent": "she gave you some glue ?", "words": ["some", "glue", "she", "you"]}, {"sent": "Im not Im not Im not going to glue until Kates finished mum .", "words": ["to", "not", "glue"]}, {"sent": "so shes taking the glue to mend the chipped vase .", "words": ["the", "so", "to", "glue"]}, {"sent": "er glue .", "words": ["glue"]}, {"sent": "and you look like youve got glue on your top as well dont you ?", "words": ["your", "like", "you", "and", "dont", "look", "on", "glue"]}, {"sent": "good maybe we should glue all the pieces todether and hang it on your wall .", "words": ["the", "all", "your", "we", "and", "it", "good", "on", "glue"]}, {"sent": "you pop glue on that side .", "words": ["pop", "you", "on", "glue", "that"]}, {"sent": "you can glue this piece of paper .", "words": ["can", "of", "paper", "you", "this", "glue"]}, {"sent": "glue the back black parts in the place on the ladybirds red body .", "words": ["the", "red", "back", "black", "in", "on", "glue"]}, {"sent": "sticky glue .", "words": ["sticky", "glue"]}, {"sent": "do you want cheese or tuna ?", "words": ["do", "tuna", "cheese", "you"]}, {"sent": "do you like tuna fish ?", "words": ["do", "like", "tuna", "you", "fish"]}, {"sent": "and apparently Ive learned this is tuna .", "words": ["this", "tuna", "is", "and"]}, {"sent": "tuna sandwiches .", "words": ["tuna"]}, {"sent": "thats tuna .", "words": ["tuna"]}, {"sent": "tuna ?", "words": ["tuna"]}, {"sent": "I dont think she would know tuna slash throw up .", "words": ["up", "think", "throw", "tuna", "dont", "would", "she"]}, {"sent": "are you smearing tuna all over this baby ?", "words": ["all", "tuna", "you", "this", "are", "over"]}, {"sent": "I dont like tuna .", "words": ["dont", "like", "tuna"]}, {"sent": "now wheres that little tin of tuna ?", "words": ["of", "tuna", "that", "little"]}, {"sent": "hm I think thats tuna .", "words": ["think", "tuna"]}, {"sent": "tuna .", "words": ["tuna"]}, {"sent": "sure we eat tuna fish .", "words": ["eat", "tuna", "fish", "we"]}, {"sent": "yeah lets get that nasty tuna out of there .", "words": ["of", "tuna", "get", "there", "out", "that"]}, {"sent": "mm tasty tuna .", "words": ["tuna"]}, {"sent": "he doesnt eat a lot of tuna just mostly because I dont .", "words": ["of", "a lot", "tuna", "eat", "dont", "he", "because", "a"]}, {"sent": "for tuna .", "words": ["for", "tuna"]}, {"sent": "tuna fish ?", "words": ["fish", "tuna"]}, {"sent": "you wanna cook some tuna ?", "words": ["tuna", "you", "cook", "wanna", "some"]}, {"sent": "pizza or tuna fish which one ?", "words": ["which", "fish", "pizza", "tuna"]}, {"sent": "tuna .", "words": ["tuna"]}, {"sent": "I hope you like tuna do you ?", "words": ["do", "tuna", "like", "you"]}, {"sent": "thats tuna .", "words": ["tuna"]}, {"sent": "I eat so much tuna .", "words": ["eat", "so", "much", "tuna"]}, {"sent": "oh yum its like a tuna salad .", "words": ["tuna", "like", "a"]}, {"sent": "yeah she wont eat tuna in water shell only eat packed in olive oil with salt in it .", "words": ["with", "tuna", "eat", "in", "salt", "it", "water", "she"]}, {"sent": "its like a cheeseburger on its only its a three cheese tuna sandwich .", "words": ["like", "tuna", "on", "cheese", "sandwich", "a"]}, {"sent": "and just had tuna and cheese ?", "words": ["tuna", "cheese", "and"]}, {"sent": "should we have tuna and cheese ?", "words": ["we", "tuna", "have", "and", "cheese"]}, {"sent": "would you like tuna fish sandwiches ?", "words": ["like", "would", "tuna", "you", "fish"]}, {"sent": "and you like tuna .", "words": ["tuna", "like", "you", "and"]}, {"sent": "like tuna sort of .", "words": ["of", "like", "tuna"]}, {"sent": "altogether with uh tuna .", "words": ["with", "tuna"]}, {"sent": "oh I think thats tuna .", "words": ["think", "tuna"]}, {"sent": "and some tuna ?", "words": ["some", "tuna", "and"]}, {"sent": "tuna fish ?", "words": ["fish", "tuna"]}, {"sent": "pizza and the tuna I dont know if I can go for that .", "words": ["the", "can", "for", "if", "go", "pizza", "tuna", "and", "dont", "that"]}, {"sent": "what can you make me with tuna ?", "words": ["can", "what", "with", "you", "tuna", "me", "make"]}, {"sent": "tuna salad lets see what else we hafta eat .", "words": ["what", "we", "tuna", "eat", "see"]}, {"sent": "Daddys gonna eat some tuna first .", "words": ["eat", "some", "first", "tuna"]}, {"sent": "oh this is kind of weird I guess this is tuna .", "words": ["of", "this", "tuna", "is"]}, {"sent": "you dont eat tuna .", "words": ["eat", "dont", "tuna", "you"]}, {"sent": "this is tuna .", "words": ["this", "tuna", "is"]}, {"sent": "tuna .", "words": ["tuna"]}, {"sent": "so like the the tuna we were both a little like hm um but everything that she has had contact with or eaten or helped me cook she knew what it was .", "words": ["the", "what", "we", "so", "was", "with", "like", "tuna", "little", "that", "cook", "it", "me", "were", "she", "but", "a"]}, {"sent": "tuna fish .", "words": ["fish", "tuna"]}, {"sent": "tuna do we need some tuna ?", "words": ["do", "need", "we", "tuna", "some"]}, {"sent": "um I mean it depends how hungry we are you could have tuna mousse and toast um and salad and that might be enough .", "words": ["how", "toast", "we", "be", "hungry", "you", "have", "tuna", "and", "it", "are", "could", "that"]}, {"sent": "and then she thought the tuna was batter because we just made a cake so but .", "words": ["the", "cake", "we", "was", "so", "tuna", "and", "but", "then", "she", "because", "a"]}, {"sent": "thats tuna fish .", "words": ["fish", "tuna"]}, {"sent": "tuna .", "words": ["tuna"]}, {"sent": "fish tuna fish .", "words": ["tuna", "fish"]}, {"sent": "with tuna there we go .", "words": ["go", "we", "with", "tuna", "there"]}, {"sent": "Im gonna eat some tuna now .", "words": ["eat", "some", "tuna"]}, {"sent": "lets cook the tuna .", "words": ["the", "cook", "tuna"]}, {"sent": "should we put tuna on the bun ?", "words": ["the", "put", "we", "tuna", "on"]}, {"sent": "thats salmon and tuna Purdie .", "words": ["tuna", "and"]}, {"sent": "tuna sandwiches .", "words": ["tuna"]}, {"sent": "tuna .", "words": ["tuna"]}, {"sent": "stir the tuna .", "words": ["the", "tuna"]}, {"sent": "its a great big tuna fish .", "words": ["tuna", "big", "fish", "a"]}, {"sent": "tuna and beets and cheese .", "words": ["cheese", "and", "tuna"]}, {"sent": "tuna .", "words": ["tuna"]}, {"sent": "I get our Conor the tuna tuna pasta .", "words": ["get", "the", "tuna", "our"]}, {"sent": "tuna .", "words": ["tuna"]}, {"sent": "would you like to put the sardines with the tuna ?", "words": ["the", "put", "to", "with", "like", "you", "tuna", "would"]}, {"sent": "alright youre cooking up some tuna .", "words": ["up", "some", "tuna"]}, {"sent": "tuna darling .", "words": ["tuna"]}, {"sent": "so I call up and I said can you ask if theres does the practice have any guidelines as to how much tuna a kid can eat ?", "words": ["up", "can", "does", "the", "how", "if", "so", "to", "any", "you", "and", "have", "tuna", "eat", "much", "a"]}, {"sent": "is this tuna ?", "words": ["this", "tuna", "is"]}, {"sent": "its tuna .", "words": ["tuna"]}, {"sent": "put tuna in there .", "words": ["in", "put", "there", "tuna"]}, {"sent": "what could we make with the tuna fish .", "words": ["the", "what", "we", "with", "tuna", "fish", "could", "make"]}, {"sent": "xxx would you eat tuna or salmon or .", "words": ["eat", "tuna", "you", "would"]}, {"sent": "water and tuna .", "words": ["tuna", "and", "water"]}, {"sent": "that looks like tuna salad .", "words": ["that", "like", "tuna"]}, {"sent": "that tuna ?", "words": ["that", "tuna"]}, {"sent": "and tuna .", "words": ["tuna", "and"]}, {"sent": "I think that might be tuna .", "words": ["be", "think", "that", "tuna"]}, {"sent": "tuna .", "words": ["tuna"]}, {"sent": "food for Purdie made with tuna fish .", "words": ["for", "with", "tuna", "food", "fish"]}, {"sent": "because she wants everyday for lunch can we have tuna ?", "words": ["can", "for", "we", "tuna", "have", "she", "because"]}, {"sent": "she has just finished tuna so well put away sardine and trout .", "words": ["put", "so", "tuna", "and", "away", "she"]}, {"sent": "some tuna .", "words": ["some", "tuna"]}, {"sent": "tuna .", "words": ["tuna"]}, {"sent": "um tuna she seemed to like but doesnt eat .", "words": ["to", "like", "tuna", "eat", "she", "but"]}, {"sent": "yeah you love tuna .", "words": ["tuna", "love", "you"]}, {"sent": "beets definitely and I dont think he recognized the tuna .", "words": ["the", "think", "and", "tuna", "dont", "he"]}, {"sent": "tuna .", "words": ["tuna"]}, {"sent": "tuna peas .", "words": ["peas", "tuna"]}, {"sent": "I think its sposta be tuna fish but its quite frightening .", "words": ["think", "be", "tuna", "fish", "but"]}, {"sent": "you know they said canned tuna has less than fresh tuna its really I guess its tuna and swordfish that your suppose to cut back on .", "words": ["your", "cut", "back", "to", "tuna", "and", "they", "you", "on", "that"]}, {"sent": "is that tuna ?", "words": ["tuna", "that", "is"]}, {"sent": "tuna I didnt know what it was .", "words": ["was", "what", "it", "tuna"]}, {"sent": "do you wanna tuna sandwich ?", "words": ["do", "tuna", "you", "wanna", "sandwich"]}, {"sent": "this is canned tuna .", "words": ["this", "tuna", "is"]}, {"sent": "theres um some tuna mousse which Sergio made which you could have either as a thing on its in as in a thing as you know as the main course or in addition to something else like sausages xxx .", "words": ["the", "to", "like", "you", "tuna", "have", "in", "on", "some", "could", "which", "a"]}, {"sent": "does Mama make you tuna ?", "words": ["does", "tuna", "make", "you"]}, {"sent": "is this tuna fish ?", "words": ["this", "tuna", "fish", "is"]}, {"sent": "whered your popsicle go ?", "words": ["popsicle", "your", "go"]}, {"sent": "popsicle molds .", "words": ["popsicle"]}, {"sent": "I just wanna eat my orange popsicle .", "words": ["my", "eat", "orange", "wanna", "popsicle"]}, {"sent": "I know you like popsicle .", "words": ["popsicle", "like", "you"]}, {"sent": "you ready for your popsicle ?", "words": ["popsicle", "for", "your", "you"]}, {"sent": "put it on the popsicle .", "words": ["the", "put", "it", "on", "popsicle"]}, {"sent": "you cant do it while youre eating your popsicle .", "words": ["do", "your", "you", "it", "popsicle"]}, {"sent": "Max knew Ruby would find a reason to throw out his popsicle she might even throw out his quack quack duck .", "words": ["throw", "to", "would", "out", "find", "his", "duck", "popsicle", "she", "a"]}, {"sent": "your popsicle ?", "words": ["popsicle", "your"]}, {"sent": "oh he eat him popsicle .", "words": ["eat", "him", "popsicle", "he"]}, {"sent": "Max put his popsicle in his shoe .", "words": ["put", "shoe", "in", "his", "popsicle"]}, {"sent": "what is the popsicle made outof ?", "words": ["the", "what", "popsicle", "is"]}, {"sent": "peanutbutter and jelly and then some icecream and then after while a popsicle .", "words": ["jelly", "and", "some", "then", "popsicle", "a"]}, {"sent": "I think you had better finish your popsicle .", "words": ["better", "think", "your", "finish", "you", "popsicle"]}, {"sent": "ready for your popsicle ?", "words": ["popsicle", "for", "your"]}, {"sent": "thats a great popsicle .", "words": ["popsicle", "a"]}, {"sent": "a popsicle ?", "words": ["popsicle", "a"]}, {"sent": "popsicle .", "words": ["popsicle"]}, {"sent": "come on dont put the popsicle on your foot sweetie .", "words": ["the", "put", "your", "dont", "foot", "on", "popsicle"]}, {"sent": "xxx have this popsicle .", "words": ["have", "this", "popsicle"]}, {"sent": "a popsicle ?", "words": ["popsicle", "a"]}, {"sent": "a popsicle was it ?", "words": ["was", "popsicle", "it", "a"]}, {"sent": "maybe Adam would like to paint his popsicle sticks .", "words": ["paint", "to", "like", "would", "his", "popsicle"]}, {"sent": "xxx popsicle hm ?", "words": ["popsicle"]}, {"sent": "do you want a popsicle or a sprinkle cookie ?", "words": ["cookie", "do", "you", "popsicle", "a"]}, {"sent": "are you eating your popsicle ?", "words": ["popsicle", "are", "your", "you"]}, {"sent": "youll hafta talk without the popsicle in your mouth .", "words": ["the", "your", "mouth", "in", "talk", "popsicle"]}, {"sent": "popsicle right .", "words": ["popsicle"]}, {"sent": "popsicle .", "words": ["popsicle"]}, {"sent": "popsicle .", "words": ["popsicle"]}, {"sent": "Jean got you a popsicle ?", "words": ["popsicle", "a", "you"]}, {"sent": "a couple of weeks ago and he had a popsicle mold set that was different .", "words": ["of", "was", "and", "he", "popsicle", "that", "a"]}, {"sent": "no it isnt from the popsicle .", "words": ["the", "it", "popsicle"]}, {"sent": "a popsicle ?", "words": ["popsicle", "a"]}, {"sent": "did you already eat your popsicle ?", "words": ["your", "you", "eat", "did", "popsicle"]}, {"sent": "can you get up here and sit down and Ill get you a popsicle .", "words": ["can", "up", "popsicle", "you", "get", "here", "and", "sit", "down", "a"]}, {"sent": "popsicle .", "words": ["popsicle"]}, {"sent": "hope your popsicle dunna melt .", "words": ["popsicle", "your"]}, {"sent": "like plastic star stick tops that went into it and she didnt know that it was popsicle mold .", "words": ["stick", "into", "star", "was", "like", "and", "it", "popsicle", "that", "she"]}, {"sent": "a new creation peanut butter and yogurt popsicle sandwich .", "words": ["yogurt", "popsicle", "and", "butter", "peanut butter", "new", "sandwich", "a"]}, {"sent": "or you want a popsicle ?", "words": ["popsicle", "a", "you"]}, {"sent": "because you were like really actually kindof turning into a popsicle .", "words": ["into", "like", "you", "popsicle", "were", "because", "a"]}, {"sent": "didnt I give you an orange popsicle ?", "words": ["you", "an", "orange", "give", "popsicle"]}, {"sent": "what happened to your popsicle ?", "words": ["to", "what", "popsicle", "your"]}, {"sent": "youre making a popsicle .", "words": ["popsicle", "a"]}, {"sent": "all the popsicle gone .", "words": ["the", "all", "popsicle"]}, {"sent": "I know you not getting no popsicle when we leave here because youre acting like you cant talk today .", "words": ["we", "like", "you", "here", "talk", "popsicle", "when", "not", "because"]}, {"sent": "wheres he putting that popsicle ?", "words": ["that", "popsicle", "he"]}, {"sent": "can we put those popsicle sticks in the um board ?", "words": ["can", "the", "put", "those", "we", "in", "popsicle"]}, {"sent": "I still dont want some no I usu Im pretty consistent I will give you a no on a popsicle almost any day of the week .", "words": ["the", "of", "any", "you", "dont", "on", "pretty", "will", "some", "give", "popsicle", "a"]}, {"sent": "Max parked his popsicle where no one would find it .", "words": ["where", "would", "it", "find", "his", "popsicle"]}, {"sent": "you want a popsicle ?", "words": ["popsicle", "a", "you"]}, {"sent": "the waters frozen like a giant silver popsicle .", "words": ["the", "popsicle", "like", "a"]}, {"sent": "its a popsicle stick .", "words": ["stick", "popsicle", "a"]}, {"sent": "or is this a chocolate popsicle ?", "words": ["is", "chocolate", "this", "popsicle", "a"]}, {"sent": "or not the PlayDoh what happened to the see what happened to the uh popsicle ?", "words": ["the", "what", "to", "see", "popsicle", "not"]}, {"sent": "we dont have two popsicle sticks .", "words": ["have", "dont", "popsicle", "we"]}, {"sent": "a popsicle .", "words": ["popsicle", "a"]}, {"sent": "but if we find some popsicle sticks we can make it .", "words": ["can", "if", "we", "it", "find", "some", "popsicle", "make", "but"]}, {"sent": "if I find it do I get a popsicle ?", "words": ["do", "if", "get", "it", "find", "popsicle", "a"]}, {"sent": "Max parked his popsicle where no one would find it .", "words": ["where", "would", "it", "find", "his", "popsicle"]}, {"sent": "because its so hot you want a popsicle ?", "words": ["so", "you", "hot", "popsicle", "because", "a"]}, {"sent": "thats a popsicle mold .", "words": ["popsicle", "a"]}, {"sent": "is that a good popsicle Abe ?", "words": ["is", "good", "popsicle", "that", "a"]}, {"sent": "you eat your popsicle still ?", "words": ["eat", "popsicle", "your", "you"]}, {"sent": "popsicle .", "words": ["popsicle"]}, {"sent": "popsicle .", "words": ["popsicle"]}, {"sent": "Max knew Ruby would find a reason to throw out his popsicle she might even throw out his .", "words": ["throw", "to", "would", "out", "find", "his", "popsicle", "she", "a"]}, {"sent": "Max parked his popsicle where no one would find it .", "words": ["where", "would", "it", "find", "his", "popsicle"]}, {"sent": "what are you gonna do with the popsicle ?", "words": ["the", "what", "do", "with", "you", "are", "popsicle"]}, {"sent": "you have popsicle .", "words": ["have", "popsicle", "you"]}, {"sent": "wheres the water asked Sam the waters frozen said Stella like a giant silver popsicle .", "words": ["the", "like", "water", "popsicle", "a"]}, {"sent": "a popsicle .", "words": ["popsicle", "a"]}, {"sent": "popsicle ?", "words": ["popsicle"]}, {"sent": "it aint no more popsicle on it Boo .", "words": ["on", "more", "it", "popsicle"]}, {"sent": "mhm does he like popsicle ?", "words": ["does", "popsicle", "he", "like"]}, {"sent": "you want a popsicle ?", "words": ["popsicle", "a", "you"]}, {"sent": "hes had a popsicle .", "words": ["popsicle", "a"]}, {"sent": "well youre not going ta have a popsicle .", "words": ["have", "popsicle", "not", "a"]}, {"sent": "yes in the popsicle didnt you ?", "words": ["the", "in", "popsicle", "you"]}, {"sent": "thats like me saying once upon a time the good fairy and the giant and the Jackinthebeanstalk and the ghost and the spaghetti and the orange popsicle .", "words": ["the", "spaghetti", "like", "and", "orange", "good", "me", "popsicle", "a"]}, {"sent": "cookie and popsicle ?", "words": ["cookie", "popsicle", "and"]}, {"sent": "and what else is the popsicle made outof ?", "words": ["the", "what", "is", "and", "popsicle"]}, {"sent": "you got a popsicle .", "words": ["popsicle", "a", "you"]}, {"sent": "oh is that like an orange popsicle ?", "words": ["is", "like", "an", "orange", "popsicle", "that"]}, {"sent": "popsicle ?", "words": ["popsicle"]}, {"sent": "thats a popsicle .", "words": ["popsicle", "a"]}, {"sent": "Max parked his popsicle where no one would find it .", "words": ["where", "would", "it", "find", "his", "popsicle"]}, {"sent": "your popsicle went down the wrong way .", "words": ["the", "down", "popsicle", "your"]}, {"sent": "the popsicle yeah .", "words": ["the", "popsicle"]}, {"sent": "popsicle .", "words": ["popsicle"]}, {"sent": "do you want a popsicle ?", "words": ["do", "popsicle", "a", "you"]}, {"sent": "you cant do it while youre eating your popsicle .", "words": ["do", "your", "you", "it", "popsicle"]}, {"sent": "put the popsicle back .", "words": ["the", "put", "popsicle", "back"]}, {"sent": "I told you you you couldnt have a popsicle until you finish your apple .", "words": ["your", "finish", "you", "have", "apple", "popsicle", "a"]}, {"sent": "Max parped parked his popsicle where no one would .", "words": ["his", "popsicle", "where", "would"]}, {"sent": "postman video has all gone ?", "words": ["all", "all gone"]}, {"sent": "avocado is all gone .", "words": ["all", "all gone", "is"]}, {"sent": "all gone now .", "words": ["all", "all gone"]}, {"sent": "look it has all gone into her tummy .", "words": ["all", "all gone", "into", "tummy", "it", "look", "her"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "ha all gone .", "words": ["all", "all gone"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "are they all gone to sleep Fraser ?", "words": ["all", "all gone", "sleep", "to", "they", "are"]}, {"sent": "all gone now .", "words": ["all", "all gone"]}, {"sent": "oh it has all gone now .", "words": ["all", "all gone", "it"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "one two all gone .", "words": ["all", "all gone"]}, {"sent": "look all gone .", "words": ["all", "all gone", "look"]}, {"sent": "it has all gone quiet .", "words": ["all", "all gone", "it", "quiet"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "something is all gone ?", "words": ["all", "all gone", "is"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "theyre all gone .", "words": ["all", "all gone"]}, {"sent": "I think that ones all gone .", "words": ["all", "think", "that", "all gone"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "its all gone now .", "words": ["all", "all gone"]}, {"sent": "his tummy has all gone .", "words": ["his", "all", "all gone", "tummy"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "its about as all gone as its gonna be without a face cloth .", "words": ["all", "all gone", "face", "be", "about", "a"]}, {"sent": "is he all gone ?", "words": ["all", "all gone", "he", "is"]}, {"sent": "mummys cup of teas all gone .", "words": ["cup", "of", "all gone", "all"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "and the policeman fireman the pilot and builder have all gone .", "words": ["the", "all", "all gone", "and", "have"]}, {"sent": "theyve all gone .", "words": ["all", "all gone"]}, {"sent": "whats all gone ?", "words": ["all", "all gone"]}, {"sent": "have they all gone under water again ?", "words": ["all", "all gone", "have", "they", "under", "water"]}, {"sent": "theyve all gone now .", "words": ["all", "all gone"]}, {"sent": "theyve all gone Thomas .", "words": ["all", "all gone"]}, {"sent": "I wonder where theyve all gone ?", "words": ["where", "all", "all gone"]}, {"sent": "its all gone now .", "words": ["all", "all gone"]}, {"sent": "say all gone .", "words": ["all", "all gone", "say"]}, {"sent": "all gone byebye .", "words": ["all", "all gone"]}, {"sent": "all gone ?", "words": ["all", "all gone"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "the dustbin man has all gone now hasnt he ?", "words": ["the", "all", "all gone", "he"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "no lollipops all gone .", "words": ["all", "all gone"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "its all gone now .", "words": ["all", "all gone"]}, {"sent": "there we are there we are all gone now give it back to Jane .", "words": ["all", "all gone", "back", "we", "to", "there", "are", "it", "give"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "look all gone .", "words": ["all", "all gone", "look"]}, {"sent": "the circus is all gone .", "words": ["the", "all", "all gone", "is"]}, {"sent": "all gone ?", "words": ["all", "all gone"]}, {"sent": "we thought it had all gone but its come back again .", "words": ["all", "all gone", "back", "we", "it", "but"]}, {"sent": "its all gone .", "words": ["all", "all gone"]}, {"sent": "the raspberries are all gone all gone .", "words": ["the", "all", "all gone", "are"]}, {"sent": "that all gone .", "words": ["all", "all gone", "that"]}, {"sent": "oh he has all gone .", "words": ["all", "all gone", "he"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "there all gone good bye shall we do them again ?", "words": ["all", "do", "all gone", "we", "them", "there", "good"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "all gone and theyre all in here .", "words": ["all", "all gone", "and", "here", "in"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "and then after youve drunk your tea and eaten your cake and it has all gone in your tummy you can come home and then what do you say ?", "words": ["can", "all", "what", "do", "your", "all gone", "cake", "say", "and", "you", "in", "it", "tummy", "then", "home"]}, {"sent": "say all gone .", "words": ["all", "all gone", "say"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "has it all gone ?", "words": ["all", "all gone", "it"]}, {"sent": "all gone now .", "words": ["all", "all gone"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "all gone byebye dirty .", "words": ["all", "all gone", "dirty"]}, {"sent": "say all gone .", "words": ["all", "all gone", "say"]}, {"sent": "dragons all gone .", "words": ["all", "all gone"]}, {"sent": "theyve all gone down havent they ?", "words": ["all", "all gone", "down", "they"]}, {"sent": "hes all gone now .", "words": ["all", "all gone"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "theyve all gone away ?", "words": ["all", "all gone", "away"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "its all gone .", "words": ["all", "all gone"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "yeah theyre all gone sweetie .", "words": ["all", "all gone"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "a spoonful of vinegar and it has all gone .", "words": ["all", "all gone", "of", "and", "it", "a"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "theyve all gone in your lap .", "words": ["all", "in", "your", "all gone"]}, {"sent": "look its all gone .", "words": ["all", "all gone", "look"]}, {"sent": "say all gone juice .", "words": ["all", "all gone", "juice", "say"]}, {"sent": "all gone .", "words": ["all", "all gone"]}, {"sent": "its all gone now .", "words": ["all", "all gone"]}, {"sent": "where do you think theyve all gone ?", "words": ["where", "do", "think", "all", "all gone", "you"]}, {"sent": "say all gone .", "words": ["all", "all gone", "say"]}, {"sent": "well theyve nearly all gone havent they ?", "words": ["all", "all gone", "they"]}, {"sent": "after that theyve all gone .", "words": ["all", "all gone", "that"]}, {"sent": "oh is it all gone ?", "words": ["all", "all gone", "it", "is"]}, {"sent": "I cant even ice skate .", "words": ["ice", "skate"]}, {"sent": "when they had finished their meal the beast asked Belle if she wanted to ice skate .", "words": ["the", "their", "if", "to", "skate", "they", "ice", "when", "she"]}, {"sent": "wheres the skate ?", "words": ["the", "skate"]}, {"sent": "can you skate on it ?", "words": ["can", "skate", "you", "it", "on"]}, {"sent": "we use we useta ice skate .", "words": ["ice", "skate", "we"]}, {"sent": "where do you go to skate ?", "words": ["where", "do", "go", "to", "skate", "you"]}, {"sent": "you know how to roller skate ?", "words": ["skate", "to", "how", "you"]}, {"sent": "ah do you like to skate ?", "words": ["do", "to", "skate", "like", "you"]}, {"sent": "well some kids think its fun to roller skate .", "words": ["to", "some", "think", "skate"]}, {"sent": "there is a skate .", "words": ["there", "skate", "a", "is"]}, {"sent": "can you do can you roller skate ?", "words": ["can", "do", "skate", "you"]}, {"sent": "the skate ?", "words": ["the", "skate"]}, {"sent": "whats the skate ?", "words": ["the", "skate"]}, {"sent": "you like to ice skate ?", "words": ["to", "skate", "like", "you", "ice"]}, {"sent": "oh that looks like a lil ice skate yeah .", "words": ["skate", "like", "ice", "that", "a"]}, {"sent": "you wanna show Manuela how you ice skate ?", "words": ["how", "show", "skate", "you", "wanna", "ice"]}, {"sent": "she reminds me of xxx xxx xxx xxx xxx and I skate regularly .", "words": ["of", "skate", "and", "me", "she"]}, {"sent": "oo a little roller skate .", "words": ["skate", "little", "a"]}, {"sent": "Ernie can skate .", "words": ["can", "skate"]}, {"sent": "Bert can skate .", "words": ["can", "skate"]}, {"sent": "theres the skate .", "words": ["the", "skate"]}, {"sent": "xxx ooh theres a nice skate one .", "words": ["skate", "nice", "a"]}, {"sent": "oh look at the ice skate .", "words": ["the", "look", "skate", "at", "ice"]}, {"sent": "now hes gonna be ready to skate .", "words": ["to", "be", "skate"]}, {"sent": "he might only have one ice skate .", "words": ["have", "ice", "he", "skate"]}, {"sent": "wheres the other ice skate ?", "words": ["the", "ice", "skate", "other"]}, {"sent": "Id rather read than skate I said without hesitation .", "words": ["skate", "read"]}, {"sent": "skate .", "words": ["skate"]}, {"sent": "do you wanna do the skate jigsaw ?", "words": ["the", "do", "skate", "you", "wanna"]}, {"sent": "yeah and she just wanted to learn how to skate fast .", "words": ["how", "fast", "to", "skate", "and", "she"]}, {"sent": "you get out on the ice the first time youre sposta skate like this .", "words": ["the", "ice", "first", "skate", "like", "you", "get", "this", "out", "on"]}, {"sent": "cause you cant learn how to skate well fast .", "words": ["how", "fast", "to", "skate", "you"]}, {"sent": "what did you learn how to skate when you were a kid ?", "words": ["what", "how", "to", "skate", "you", "did", "when", "were", "a"]}, {"sent": "and you went out and played xxx hey kids used to roller skate here in New York .", "words": ["to", "skate", "you", "and", "here", "in", "out"]}, {"sent": "oops a roller skate crash .", "words": ["skate", "a"]}, {"sent": "well I dont know how to roller skate .", "words": ["to", "dont", "how", "skate"]}, {"sent": "what you dont know how to roller skate ?", "words": ["what", "how", "to", "skate", "you", "dont"]}, {"sent": "maybe since you know how to roller skate .", "words": ["skate", "to", "how", "you"]}, {"sent": "when I skate ?", "words": ["skate", "when"]}, {"sent": "when I useta skate I didnt fall very often but I havent skated in a long time .", "words": ["skate", "fall", "long", "in", "when", "but", "a"]}, {"sent": "oh I told Robert get a skate a bottle anything .", "words": ["get", "skate", "bottle", "a"]}, {"sent": "thats pretty good skate on them .", "words": ["skate", "them", "pretty", "good", "on"]}, {"sent": "because if you can skate like you did last night you should be able to ride your bike .", "words": ["can", "your", "if", "be", "skate", "last", "to", "like", "you", "did", "ride", "because"]}, {"sent": "you dont skate ?", "words": ["dont", "skate", "you"]}, {"sent": "you skate over there ?", "words": ["there", "skate", "over", "you"]}, {"sent": "where would you like to roller skate ?", "words": ["where", "to", "skate", "like", "you", "would"]}, {"sent": "roller skate ?", "words": ["skate"]}, {"sent": "are you pretending to roller skate are you ?", "words": ["to", "skate", "are", "you"]}, {"sent": "have you ever seen Squigley roller skate ?", "words": ["have", "skate", "you"]}, {"sent": "well why do you think they did not want him to skate there ?", "words": ["do", "think", "to", "skate", "you", "they", "him", "there", "did", "not", "why"]}, {"sent": "he was able to skate by himself .", "words": ["to", "was", "skate", "by", "he"]}, {"sent": "do they ice skate ?", "words": ["ice", "do", "skate", "they"]}, {"sent": "anything to skate across the floor huh ?", "words": ["the", "to", "skate"]}, {"sent": "I give you a piece of gum for that .", "words": ["for", "of", "you", "gum", "give", "that", "a"]}, {"sent": "oh ArtooDeetoo gum .", "words": ["gum"]}, {"sent": "the rest if you dont take gum .", "words": ["the", "if", "you", "dont", "gum", "take"]}, {"sent": "we do we have the banana one we dont have the bubble gum .", "words": ["the", "do", "we", "have", "banana", "dont", "gum"]}, {"sent": "let Mommy get rid of her gum .", "words": ["get", "of", "her", "gum"]}, {"sent": "by gum .", "words": ["gum", "by"]}, {"sent": "dos that feel good to have Mommy rub on your gum ?", "words": ["your", "to", "have", "good", "gum", "on", "that"]}, {"sent": "what kind of gum ?", "words": ["of", "what", "gum"]}, {"sent": "ya cant sing with gum in your mouth anyway .", "words": ["your", "mouth", "with", "sing", "in", "gum"]}, {"sent": "gum would you like a piece of candy ?", "words": ["of", "like", "you", "gum", "would", "candy", "a"]}, {"sent": "dont put your gum in it .", "words": ["put", "your", "dont", "in", "it", "gum"]}, {"sent": "um when shes not that interested in them other than to like gum on them and then throw them down so .", "words": ["when", "throw", "to", "so", "them", "like", "other", "and", "in", "gum", "on", "then", "down", "that", "not"]}, {"sent": "bubble gum ?", "words": ["gum"]}, {"sent": "oh are you sure this is bubble gum ?", "words": ["is", "you", "this", "are", "gum"]}, {"sent": "you want ta throw your gum away ?", "words": ["your", "throw", "you", "away", "gum"]}, {"sent": "and well have this fantastic bubble gum machine Jeannine .", "words": ["have", "this", "gum", "and"]}, {"sent": "oh wheres that gum ?", "words": ["that", "gum"]}, {"sent": "this is like bubble gum pink and thats .", "words": ["is", "like", "and", "this", "gum"]}, {"sent": "is that gum good ?", "words": ["that", "good", "gum", "is"]}, {"sent": "you like chewing gum ?", "words": ["like", "gum", "you"]}, {"sent": "he wants real gum .", "words": ["he", "gum"]}, {"sent": "whered I find a piece a your gum ?", "words": ["gum", "your", "a", "find"]}, {"sent": "should we check to see if theres any bubble gum inside ?", "words": ["if", "we", "to", "inside", "any", "gum", "see"]}, {"sent": "just to get one little piece of gum .", "words": ["of", "to", "little", "get", "gum"]}, {"sent": "can I have some chewing gum ?", "words": ["can", "some", "have", "gum"]}, {"sent": "why didnt we have gum ?", "words": ["have", "gum", "why", "we"]}, {"sent": "we dont have any gum .", "words": ["we", "any", "have", "dont", "gum"]}, {"sent": "oh youre gonna buy some gum .", "words": ["some", "buy", "gum"]}, {"sent": "but they mustnt have sliced my gum .", "words": ["my", "have", "they", "gum", "but"]}, {"sent": "gum .", "words": ["gum"]}, {"sent": "you wanna know if its time to get green gum .", "words": ["if", "to", "you", "get", "wanna", "green", "gum"]}, {"sent": "little people just get pretend gum ?", "words": ["get", "pretend", "gum", "little"]}, {"sent": "dat gum .", "words": ["gum"]}, {"sent": "but by gum you could crawl around .", "words": ["you", "by", "gum", "around", "could", "but"]}, {"sent": "some chewing gum ?", "words": ["some", "gum"]}, {"sent": "that might cut your gum again if you do that .", "words": ["do", "your", "cut", "if", "you", "gum", "that"]}, {"sent": "you had gum at the store ?", "words": ["the", "store", "you", "gum", "at"]}, {"sent": "oh we hafta be yeah shes stepped in some gum or something and has it on the bottom of her shoe .", "words": ["the", "bottom", "we", "of", "be", "and", "shoe", "in", "it", "gum", "on", "some", "her"]}, {"sent": "gum gumbaya I know .", "words": ["gum"]}, {"sent": "it grows up in the gum .", "words": ["up", "the", "in", "it", "gum"]}, {"sent": "pretend gum ?", "words": ["pretend", "gum"]}, {"sent": "your piece of gum .", "words": ["of", "your", "gum"]}, {"sent": "you really want your gum .", "words": ["your", "gum", "you"]}, {"sent": "hold your gum ?", "words": ["your", "hold", "gum"]}, {"sent": "I said you could have two pieces of green gum .", "words": ["of", "you", "have", "green", "gum", "could"]}, {"sent": "and he gives you the gum .", "words": ["the", "you", "and", "he", "gum"]}, {"sent": "coming through the gum .", "words": ["the", "gum"]}, {"sent": "oh goody goody gum .", "words": ["gum"]}, {"sent": "and I this useta be filled with gum .", "words": ["be", "with", "and", "this", "gum"]}, {"sent": "so ya can save up and get gum .", "words": ["can", "up", "so", "and", "get", "gum"]}, {"sent": "take this gum out of my mouth .", "words": ["my", "of", "mouth", "this", "out", "gum", "take"]}, {"sent": "my loverly bubbly friend gimme a bubble gum .", "words": ["my", "gum", "a"]}, {"sent": "a bubble gum too .", "words": ["too", "gum", "a"]}, {"sent": "look at your gum .", "words": ["at", "your", "gum", "look"]}, {"sent": "I got chewing gum .", "words": ["gum"]}, {"sent": "I think youre not allowed to chew chewing gum in school are you ?", "words": ["think", "to", "school", "you", "in", "are", "gum", "not"]}, {"sent": "no I dont have any gum .", "words": ["have", "dont", "any", "gum"]}, {"sent": "and gum ?", "words": ["gum", "and"]}, {"sent": "you can have one piece of green gum .", "words": ["can", "of", "you", "have", "green", "gum"]}, {"sent": "heres your bubble gum .", "words": ["your", "gum"]}, {"sent": "taking the gum out and all over the place .", "words": ["the", "all", "and", "out", "gum", "over"]}, {"sent": "this guy eat his chewing gum .", "words": ["eat", "this", "gum", "his"]}, {"sent": "does one eat gum after breakfast ?", "words": ["eat", "does", "gum"]}, {"sent": "the xxx based water gum to take home .", "words": ["the", "to", "gum", "water", "take", "home"]}, {"sent": "no gum .", "words": ["gum"]}, {"sent": "I dont think we have any more gum sweetie .", "words": ["think", "we", "any", "have", "dont", "more", "gum"]}, {"sent": "leaves from the gum gummy trees or some kind of .", "words": ["the", "some", "of", "gum"]}, {"sent": "it looks about like bubble gum .", "words": ["gum", "it", "like", "about"]}, {"sent": "I want gum .", "words": ["gum"]}, {"sent": "Max rescued his very favorite piece of gum on a string .", "words": ["of", "on", "gum", "his", "a"]}, {"sent": "why did you get gum because I bough it for you .", "words": ["for", "because", "you", "get", "it", "did", "gum", "why"]}, {"sent": "bubble gum .", "words": ["gum"]}, {"sent": "let me take the gum .", "words": ["the", "me", "take", "gum"]}, {"sent": "by gum therell be trouble .", "words": ["be", "gum", "by"]}, {"sent": "xxx take that gum outof your mouth .", "words": ["your", "mouth", "gum", "take", "that"]}, {"sent": "gum drops those are gum drops right ?", "words": ["are", "those", "gum"]}, {"sent": "gum .", "words": ["gum"]}, {"sent": "thats old and dirty gum throw it away .", "words": ["old", "throw", "and", "it", "away", "dirty", "gum"]}, {"sent": "some gum ?", "words": ["some", "gum"]}, {"sent": "then Mrs Fletcher will get angry with you if you chew chewing gum .", "words": ["if", "with", "you", "get", "gum", "will", "then"]}, {"sent": "thats not a gum .", "words": ["not", "gum", "a"]}, {"sent": "kids love bubble gum .", "words": ["gum", "love"]}, {"sent": "where ya find the gum ?", "words": ["where", "the", "gum", "find"]}, {"sent": "the many shapes of chewing gum .", "words": ["the", "of", "gum"]}, {"sent": "you cant have gum in preschool .", "words": ["have", "in", "gum", "you"]}, {"sent": "wheres Dadas gum ?", "words": ["gum"]}, {"sent": "your gum .", "words": ["your", "gum"]}, {"sent": "its chewy gum ?", "words": ["gum"]}, {"sent": "give him half a a piece of gum .", "words": ["of", "him", "gum", "give", "a"]}, {"sent": "take the gum outof your mouth .", "words": ["the", "your", "mouth", "gum", "take"]}, {"sent": "the gum is ?", "words": ["the", "gum", "is"]}, {"sent": "wait what if I thought this was gum ?", "words": ["what", "wait", "if", "was", "this", "gum"]}, {"sent": "is it time to get green gum .", "words": ["is", "to", "get", "it", "green", "gum"]}, {"sent": "you tired of that gum ?", "words": ["of", "tired", "you", "gum", "that"]}, {"sent": "gum ?", "words": ["gum"]}, {"sent": "I wont give you any chewing gum xxx see thats a banana peel and xxx .", "words": ["any", "you", "and", "banana", "gum", "see", "give", "a"]}, {"sent": "why did she give you the gum ?", "words": ["the", "you", "did", "gum", "give", "she", "why"]}, {"sent": "these are all going in the basement .", "words": ["the", "all", "these", "basement", "in", "are"]}, {"sent": "basement door is shut .", "words": ["basement", "is", "door"]}, {"sent": "please go down into the basement and have your tantrum there because I am not interested .", "words": ["the", "basement", "your", "into", "go", "and", "have", "there", "am", "down", "not", "because"]}, {"sent": "he has a toy box in our basement .", "words": ["basement", "box", "our", "in", "toy", "he", "a"]}, {"sent": "in the basement ?", "words": ["the", "in", "basement"]}, {"sent": "so were not downstairs in in sort of basement as much as you know we have a finished basement .", "words": ["basement", "we", "of", "so", "much", "you", "have", "in", "were", "not", "a"]}, {"sent": "in up here where are why are they out of the basement ?", "words": ["up", "where", "the", "basement", "of", "here", "they", "in", "are", "out", "why"]}, {"sent": "theres some paper in the basement yeah I dont know if I wanna go down in the basement .", "words": ["the", "basement", "if", "go", "paper", "dont", "in", "wanna", "some", "down"]}, {"sent": "Ill go look in the basement .", "words": ["the", "basement", "go", "in", "look"]}, {"sent": "its probably in the basement .", "words": ["the", "in", "basement"]}, {"sent": "Thomas the basement is underneath .", "words": ["the", "basement", "is"]}, {"sent": "uh last time I saw Doggy Woggy he was on the stairs that go the basement .", "words": ["the", "basement", "go", "was", "last", "he", "on", "stairs", "that"]}, {"sent": "Im going to go in the basement Eve .", "words": ["the", "basement", "go", "to", "in"]}, {"sent": "I think we should go to the basement .", "words": ["the", "basement", "think", "go", "we", "to"]}, {"sent": "Im going to go to the basement .", "words": ["the", "to", "basement", "go"]}, {"sent": "its in the basement .", "words": ["the", "in", "basement"]}, {"sent": "look next to the basement door .", "words": ["the", "basement", "to", "look", "door"]}, {"sent": "mommy can I go play in the basement ?", "words": ["can", "the", "basement", "go", "play", "in"]}, {"sent": "we have a basement .", "words": ["have", "basement", "a", "we"]}, {"sent": "Naima I hafta go to the basement will you come with me ?", "words": ["the", "basement", "go", "to", "with", "you", "will", "me"]}, {"sent": "yeah and the kitchen my ah our play kitchens in the basement .", "words": ["the", "basement", "my", "play", "our", "and", "in", "kitchen"]}, {"sent": "but its been put in the basement .", "words": ["the", "basement", "put", "in", "but"]}, {"sent": "um we have like a a finished basement she has all her food toys are downstairs but her dolls are upstairs .", "words": ["basement", "all", "we", "like", "have", "food", "are", "her", "she", "but", "a"]}, {"sent": "do you wanna go down and see Daddy in his basement office ?", "words": ["basement", "do", "go", "you", "and", "in", "wanna", "his", "see", "down"]}, {"sent": "thats so funny it was caught in there and I was like oh my gosh caught and I told Don and then later that day he was like I think I found it in the basement .", "words": ["the", "basement", "my", "think", "was", "so", "like", "and", "there", "in", "it", "he", "then", "that"]}, {"sent": "is the spoon in the basement ?", "words": ["the", "basement", "is", "in", "spoon"]}, {"sent": "whats what what are we gonna have in the basement whats gonna happen in the basement once were finished fixing it ?", "words": ["the", "what", "basement", "we", "have", "in", "are", "it", "were"]}, {"sent": "lets turn off the light and shut the basement door .", "words": ["off", "the", "basement", "light", "and", "door"]}, {"sent": "my brother came up the basement stairs lugging the kids table and folding chairs .", "words": ["up", "the", "my", "basement", "table", "and", "stairs"]}, {"sent": "we have a little kitchen area downstairs in the basement and a play area .", "words": ["the", "basement", "we", "play", "little", "have", "and", "in", "kitchen", "a"]}, {"sent": "and without waiting another minute the guard snapped on his flash light and hurried downstairs to the basement .", "words": ["the", "basement", "light", "to", "and", "another", "his", "on"]}, {"sent": "and the choochoo train down in the basement .", "words": ["the", "basement", "train", "and", "in", "down"]}, {"sent": "she doesnt have it its in the basement .", "words": ["the", "basement", "have", "in", "it", "she"]}, {"sent": "hes practically drowning in the basement .", "words": ["the", "in", "basement"]}, {"sent": "if theres something underneath the ground its the basement .", "words": ["the", "basement", "if"]}, {"sent": "oh this must have been in the basement .", "words": ["basement", "the", "have", "this", "in"]}, {"sent": "mine the basement .", "words": ["the", "mine", "basement"]}, {"sent": "which were kept in the basement of the museum .", "words": ["the", "basement", "of", "in", "which", "were"]}, {"sent": "thats gonna the basement .", "words": ["the", "basement"]}, {"sent": "basement cellar .", "words": ["basement"]}, {"sent": "is she in the basement ?", "words": ["the", "basement", "is", "in", "she"]}, {"sent": "do you know how to go out the basement ?", "words": ["the", "do", "how", "basement", "go", "to", "you", "out"]}, {"sent": "hey really I didnt think tile the basement .", "words": ["the", "think", "basement"]}, {"sent": "Papas going to go to the basement with the movers .", "words": ["the", "basement", "go", "to", "with"]}, {"sent": "how many times did I tell you not to ring basement ?", "words": ["basement", "how", "to", "you", "did", "not"]}, {"sent": "when we were in the lift in Spain we useta get in the lift and we hadta press b for basement .", "words": ["the", "basement", "for", "we", "and", "get", "in", "when", "were"]}, {"sent": "lets go look in the basement .", "words": ["the", "basement", "go", "in", "look"]}, {"sent": "no you cant open the basement door .", "words": ["the", "basement", "open", "you", "door"]}, {"sent": "like our kitchen like the kitchen in the basement ?", "words": ["the", "basement", "like", "our", "in", "kitchen"]}, {"sent": "oh hes gonna work down in the basement ?", "words": ["the", "basement", "work", "in", "down"]}, {"sent": "basement .", "words": ["basement"]}, {"sent": "in the basement .", "words": ["the", "in", "basement"]}, {"sent": "were not going to the basement right now baby .", "words": ["the", "basement", "to", "were", "not"]}, {"sent": "basement ?", "words": ["basement"]}, {"sent": "Im going to the basement and Ill be right back .", "words": ["the", "basement", "back", "to", "be", "and"]}, {"sent": "whats going on in the basement ?", "words": ["on", "the", "in", "basement"]}, {"sent": "hes in the basement .", "words": ["the", "in", "basement"]}, {"sent": "uh whats xxx about the basement and the xxx xxx restroom ?", "words": ["the", "basement", "about", "and"]}, {"sent": "yeah I can see that part of the reason why theres so much stuff over here is youre going down into the basement and taking bringing things up here .", "words": ["can", "the", "basement", "up", "into", "is", "of", "so", "much", "and", "here", "over", "see", "down", "that", "why"]}, {"sent": "and Ill run down to the basement and grab some barley cereal .", "words": ["the", "basement", "to", "and", "run", "some", "down", "cereal"]}, {"sent": "but thats really not the whole reason its in the basement is so that its not here .", "words": ["the", "basement", "is", "so", "here", "in", "that", "not", "but"]}, {"sent": "Ill get some out of the basement .", "words": ["the", "basement", "of", "get", "out", "some"]}, {"sent": "Mommy is gonna get a drink of water and then well go down to the basement .", "words": ["the", "basement", "drink", "is", "of", "go", "to", "and", "get", "water", "then", "down", "a"]}, {"sent": "lets go to the basement then .", "words": ["the", "basement", "go", "to", "then"]}, {"sent": "books in the basement ?", "words": ["the", "in", "basement"]}, {"sent": "xxx the cats in the basement .", "words": ["the", "in", "basement"]}, {"sent": "okay all this stuff going in the basement .", "words": ["the", "all", "basement", "this", "in"]}, {"sent": "who often came to the basement to set traps for mice .", "words": ["the", "basement", "for", "to", "who"]}, {"sent": "oh whats going on in the basement ?", "words": ["on", "the", "in", "basement"]}, {"sent": "we have play food and we have a a large playroom in our basement and she and we have play food down there and she always goes straight to that .", "words": ["basement", "we", "to", "play", "and", "have", "our", "food", "in", "there", "down", "that", "she", "a"]}, {"sent": "down the basement .", "words": ["the", "down", "basement"]}, {"sent": "remember the garage in the basement ?", "words": ["the", "in", "garage", "basement"]}, {"sent": "well youve got to go pick up Josephs pictures help Daddy look for a new tool we need a new machine for the basement and look for a couple shirts for you maybe if you need some more for your uniforms .", "words": ["the", "basement", "help", "go", "more", "some", "up", "for", "if", "pick", "to", "and", "you", "new", "a", "need", "your", "we", "look"]}, {"sent": "I think Ill do the treadmill in the basement .", "words": ["the", "do", "think", "basement", "in"]}, {"sent": "Bill and Judy Brady probably xxx down in the basement .", "words": ["the", "basement", "and", "in", "down"]}, {"sent": "basement cellar .", "words": ["basement"]}, {"sent": "oh well then youll hafta put it in youre ah basement jacket pocket .", "words": ["basement", "put", "in", "it", "then", "jacket"]}, {"sent": "thats the basement ?", "words": ["the", "basement"]}, {"sent": "I trying to figure out where this came from and its the basement .", "words": ["where", "the", "basement", "to", "and", "this", "out"]}, {"sent": "take this stuff downstairs in the basement .", "words": ["the", "basement", "this", "in", "take"]}, {"sent": "what are you saying about the basement though ?", "words": ["the", "what", "basement", "you", "are", "about"]}, {"sent": "and he killed a bumble bee in the basement .", "words": ["the", "basement", "bee", "and", "in", "he", "a"]}, {"sent": "oh thats in the basement .", "words": ["the", "in", "basement"]}, {"sent": "we find a great big treasure trunk in the basement .", "words": ["the", "basement", "we", "in", "find", "big", "a"]}, {"sent": "but yes where did those books come from besides the basement ?", "words": ["where", "the", "basement", "those", "did", "but"]}, {"sent": "and he crashes down to the basement .", "words": ["the", "basement", "to", "and", "he", "down"]}, {"sent": "and he has his own kitchen in the basement and uh .", "words": ["the", "basement", "and", "in", "he", "kitchen", "his"]}, {"sent": "having a quick look in the toy store in the basement .", "words": ["the", "basement", "store", "toy", "in", "look", "a"]}, {"sent": "Ill get some more out of the basement .", "words": ["the", "basement", "of", "get", "more", "out", "some"]}, {"sent": "we play in the basement where its nice and cool .", "words": ["the", "basement", "where", "we", "play", "nice", "and", "in"]}, {"sent": "hes right next to the basement door .", "words": ["the", "to", "basement", "door"]}, {"sent": "down in the basement ?", "words": ["the", "in", "down", "basement"]}, {"sent": "he is in the basement .", "words": ["the", "basement", "is", "in", "he"]}, {"sent": "oh there they are theres the kitty going down to the basement .", "words": ["the", "basement", "to", "there", "they", "are", "kitty", "down"]}, {"sent": "things that Im going to put in the basement .", "words": ["the", "basement", "put", "to", "in", "that"]}, {"sent": "is it we could make that could be the basement down there .", "words": ["the", "basement", "we", "is", "be", "there", "that", "it", "could", "down", "make"]}, {"sent": "some more french fries ?", "words": ["some", "more", "french fries"]}, {"sent": "minty fresh french fries .", "words": ["french fries"]}, {"sent": "french fries .", "words": ["french fries"]}, {"sent": "hamburger and french fries ?", "words": ["french fries", "and", "hamburger"]}, {"sent": "you like real french fries ?", "words": ["french fries", "like", "you"]}, {"sent": "hamburger french fries .", "words": ["french fries", "hamburger"]}, {"sent": "Im making you some purple french fries .", "words": ["some", "french fries", "you"]}, {"sent": "or you dont like playdough french fries ?", "words": ["dont", "french fries", "like", "you"]}, {"sent": "you want your french fries wet ?", "words": ["wet", "french fries", "your", "you"]}, {"sent": "french fries .", "words": ["french fries"]}, {"sent": "what else do you get besides hamburgers and french fries ?", "words": ["what", "do", "french fries", "and", "get", "you"]}, {"sent": "did you have a Bigmac and french fries and a thick shake ?", "words": ["french fries", "shake", "and", "have", "you", "did", "a"]}, {"sent": "french fries ?", "words": ["french fries"]}, {"sent": "Im going to make you some french fries .", "words": ["french fries", "to", "you", "some", "make"]}, {"sent": "yeah Pooh Bear loves french fries .", "words": ["french fries"]}, {"sent": "xxx get a Tictac up your ass and get a xxx french fries and xxx xxx talking into that .", "words": ["up", "french fries", "your", "into", "and", "get", "that", "a"]}, {"sent": "because I just put french fries in .", "words": ["in", "put", "french fries", "because"]}, {"sent": "no no those are french fries .", "words": ["french fries", "are", "those"]}, {"sent": "you like french fries huh .", "words": ["french fries", "like", "you"]}, {"sent": "how come you didnt get no french fries ?", "words": ["get", "french fries", "how", "you"]}, {"sent": "what are you making french fries ?", "words": ["what", "are", "french fries", "you"]}, {"sent": "he got grilled cheese and french fries too .", "words": ["french fries", "and", "he", "too", "cheese"]}, {"sent": "youll hafta get some french fries .", "words": ["get", "some", "french fries"]}, {"sent": "why dont you finish your french fries .", "words": ["french fries", "your", "finish", "you", "dont", "why"]}, {"sent": "you didnt have french fries ?", "words": ["have", "french fries", "you"]}, {"sent": "do you want some ketchup on your french fries ?", "words": ["do", "french fries", "your", "you", "on", "some"]}, {"sent": "how come you dont get french fries for xxx ?", "words": ["french fries", "how", "for", "you", "get", "dont"]}, {"sent": "wanna make french fries ?", "words": ["french fries", "make", "wanna"]}, {"sent": "french fries tuna ?", "words": ["french fries", "tuna"]}, {"sent": "Im even gonna take your french fries .", "words": ["your", "french fries", "take"]}, {"sent": "there are no more french fries darling .", "words": ["there", "more", "are", "french fries"]}, {"sent": "those are french fries .", "words": ["french fries", "are", "those"]}, {"sent": "okay so you do need lotsof french fries so well just get those .", "words": ["do", "french fries", "need", "those", "so", "you", "get"]}, {"sent": "french fries .", "words": ["french fries"]}, {"sent": "he doesnt eat potatoes and he doesnt he thinks theyre different than like french fries .", "words": ["french fries", "like", "and", "eat", "he"]}, {"sent": "oh big french fries .", "words": ["french fries", "big"]}, {"sent": "french fries ?", "words": ["french fries"]}, {"sent": "I want some more french fries .", "words": ["some", "more", "french fries"]}, {"sent": "oh these are french fries .", "words": ["these", "are", "french fries"]}, {"sent": "Emmas getting your meatloaf and your french fries too .", "words": ["french fries", "too", "your", "and"]}, {"sent": "have you ever seen french fries that look like that ?", "words": ["french fries", "like", "you", "have", "look", "that"]}, {"sent": "french fries .", "words": ["french fries"]}, {"sent": "oh thats for making french fries .", "words": ["french fries", "for"]}, {"sent": "ah a big french fries .", "words": ["french fries", "big", "a"]}, {"sent": "I had steak french fries .", "words": ["french fries"]}, {"sent": "do you like french fries ?", "words": ["do", "french fries", "like", "you"]}, {"sent": "did you eat a did you eat french fries ?", "words": ["french fries", "you", "eat", "did", "a"]}, {"sent": "better than french fries .", "words": ["better", "french fries"]}, {"sent": "do you like french fries ?", "words": ["do", "french fries", "like", "you"]}, {"sent": "pork chops and french fries and lettuce and tomatoes .", "words": ["french fries", "and"]}, {"sent": "a big french fries and what else ?", "words": ["what", "french fries", "and", "big", "a"]}, {"sent": "youre going to fix my french fries xxx .", "words": ["to", "my", "french fries", "fix"]}, {"sent": "come on french fries tuna .", "words": ["on", "french fries", "tuna"]}, {"sent": "here Violet lets make the french fries you wanna make the french fries ?", "words": ["the", "french fries", "you", "here", "wanna", "make"]}, {"sent": "Julia are those french fries as good as the other kind ?", "words": ["the", "french fries", "those", "other", "are", "good"]}, {"sent": "where do you um where do you get french fries from ?", "words": ["where", "do", "french fries", "you", "get"]}, {"sent": "now Dexter if you wanted the french fries you were to bring them back here .", "words": ["the", "french fries", "if", "back", "to", "them", "bring", "you", "here", "were"]}, {"sent": "french fries .", "words": ["french fries"]}, {"sent": "he he gave me french fries when we would come here .", "words": ["french fries", "we", "here", "he", "would", "me", "when"]}, {"sent": "can I have some m m more of them soggy french fries ?", "words": ["can", "french fries", "of", "them", "have", "more", "some"]}, {"sent": "he gave you french fries ?", "words": ["french fries", "he", "you"]}, {"sent": "no this ones the french fries right .", "words": ["the", "this", "french fries"]}, {"sent": "oh french fries .", "words": ["french fries"]}, {"sent": "oh thank you would you like some french fries ?", "words": ["french fries", "like", "you", "would", "some"]}, {"sent": "french fries ?", "words": ["french fries"]}, {"sent": "look just let me do this get french fries xxx you get a box and then well share it .", "words": ["do", "french fries", "box", "share", "you", "get", "this", "and", "it", "look", "me", "then", "a"]}, {"sent": "Id kinda like some french fries .", "words": ["some", "french fries", "like"]}, {"sent": "I think your hippos going to eat your french fries .", "words": ["french fries", "think", "your", "to", "eat"]}, {"sent": "did you help me eat my french fries ?", "words": ["french fries", "help", "my", "you", "eat", "did", "me"]}, {"sent": "looks like french fries .", "words": ["french fries", "like"]}, {"sent": "wouldja like to give some french fries to Pooh Bear ?", "words": ["french fries", "to", "like", "some", "give"]}, {"sent": "french fries ?", "words": ["french fries"]}, {"sent": "Dexter take those french fries .", "words": ["french fries", "take", "those"]}, {"sent": "well the french fries are not ready .", "words": ["the", "french fries", "are", "not"]}, {"sent": "chicken and french fries okay .", "words": ["chicken", "french fries", "and"]}, {"sent": "well Im gonna make some french fries .", "words": ["some", "french fries", "make"]}, {"sent": "but where do french fries come from ?", "words": ["where", "do", "french fries", "but"]}, {"sent": "you always want french fries Jilly .", "words": ["french fries", "you"]}, {"sent": "cause you need dog ketchup with your other french fries .", "words": ["french fries", "your", "need", "with", "you", "other", "dog"]}, {"sent": "but where where does Mcdonalds get french fries ?", "words": ["where", "does", "french fries", "get", "but"]}, {"sent": "I love those french fries .", "words": ["french fries", "those", "love"]}, {"sent": "do you not like french fries ?", "words": ["do", "french fries", "like", "you", "not"]}, {"sent": "french fries those are potatoes huh cut up .", "words": ["up", "french fries", "those", "cut", "are"]}, {"sent": "I was gonna make you some french fries .", "words": ["french fries", "was", "you", "some", "make"]}, {"sent": "eat your french fries and your cucumbers .", "words": ["eat", "french fries", "your", "and"]}, {"sent": "these are french fries .", "words": ["these", "are", "french fries"]}, {"sent": "got any french fries to go with it ?", "words": ["french fries", "go", "to", "any", "with", "it"]}, {"sent": "you had french fries ?", "words": ["french fries", "you"]}, {"sent": "here I made french fries .", "words": ["here", "french fries"]}, {"sent": "he says would you like some french fries ?", "words": ["french fries", "like", "you", "he", "would", "some"]}, {"sent": "fine you dont like my french fries ?", "words": ["french fries", "my", "fine", "like", "you", "dont"]}, {"sent": "hey could I buy a french fries ?", "words": ["french fries", "could", "buy", "a"]}, {"sent": "no he is not eating french fries .", "words": ["french fries", "he", "not", "is"]}, {"sent": "these look like french fries .", "words": ["these", "french fries", "like", "look"]}, {"sent": "blue french fries ?", "words": ["french fries", "blue"]}, {"sent": "and I want a hamburger ad french fries .", "words": ["hamburger", "french fries", "a", "and"]}, {"sent": "not french fries either those would also get cold .", "words": ["french fries", "those", "cold", "get", "would", "not"]}, {"sent": "do you like french fries ?", "words": ["do", "french fries", "like", "you"]}, {"sent": "she doesnt like starch she really doesnt like french fries .", "words": ["french fries", "she", "like"]}, {"sent": "right Alec has a penis but he has a tushie too .", "words": ["penis", "he", "too", "but", "a"]}, {"sent": "well hes got a penis .", "words": ["penis", "a"]}, {"sent": "thats right he puts his penis in .", "words": ["his", "in", "he", "penis"]}, {"sent": "is see his penis .", "words": ["his", "see", "penis", "is"]}, {"sent": "yes Matthew has a penis .", "words": ["penis", "a"]}, {"sent": "he has a penis .", "words": ["he", "penis", "a"]}, {"sent": "you dont play with your penis at the table .", "words": ["the", "your", "table", "play", "with", "penis", "you", "dont", "at"]}, {"sent": "a tush and a penis ?", "words": ["penis", "and", "a"]}, {"sent": "and daddy has a penis right .", "words": ["penis", "a", "and"]}, {"sent": "right you were naked and Alec has a penis .", "words": ["penis", "and", "you", "were", "a"]}, {"sent": "right boys have a penis and a tushie .", "words": ["have", "penis", "and", "a"]}, {"sent": "more penis .", "words": ["more", "penis"]}, {"sent": "look at your machine thats a penis .", "words": ["look", "your", "penis", "at", "a"]}, {"sent": "and your penis sticking out .", "words": ["your", "penis", "out", "and"]}, {"sent": "ha and the penis has on that thing and youre peeing .", "words": ["the", "penis", "and", "on", "that"]}, {"sent": "thats the cows penis .", "words": ["the", "penis"]}, {"sent": "you cant make an ax outof a penis .", "words": ["penis", "you", "an", "make", "a"]}, {"sent": "you youre holding your penis .", "words": ["your", "penis", "you"]}, {"sent": "no theyre part of your penis .", "words": ["of", "your", "penis"]}, {"sent": "you have testicles and a penis .", "words": ["penis", "and", "have", "you", "a"]}, {"sent": "now were going to wash the penis .", "words": ["the", "to", "penis", "wash", "were"]}, {"sent": "wash Nathaniels penis .", "words": ["wash", "penis"]}, {"sent": "your penis feels wet sometimes after youve gone peepee .", "words": ["wet", "your", "penis"]}, {"sent": "theres a penis .", "words": ["penis", "a"]}, {"sent": "thats not the penis .", "words": ["the", "penis", "not"]}, {"sent": "it oh can you pull down his pants so I can see his penis ?", "words": ["can", "so", "penis", "pull", "you", "it", "pants", "his", "see", "down"]}, {"sent": "so I can see his penis .", "words": ["can", "so", "penis", "his", "see"]}, {"sent": "little penis .", "words": ["penis", "little"]}, {"sent": "boys penis goes into a girls vagina .", "words": ["vagina", "penis", "into", "a"]}, {"sent": "can I see his penis now ?", "words": ["can", "see", "penis", "his"]}, {"sent": "Im getting tired of changing penis all the time .", "words": ["the", "all", "of", "penis", "tired"]}, {"sent": "I do saw a penis again .", "words": ["do", "penis", "a"]}, {"sent": "you mean you could see the penis of Awww ?", "words": ["the", "of", "penis", "you", "see", "could"]}, {"sent": "oh you mean will you ever have a penis ?", "words": ["penis", "you", "have", "will", "a"]}, {"sent": "yes because Jack has a penis .", "words": ["penis", "because", "a"]}, {"sent": "you think thats funny you can laugh with a toe in your mouth huh ?", "words": ["can", "think", "your", "toe", "mouth", "with", "you", "in", "a"]}, {"sent": "and as youve been running around your toe has come through .", "words": ["toe", "around", "your", "and"]}, {"sent": "big toe .", "words": ["toe", "big"]}, {"sent": "Thomas poorly toe ?", "words": ["toe"]}, {"sent": "do you point your toe ?", "words": ["toe", "do", "your", "you"]}, {"sent": "is that the big toe ?", "words": ["the", "is", "toe", "that", "big"]}, {"sent": "thats your toe .", "words": ["toe", "your"]}, {"sent": "and theres a toe .", "words": ["toe", "a", "and"]}, {"sent": "this keeps getting in your toe .", "words": ["toe", "this", "in", "your"]}, {"sent": "well you can kiss the dollys toe this time .", "words": ["can", "the", "toe", "you", "this", "kiss"]}, {"sent": "you big toe ?", "words": ["toe", "big", "you"]}, {"sent": "shall we draw some toe nails ?", "words": ["toe", "some", "draw", "we"]}, {"sent": "your toe ?", "words": ["toe", "your"]}, {"sent": "this is your toe .", "words": ["toe", "this", "your", "is"]}, {"sent": "hell bite your toe .", "words": ["bite", "toe", "your"]}, {"sent": "I said big toe .", "words": ["toe", "big"]}, {"sent": "on your toe .", "words": ["on", "toe", "your"]}, {"sent": "you biting your toe .", "words": ["toe", "your", "you"]}, {"sent": "its your big toe .", "words": ["toe", "big", "your"]}, {"sent": "whose toe is it ?", "words": ["toe", "it", "is"]}, {"sent": "lets have a look at your toe .", "words": ["your", "toe", "have", "at", "look", "a"]}, {"sent": "you just banged your little toe did you ?", "words": ["your", "toe", "little", "you", "did"]}, {"sent": "a toe its a toe .", "words": ["toe", "a"]}, {"sent": "dont bite my toe .", "words": ["bite", "dont", "my", "toe"]}, {"sent": "you have a booboo on your toe honey ?", "words": ["your", "toe", "you", "have", "on", "a"]}, {"sent": "that big toe ?", "words": ["toe", "big", "that"]}, {"sent": "how have you hurt your toe ?", "words": ["how", "your", "hurt", "toe", "you", "have"]}, {"sent": "toe \u662f \u4ec0\u4e48 ?", "words": ["toe"]}, {"sent": "are you putting toe cream on my toe ?", "words": ["my", "toe", "you", "are", "on"]}, {"sent": "going to crawl up your toe .", "words": ["up", "to", "toe", "your"]}, {"sent": "oh is it your finger or your toe ?", "words": ["your", "is", "toe", "finger", "it"]}, {"sent": "thats a big toe .", "words": ["toe", "big", "a"]}, {"sent": "wheres the big toe ?", "words": ["the", "toe", "big"]}, {"sent": "oh theres a toe and theres a toe and theres a toe theres a toe and theres a toe .", "words": ["toe", "and", "a"]}, {"sent": "tiptoe tip toe tip toe .", "words": ["toe"]}, {"sent": "did that doggy lick your toe ?", "words": ["your", "toe", "that", "did", "lick"]}, {"sent": "tickle toe .", "words": ["toe", "tickle"]}, {"sent": "you hurt your toe ?", "words": ["hurt", "toe", "your", "you"]}, {"sent": "fits on your toe .", "words": ["on", "toe", "your"]}, {"sent": "is it pretty good that big toe ?", "words": ["is", "toe", "big", "it", "pretty", "good", "that"]}, {"sent": "measure your toe ?", "words": ["toe", "your"]}, {"sent": "whats wrong with that toe ?", "words": ["toe", "that", "with"]}, {"sent": "this is going on your toe .", "words": ["your", "is", "toe", "this", "on"]}, {"sent": "we cut your toe nails last night didnt we ?", "words": ["your", "cut", "we", "toe", "last"]}, {"sent": "you stand on my toe .", "words": ["my", "toe", "stand", "you", "on"]}, {"sent": "on your big toe or your little toe ?", "words": ["your", "toe", "little", "on", "big"]}, {"sent": "uhoh he got my toe .", "words": ["toe", "my", "he"]}, {"sent": "round the big toe .", "words": ["the", "toe", "big"]}, {"sent": "I uh xxx fell on my toe and a piece a of meat was on my toe .", "words": ["my", "toe", "of", "was", "meat", "and", "on", "a"]}, {"sent": "that was Mommys toe .", "words": ["toe", "was", "that"]}, {"sent": "your yeah thats your toe your feet your tongue in your mouth .", "words": ["your", "toe", "mouth", "in", "tongue"]}, {"sent": "your toe nails are dirty .", "words": ["toe", "are", "your", "dirty"]}, {"sent": "toe \u5728 \u54ea\u513f ?", "words": ["toe"]}, {"sent": "do you think it might go on your toe ?", "words": ["do", "think", "your", "go", "toe", "you", "it", "on"]}, {"sent": "dont drop it on your toe .", "words": ["your", "toe", "dont", "it", "drop", "on"]}, {"sent": "hurt your toe ?", "words": ["hurt", "toe", "your"]}, {"sent": "wait for the bathroom and wash head to toe .", "words": ["the", "wait", "for", "toe", "to", "head", "and", "bathroom", "wash"]}, {"sent": "crash into your toe ?", "words": ["toe", "your", "into"]}, {"sent": "what happened to my toe ?", "words": ["to", "what", "my", "toe"]}, {"sent": "toe .", "words": ["toe"]}, {"sent": "toe .", "words": ["toe"]}, {"sent": "what happened to your little toe ?", "words": ["what", "your", "toe", "to", "little"]}, {"sent": "watch the toe .", "words": ["the", "watch", "toe"]}, {"sent": "but I wont wear my open toe shoes .", "words": ["open", "my", "toe", "but"]}, {"sent": "dont bite that toe .", "words": ["bite", "dont", "toe", "that"]}, {"sent": "is it to toe eating time ?", "words": ["toe", "to", "it", "is"]}, {"sent": "well I just saw you bang your toe .", "words": ["toe", "your", "you"]}, {"sent": "be careful because if that drops on your toe its gonna hurt .", "words": ["your", "if", "toe", "be", "hurt", "careful", "on", "that", "because"]}, {"sent": "theres a toe theres a toe and theres a toe .", "words": ["toe", "and", "a"]}, {"sent": "a poorly toe has she ?", "words": ["toe", "she", "a"]}, {"sent": "licked your toe doggy ?", "words": ["toe", "your"]}, {"sent": "well hang it by the toe .", "words": ["the", "it", "toe", "by"]}, {"sent": "oh it looked like he was nibbling on your toe .", "words": ["your", "toe", "was", "like", "it", "he", "on"]}, {"sent": "oh a toe there and a toe here .", "words": ["toe", "and", "here", "there", "a"]}, {"sent": "by his toe .", "words": ["his", "toe", "by"]}, {"sent": "ouch on my toe .", "words": ["on", "my", "toe"]}, {"sent": "like toe ?", "words": ["toe", "like"]}, {"sent": "my toe .", "words": ["toe", "my"]}, {"sent": "you told Daddy about that toe and what did Daddy say ?", "words": ["what", "toe", "say", "and", "you", "did", "about", "that"]}, {"sent": "did Mummy bite your toe ?", "words": ["bite", "toe", "your", "did"]}, {"sent": "who had your toe ?", "words": ["toe", "your", "who"]}, {"sent": "and wheres the baby toe ?", "words": ["the", "toe", "and"]}, {"sent": "wheres your toe ?", "words": ["toe", "your"]}, {"sent": "its Thomass toe .", "words": ["toe"]}, {"sent": "is that one for your toe ?", "words": ["for", "your", "is", "toe", "that"]}, {"sent": "got your toe .", "words": ["toe", "your"]}, {"sent": "did the fish bite your toe ?", "words": ["the", "your", "toe", "did", "fish", "bite"]}, {"sent": "big toe .", "words": ["toe", "big"]}, {"sent": "do you have a little toe ?", "words": ["do", "toe", "little", "have", "you", "a"]}, {"sent": "is it on your toe ?", "words": ["your", "is", "toe", "it", "on"]}, {"sent": "does your toe hurt ?", "words": ["toe", "does", "your", "hurt"]}, {"sent": "what have you done with your toe ?", "words": ["what", "your", "toe", "with", "you", "have"]}, {"sent": "you hurt my toe .", "words": ["hurt", "my", "toe", "you"]}, {"sent": "youve hurt your toe ?", "words": ["hurt", "toe", "your"]}, {"sent": "he said you went out to get the paper and there was a big dog and the dog licked your toe and you said other toe .", "words": ["the", "your", "toe", "to", "paper", "was", "you", "get", "and", "there", "other", "he", "out", "big", "dog", "a"]}, {"sent": "yeah overlap above .", "words": ["above"]}, {"sent": "youve got a bruise right above your nose .", "words": ["above", "nose", "your", "a"]}, {"sent": "you also want the zucchini and broccoli thats above the drawer ?", "words": ["the", "and", "you", "above", "drawer"]}, {"sent": "the other one above the boat .", "words": ["the", "boat", "above", "other"]}, {"sent": "yeah xxx above the chimney .", "words": ["above", "the"]}, {"sent": "you needta find the number above the beach ball is twenty seven .", "words": ["the", "is", "you", "ball", "beach", "find", "above"]}, {"sent": "what is that above her bed ?", "words": ["what", "is", "bed", "that", "above", "her"]}, {"sent": "that bird thats got a nest above my window .", "words": ["my", "bird", "above", "window", "that", "a"]}, {"sent": "above .", "words": ["above"]}, {"sent": "and his glory above the nations .", "words": ["his", "the", "above", "and"]}, {"sent": "she has got a little scratch above her eye .", "words": ["eye", "little", "above", "her", "she", "a"]}, {"sent": "up above the world so high .", "words": ["up", "the", "so", "high", "above"]}, {"sent": "ideas above the station .", "words": ["above", "the"]}, {"sent": "whats the state right above Texas ?", "words": ["the", "above"]}, {"sent": "anything above that .", "words": ["above", "that"]}, {"sent": "youre getting it above your head .", "words": ["above", "head", "it", "your"]}, {"sent": "there began to be other sounds up above .", "words": ["up", "to", "be", "other", "there", "above"]}, {"sent": "above the boat .", "words": ["above", "the", "boat"]}, {"sent": "above it would be a little piece of metal and the shopkeeper would put the price on the metal .", "words": ["the", "put", "of", "be", "little", "and", "on", "it", "would", "above", "a"]}, {"sent": "how many are above the ground ?", "words": ["above", "the", "how", "are"]}, {"sent": "draw the feelers above your caterpillars head .", "words": ["the", "your", "head", "draw", "above"]}, {"sent": "the one above Daddys bed is on .", "words": ["the", "is", "bed", "on", "above"]}, {"sent": "lets see what else do you remember this one above Florida right above Florida ?", "words": ["what", "do", "you", "this", "above", "see"]}, {"sent": "youre lifting it high above your head ?", "words": ["your", "head", "it", "high", "above"]}, {"sent": "oh youre lifting them above your head and said big high up .", "words": ["up", "your", "head", "them", "and", "high", "above", "big"]}, {"sent": "up above the world so high .", "words": ["up", "the", "so", "high", "above"]}, {"sent": "up above the world so high .", "words": ["up", "the", "so", "high", "above"]}, {"sent": "that piece goes above here .", "words": ["above", "here", "that"]}, {"sent": "above the world so high .", "words": ["above", "the", "so", "high"]}, {"sent": "up above the world so high .", "words": ["up", "the", "so", "high", "above"]}, {"sent": "the Lord is high above the heavens .", "words": ["the", "above", "high", "is"]}, {"sent": "above below .", "words": ["above"]}, {"sent": "youre holding the talcum powder high above your head .", "words": ["the", "your", "head", "high", "above"]}, {"sent": "wheres the bit that goes above it ?", "words": ["the", "it", "that", "above"]}, {"sent": "youre about super above super happy .", "words": ["above", "happy", "about"]}, {"sent": "but when the guard held a piece of cheese above the hole Norman sniffed it .", "words": ["the", "of", "it", "above", "cheese", "when", "but", "a"]}, {"sent": "one up above and one below .", "words": ["up", "above", "and"]}, {"sent": "up above the world so high .", "words": ["up", "the", "so", "high", "above"]}, {"sent": "hes up above my head .", "words": ["up", "my", "above", "head"]}, {"sent": "above our house ?", "words": ["above", "house", "our"]}, {"sent": "and when she comes home above her bed she has got a picture of a girl riding a horse .", "words": ["of", "picture", "her", "bed", "and", "horse", "above", "when", "she", "a", "home"]}, {"sent": "up above the world so high .", "words": ["up", "the", "so", "high", "above"]}, {"sent": "this says from three and above and I think youre brilliant .", "words": ["above", "this", "think", "and"]}, {"sent": "and then I shall then write above it .", "words": ["write", "and", "it", "above", "then"]}, {"sent": "so do you think that will probably go there above those penguins ?", "words": ["do", "think", "those", "go", "so", "above", "you", "there", "will", "that"]}, {"sent": "above below .", "words": ["above"]}, {"sent": "above it .", "words": ["above", "it"]}, {"sent": "heaven above and the sea below .", "words": ["above", "the", "and"]}, {"sent": "it goes on the above the red one .", "words": ["the", "red", "it", "on", "above"]}, {"sent": "put the children above .", "words": ["the", "put", "above"]}, {"sent": "think it goes above the pirate .", "words": ["above", "the", "think", "it"]}, {"sent": "thats above and thats below .", "words": ["above", "and"]}, {"sent": "up above the world so high .", "words": ["up", "the", "so", "high", "above"]}, {"sent": "above the nest .", "words": ["above", "the"]}, {"sent": "above below .", "words": ["above"]}, {"sent": "I wanna see a dragon who can fly above my head .", "words": ["can", "my", "head", "wanna", "who", "above", "see", "a"]}, {"sent": "everybody above the fences .", "words": ["above", "the"]}, {"sent": "and hes standing on the dock above the water .", "words": ["the", "and", "water", "on", "above"]}, {"sent": "above below .", "words": ["above"]}, {"sent": "its above the mouse .", "words": ["above", "the", "mouse"]}, {"sent": "its on the mantlepiece above the fire place .", "words": ["on", "the", "above"]}, {"sent": "happy sad above below .", "words": ["above", "sad", "happy"]}, {"sent": "it was hovering above the factory .", "words": ["above", "was", "it", "the"]}, {"sent": "the Lord is high above the heavens .", "words": ["the", "above", "high", "is"]}, {"sent": "for as high as the heavens above .", "words": ["the", "for", "above", "high"]}, {"sent": "half of Wendys head a bit above Wendys head ahhah .", "words": ["of", "head", "above", "a"]}, {"sent": "look I see a plug right above your head .", "words": ["your", "head", "look", "above", "see", "a"]}, {"sent": "just above Sues house .", "words": ["above", "house"]}, {"sent": "can you put the bird above the man whos waving .", "words": ["can", "the", "put", "bird", "you", "above"]}, {"sent": "oh worship the king all glorious above .", "words": ["the", "all", "above"]}, {"sent": "because when he swooped off with my sandwich he flew right above your head didnt he ?", "words": ["off", "my", "your", "sandwich", "head", "with", "he", "above", "when", "because"]}, {"sent": "pretty high up on the wall too like I was thinking about it I dont I just dont even know if its in his range of vision most of the time hes like on the floor or on the couch and thats still like another three or four feet above him so .", "words": ["the", "of", "like", "about", "high", "another", "it", "on", "up", "if", "was", "so", "and", "dont", "him", "too", "above", "couch", "in", "pretty", "his"]}, {"sent": "and whats this one right above Texas ?", "words": ["above", "this", "and"]}, {"sent": "wow theyre really high up above the houses arent they ?", "words": ["up", "the", "they", "high", "above"]}, {"sent": "and shes towering above some of the POnes youknow ?", "words": ["the", "of", "and", "above", "some"]}, {"sent": "and theres a rainbow glowing above it .", "words": ["above", "it", "a", "and"]}, {"sent": "they could not see from so high above that the village had been through many hard times .", "words": ["the", "so", "they", "hard", "high", "above", "see", "could", "that", "not"]}, {"sent": "good Lord above .", "words": ["above", "good"]}, {"sent": "shes above your shelf there .", "words": ["above", "there", "your"]}, {"sent": "the night watchman was going on his rounds on the floor above .", "words": ["the", "was", "his", "on", "above"]}, {"sent": "slowly Thomas opened his eyes he looked at the blue sky above and the green mountains all around .", "words": ["the", "all", "around", "sky", "blue", "and", "he", "green", "at", "his", "above"]}, {"sent": "now you look just above Sues house .", "words": ["above", "look", "house", "you"]}, {"sent": "and then that one goes above Percys face .", "words": ["face", "and", "above", "then", "that"]}, {"sent": "actually her head circumference is just above the fiftieth centile xxx .", "words": ["the", "is", "head", "above", "her"]}, {"sent": "and his glory above the nations .", "words": ["his", "the", "above", "and"]}, {"sent": "is he above or below the cloud .", "words": ["the", "is", "he", "cloud", "above"]}, {"sent": "twelve is above the mouse .", "words": ["above", "the", "mouse", "is"]}, {"sent": "look theres a little bubble coming about above ah above the boys head and he can just see a picture in his mind of his teddy on the seat and what does he say ?", "words": ["the", "what", "of", "about", "can", "does", "on", "head", "little", "and", "see", "above", "a", "picture", "say", "in", "he", "look", "his"]}, {"sent": "put it on on the mantlepiece above the fire .", "words": ["the", "put", "it", "on", "above"]}, {"sent": "propeller blades above my nose and runners instead of toes .", "words": ["my", "of", "and", "above", "nose"]}, {"sent": "how many are above the ground ?", "words": ["above", "the", "how", "are"]}, {"sent": "can you see just above there ?", "words": ["can", "you", "there", "above", "see"]}, {"sent": "because there was a lot of things happening above werent there ?", "words": ["of", "was", "a lot", "there", "above", "because", "a"]}, {"sent": "oh heavens above .", "words": ["above"]}, {"sent": "can you see whats above the dining room table on this picture ?", "words": ["can", "the", "table", "picture", "you", "this", "on", "room", "above", "see"]}, {"sent": "he took the scrubbing brush and he buried it in the backyard .", "words": ["the", "backyard", "brush", "and", "in", "it", "he"]}, {"sent": "Lily come look in the backyard .", "words": ["the", "backyard", "in", "look"]}, {"sent": "alright were gonna go this way gonna go in the backyard William were not gonna go out the way this mans going .", "words": ["the", "backyard", "go", "this", "in", "out", "were", "not"]}, {"sent": "hey did we see any squirrels in the backyard ?", "words": ["the", "backyard", "we", "any", "in", "did", "see"]}, {"sent": "oh you found some grapes in our backyard .", "words": ["backyard", "our", "you", "in", "grapes", "some"]}, {"sent": "xxx in the backyard .", "words": ["the", "backyard", "in"]}, {"sent": "tell her were in the backyard .", "words": ["the", "backyard", "in", "her", "were"]}, {"sent": "its not Grandmas backyard ?", "words": ["backyard", "not"]}, {"sent": "the lady woke up early in the morning and there was a moose in her backyard .", "words": ["the", "up", "backyard", "was", "and", "moose", "there", "in", "her", "a"]}, {"sent": "yeah its in our backyard remember ?", "words": ["backyard", "in", "our"]}, {"sent": "yes I can hear you all the way in the backyard .", "words": ["can", "all", "the", "backyard", "you", "hear", "in"]}, {"sent": "and our backyard .", "words": ["our", "backyard", "and"]}, {"sent": "going in xxx the backyard .", "words": ["the", "backyard", "in"]}, {"sent": "you see that in the backyard dont you ?", "words": ["the", "backyard", "you", "dont", "in", "see", "that"]}, {"sent": "whats in the backyard ?", "words": ["the", "backyard", "in"]}, {"sent": "what have we had that keep going in our backyard that we sometimes see in the back backyard ?", "words": ["the", "what", "backyard", "back", "we", "our", "have", "in", "see", "that"]}, {"sent": "well go outside to our messy backyard .", "words": ["backyard", "go", "to", "outside", "our"]}, {"sent": "well probably not sun in the backyard .", "words": ["the", "backyard", "sun", "in", "not"]}, {"sent": "in the backyard .", "words": ["the", "backyard", "in"]}, {"sent": "oh you mean like out in the backyard ?", "words": ["the", "backyard", "like", "you", "in", "out"]}, {"sent": "where did you swim in in our backyard ?", "words": ["where", "backyard", "swim", "you", "our", "in", "did"]}, {"sent": "yeah youre stuck in the backyard now Violet .", "words": ["the", "backyard", "in", "stuck"]}, {"sent": "but I we be in the backyard with my son .", "words": ["the", "backyard", "my", "we", "be", "with", "in", "but"]}, {"sent": "I thought grandma grew it in her backyard .", "words": ["backyard", "in", "it", "her"]}, {"sent": "Minnie and Moo hid the Easter eggs in the farmers backyard .", "words": ["the", "backyard", "in", "and"]}, {"sent": "I think hes talking about something in the backyard .", "words": ["the", "backyard", "think", "in", "about"]}, {"sent": "yeah you can also catch those meat things jumping around in your backyard xxx youll should do get a bottle full of chocolate and grab one xxx .", "words": ["can", "backyard", "do", "those", "your", "full", "of", "catch", "chocolate", "meat", "you", "get", "and", "in", "around", "bottle", "a"]}, {"sent": "that was Daddy in the backyard with Jwww .", "words": ["the", "backyard", "was", "with", "in", "that"]}, {"sent": "I said when you get married Ill have your wedding in my backyard .", "words": ["backyard", "my", "your", "you", "get", "have", "in", "when"]}, {"sent": "or a playground to put in the backyard for you all or something like that .", "words": ["the", "backyard", "put", "for", "all", "to", "like", "you", "in", "that", "a"]}, {"sent": "and whoosh it swizzled down so hard and bloop it bounced in his backyard and goink it landed on his toe .", "words": ["backyard", "toe", "so", "and", "in", "it", "on", "hard", "his", "down"]}, {"sent": "we gonna put em in the backyard ?", "words": ["the", "backyard", "put", "we", "in"]}, {"sent": "come climb up on the chair and tell me what you see in the backyard Honey .", "words": ["up", "the", "what", "backyard", "see", "chair", "and", "you", "in", "on", "me", "climb"]}, {"sent": "um of course the dog and the cat and all the ducks on the pond and the bunnies in the backyard he just absolutely has a .", "words": ["the", "all", "backyard", "of", "and", "cat", "in", "he", "on", "dog", "a"]}, {"sent": "in the backyard .", "words": ["the", "backyard", "in"]}, {"sent": "what a fabulous backyard .", "words": ["what", "backyard", "a"]}, {"sent": "when we were playing in the playhouse in the backyard he was here .", "words": ["the", "backyard", "we", "was", "here", "in", "he", "when", "were"]}, {"sent": "now that we xxx in the backyard .", "words": ["the", "backyard", "we", "in", "that"]}, {"sent": "do you wanna ride around the backyard in your car ?", "words": ["the", "do", "backyard", "your", "car", "you", "in", "wanna", "around", "ride"]}, {"sent": "in the backyard ?", "words": ["the", "backyard", "in"]}, {"sent": "honey the backyard is very cold were not going outside .", "words": ["the", "backyard", "is", "outside", "not", "were", "cold"]}, {"sent": "well hes burying the scrub brush in the backyard .", "words": ["the", "backyard", "in", "brush"]}, {"sent": "thats right do you see how many leaves we have in the backyard ?", "words": ["the", "do", "how", "backyard", "we", "you", "have", "in", "see"]}, {"sent": "okay stay in the backyard promise .", "words": ["the", "stay", "in", "backyard"]}, {"sent": "let me open the curtains and you can see out the backyard .", "words": ["the", "can", "backyard", "open", "see", "you", "and", "out", "me"]}, {"sent": "oh no no no Gloria cant see you in the backyard .", "words": ["the", "backyard", "you", "in", "see"]}, {"sent": "oh yeah the backyard .", "words": ["the", "backyard"]}, {"sent": "its raining out you cant go in the backyard because its .", "words": ["the", "backyard", "go", "you", "in", "out", "because"]}, {"sent": "you know the little house she has in her backyard ?", "words": ["the", "backyard", "little", "you", "in", "house", "her", "she"]}, {"sent": "is she on tv or does she live in your backyard ?", "words": ["does", "backyard", "your", "is", "tv", "in", "on", "she"]}, {"sent": "do they have a backyard ?", "words": ["do", "backyard", "have", "they", "a"]}, {"sent": "ya we have a little one in the backyard .", "words": ["the", "backyard", "we", "little", "have", "in", "a"]}, {"sent": "is this a deer like in our backyard ?", "words": ["backyard", "is", "deer", "like", "our", "this", "in", "a"]}, {"sent": "do you slide out in the backyard ?", "words": ["the", "do", "backyard", "slide", "you", "in", "out"]}, {"sent": "wanna go in the backyard again ?", "words": ["the", "backyard", "go", "in", "wanna"]}, {"sent": "I dont know where she went well just hafta see maybe shes in the backyard .", "words": ["where", "the", "backyard", "dont", "in", "see", "she"]}, {"sent": "shed get down in the backyard Carol .", "words": ["the", "backyard", "get", "in", "down"]}, {"sent": "lets go in the backyard please .", "words": ["the", "backyard", "in", "go"]}, {"sent": "here you go we can go out and look at the backyard first .", "words": ["can", "the", "backyard", "go", "we", "first", "you", "here", "and", "at", "out", "look"]}, {"sent": "go in the backyard .", "words": ["the", "backyard", "in", "go"]}, {"sent": "I see white cat and he even goes in our backyard .", "words": ["backyard", "and", "our", "cat", "in", "he", "see", "white"]}, {"sent": "whada they do in the backyard ?", "words": ["the", "do", "backyard", "they", "in"]}, {"sent": "lets go in the backyard .", "words": ["the", "backyard", "in", "go"]}, {"sent": "how many are in your backyard ?", "words": ["backyard", "how", "your", "in", "are"]}, {"sent": "does she try to do that at home in the backyard ?", "words": ["the", "does", "do", "backyard", "to", "that", "in", "at", "try to", "try", "she", "home"]}, {"sent": "the backyard .", "words": ["the", "backyard"]}, {"sent": "okay were gonna go the backyard love baby .", "words": ["the", "backyard", "go", "love", "were"]}, {"sent": "she came in fast from that backyard last night didnt she ?", "words": ["backyard", "fast", "last", "in", "that", "she"]}, {"sent": "well we have something in our backyard now that you lay in .", "words": ["backyard", "we", "our", "have", "you", "in", "that"]}, {"sent": "what did we haul off in the backyard this weekend ?", "words": ["off", "what", "the", "backyard", "we", "this", "in", "did"]}, {"sent": "you could probably even play outside with Phoebe and Trevor today maybe in Aunties backyard .", "words": ["backyard", "outside", "play", "with", "you", "and", "in", "could"]}, {"sent": "theyre upstairs talking about the construction of the Ewok village type idea that theyre planning for the summer in the backyard .", "words": ["the", "backyard", "for", "of", "in", "about", "that"]}, {"sent": "see this is great now hes gonna be in my backyard .", "words": ["backyard", "my", "is", "be", "this", "in", "see"]}, {"sent": "he usually does the front of the house first and then hell go in the backyard .", "words": ["the", "does", "backyard", "go", "of", "first", "and", "in", "house", "he", "then"]}, {"sent": "were you playing with Elmo in the backyard the other day ?", "words": ["the", "backyard", "with", "you", "other", "in", "were"]}, {"sent": "we planted some things in the backyard didnt we ?", "words": ["the", "backyard", "we", "in", "some"]}, {"sent": "you made a big snowman in our backyard .", "words": ["backyard", "you", "our", "in", "snowman", "big", "a"]}, {"sent": "let me make a sign sothat Marni will know were in the backyard Bo .", "words": ["the", "backyard", "in", "will", "me", "make", "were", "a"]}, {"sent": "youre in the backyard being a cool dude ?", "words": ["the", "backyard", "in", "a"]}, {"sent": "grandma and grandpa have lots of these in the backyard .", "words": ["the", "these", "backyard", "of", "and", "have", "in"]}, {"sent": "come look in the backyard and tell me what you see .", "words": ["the", "backyard", "what", "see", "and", "you", "in", "look", "me"]}, {"sent": "but we have one in our backyard dont we ?", "words": ["backyard", "we", "our", "have", "dont", "in", "but"]}, {"sent": "thats the door to go outside in the backyard or the front yard ?", "words": ["the", "backyard", "go", "to", "outside", "in", "door"]}, {"sent": "you lost them in you backyard ?", "words": ["backyard", "in", "them", "you"]}, {"sent": "whats that orange thing out in the backyard ?", "words": ["the", "backyard", "in", "orange", "out", "that"]}, {"sent": "now lets just say I filled up my whole backyard with these trees .", "words": ["up", "backyard", "my", "these", "say", "with"]}, {"sent": "so you wont be able to get out of the backyard .", "words": ["the", "backyard", "of", "to", "so", "be", "you", "get", "out"]}, {"sent": "other animals alright maybe Ill start a new kinda thing another thing because you know I have a very big backyard so I could probably put up a couple things okay .", "words": ["up", "backyard", "put", "so", "other", "another", "you", "have", "could", "new", "big", "because", "a"]}, {"sent": "yeah you got your white ones but youre not gonna wear these in the backyard .", "words": ["the", "these", "backyard", "your", "you", "in", "white", "not", "but"]}, {"sent": "in the backyard .", "words": ["the", "backyard", "in"]}, {"sent": "oh is it their backyard ?", "words": ["backyard", "it", "their", "is"]}, {"sent": "were gonna go in the backyard now .", "words": ["the", "backyard", "go", "in", "were"]}, {"sent": "you got a cold in your backyard ?", "words": ["backyard", "your", "you", "in", "cold", "a"]}, {"sent": "*VI1: you be in uh in our backyard right ?", "words": ["backyard", "be", "our", "you", "in"]}, {"sent": "we have these in our backyard .", "words": ["these", "backyard", "we", "our", "have", "in"]}, {"sent": "do you wanna put peanutbutter with the melon ?", "words": ["the", "do", "put", "with", "you", "wanna", "melon"]}, {"sent": "and Purdie loves melon doesnt she ?", "words": ["melon", "she", "and"]}, {"sent": "you have the melon .", "words": ["have", "the", "melon", "you"]}, {"sent": "melon .", "words": ["melon"]}, {"sent": "I do like melon .", "words": ["do", "melon", "like"]}, {"sent": "whats whats that um melon called ?", "words": ["that", "melon"]}, {"sent": "thats melon .", "words": ["melon"]}, {"sent": "Mummys going to have a slice of melon now .", "words": ["of", "to", "have", "melon", "a"]}, {"sent": "and a slice of melon .", "words": ["of", "melon", "a", "and"]}, {"sent": "would you like to try some melon ?", "words": ["to", "like", "you", "melon", "would", "some", "try"]}, {"sent": "you know what this melon is called ?", "words": ["what", "is", "you", "this", "melon"]}, {"sent": "Goofy eats a melon in your video .", "words": ["your", "in", "melon", "a"]}, {"sent": "whats this melon called ?", "words": ["this", "melon"]}, {"sent": "try some melon ?", "words": ["some", "try", "melon"]}, {"sent": "oh and he called this melon .", "words": ["this", "he", "melon", "and"]}, {"sent": "one melon .", "words": ["melon"]}, {"sent": "you definitely dont like melon do you ?", "words": ["do", "like", "you", "dont", "melon"]}, {"sent": "melon .", "words": ["melon"]}, {"sent": "melon .", "words": ["melon"]}, {"sent": "do you like melon ?", "words": ["do", "melon", "like", "you"]}, {"sent": "its melon .", "words": ["melon"]}, {"sent": "having the melon ?", "words": ["the", "melon"]}, {"sent": "melon .", "words": ["melon"]}, {"sent": "xxx melon .", "words": ["melon"]}, {"sent": "oh the melon ?", "words": ["the", "melon"]}, {"sent": "honeydew melon ?", "words": ["melon"]}, {"sent": "you dont like melon .", "words": ["dont", "melon", "like", "you"]}, {"sent": "melon ?", "words": ["melon"]}, {"sent": "okay I dont know if you should roll it too far because this is just a melon and it might kind of xxx xxx down .", "words": ["if", "is", "of", "you", "and", "dont", "this", "it", "too", "down", "melon", "because", "a"]}, {"sent": "wheres the melon ?", "words": ["the", "melon"]}, {"sent": "a grape apple melon .", "words": ["apple", "melon", "a"]}, {"sent": "you cut melon .", "words": ["melon", "cut", "you"]}, {"sent": "broccoli spinach cauliflower melon mango kiwi papaya apricots grapefruit grape halfs strawberries tomatos pasta .", "words": ["melon"]}, {"sent": "you dont make melon .", "words": ["dont", "make", "melon", "you"]}, {"sent": "and some melon .", "words": ["some", "melon", "and"]}, {"sent": "or half a melon .", "words": ["melon", "a"]}, {"sent": "melon .", "words": ["melon"]}, {"sent": "honeydew melon .", "words": ["melon"]}, {"sent": "melon .", "words": ["melon"]}, {"sent": "thats melon .", "words": ["melon"]}, {"sent": "do you like melon ?", "words": ["do", "melon", "like", "you"]}, {"sent": "would you like to share Mummys melon ?", "words": ["share", "to", "like", "you", "would", "melon"]}, {"sent": "oh you mean on the melon .", "words": ["on", "the", "melon", "you"]}, {"sent": "melon .", "words": ["melon"]}, {"sent": "wheres melon ?", "words": ["melon"]}, {"sent": "because when NinNin had finished those melon skins she said Im going to put them in the compost bin .", "words": ["the", "put", "those", "to", "them", "in", "melon", "when", "she", "because"]}, {"sent": "I think its a melon .", "words": ["think", "melon", "a"]}, {"sent": "and were having melon later .", "words": ["were", "melon", "and"]}, {"sent": "remember thats the melon .", "words": ["the", "melon"]}, {"sent": "and Purdie loves melon doesnt she ?", "words": ["melon", "she", "and"]}, {"sent": "its a melon .", "words": ["melon", "a"]}, {"sent": "its a melon .", "words": ["melon", "a"]}, {"sent": "other melon .", "words": ["melon", "other"]}, {"sent": "honeydew melon .", "words": ["melon"]}, {"sent": "thats melon .", "words": ["melon"]}, {"sent": "can you say melon ?", "words": ["can", "say", "melon", "you"]}, {"sent": "a melon .", "words": ["melon", "a"]}, {"sent": "mm melon .", "words": ["melon"]}, {"sent": "and a piece of melon .", "words": ["of", "melon", "a", "and"]}, {"sent": "ask them again ask them ask them if they want some um melon .", "words": ["if", "them", "they", "some", "melon"]}, {"sent": "honeydew melon .", "words": ["melon"]}, {"sent": "thats a melon .", "words": ["melon", "a"]}, {"sent": "that ones called melon .", "words": ["that", "melon"]}, {"sent": "well this is a honeydew melon .", "words": ["this", "melon", "a", "is"]}, {"sent": "melon .", "words": ["melon"]}, {"sent": "a melon .", "words": ["melon", "a"]}, {"sent": "then Im going to take this melon outside .", "words": ["to", "outside", "this", "melon", "then", "take"]}, {"sent": "oh thats a big melon look at that .", "words": ["look", "that", "melon", "at", "big", "a"]}, {"sent": "thats right wheres the melon coming from ?", "words": ["the", "melon"]}, {"sent": "yellow hippo has a yellow melon .", "words": ["yellow", "melon", "a"]}, {"sent": "some nice juicy melon .", "words": ["some", "nice", "melon"]}, {"sent": "melon .", "words": ["melon"]}, {"sent": "and a slice a slice of melon .", "words": ["of", "melon", "a", "and"]}, {"sent": "melon .", "words": ["melon"]}, {"sent": "have a look whats inside this melon .", "words": ["inside", "have", "this", "look", "melon", "a"]}, {"sent": "thats not melon .", "words": ["melon", "not"]}, {"sent": "no more melon after that darling .", "words": ["that", "more", "melon"]}, {"sent": "what goes with melon ?", "words": ["what", "with", "melon"]}, {"sent": "his head is a melon thats why they call him ProfessorHoneydew .", "words": ["is", "head", "they", "him", "his", "melon", "why", "a"]}, {"sent": "whenever we have melon we hafta cut up some .", "words": ["up", "cut", "we", "have", "some", "melon"]}, {"sent": "yeah thats melon yep .", "words": ["melon"]}, {"sent": "youre gonna have your dinner soon then you can have melon afterwards .", "words": ["can", "your", "you", "have", "then", "melon"]}, {"sent": "and thats five slices of melon .", "words": ["of", "melon", "and"]}, {"sent": "okay we hafta match this one w w water melon windmill to the windmill thats on the card .", "words": ["the", "we", "to", "this", "water", "on", "melon"]}, {"sent": "yellow hippo has a yellow melon .", "words": ["yellow", "melon", "a"]}, {"sent": "its called honeydew melon .", "words": ["melon"]}, {"sent": "that should be a melon .", "words": ["be", "that", "melon", "a"]}, {"sent": "it looks a bit like melon .", "words": ["it", "melon", "like", "a"]}, {"sent": "what do you feel sick from eating too much cheese and melon ?", "words": ["what", "do", "you", "and", "sick", "melon", "too", "cheese", "much"]}, {"sent": "what kind of melon ?", "words": ["of", "what", "melon"]}, {"sent": "oh melon .", "words": ["melon"]}, {"sent": "ask them if they want some melon .", "words": ["if", "them", "they", "some", "melon"]}, {"sent": "melon .", "words": ["melon"]}, {"sent": "its a melon isnt it ?", "words": ["it", "melon", "a"]}, {"sent": "this is called honeydew melon .", "words": ["this", "melon", "is"]}, {"sent": "youve had enough now of melon .", "words": ["of", "melon"]}, {"sent": "you can have some melon .", "words": ["can", "you", "have", "some", "melon"]}, {"sent": "bitter melon .", "words": ["melon"]}, {"sent": "our muffin pan .", "words": ["muffin", "our"]}, {"sent": "did the muffin man come ?", "words": ["the", "muffin", "did"]}, {"sent": "muffin .", "words": ["muffin"]}, {"sent": "how bout how bout eating your muffin ?", "words": ["muffin", "how", "your"]}, {"sent": "ah heres the other muffin .", "words": ["the", "muffin", "other"]}, {"sent": "now shall we use these little white muffin .", "words": ["muffin", "these", "we", "little", "white"]}, {"sent": "the icecube tray has more than twelve and the muffin tin has twelve places for twelve muffins .", "words": ["the", "muffin", "for", "and", "more", "tray"]}, {"sent": "the muffin man .", "words": ["the", "muffin"]}, {"sent": "Im gonna give you this big muffin tin .", "words": ["muffin", "you", "this", "give", "big"]}, {"sent": "or the muffin .", "words": ["the", "muffin"]}, {"sent": "Carolines got the big muffin .", "words": ["the", "muffin", "big"]}, {"sent": "Steve lets go fix an english muffin okay ?", "words": ["muffin", "fix", "an", "go"]}, {"sent": "can mommy have a muffin ?", "words": ["can", "have", "muffin", "a"]}, {"sent": "muffin ?", "words": ["muffin"]}, {"sent": "you hafta put the muffin in .", "words": ["the", "muffin", "put", "you", "in"]}, {"sent": "thats the last muffin .", "words": ["the", "last", "muffin"]}, {"sent": "oh yes I know the muffin man who lives in Drury lane .", "words": ["the", "in", "muffin", "who"]}, {"sent": "yes I know the muffin man .", "words": ["the", "muffin"]}, {"sent": "I broke my muffin in half .", "words": ["muffin", "in", "my"]}, {"sent": "you eat your muffin .", "words": ["eat", "muffin", "your", "you"]}, {"sent": "muffin ?", "words": ["muffin"]}, {"sent": "is that a muffin ?", "words": ["muffin", "that", "a", "is"]}, {"sent": "muffin ?", "words": ["muffin"]}, {"sent": "the muffin man the muffin man ?", "words": ["the", "muffin"]}, {"sent": "this is not the other muffin man .", "words": ["the", "muffin", "is", "other", "this", "not"]}, {"sent": "and a muffin pan .", "words": ["muffin", "a", "and"]}, {"sent": "did you wanna eat a muffin when theyre done ?", "words": ["muffin", "you", "eat", "did", "wanna", "when", "a"]}, {"sent": "a muffin ?", "words": ["muffin", "a"]}, {"sent": "he called them muffins which Ive never heard him say muffin before .", "words": ["muffin", "say", "them", "him", "he", "which"]}, {"sent": "would you like another muffin ?", "words": ["muffin", "like", "you", "another", "would"]}, {"sent": "oh yes I know the muffin man the muffin man .", "words": ["the", "muffin"]}, {"sent": "she cut you more muffin ?", "words": ["muffin", "cut", "you", "more", "she"]}, {"sent": "its only for the muffin .", "words": ["the", "for", "muffin"]}, {"sent": "you could get maybe you could get like a muffin or a cookie .", "words": ["muffin", "cookie", "like", "you", "get", "could", "a"]}, {"sent": "muffin yeah muffin .", "words": ["muffin"]}, {"sent": "er muffin .", "words": ["muffin"]}, {"sent": "or a muffin .", "words": ["muffin", "a"]}, {"sent": "no the bee cant have a muffin ?", "words": ["the", "muffin", "bee", "have", "a"]}, {"sent": "now Mummy I just ate half a muffin .", "words": ["muffin", "a"]}, {"sent": "ooh theres the last muffin .", "words": ["the", "last", "muffin"]}, {"sent": "eat the muffin .", "words": ["eat", "the", "muffin"]}, {"sent": "can you say muffin ?", "words": ["can", "muffin", "say", "you"]}, {"sent": "hah oh I found the other muffin .", "words": ["the", "muffin", "other"]}, {"sent": "muffin ?", "words": ["muffin"]}, {"sent": "that my little love muffin threw every which way .", "words": ["muffin", "my", "which", "little", "every", "love", "that"]}, {"sent": "or a muffin ?", "words": ["muffin", "a"]}, {"sent": "its a little muffin pan .", "words": ["muffin", "little", "a"]}, {"sent": "can you get the muffin ?", "words": ["can", "the", "muffin", "you", "get"]}, {"sent": "I like the muffin .", "words": ["the", "muffin", "like"]}, {"sent": "mm I like the muffin .", "words": ["the", "muffin", "like"]}, {"sent": "oh do you know the muffin man who lives in Drury lane ?", "words": ["the", "do", "muffin", "you", "in", "who"]}, {"sent": "every time were there I always buy her like a muffin or a croissant or a bagel or something so .", "words": ["muffin", "so", "like", "there", "every", "her", "buy", "were", "a"]}, {"sent": "I think we have a muffin tin that has twelve .", "words": ["muffin", "think", "we", "have", "that", "a"]}, {"sent": "the muffin man the muffin man .", "words": ["the", "muffin"]}, {"sent": "go ahead and eat the muffin now okay ?", "words": ["the", "muffin", "go", "and", "eat"]}, {"sent": "Im gonna give you another muffin tin to put them into .", "words": ["muffin", "put", "into", "to", "them", "you", "another", "give"]}, {"sent": "yummy muffin .", "words": ["muffin"]}, {"sent": "whats she having a muffin ?", "words": ["muffin", "she", "a"]}, {"sent": "do you know the muffin man that lives in Drury lane ?", "words": ["the", "do", "muffin", "you", "in", "that"]}, {"sent": "oh do you know the muffin man .", "words": ["the", "do", "muffin", "you"]}, {"sent": "muffin cases wont we ?", "words": ["muffin", "we"]}, {"sent": "oh okay one muffin per burner .", "words": ["muffin"]}, {"sent": "gonna finish your muffin ?", "words": ["muffin", "your", "finish"]}, {"sent": "thats how an English muffin is made .", "words": ["muffin", "how", "is", "an"]}, {"sent": "where does the muffin man live ?", "words": ["where", "does", "muffin", "the"]}, {"sent": "we have some muffin pans .", "words": ["have", "some", "muffin", "we"]}, {"sent": "you eat the muffin too .", "words": ["the", "muffin", "you", "eat", "too"]}, {"sent": "did you a muffin xxx ?", "words": ["muffin", "did", "a", "you"]}, {"sent": "oh here you wanna give her a muffin to eat ?", "words": ["muffin", "to", "you", "here", "eat", "wanna", "give", "her", "a"]}, {"sent": "the muffin song ?", "words": ["the", "muffin"]}, {"sent": "it was very much like a muffin tin youknow its it was like six little things like a muffin tin like thing that had stars .", "words": ["muffin", "was", "like", "little", "that", "it", "much", "a"]}, {"sent": "can you get the muffin and well put it back in the pan .", "words": ["can", "the", "muffin", "put", "back", "you", "get", "and", "in", "it"]}, {"sent": "I dont think the muffin will fit in the hole .", "words": ["the", "muffin", "think", "fit", "dont", "in", "will"]}, {"sent": "does that muffin goes in ?", "words": ["muffin", "does", "in", "that"]}, {"sent": "oh do you know the muffin man the muffin man the muffin man .", "words": ["the", "do", "muffin", "you"]}, {"sent": "oh okay well we already put the dough into those a muffin tins .", "words": ["the", "muffin", "put", "those", "into", "we", "a"]}, {"sent": "a muffin ?", "words": ["muffin", "a"]}, {"sent": "oh hes on the muffin .", "words": ["on", "the", "muffin"]}, {"sent": "do you know the muffin man who lives on Drury Lane ?", "words": ["the", "do", "muffin", "you", "who", "on"]}, {"sent": "would you like a muffin ?", "words": ["muffin", "like", "you", "would", "a"]}, {"sent": "why dont you try the muffin .", "words": ["the", "muffin", "you", "dont", "try", "why"]}, {"sent": "does it look like a muffin to you ?", "words": ["muffin", "does", "to", "like", "you", "it", "look", "a"]}, {"sent": "yeah from it looks like a muffin .", "words": ["muffin", "it", "like", "a"]}, {"sent": "now you gonna give me another muffin .", "words": ["muffin", "you", "another", "me", "give"]}, {"sent": "heres theres the other muffin .", "words": ["the", "muffin", "other"]}, {"sent": "but you just hafta wait till the muffin pops up .", "words": ["the", "muffin", "wait", "up", "you", "but"]}, {"sent": "thats a muffin .", "words": ["muffin", "a"]}, {"sent": "Ill see if we can make a blue muffin .", "words": ["can", "muffin", "if", "we", "blue", "see", "make", "a"]}, {"sent": "muffin ?", "words": ["muffin"]}, {"sent": "if we run out Ive got some tiny tiny muffin cases that well put some extra in .", "words": ["tiny", "muffin", "put", "if", "we", "run", "in", "out", "some", "that"]}, {"sent": "Dwwws licking the muffin .", "words": ["the", "muffin"]}, {"sent": "and he wanted to put them in the muffin tin .", "words": ["the", "muffin", "put", "to", "them", "and", "in", "he"]}, {"sent": "muffin .", "words": ["muffin"]}, {"sent": "theyre going to be very fiddly putting in these tiny muffin cases arent they ?", "words": ["tiny", "these", "muffin", "to", "be", "they", "in"]}, {"sent": "do you know the muffin man .", "words": ["the", "do", "muffin", "you"]}, {"sent": "do you know the muffin man ?", "words": ["the", "do", "muffin", "you"]}, {"sent": "did you eat a muffin ?", "words": ["muffin", "you", "eat", "did", "a"]}, {"sent": "now sweep .", "words": ["sweep"]}, {"sent": "are you going to sweep the floor for me ?", "words": ["the", "for", "to", "sweep", "you", "are", "me"]}, {"sent": "do you sweep ?", "words": ["do", "sweep", "you"]}, {"sent": "sweep sweep sweep .", "words": ["sweep"]}, {"sent": "I dont want you to sweep me up .", "words": ["up", "to", "sweep", "you", "dont", "me"]}, {"sent": "I shall hafta sweep the floor now .", "words": ["the", "sweep"]}, {"sent": "here why dont you get up and let Mommy sweep this floor ?", "words": ["up", "sweep", "you", "here", "dont", "get", "and", "this", "why"]}, {"sent": "Ill just sweep up this mess .", "words": ["up", "this", "sweep"]}, {"sent": "sweep sweep sweep sweep sweep .", "words": ["sweep"]}, {"sent": "sweep the floor ?", "words": ["the", "sweep"]}, {"sent": "we sweep sweep sweep .", "words": ["sweep", "we"]}, {"sent": "yeah oh sweep the floor .", "words": ["the", "sweep"]}, {"sent": "is he hungry or is he going to sweep the floor ?", "words": ["the", "is", "to", "sweep", "hungry", "he"]}, {"sent": "Mommy cant sweep ?", "words": ["sweep"]}, {"sent": "sweep that under the chair .", "words": ["the", "sweep", "chair", "under", "that"]}, {"sent": "whos going to sweep it up ?", "words": ["up", "to", "it", "sweep"]}, {"sent": "this is the way we sweep the floor sweep the floor this is the way we sweep the floor .", "words": ["the", "we", "is", "sweep", "this"]}, {"sent": "sweep sweep sweep sweep .", "words": ["sweep"]}, {"sent": "they useta sweep the footpaths and the road .", "words": ["the", "they", "sweep", "and"]}, {"sent": "you sweep sweep .", "words": ["sweep", "you"]}, {"sent": "why dont you sweep this floor ?", "words": ["sweep", "you", "dont", "this", "why"]}, {"sent": "you needta sweep it into the into the pan dont you ?", "words": ["the", "into", "sweep", "you", "dont", "it"]}, {"sent": "hours later Cinderella began to sweep and scrub the endless stone floor .", "words": ["the", "to", "sweep", "and", "stone"]}, {"sent": "if you make a mess on my patio and I hafta sweep you up .", "words": ["up", "my", "if", "sweep", "you", "and", "on", "make", "a"]}, {"sent": "come on sweep the floor now .", "words": ["on", "the", "sweep"]}, {"sent": "once you let your ball go the current will sweep it down there .", "words": ["the", "your", "go", "sweep", "you", "ball", "there", "it", "will", "down"]}, {"sent": "just sweep this little bit .", "words": ["this", "sweep", "little"]}, {"sent": "or you sweep the floors ?", "words": ["the", "sweep", "you"]}, {"sent": "dont sweep it up .", "words": ["up", "dont", "it", "sweep"]}, {"sent": "sweep .", "words": ["sweep"]}, {"sent": "and sweep up ?", "words": ["up", "sweep", "and"]}, {"sent": "sweep sweep sweep the floor .", "words": ["the", "sweep"]}, {"sent": "sweep with the broom .", "words": ["the", "broom", "sweep", "with"]}, {"sent": "I gotta sweep up one more little spot .", "words": ["up", "more", "sweep", "little"]}, {"sent": "I hafta sweep up your bottom and your willie and your nips and your belly ?", "words": ["up", "bottom", "your", "sweep", "and"]}, {"sent": "sweep .", "words": ["sweep"]}, {"sent": "she can plug it in and sweep .", "words": ["can", "sweep", "and", "in", "it", "she"]}, {"sent": "sweep it up .", "words": ["up", "it", "sweep"]}, {"sent": "I sweep .", "words": ["sweep"]}, {"sent": "its always full of fallen leaves and you hafta sweep it .", "words": ["full", "of", "sweep", "you", "and", "it"]}, {"sent": "sweep sweep .", "words": ["sweep"]}, {"sent": "oh its sometimes easier to sweep it up then wipe it up isnt it ?", "words": ["up", "to", "sweep", "it", "then", "wipe"]}, {"sent": "if I spill it you can sweep it up .", "words": ["can", "up", "if", "sweep", "you", "it", "spill"]}, {"sent": "sweep .", "words": ["sweep"]}, {"sent": "what do you sweep ?", "words": ["what", "do", "sweep", "you"]}, {"sent": "are you going to come and sweep up ?", "words": ["up", "to", "sweep", "and", "you", "are"]}, {"sent": "if you smashed through the window and fell in the road wed hafta sweep you up .", "words": ["the", "up", "if", "sweep", "and", "you", "in", "window"]}, {"sent": "sweep the floor sweep .", "words": ["the", "sweep"]}, {"sent": "sweep sweep sweep .", "words": ["sweep"]}, {"sent": "why dont you sweep the kitchen ?", "words": ["the", "sweep", "you", "dont", "kitchen", "why"]}, {"sent": "sweep sweep .", "words": ["sweep"]}, {"sent": "sweep .", "words": ["sweep"]}, {"sent": "hes a chimney sweep .", "words": ["sweep", "a"]}, {"sent": "therell be trouble if I hafta sweep bits of Thomas because he has fallen through the window .", "words": ["the", "if", "of", "be", "sweep", "he", "window", "because"]}, {"sent": "sweep sweep sweep like that .", "words": ["that", "sweep", "like"]}, {"sent": "okay Bobby youre gonna sweep for awhile huh ?", "words": ["for", "sweep"]}, {"sent": "so I think hes going to sweep those ashes up and perhaps put them in the dustbin .", "words": ["up", "the", "think", "put", "those", "to", "so", "sweep", "them", "and", "in"]}, {"sent": "you can sweep the floor .", "words": ["can", "the", "sweep", "you"]}, {"sent": "sweep over them .", "words": ["over", "sweep", "them"]}, {"sent": "an egg and a sweep .", "words": ["egg", "sweep", "and", "an", "a"]}, {"sent": "thats okay if they fall Mommy will sweep them up later okay ?", "words": ["up", "if", "sweep", "fall", "them", "they", "will"]}, {"sent": "this is the way we sweep the floor .", "words": ["the", "we", "is", "sweep", "this"]}, {"sent": "shall we get the brush out and sweep some of these crumbs up ?", "words": ["the", "these", "up", "we", "brush", "of", "sweep", "and", "get", "out", "some"]}, {"sent": "like when we sweep the floor ?", "words": ["the", "we", "sweep", "like", "when"]}, {"sent": "sweep sweep sweep .", "words": ["sweep"]}, {"sent": "youve been using that brush to sweep all that rubbish up .", "words": ["up", "all", "brush", "to", "sweep", "that"]}, {"sent": "sweep .", "words": ["sweep"]}, {"sent": "sweep the floor .", "words": ["the", "sweep"]}, {"sent": "weve gotta weve gotta finish the sweep sweep first .", "words": ["the", "first", "sweep", "finish"]}, {"sent": "we should sweep them up .", "words": ["up", "them", "sweep", "we"]}, {"sent": "you gonna help Mommy sweep the floor ?", "words": ["the", "help", "sweep", "you"]}, {"sent": "sweep up all the blossom when that has blown down .", "words": ["up", "all", "the", "sweep", "that", "down", "when"]}, {"sent": "okay now you can sweep the floor .", "words": ["can", "the", "sweep", "you"]}, {"sent": "are you going to sweep up ?", "words": ["up", "to", "sweep", "you", "are"]}, {"sent": "and theyre going to sweep in the kitchen yeah ?", "words": ["the", "to", "sweep", "and", "in", "kitchen"]}, {"sent": "a sweep ?", "words": ["sweep", "a"]}, {"sent": "yeah but what room is he going to sweep though ?", "words": ["what", "is", "to", "sweep", "room", "he", "but"]}, {"sent": "sweep sweep sweep .", "words": ["sweep"]}, {"sent": "Ill sweep it up in a minute .", "words": ["up", "sweep", "in", "it", "a"]}, {"sent": "what do you think shes going to sweep ?", "words": ["what", "do", "think", "to", "sweep", "you"]}, {"sent": "wait a minute Grannyll sweep it up .", "words": ["up", "wait", "sweep", "it", "a"]}, {"sent": "lets sweep it up .", "words": ["up", "it", "sweep"]}, {"sent": "youd like the brush to sweep it up ?", "words": ["the", "up", "brush", "to", "sweep", "like", "it"]}, {"sent": "you sweep the walls ?", "words": ["the", "sweep", "you"]}, {"sent": "youd better help me sweep this floor when youre done .", "words": ["better", "help", "sweep", "this", "me", "when"]}, {"sent": "sweep it up .", "words": ["up", "it", "sweep"]}, {"sent": "now can you just sit very still while I get a brush to sweep it up ?", "words": ["can", "up", "brush", "to", "sweep", "you", "get", "it", "sit", "a"]}, {"sent": "sweep .", "words": ["sweep"]}, {"sent": "you were going to sweep it better ?", "words": ["better", "to", "sweep", "you", "it", "were"]}, {"sent": "sweep sweep .", "words": ["sweep"]}, {"sent": "sweep sweep sweep .", "words": ["sweep"]}, {"sent": "would you like to sweep it up ?", "words": ["up", "to", "sweep", "like", "you", "it", "would"]}, {"sent": "let me sweep some of this dirt up .", "words": ["up", "of", "sweep", "this", "me", "some"]}, {"sent": "Alex can you say ankle ?", "words": ["can", "ankle", "say", "you"]}, {"sent": "Im gonna make a bow on Naimas ankle .", "words": ["on", "ankle", "make", "a"]}, {"sent": "she hurt her ankle .", "words": ["hurt", "ankle", "her", "she"]}, {"sent": "you washing her ankle for her ?", "words": ["ankle", "for", "her", "you"]}, {"sent": "does she have a dirty ankle ?", "words": ["does", "have", "ankle", "dirty", "she", "a"]}, {"sent": "look at your pretty oh shall we put it on your ankle ?", "words": ["put", "look", "your", "we", "it", "pretty", "ankle", "at", "on"]}, {"sent": "wheres your ankle ?", "words": ["ankle", "your"]}, {"sent": "theres your ankle .", "words": ["ankle", "your"]}, {"sent": "wheres your ankle ?", "words": ["ankle", "your"]}, {"sent": "wheres Mamas ankle darling ?", "words": ["ankle"]}, {"sent": "what happened to your ankle ?", "words": ["to", "what", "ankle", "your"]}, {"sent": "mommy today my kneecap mommy today my ankle was hurting because I was running and then I slipped on that rock and the rock hit my kneecap and my kneecap started hurting .", "words": ["the", "my", "was", "and", "ankle", "on", "then", "hit", "that", "because", "rock"]}, {"sent": "my ankle is swollen outof this world .", "words": ["this", "my", "ankle", "is"]}, {"sent": "round the ankle and what are those one the front ?", "words": ["the", "what", "those", "and", "are", "ankle"]}, {"sent": "it could be an ankle bracelet for her .", "words": ["for", "be", "an", "it", "ankle", "could", "her"]}, {"sent": "your mommy had a hurt ankle and she hadta get some crutches too ?", "words": ["your", "hurt", "and", "get", "ankle", "too", "some", "she", "a"]}, {"sent": "I went to stand up in Maggies my ankle took big shooting pains in my ankle .", "words": ["up", "my", "to", "stand", "in", "ankle", "big"]}, {"sent": "and that ankle was just going .", "words": ["was", "ankle", "that", "and"]}, {"sent": "youre gonna hitcha ankle again .", "words": ["ankle"]}, {"sent": "Daddy told me she sprained her ankle .", "words": ["me", "ankle", "her", "she"]}, {"sent": "oh and you had a bangle on your ankle also .", "words": ["your", "you", "and", "ankle", "on", "a"]}, {"sent": "you had that bangle on your ankle too .", "words": ["your", "you", "ankle", "too", "on", "that"]}, {"sent": "and an ankle bangle like you had at the hospital .", "words": ["the", "like", "you", "and", "an", "ankle", "at"]}, {"sent": "its your ankle isnt it ?", "words": ["ankle", "it", "your"]}, {"sent": "youre standing right on my ankle .", "words": ["on", "my", "ankle"]}, {"sent": "thats her ankle .", "words": ["ankle", "her"]}, {"sent": "is that your ankle ?", "words": ["your", "ankle", "that", "is"]}, {"sent": "wheres your ankle ?", "words": ["ankle", "your"]}, {"sent": "wheres your ankle ?", "words": ["ankle", "your"]}, {"sent": "ankle .", "words": ["ankle"]}, {"sent": "theres Awwws ankle there .", "words": ["there", "ankle"]}, {"sent": "theres your ankle there .", "words": ["there", "ankle", "your"]}, {"sent": "wheres Mamas ankle darling ?", "words": ["ankle"]}, {"sent": "wheres your ankle ?", "words": ["ankle", "your"]}, {"sent": "ankle ?", "words": ["ankle"]}, {"sent": "theres Eleanors ankle .", "words": ["ankle"]}, {"sent": "wheres your ankle ?", "words": ["ankle", "your"]}, {"sent": "ankle .", "words": ["ankle"]}, {"sent": "its your ankle .", "words": ["ankle", "your"]}, {"sent": "theres the other ankle .", "words": ["the", "ankle", "other"]}, {"sent": "Eleanor youll hurt your ankle .", "words": ["hurt", "ankle", "your"]}, {"sent": "ankle ?", "words": ["ankle"]}, {"sent": "you gotta push the ankle .", "words": ["the", "ankle", "push", "you"]}, {"sent": "push the ankle .", "words": ["the", "ankle", "push"]}, {"sent": "tickle tickle on the ankle .", "words": ["on", "the", "ankle", "tickle"]}, {"sent": "tickle tickle on the ankle .", "words": ["on", "the", "ankle", "tickle"]}, {"sent": "I the ankle .", "words": ["the", "ankle"]}, {"sent": "thats not the ankle .", "words": ["the", "ankle", "not"]}, {"sent": "is the ankle .", "words": ["the", "ankle", "is"]}, {"sent": "which is your ankle Daniel ?", "words": ["your", "ankle", "which", "is"]}, {"sent": "wheres the ankle ?", "words": ["the", "ankle"]}, {"sent": "ankle no thats your elbow .", "words": ["your", "ankle"]}, {"sent": "look at your ankle .", "words": ["ankle", "at", "your", "look"]}, {"sent": "ankle is down by your socks .", "words": ["your", "is", "by", "ankle", "down"]}, {"sent": "Julia show Daniel what his ankle is .", "words": ["what", "is", "show", "ankle", "his"]}, {"sent": "shes showing you the ankle .", "words": ["the", "ankle", "you"]}, {"sent": "touch you ankle thats the ankle .", "words": ["the", "ankle", "touch", "you"]}, {"sent": "show him the ankle .", "words": ["show", "the", "him", "ankle"]}, {"sent": "your left knee your left ankle .", "words": ["ankle", "knee", "your"]}, {"sent": "why did I hafta say I was doing this Im not I didnt feel well in the first place and my leg hurts and my ankle hurts now .", "words": ["the", "my", "was", "first", "say", "and", "this", "in", "ankle", "did", "leg", "not", "why"]}, {"sent": "whats that thing on your ankle ?", "words": ["on", "ankle", "that", "your"]}, {"sent": "look at my ankle .", "words": ["my", "at", "ankle", "look"]}, {"sent": "and she hurt her ankle .", "words": ["hurt", "and", "ankle", "her", "she"]}, {"sent": "cause her ankle hurt too bad .", "words": ["hurt", "bad", "ankle", "too", "her"]}, {"sent": "mind mind my ankle .", "words": ["my", "ankle"]}, {"sent": "your jelly shoes arent as important as my ankle .", "words": ["jelly", "my", "your", "ankle"]}, {"sent": "she fell down three steps and hurt her ankle didnt she ?", "words": ["hurt", "and", "ankle", "down", "her", "she"]}, {"sent": "I thought I was going to hurt my ankle .", "words": ["my", "hurt", "to", "was", "ankle"]}, {"sent": "youd fall over and break your ankle wouldnt you ?", "words": ["your", "fall", "and", "you", "ankle", "over", "break"]}, {"sent": "your ankle .", "words": ["ankle", "your"]}, {"sent": "have we got the ankle one ?", "words": ["have", "the", "ankle", "we"]}, {"sent": "ow this is even a badder cut than my sprained ankle .", "words": ["my", "cut", "is", "this", "ankle", "a"]}, {"sent": "after breakfast he put on his snowsuit and ran outside .", "words": ["put", "outside", "and", "his", "snowsuit", "he", "on"]}, {"sent": "and her snowsuit .", "words": ["snowsuit", "her", "and"]}, {"sent": "oh show her her pink fur snowsuit just like yours .", "words": ["show", "snowsuit", "her", "like"]}, {"sent": "the snowsuit you mean ?", "words": ["the", "snowsuit", "you"]}, {"sent": "a snowsuit .", "words": ["snowsuit", "a"]}, {"sent": "a xxx from your snowsuit .", "words": ["snowsuit", "your", "a"]}, {"sent": "oh is that the color of your snowsuit jacket ?", "words": ["the", "your", "is", "of", "snowsuit", "that", "jacket"]}, {"sent": "raincoat or your snowsuit jacket ?", "words": ["snowsuit", "your", "jacket"]}, {"sent": "we better put your snowsuit jacket on .", "words": ["better", "put", "your", "we", "snowsuit", "on", "jacket"]}, {"sent": "I have a little boy all dressed up in a snowsuit .", "words": ["up", "all", "little", "have", "snowsuit", "in", "a"]}, {"sent": "its a winter snowsuit .", "words": ["snowsuit", "a"]}, {"sent": "that is a snowsuit .", "words": ["snowsuit", "that", "a", "is"]}, {"sent": "it is a bear in a snowsuit .", "words": ["is", "bear", "snowsuit", "in", "it", "a"]}, {"sent": "that is the bear in the snowsuit .", "words": ["the", "is", "bear", "snowsuit", "in", "that"]}, {"sent": "Im gonna the applesauce first .", "words": ["the", "first", "applesauce"]}, {"sent": "mm is applesauce yummy ?", "words": ["applesauce", "is"]}, {"sent": "you want some applesauce ?", "words": ["some", "applesauce", "you"]}, {"sent": "would you like some pear sauce or some applesauce ?", "words": ["sauce", "like", "you", "would", "some", "applesauce"]}, {"sent": "did you have applesauce ?", "words": ["have", "applesauce", "did", "you"]}, {"sent": "applesauce cools stuff off .", "words": ["off", "applesauce"]}, {"sent": "mm applesauce .", "words": ["applesauce"]}, {"sent": "were pouring you some applesauce Laura .", "words": ["applesauce", "some", "were", "you"]}, {"sent": "you like applesauce in yours ?", "words": ["in", "applesauce", "like", "you"]}, {"sent": "say applesauce .", "words": ["applesauce", "say"]}, {"sent": "so you dont like applesauce and you are allergic to it ?", "words": ["so", "to", "like", "you", "and", "dont", "are", "it", "applesauce"]}, {"sent": "some applesauce .", "words": ["some", "applesauce"]}, {"sent": "maybe some applesauce later .", "words": ["some", "applesauce"]}, {"sent": "think thats right heres some pear and applesauce .", "words": ["some", "think", "applesauce", "and"]}, {"sent": "cottagecheese and applesauce .", "words": ["applesauce", "and"]}, {"sent": "going to feed her applesauce .", "words": ["applesauce", "to", "her", "feed"]}, {"sent": "yogurt with applesauce and almond butter .", "words": ["yogurt", "with", "and", "butter", "applesauce"]}, {"sent": "try some applesauce .", "words": ["applesauce", "some", "try"]}, {"sent": "want some applesauce ?", "words": ["some", "applesauce"]}, {"sent": "you want some applesauce ?", "words": ["some", "applesauce", "you"]}, {"sent": "some applesauce and some milk ?", "words": ["milk", "some", "applesauce", "and"]}, {"sent": "more applesauce ?", "words": ["more", "applesauce"]}, {"sent": "thats applesauce .", "words": ["applesauce"]}, {"sent": "Ill give you some applesauce first .", "words": ["first", "you", "some", "give", "applesauce"]}, {"sent": "and applesauce .", "words": ["applesauce", "and"]}, {"sent": "are you gonna stay here and finish your applesauce ?", "words": ["your", "finish", "and", "here", "you", "are", "stay", "applesauce"]}, {"sent": "maybe well have a big jar of squash and we wont have any applesauce .", "words": ["we", "of", "any", "and", "have", "big", "applesauce", "jar", "a"]}, {"sent": "do you wanna eat some applesauce Naima ?", "words": ["do", "you", "eat", "wanna", "some", "applesauce"]}, {"sent": "is applesauce yummy ?", "words": ["applesauce", "is"]}, {"sent": "this is the applesauce right ?", "words": ["the", "this", "applesauce", "is"]}, {"sent": "applesauce cake .", "words": ["applesauce", "cake"]}, {"sent": "want some applesauce ?", "words": ["some", "applesauce"]}, {"sent": "Ill just give you some applesauce by itself okay ?", "words": ["by", "you", "some", "give", "applesauce"]}, {"sent": "you made applesauce at school ?", "words": ["school", "at", "applesauce", "you"]}, {"sent": "you want more applesauce ?", "words": ["more", "applesauce", "you"]}, {"sent": "is that good applesauce ?", "words": ["applesauce", "that", "good", "is"]}, {"sent": "and some applesauce .", "words": ["some", "applesauce", "and"]}, {"sent": "and a little bit of applesauce .", "words": ["of", "little", "and", "applesauce", "a"]}, {"sent": "did you say applesauce fry waffle ?", "words": ["applesauce", "say", "did", "you"]}, {"sent": "you want the plain applesauce ?", "words": ["the", "applesauce", "you"]}, {"sent": "right would you like applesauce ?", "words": ["applesauce", "like", "you", "would"]}, {"sent": "you want applesauce ?", "words": ["applesauce", "you"]}, {"sent": "Laura the applesauce cools the stuff off .", "words": ["the", "applesauce", "off"]}, {"sent": "got some applesauce coming too .", "words": ["some", "applesauce", "too"]}, {"sent": "thats some applesauce with yogurt .", "words": ["some", "yogurt", "applesauce", "with"]}, {"sent": "I think you have applesauce .", "words": ["have", "think", "applesauce", "you"]}, {"sent": "its applesauce .", "words": ["applesauce"]}, {"sent": "Im gonna give you a little bit of the cold applesauce .", "words": ["the", "of", "little", "you", "give", "applesauce", "cold", "a"]}, {"sent": "youre having sesame with applesauce mm .", "words": ["applesauce", "with"]}, {"sent": "well I dont think I have more applesauce but we can go to the supermarket and buy more later .", "words": ["can", "the", "think", "we", "go", "to", "and", "have", "dont", "more", "applesauce", "buy", "but"]}, {"sent": "Brett maybe Carl would like some more applesauce .", "words": ["like", "more", "would", "some", "applesauce"]}, {"sent": "anything else with your applesauce ?", "words": ["your", "with", "applesauce"]}, {"sent": "or applesauce .", "words": ["applesauce"]}, {"sent": "yummy applesauce .", "words": ["applesauce"]}, {"sent": "theres the applesauce .", "words": ["the", "applesauce"]}, {"sent": "so you want pears peaches or applesauce ?", "words": ["so", "applesauce", "you"]}, {"sent": "Ive never heard of an applesauce fry waffle .", "words": ["of", "an", "applesauce"]}, {"sent": "I might need the applesauce .", "words": ["the", "need", "applesauce"]}, {"sent": "um she had the applesauce but Im not certain she knows what this is .", "words": ["the", "what", "is", "this", "she", "applesauce", "not", "but"]}, {"sent": "and applesauce .", "words": ["applesauce", "and"]}, {"sent": "and applesauce and .", "words": ["applesauce", "and"]}, {"sent": "bought you some applesauce yesterday .", "words": ["some", "applesauce", "you"]}, {"sent": "Dwww give you some applesauce .", "words": ["some", "give", "applesauce", "you"]}, {"sent": "Naima Boo theres more applesauce .", "words": ["more", "applesauce"]}, {"sent": "you wanna try some applesauce ?", "words": ["try", "you", "wanna", "some", "applesauce"]}, {"sent": "little bit of applesauce .", "words": ["of", "applesauce", "little"]}, {"sent": "now if you wanna eat the applesauce and feta cheese .", "words": ["the", "if", "you", "eat", "and", "wanna", "cheese", "applesauce"]}, {"sent": "that is a messy way to eat applesauce sweetie .", "words": ["is", "to", "eat", "applesauce", "that", "a"]}, {"sent": "did you applesauce you both liked .", "words": ["applesauce", "did", "you"]}, {"sent": "eat your applesauce .", "words": ["eat", "your", "applesauce"]}, {"sent": "what else did you have besides applesauce ?", "words": ["what", "you", "have", "did", "applesauce"]}, {"sent": "see thats the applesauce Im gonna mix it in .", "words": ["the", "in", "it", "see", "applesauce"]}, {"sent": "you want some applesauce ?", "words": ["some", "applesauce", "you"]}, {"sent": "xxx why dont we give her some applesauce to cool if off and make it taste better ?", "words": ["off", "better", "if", "we", "to", "taste", "and", "dont", "make", "it", "applesauce", "some", "give", "her", "why"]}, {"sent": "Nathaniel has some applesauce ?", "words": ["some", "applesauce"]}, {"sent": "and you like applesauce .", "words": ["applesauce", "like", "you", "and"]}, {"sent": "applesauce cake .", "words": ["applesauce", "cake"]}, {"sent": "Laura do you want applesauce in yours ?", "words": ["do", "in", "applesauce", "you"]}, {"sent": "chicken noodle or applesauce .", "words": ["chicken", "applesauce"]}, {"sent": "like applesauce ?", "words": ["applesauce", "like"]}, {"sent": "Ill get you some applesauce .", "words": ["get", "some", "applesauce", "you"]}, {"sent": "Ill mix more applesauce into this bite .", "words": ["into", "this", "more", "bite", "applesauce"]}, {"sent": "applesauce remember ?", "words": ["applesauce"]}, {"sent": "do you want applesauce too ?", "words": ["do", "too", "applesauce", "you"]}, {"sent": "applesauce crackers .", "words": ["applesauce"]}, {"sent": "more applesauce ?", "words": ["more", "applesauce"]}, {"sent": "oh is that good applesauce it looks like it .", "words": ["is", "like", "that", "it", "good", "applesauce"]}, {"sent": "little bit of the applesauce .", "words": ["of", "the", "applesauce", "little"]}, {"sent": "oh an applesauce fry waffle .", "words": ["an", "applesauce"]}, {"sent": "applesauce covered noodles there .", "words": ["noodles", "there", "applesauce"]}, {"sent": "but you like carrots and applesauce but you didnt like liver .", "words": ["like", "and", "you", "carrots", "applesauce", "but"]}, {"sent": "Mommy got you some applesauce too .", "words": ["some", "too", "applesauce", "you"]}, {"sent": "Naima would you like to eat some applesauce or some carrots ?", "words": ["to", "like", "you", "eat", "carrots", "would", "some", "applesauce"]}, {"sent": "how about we get you some applesauce ?", "words": ["how", "we", "you", "get", "about", "some", "applesauce"]}, {"sent": "maybe applesauce for later .", "words": ["for", "applesauce"]}, {"sent": "like applesauce ?", "words": ["applesauce", "like"]}, {"sent": "Laura eat the applesauce in your bowl please .", "words": ["the", "bowl", "your", "eat", "in", "applesauce"]}, {"sent": "do you like applesauce ?", "words": ["do", "applesauce", "like", "you"]}, {"sent": "are you applesauce .", "words": ["are", "applesauce", "you"]}, {"sent": "the donut .", "words": ["the", "donut"]}, {"sent": "donut ?", "words": ["donut"]}, {"sent": "that is the donut .", "words": ["the", "donut", "that", "is"]}, {"sent": "donut .", "words": ["donut"]}, {"sent": "donut .", "words": ["donut"]}, {"sent": "mm good donut huh ?", "words": ["donut", "good"]}, {"sent": "a donut ?", "words": ["donut", "a"]}, {"sent": "Ill eat my donut .", "words": ["eat", "my", "donut"]}, {"sent": "good donut .", "words": ["donut", "good"]}, {"sent": "youre getting donut all over .", "words": ["donut", "all", "over"]}, {"sent": "youre getting donut all over .", "words": ["donut", "all", "over"]}, {"sent": "you want a donut ?", "words": ["donut", "a", "you"]}, {"sent": "you want another donut ?", "words": ["another", "donut", "you"]}, {"sent": "whada you want for lunch a donut .", "words": ["donut", "for", "a", "you"]}, {"sent": "a donut wow .", "words": ["donut", "a"]}, {"sent": "how is that donut ?", "words": ["donut", "how", "that", "is"]}, {"sent": "yummy whats on that donut ?", "words": ["on", "donut", "that"]}, {"sent": "it kinduv tastes like a jelly donut .", "words": ["jelly", "donut", "like", "it", "a"]}, {"sent": "time for a donut .", "words": ["donut", "for", "a"]}, {"sent": "its a small donut ?", "words": ["donut", "a"]}, {"sent": "a sprinkle donut .", "words": ["donut", "a"]}, {"sent": "a sprinkle donut ?", "words": ["donut", "a"]}, {"sent": "okay when were all done playing would you like to take a walk and go get a donut ?", "words": ["donut", "all", "go", "to", "like", "you", "and", "get", "walk", "would", "take", "when", "were", "a"]}, {"sent": "thats not a donut thats a dont you put that in your mouth .", "words": ["donut", "put", "your", "mouth", "you", "dont", "in", "that", "not", "a"]}, {"sent": "its a donut .", "words": ["donut", "a"]}, {"sent": "this looks like a purple donut .", "words": ["donut", "this", "like", "a"]}, {"sent": "that ones yucky this one looks like a donut .", "words": ["donut", "yucky", "like", "this", "that", "a"]}, {"sent": "oh so it looks like a donut .", "words": ["donut", "so", "like", "it", "a"]}, {"sent": "and heres half a donut .", "words": ["donut", "a", "and"]}, {"sent": "and theres a whole donut .", "words": ["donut", "a", "and"]}, {"sent": "a donut ?", "words": ["donut", "a"]}, {"sent": "a pink donut ?", "words": ["donut", "a"]}, {"sent": "you got the donut ?", "words": ["the", "donut", "you"]}, {"sent": "a donut ?", "words": ["donut", "a"]}, {"sent": "and this donut is a circle .", "words": ["donut", "is", "and", "this", "a"]}, {"sent": "donut .", "words": ["donut"]}, {"sent": "donut .", "words": ["donut"]}, {"sent": "can you go get the cupcake and the donut ?", "words": ["can", "the", "donut", "go", "you", "get", "and"]}, {"sent": "it is a donut .", "words": ["donut", "it", "a", "is"]}, {"sent": "haha is that a good donut ?", "words": ["donut", "is", "good", "that", "a"]}, {"sent": "thats a donut .", "words": ["donut", "a"]}, {"sent": "you gonna eat a donut ?", "words": ["eat", "donut", "a", "you"]}, {"sent": "donut is this a donut ?", "words": ["donut", "this", "a", "is"]}, {"sent": "donut ?", "words": ["donut"]}, {"sent": "I think this is a donut but Im not sure whats on top .", "words": ["donut", "think", "is", "this", "on", "not", "but", "a"]}, {"sent": "that was a donut .", "words": ["donut", "was", "that", "a"]}, {"sent": "yeah he thought the uh donut was a scone because when Mommy goes to Starbucks I get him a little scone .", "words": ["the", "donut", "was", "to", "little", "get", "him", "he", "when", "because", "a"]}, {"sent": "thats a donut .", "words": ["donut", "a"]}, {"sent": "and a donut .", "words": ["donut", "a", "and"]}, {"sent": "yeah a donut .", "words": ["donut", "a"]}, {"sent": "yeah you have a donut ?", "words": ["have", "donut", "a", "you"]}, {"sent": "a donut and a spare rib .", "words": ["donut", "and", "a"]}, {"sent": "donut .", "words": ["donut"]}, {"sent": "like a donut .", "words": ["donut", "like", "a"]}, {"sent": "donut .", "words": ["donut"]}, {"sent": "a donut ?", "words": ["donut", "a"]}, {"sent": "I think she pretty much knew oh well maybe with the donut .", "words": ["the", "donut", "think", "with", "pretty", "much", "she"]}, {"sent": "she didnt know what the donut was .", "words": ["the", "what", "donut", "was", "she"]}, {"sent": "mommy get some donut too ?", "words": ["get", "some", "donut", "too"]}, {"sent": "yeah its a donut .", "words": ["donut", "a"]}, {"sent": "youre adding a donut to it too ?", "words": ["donut", "to", "it", "too", "a"]}, {"sent": "its a donut .", "words": ["donut", "a"]}, {"sent": "a donut .", "words": ["donut", "a"]}, {"sent": "shes seen donuts but shes I dont know if shes really had donuts shes had more of the munchkins youknow not a real a true donut .", "words": ["the", "donut", "if", "of", "dont", "more", "not", "but", "a"]}, {"sent": "you want coffee cake or a donut ?", "words": ["donut", "cake", "you", "coffee", "a"]}, {"sent": "mom Im gonna have a donut .", "words": ["have", "donut", "a"]}, {"sent": "no donut .", "words": ["donut"]}, {"sent": "xxx donut and milk .", "words": ["donut", "milk", "and"]}, {"sent": "it sortof looks like a donut .", "words": ["donut", "it", "like", "a"]}, {"sent": "it was a mistake giving you a donut last week wasnt it ?", "words": ["donut", "was", "last", "you", "it", "a"]}, {"sent": "they said Lina would you like a donut ?", "words": ["donut", "like", "you", "they", "would", "a"]}, {"sent": "here comes a donut .", "words": ["here", "donut", "a"]}, {"sent": "now is that donut ?", "words": ["donut", "that", "is"]}, {"sent": "wheres your donut ?", "words": ["donut", "your"]}, {"sent": "take a taste of the donut .", "words": ["the", "donut", "of", "taste", "take", "a"]}, {"sent": "donut yeah .", "words": ["donut"]}, {"sent": "its a donut .", "words": ["donut", "a"]}, {"sent": "what do you do with a donut ?", "words": ["donut", "what", "do", "with", "you", "a"]}, {"sent": "the donut ?", "words": ["the", "donut"]}, {"sent": "donut .", "words": ["donut"]}, {"sent": "CookieMonsters eating the donut too ?", "words": ["the", "donut", "too"]}, {"sent": "a donut .", "words": ["donut", "a"]}, {"sent": "a donut right ?", "words": ["donut", "a"]}, {"sent": "do you think its a donut ?", "words": ["donut", "do", "think", "you", "a"]}, {"sent": "it looks like a donut hole ?", "words": ["donut", "it", "like", "a"]}, {"sent": "it looks like a big donut .", "words": ["donut", "like", "it", "big", "a"]}, {"sent": "its a donut truck ?", "words": ["donut", "truck", "a"]}, {"sent": "good donut .", "words": ["donut", "good"]}, {"sent": "is that a good donut .", "words": ["donut", "is", "good", "that", "a"]}, {"sent": "you got so much donut in your mouth .", "words": ["donut", "your", "so", "mouth", "you", "in", "much"]}, {"sent": "you trying to hand Daddy a donut ?", "words": ["donut", "hand", "to", "you", "a"]}, {"sent": "is that a real potato chip or a play potato chip ?", "words": ["is", "play", "potato chip", "that", "a", "potato"]}, {"sent": "thats a hamburger I dont want the hamburger potato chip .", "words": ["the", "hamburger", "dont", "potato chip", "a", "potato"]}, {"sent": "see that potato chip over there ?", "words": ["potato chip", "there", "that", "see", "over", "potato"]}, {"sent": "oh theres that potato chip .", "words": ["potato chip", "that", "potato"]}, {"sent": "not the potato chip you silly .", "words": ["the", "you", "potato chip", "not", "potato"]}, {"sent": "its piercing like shell take a fork and try and like cut like you know a potato chip in half or something .", "words": ["try", "cut", "potato chip", "like", "and", "you", "in", "fork", "take", "potato", "a"]}, {"sent": "ha you eat the potato chip .", "words": ["the", "you", "eat", "potato chip", "potato"]}, {"sent": "potato chip .", "words": ["potato chip", "potato"]}, {"sent": "uh you give her potato chip .", "words": ["her", "you", "give", "potato chip", "potato"]}, {"sent": "how about maybe give the puppet some cheese or maybe a potato chip ?", "words": ["the", "how", "potato chip", "about", "some", "give", "cheese", "potato", "a"]}, {"sent": "a play potato chip .", "words": ["play", "potato chip", "potato", "a"]}, {"sent": "hey you I see a potato chip can you hand me the potato chip ?", "words": ["can", "the", "hand", "me", "you", "see", "potato chip", "a", "potato"]}, {"sent": "no nothing to drink see the potato chip ?", "words": ["the", "drink", "to", "see", "potato chip", "potato"]}, {"sent": "can I have a potato chip ?", "words": ["can", "have", "potato chip", "a", "potato"]}, {"sent": "look in your box and look for the potato chip mister .", "words": ["the", "for", "box", "your", "and", "in", "look", "potato chip", "potato"]}, {"sent": "wheres my potato chip ?", "words": ["my", "potato chip", "potato"]}, {"sent": "thats a hamburger look in that big box over there and see if you can find a potato chip .", "words": ["can", "box", "if", "potato chip", "find", "hamburger", "and", "there", "in", "big", "you", "look", "over", "see", "that", "potato", "a"]}, {"sent": "thats not my potato chip though do you see any potato chips in there ?", "words": ["do", "my", "any", "you", "there", "in", "see", "potato chip", "not", "potato"]}, {"sent": "is that a potato chip ?", "words": ["is", "that", "potato chip", "a", "potato"]}, {"sent": "xxx is different from a vagina .", "words": ["vagina", "a", "is"]}, {"sent": "well girls have tushies and girls also have a vagina .", "words": ["have", "vagina", "a", "and"]}, {"sent": "Prudence has a tushie and Prudence has a vagina .", "words": ["vagina", "and", "a"]}, {"sent": "the vagina is almost here at last .", "words": ["the", "is", "last", "here", "vagina", "at"]}, {"sent": "I said vagina you silly .", "words": ["you", "vagina"]}, {"sent": "not badinas badian ahz I said vagina .", "words": ["not", "vagina"]}, {"sent": "outof the vagina .", "words": ["the", "vagina"]}] diff --git a/evaluation-pipeline/aoa_data/word_list.csv b/evaluation-pipeline/aoa_data/word_list.csv new file mode 100644 index 0000000000000000000000000000000000000000..5b2b3b082f82091b90fe3dc9e72d33f473a97105 --- /dev/null +++ b/evaluation-pipeline/aoa_data/word_list.csv @@ -0,0 +1,592 @@ +language,uni_lemma,lexical_category,category,definition,word_clean,aoa,concreteness,frequency +English (American),a,function_words,quantifiers,a,a,26.75657271813926,1.46,301773 +English (American),a,function_words,quantifiers,an,an,26.75657271813926,1.46,13508 +English (American),a lot,function_words,quantifiers,a lot,a lot,29.354943884668234,2.37,4807 +English (American),about,function_words,locations,about,about,33.56541793707104,1.77,27478 +English (American),above,function_words,locations,above,above,32.94888134846138,3.33,332 +English (American),airplane,nouns,vehicles,airplane,airplane,20.14529714809586,4.96,1192 +English (American),all,function_words,quantifiers,all,all,26.31156607337898,2.27,49585 +English (American),all gone,predicates,descriptive_words,all gone,all gone,19.106125409049845,2.04,779 +English (American),alligator,nouns,animals,alligator,alligator,25.74050182260553,4.96,404 +English (American),am,predicates,helping_predicates,am,am,29.39866638780256,1.96,4070 +English (American),and,function_words,connecting_words,and,and,27.669136574125037,1.52,217485 +English (American),animal,nouns,animals,animal,animal,24.531890826243654,4.61,1523 +English (American),ankle,nouns,body_parts,ankle,ankle,29.54373992549591,4.81,75 +English (American),another,function_words,quantifiers,another,another,28.49498165323644,2.69,14222 +English (American),ant,nouns,animals,ant,ant,24.632024173206105,4.86,182 +English (American),any,function_words,quantifiers,any,any,30.168894359149128,1.72,12098 +English (American),apple,nouns,food_drink,apple,apple,18.316415297779947,5,3736 +English (American),applesauce,nouns,food_drink,applesauce,applesauce,25.18708208845118,4.96,127 +English (American),are,function_words,helping_predicates,are,are,30.400848330016352,1.96,106583 +English (American),arm,nouns,body_parts,arm,arm,21.679117916792883,4.96,1528 +English (American),around,function_words,locations,around,around,29.73724069586142,1.96,6835 +English (American),asleep,predicates,descriptive_words,asleep,asleep,24.95132838125333,3.71,1365 +English (American),at,function_words,locations,at,at,28.44395214386389,2.07,52573 +English (American),awake,predicates,descriptive_words,awake,awake,26.406667306484906,3.32,362 +English (American),away,function_words,locations,away,away,27.268694284819865,2.23,11014 +English (American),back (location),function_words,locations,back,back,26.041013448918488,4.33,26533 +English (American),bad,predicates,descriptive_words,bad,bad,24.376033345405514,1.68,2576 +English (American),ball,nouns,toys,ball,ball,14.051831559140178,5,8382 +English (American),balloon,nouns,toys,balloon,balloon,17.252991342080126,4.92,1810 +English (American),banana,nouns,food_drink,banana,banana,16.787250784202772,5,1615 +English (American),basement,nouns,furniture_rooms,basement,basement,31.403406659378565,4.89,121 +English (American),basket,nouns,household,basket,basket,25.32137110090939,5,1734 +English (American),bat,nouns,toys,bat,bat,25.94740000625428,5,384 +English (American),bathroom,nouns,furniture_rooms,bathroom,bathroom,23.22834292081228,4.52,1059 +English (American),bathtub,nouns,furniture_rooms,bathtub,bathtub,21.86077443068652,4.92,410 +English (American),be,function_words,helping_predicates,be,be,30.344096265123305,1.85,41097 +English (American),beach,nouns,outside,beach,beach,26.078273320304067,4.79,991 +English (American),beads,nouns,clothing,beads,beads,29.819752587655973,4.9,546 +English (American),beans,nouns,food_drink,beans,beans,24.532090787380607,5,1249 +English (American),bear,nouns,animals,bear,bear,19.290426322598368,4.88,5220 +English (American),because,function_words,connecting_words,because,because,30.412366644638137,1.22,31730 +English (American),bed,nouns,furniture_rooms,bed,bed,20.494046155718603,5,7584 +English (American),bedroom,nouns,furniture_rooms,bedroom,bedroom,25.645644681903754,4.9,1119 +English (American),bee,nouns,animals,bee,bee,21.306296871726428,4.88,1475 +English (American),behind,function_words,locations,behind,behind,29.581051780523048,3.48,2954 +English (American),belly button,nouns,body_parts,belly button,belly button,20.62619109370771,5,90 +English (American),belt,nouns,clothing,belt,belt,25.39428676791078,4.9,307 +English (American),bench,nouns,furniture_rooms,bench,bench,31.122752354523378,4.87,241 +English (American),beside,function_words,locations,beside,beside,33.68775817791739,2.59,304 +English (American),better,predicates,descriptive_words,better,better,27.197801378663197,1.91,9742 +English (American),bib,nouns,clothing,bib,bib,23.648183674597796,4.79,639 +English (American),bicycle,nouns,vehicles,bicycle,bicycle,21.77871249104521,4.89,660 +English (American),big,predicates,descriptive_words,big,big,22.25931777062911,3.66,25182 +English (American),bird,nouns,animals,bird,bird,17.224492119324236,5,2517 +English (American),bite,predicates,action_words,bite,bite,21.93843222383973,4.44,2541 +English (American),black,predicates,descriptive_words,black,black,26.65032453394826,3.76,2775 +English (American),blanket,nouns,household,blanket,blanket,20.364584241108936,5,957 +English (American),block,nouns,toys,block,block,21.47767824557252,4.48,1117 +English (American),blow,predicates,action_words,blow,blow,24.100630216525534,3.74,2110 +English (American),blue,predicates,descriptive_words,blue,blue,23.520465305593365,3.76,9232 +English (American),boat,nouns,vehicles,boat,boat,20.384388304955454,4.93,2544 +English (American),book,nouns,toys,book,book,16.378840412498462,4.9,15413 +English (American),boots,nouns,clothing,boots,boots,22.762953333636315,5,792 +English (American),bottle,nouns,household,bottle,bottle,18.109441114425163,4.91,3023 +English (American),bowl,nouns,household,bowl,bowl,22.67450309415727,4.87,1967 +English (American),box,nouns,household,box,box,22.36283488674821,4.9,8714 +English (American),bread,nouns,food_drink,bread,bread,21.714838666180317,4.92,2959 +English (American),break,predicates,action_words,break,break,24.498575833372556,3.71,2940 +English (American),bring,predicates,action_words,bring,bring,27.00190998469981,2.55,5164 +English (American),broken,predicates,descriptive_words,broken,broken,23.262899073559634,4.11,2733 +English (American),broom,nouns,household,broom,broom,23.51056906797502,4.89,282 +English (American),brown,predicates,descriptive_words,brown,brown,27.44722287259727,4.48,2294 +English (American),brush (object),nouns,household,brush,brush,21.812402640756495,4.54,2162 +English (American),bubbles,nouns,toys,bubbles,bubbles,18.447326266810368,4.6,1229 +English (American),bucket,nouns,household,bucket,bucket,26.11855610121108,4.96,896 +English (American),bug,nouns,animals,bug,bug,21.45346928441042,5,653 +English (American),build,predicates,action_words,build,build,27.561167006392587,3.71,2552 +English (American),bump,predicates,action_words,bump,bump,25.507631046994266,4.1,1008 +English (American),bunny,nouns,animals,bunny,bunny,20.465056141535673,4.97,2386 +English (American),bus,nouns,vehicles,bus,bus,21.261211792115443,4.9,3774 +English (American),but,function_words,connecting_words,but,but,32.30802214785319,2.04,34537 +English (American),butter,nouns,food_drink,butter,butter,24.566501758151283,4.9,1358 +English (American),butterfly,nouns,animals,butterfly,butterfly,23.47450828971274,4.93,882 +English (American),button,nouns,clothing,button,button,22.084011296770807,4.96,1602 +English (American),buy,predicates,action_words,buy,buy,27.61820820612325,3.35,3593 +English (American),by,function_words,locations,by,by,28.916261166336813,1.55,5847 +English (American),cake,nouns,food_drink,cake,cake,21.93745596797483,4.81,3506 +English (American),camera,nouns,household,camera,camera,25.73750398456283,5,1185 +English (American),can (auxiliary),function_words,helping_predicates,can (auxiliary),can,27.82698008632811,4.55,175734 +English (American),can (object),nouns,household,can (object),can,26.84891293682625,4.55,175734 +English (American),candy,nouns,food_drink,candy,candy,22.10696542716163,4.83,704 +English (American),car,nouns,vehicles,car,car,17.53434303192908,4.89,13476 +English (American),careful,predicates,descriptive_words,careful,careful,26.43412690050291,1.86,6107 +English (American),carrots,nouns,food_drink,carrots,carrots,24.037122543158606,5,802 +English (American),carry,predicates,action_words,carry,carry,25.681963506798628,4.04,1148 +English (American),cat,nouns,animals,cat,cat,17.67814164845057,4.86,5835 +English (American),catch,predicates,action_words,catch,catch,25.556075994667953,4.11,2813 +English (American),cereal,nouns,food_drink,cereal,cereal,22.086966073355658,4.83,1034 +English (American),chair,nouns,furniture_rooms,chair,chair,20.747062695367397,4.58,6382 +English (American),chalk,nouns,toys,chalk,chalk,27.572948668482248,4.9,246 +English (American),chase,predicates,action_words,chase,chase,28.40000722125352,3.48,251 +English (American),cheek,nouns,body_parts,cheek,cheek,23.639051249460564,4.83,159 +English (American),cheerios,nouns,food_drink,cheerios,cheerios,23.30250911956345,4.35,59 +English (American),cheese,nouns,food_drink,cheese,cheese,18.35624365220089,4.7,4215 +English (American),chicken (animal),nouns,animals,chicken (animal),chicken,22.94116221978306,4.8,5636 +English (American),chicken (food),nouns,food_drink,chicken (food),chicken,22.755311666095626,4.8,5636 +English (American),chin,nouns,body_parts,chin,chin,23.932790792488237,4.89,444 +English (American),chips,nouns,food_drink,potato chip,potato chip,23.914163430448284,4.9,23 +English (American),chocolate,nouns,food_drink,chocolate,chocolate,25.207246056049254,4.72,2469 +English (American),church,nouns,outside,church*,church,26.377623217552873,4.9,547 +English (American),clap,predicates,action_words,clap,clap,23.57129610232303,4.16,1106 +English (American),clean (action),predicates,action_words,clean (action),clean,24.56600009616096,3.07,7338 +English (American),clean (description),predicates,descriptive_words,clean (description),clean,24.16485158037349,3.07,7338 +English (American),climb,predicates,action_words,climb,climb,25.941366782032414,4.11,1076 +English (American),clock,nouns,household,clock,clock,22.729118690605997,5,1709 +English (American),close,predicates,action_words,close,close,24.612899480281406,3.2,3138 +English (American),closet,nouns,furniture_rooms,closet,closet,26.33273273898802,4.83,289 +English (American),cloud,nouns,outside,cloud,cloud,25.68327028006053,4.54,273 +English (American),coat,nouns,clothing,coat,coat,22.77758807471102,4.97,1191 +English (American),coffee,nouns,food_drink,coffee,coffee,24.733868049413292,4.81,2249 +English (American),coke,nouns,food_drink,coke,coke,28.250686610987113,4.83,184 +English (American),cold,predicates,descriptive_words,cold,cold,20.986347269894612,3.85,3891 +English (American),comb (object),nouns,household,comb,comb,24.521530115219505,5,639 +English (American),cook,predicates,action_words,cook,cook,25.20950610885458,4.32,1634 +English (American),cookie,nouns,food_drink,cookie,cookie,18.05879104299356,5,2651 +English (American),corn,nouns,food_drink,corn,corn,23.75808620462897,4.96,769 +English (American),couch,nouns,furniture_rooms,couch,couch,24.81633921615295,4.71,505 +English (American),could,function_words,helping_predicates,could,could,33.0915181485895,1.34,12880 +English (American),cover (action),predicates,action_words,cover,cover,28.093896291699608,4.23,902 +English (American),cow,nouns,animals,cow,cow,20.23330616534318,4.96,3496 +English (American),cracker,nouns,food_drink,cracker,cracker,19.128739975548182,4.96,471 +English (American),crayon,nouns,toys,crayon,crayon,22.958268446308498,4.87,1010 +English (American),crib,nouns,furniture_rooms,crib,crib,25.54633696377014,4.86,306 +English (American),cry,predicates,action_words,cry,cry,22.833033348617004,4,1408 +English (American),cup,nouns,household,cup,cup,19.458761353171624,5,5361 +English (American),cut,predicates,action_words,cut,cut,26.834383653830635,4.55,3650 +English (American),cute,predicates,descriptive_words,cute,cute,26.878579973455473,2.76,1371 +English (American),dance,predicates,action_words,dance,dance,23.594173710650814,4.32,1107 +English (American),dark,predicates,descriptive_words,dark,dark,26.374254558849202,4.29,1494 +English (American),deer,nouns,animals,deer,deer,26.28895723853603,4.86,374 +English (American),diaper,nouns,clothing,diaper,diaper,19.45888352722782,4.82,1072 +English (American),did,function_words,helping_predicates,did/did ya,did,28.236905169713484,2.45,57888 +English (American),dirty,predicates,descriptive_words,dirty,dirty,22.183512254438906,4.23,2706 +English (American),dish,nouns,household,dish,dish,26.369638755777036,4.9,312 +English (American),do,function_words,helping_predicates,do,do,24.418127084458625,2.46,157341 +English (American),does,function_words,helping_predicates,does,does,31.321420112088582,2.24,30079 +English (American),dog,nouns,animals,dog,dog,14.594052719227676,4.85,5509 +English (American),doll,nouns,toys,doll,doll,21.41564709887175,5,1830 +English (American),don't,function_words,helping_predicates,don't,dont,25.29034229368797,2.46,89665 +English (American),donkey,nouns,animals,donkey,donkey,27.89414096656381,5,575 +English (American),donut,nouns,food_drink,donut,donut,26.088004339085252,4.93,98 +English (American),door,nouns,furniture_rooms,door,door,20.2792407612697,4.81,6193 +English (American),down,function_words,locations,down,down,19.778758085736765,3.52,31240 +English (American),draw,predicates,action_words,draw,draw,26.127344086880825,3.97,4917 +English (American),drawer,nouns,furniture_rooms,drawer,drawer,27.477050929051146,4.67,511 +English (American),dress (object),nouns,clothing,dress (object),dress,25.175475858275057,4.93,2087 +English (American),drink (action),predicates,action_words,drink (action),drink,21.405832133186447,3.77,12546 +English (American),drink (beverage),nouns,food_drink,drink (beverage),drink,21.12191463243953,4.76,12546 +English (American),drive,predicates,action_words,drive,drive,24.75429950810191,3.86,1883 +English (American),drop,predicates,action_words,drop,drop,26.14791694487336,4.21,1105 +English (American),dry (action),predicates,action_words,dry (action),dry,26.69818180786187,3.77,3410 +English (American),dry (description),predicates,descriptive_words,dry (description),dry,26.486133398623007,3.77,3410 +English (American),dryer,nouns,furniture_rooms,dryer,dryer,27.940244672930877,4.79,93 +English (American),duck,nouns,animals,duck,duck,17.185401989712528,4.86,2872 +English (American),dump,predicates,action_words,dump,dump,29.574191643049147,3.93,626 +English (American),each,function_words,quantifiers,each,each,34.433173955126186,2.03,1275 +English (American),ear,nouns,body_parts,ear,ear,18.42450761164981,5,1260 +English (American),eat,predicates,action_words,eat,eat,19.996890077067878,4.44,18091 +English (American),egg,nouns,food_drink,egg,egg,22.122988028871536,4.97,2550 +English (American),elephant,nouns,animals,elephant,elephant,23.021554163445344,5,2858 +English (American),empty,predicates,descriptive_words,empty,empty,26.56355703829002,3.43,1237 +English (American),every,function_words,quantifiers,every,every,32.62625782784631,2.28,1932 +English (American),eye,nouns,body_parts,eye,eye,17.19831986641664,4.9,1696 +English (American),face,nouns,body_parts,face,face,23.386576805200583,4.87,4356 +English (American),fall,predicates,action_words,fall,fall,23.05658646359307,4.04,4547 +English (American),fast,predicates,descriptive_words,fast,fast,26.397899335243192,3.32,1814 +English (American),feed,predicates,action_words,feed,feed,27.04318304347576,4.17,1696 +English (American),find,predicates,action_words,find,find,25.911291431365097,2.63,11096 +English (American),fine,predicates,descriptive_words,fine,fine,30.348259169087953,2.61,1791 +English (American),finger,nouns,body_parts,finger,finger,21.846032665491855,5,2157 +English (American),finish,predicates,action_words,finish,finish,28.14761497515676,2.89,2774 +English (American),firetruck,nouns,vehicles,firetruck,firetruck,24.716233162853985,5,149 +English (American),first,predicates,descriptive_words,first,first,28.530149212482925,2.76,8389 +English (American),fish (animal),nouns,animals,fish (animal),fish,18.978116085220503,5,9298 +English (American),fish (food),nouns,food_drink,fish (food),fish,22.085905665472275,5,9298 +English (American),fit,predicates,action_words,fit,fit,29.08064200730688,2.7,3997 +English (American),fix,predicates,action_words,fix,fix,25.32658824937031,2.93,2929 +English (American),flag,nouns,outside,flag,flag,26.39411115381629,4.79,236 +English (American),flower,nouns,outside,flower,flower,20.519981351232357,5,1361 +English (American),food,nouns,food_drink,food,food,23.490400956729655,4.8,6233 +English (American),foot,nouns,body_parts,foot,foot,20.351627730476352,4.9,3505 +English (American),for,function_words,locations,for,for,29.21964285739064,1.63,60043 +English (American),fork,nouns,household,fork,fork,21.912707160579416,4.9,782 +English (American),french fries,nouns,food_drink,french fries,french fries,22.485372598777225,4.87,111 +English (American),frog,nouns,animals,frog,frog,22.53087514553091,5,1734 +English (American),full,predicates,descriptive_words,full,full,27.307650021326054,3.59,1585 +English (American),game,nouns,toys,game,game,26.558324042448465,4.5,2813 +English (American),garage,nouns,furniture_rooms,garage,garage,26.95193574573295,4.96,1135 +English (American),garbage,nouns,household,garbage,garbage,25.263290629707857,4.69,493 +English (American),garden,nouns,outside,garden,garden,28.814903721740233,4.73,1711 +English (American),gentle,predicates,descriptive_words,gentle,gentle,27.53063486592105,2.53,955 +English (American),get,predicates,action_words,get,get,24.203818250784888,2.38,55665 +English (American),giraffe,nouns,animals,giraffe,giraffe,24.848340134105648,4.73,1156 +English (American),give,predicates,action_words,give,give,26.131793055183277,2.83,13726 +English (American),glass,nouns,household,glass,glass,25.377947756405593,4.82,1069 +English (American),glasses,nouns,household,glasses,glasses,23.382539516196495,4.9,985 +English (American),gloves,nouns,clothing,gloves,gloves,27.24887506361188,4.97,339 +English (American),glue,nouns,toys,glue,glue,29.779166021342032,4.65,672 +English (American),go,predicates,action_words,go,go,19.73419022328145,3.15,83559 +English (American),good,predicates,descriptive_words,good,good,23.555059836026327,1.64,42780 +English (American),goose,nouns,animals,goose,goose,26.868415336421975,4.81,522 +English (American),grapes,nouns,food_drink,grapes,grapes,21.87000053388467,5,1148 +English (American),grass,nouns,outside,grass,grass,23.421522474544503,4.93,1407 +English (American),green,predicates,descriptive_words,green,green,24.864917218093773,4.07,8568 +English (American),green beans,nouns,food_drink,green beans,green beans,28.18246325512689,4.9,224 +English (American),gum,nouns,food_drink,gum,gum,25.6705889024626,4.89,374 +English (American),hair,nouns,body_parts,hair,hair,19.550956015647447,4.97,6378 +English (American),hamburger,nouns,food_drink,hamburger,hamburger,24.060486137223638,5,565 +English (American),hammer,nouns,household,hammer,hammer,25.908134489269617,4.77,860 +English (American),hand,nouns,body_parts,hand,hand,20.888392084906826,4.72,6426 +English (American),happy,predicates,descriptive_words,happy,happy,24.111067699312272,2.56,3476 +English (American),hard,predicates,descriptive_words,hard,hard,28.34690319460537,3.76,4563 +English (American),hat,nouns,clothing,hat,hat,18.800006578327608,4.88,5643 +English (American),hate,predicates,action_words,hate,hate,32.742713404389164,1.97,285 +English (American),have,predicates,action_words,have,have,26.383343346266226,2.18,80189 +English (American),he,function_words,pronouns,he,he,28.39328352610605,3.93,85599 +English (American),head,nouns,body_parts,head,head,21.52867556120851,4.75,7286 +English (American),hear,predicates,action_words,hear,hear,27.103048512621903,3.66,6278 +English (American),heavy,predicates,descriptive_words,heavy,heavy,25.14874997804902,3.37,1182 +English (American),helicopter,nouns,vehicles,helicopter,helicopter,24.90634156578962,4.62,729 +English (American),help,predicates,action_words,help,help,22.819438880083318,2.56,8981 +English (American),hen,nouns,animals,hen,hen,29.623281978610162,4.9,378 +English (American),her,function_words,pronouns,her,her,30.0706499832329,3,32518 +English (American),here,function_words,locations,here,here,25.0324996525398,3.13,74502 +English (American),hers,function_words,pronouns,hers,hers,31.577356326543732,2.61,401 +English (American),hide,predicates,action_words,hide,hide,25.6831882822722,3.21,1461 +English (American),high,predicates,descriptive_words,high,high,27.021723647237135,3.46,1690 +English (American),high chair,nouns,furniture_rooms,high chair,high chair,25.143576833379374,4.83,177 +English (American),him,function_words,pronouns,him,him,30.171545513305684,3.54,24073 +English (American),his,function_words,pronouns,his,his,30.86529830099037,3.14,25985 +English (American),hit,predicates,action_words,hit,hit,24.176504122997176,4.11,2339 +English (American),hold,predicates,action_words,hold,hold,25.28951191411586,3.68,6702 +English (American),home,nouns,outside,home,home,21.55292850686028,4.11,9648 +English (American),horse,nouns,animals,horse,horse,20.598917207238273,5,4162 +English (American),hose,nouns,outside,hose,hose,26.925240566139266,4.87,264 +English (American),hot,predicates,descriptive_words,hot,hot,17.886174346828543,4.31,4425 +English (American),house,nouns,outside,house,house,22.949798098688984,5,12111 +English (American),how,function_words,question_words,how,how,30.185466664457547,1.35,31682 +English (American),hug,predicates,action_words,hug,hug,22.04054246795061,4.14,1172 +English (American),hungry,predicates,descriptive_words,hungry,hungry,24.529656637397714,2.9,2717 +English (American),hurry,predicates,action_words,hurry,hurry,27.48425052166916,2.63,865 +English (American),hurt (description),predicates,descriptive_words,hurt,hurt,24.88161558121455,3.61,4613 +English (American),i,function_words,pronouns,I,i,23.977441347865177,3.93,786 +English (American),ice,nouns,food_drink,ice,ice,22.358270283361815,4.89,2197 +English (American),ice cream,nouns,food_drink,ice cream,ice cream,21.580952159819738,5,1557 +English (American),if,function_words,connecting_words,if,if,34.068154850176526,1.19,31526 +English (American),in,function_words,locations,in,in,24.07962150691501,3,276036 +English (American),in,function_words,locations,inside/in,inside,24.07962150691501,3,7888 +English (American),inside,function_words,locations,inside,inside,28.910801367846076,3.67,7888 +English (American),into,function_words,locations,into,into,32.328072400666066,2.3,7862 +English (American),is,function_words,helping_predicates,is,is,28.508756906338796,1.59,183357 +English (American),it,function_words,pronouns,it,it,25.985519116000557,2.81,297679 +English (American),jacket,nouns,clothing,jacket,jacket,23.764267757552943,4.86,655 +English (American),jar,nouns,household,jar,jar,30.036814769417507,5,389 +English (American),jeans,nouns,clothing,jeans,jeans,27.85718213630826,5,160 +English (American),jelly,nouns,food_drink,jelly,jelly,26.459272534245788,4.93,851 +English (American),juice,nouns,food_drink,juice,juice,17.663424879988565,4.89,6380 +English (American),jump,predicates,action_words,jump,jump,23.05187845463102,4.52,2396 +English (American),keys,nouns,household,keys,keys,19.821178140120676,4.89,1273 +English (American),kick,predicates,action_words,kick,kick,24.607244736468125,4.33,925 +English (American),kiss,predicates,action_words,kiss,kiss,20.97729762630762,4.48,3370 +English (American),kitchen,nouns,furniture_rooms,kitchen,kitchen,23.721678813935878,4.97,2439 +English (American),kitty,nouns,animals,kitty,kitty,17.76353044822186,4.97,2807 +English (American),knee,nouns,body_parts,knee,knee,22.83341234944545,5,1252 +English (American),knife,nouns,household,knife,knife,25.180125523737622,4.9,749 +English (American),knock,predicates,action_words,knock,knock,25.65520970263338,4.24,1500 +English (American),ladder,nouns,outside,ladder,ladder,27.453139889383696,5,836 +English (American),lamb,nouns,animals,lamb,lamb,26.77611745254101,4.97,762 +English (American),lamp,nouns,household,lamp,lamp,27.707727325952707,4.97,286 +English (American),last,predicates,descriptive_words,last,last,31.94810812999018,3.04,6041 +English (American),lawn mower,nouns,outside,lawn mower,lawn mower,26.25206641047887,4.93,37 +English (American),leg,nouns,body_parts,leg,leg,22.786751225180204,4.83,1696 +English (American),lick,predicates,action_words,lick,lick,27.885918786410006,4.52,443 +English (American),light (object),nouns,household,light,light,20.167269067810214,4.21,2752 +English (American),like (action),predicates,action_words,like,like,25.712763627585442,1.89,80659 +English (American),lion,nouns,animals,lion,lion,23.292560818293822,4.96,1972 +English (American),listen,predicates,action_words,listen,listen,28.120047324042673,3.47,3194 +English (American),little (description),predicates,descriptive_words,little (description),little,25.166304259548355,3.67,39012 +English (American),living room,nouns,furniture_rooms,living room,living room,27.67343106925516,4.7,431 +English (American),lollipop,nouns,food_drink,lollipop,lollipop,27.20359032793206,4.96,276 +English (American),long,predicates,descriptive_words,long,long,30.43920077538411,3.18,5656 +English (American),look,predicates,action_words,look,look,24.07200126089267,2.96,68469 +English (American),loud,predicates,descriptive_words,loud,loud,27.096587104346657,3.73,1100 +English (American),love,predicates,action_words,love,love,22.97930051104563,2.07,5139 +English (American),mad,predicates,descriptive_words,mad,mad,28.322211878333118,2.76,839 +English (American),make,predicates,action_words,make,make,27.26683722139889,2.67,22678 +English (American),me,function_words,pronouns,me,me,21.782136908963487,4.33,58317 +English (American),meat,nouns,food_drink,meat,meat,25.21618096142793,4.9,986 +English (American),medicine,nouns,household,medicine,medicine,24.410637544584084,4.79,915 +English (American),melon,nouns,food_drink,melon,melon,26.929946230882774,4.78,141 +English (American),milk,nouns,food_drink,milk,milk,18.304538681163276,4.92,5537 +English (American),mine,function_words,pronouns,mine,mine,20.0092783421445,3.56,2670 +English (American),mittens,nouns,clothing,mittens,mittens,27.13676422394422,4.89,267 +English (American),money,nouns,household,money,money,22.84702188801892,4.54,3251 +English (American),monkey,nouns,animals,monkey,monkey,21.75146876358306,4.9,2316 +English (American),moon,nouns,outside,moon,moon,21.064582625649372,4.9,1461 +English (American),moose,nouns,animals,moose,moose,30.726348818983883,4.97,324 +English (American),mop,nouns,household,mop,mop,28.697523528698856,4.97,158 +English (American),more,function_words,quantifiers,more,more,19.934125196097916,2.37,23403 +English (American),motorcycle,nouns,vehicles,motorcycle,motorcycle,25.424516980060698,4.97,366 +English (American),mouse,nouns,animals,mouse,mouse,23.186364426729803,4.83,1784 +English (American),mouth,nouns,body_parts,mouth,mouth,20.385274871125226,4.74,6017 +English (American),much,function_words,quantifiers,much,much,31.135471437484984,1.69,7404 +English (American),muffin,nouns,food_drink,muffin,muffin,26.053662526327358,4.78,219 +English (American),my,function_words,pronouns,my,my,24.67561156823274,2.42,30474 +English (American),myself,function_words,pronouns,myself,myself,30.826981734088697,2.97,674 +English (American),nail (object),nouns,household,nail,nail,27.745919885712205,4.93,268 +English (American),napkin,nouns,household,napkin,napkin,24.71066092570806,4.93,393 +English (American),naughty,predicates,descriptive_words,naughty,naughty,30.45927863451864,2.04,2904 +English (American),necklace,nouns,clothing,necklace,necklace,25.67583784602768,4.96,341 +English (American),need,function_words,helping_predicates,need/need to,need,28.517326473959855,1.69,11101 +English (American),new,predicates,descriptive_words,new,new,28.242927764036946,2.81,5522 +English (American),nice,predicates,descriptive_words,nice,nice,24.731878757128417,2.18,20152 +English (American),noisy,predicates,descriptive_words,noisy,noisy,28.238292864810433,3.56,903 +English (American),none,function_words,quantifiers,none,none,30.075465085938674,2.59,494 +English (American),nose,nouns,body_parts,nose,nose,17.50000892567104,4.89,5607 +English (American),not,function_words,quantifiers,not,not,28.331503696333467,2.08,62788 +English (American),of,function_words,locations,of,of,32.135421130600605,1.67,72422 +English (American),off,function_words,locations,off,off,21.76331515371575,2.79,21580 +English (American),old,predicates,descriptive_words,old,old,29.745205894423886,2.72,3975 +English (American),on,function_words,locations,on,on,22.07453703332649,3.25,130164 +English (American),open (action),predicates,action_words,open,open,21.70959024951605,3.21,8246 +English (American),orange (description),predicates,descriptive_words,orange (description),orange,24.48307542955755,4.66,10036 +English (American),orange (food),nouns,food_drink,orange (food),orange,22.862362842722305,4.66,10036 +English (American),other,function_words,quantifiers,other,other,29.008003039766972,2.04,15644 +English (American),our,function_words,pronouns,our,our,31.879580552349164,1.97,6116 +English (American),out,function_words,locations,out,out,21.787538623492427,2.73,34288 +English (American),outside,nouns,outside,outside,outside,20.10026304783487,4.25,5164 +English (American),oven,nouns,furniture_rooms,oven,oven,26.959440376130473,4.97,718 +English (American),over,function_words,locations,over,over,27.81126335036735,2.46,23641 +English (American),owie,nouns,body_parts,owie/boo boo,owie,19.225364028172503,3.61,101 +English (American),owl,nouns,animals,owl,owl,24.378811443862723,4.93,737 +English (American),paint (action),predicates,action_words,paint,paint,26.7264001349599,4.79,1346 +English (American),pajamas,nouns,clothing,pajamas,pajamas,23.01441672978766,4.73,776 +English (American),pancake,nouns,food_drink,pancake,pancake,24.195947688413984,4.86,201 +English (American),pants,nouns,clothing,pants,pants,21.997653199154513,4.86,2028 +English (American),paper,nouns,household,paper,paper,22.40794784536237,4.93,4643 +English (American),park,nouns,outside,park,park,23.44395480659095,4.74,1855 +English (American),party,nouns,outside,party,party,26.48232416632208,3.89,2705 +English (American),pasta,nouns,food_drink,noodles,noodles,22.2125626020018,4.86,209 +English (American),pasta,nouns,food_drink,spaghetti,spaghetti,22.2125626020018,4.86,1466 +English (American),peanut butter,nouns,food_drink,peanut butter,peanut butter,24.48646845168075,4.93,323 +English (American),peas,nouns,food_drink,peas,peas,24.557921410443495,4.9,2307 +English (American),pen,nouns,toys,pen,pen,23.51987496977609,4.92,1411 +English (American),pencil,nouns,toys,pencil,pencil,25.06258390684458,4.88,1040 +English (American),penguin,nouns,animals,penguin,penguin,27.293224141508716,5,658 +English (American),penis,nouns,body_parts,penis*,penis,26.231902171762034,5,35 +English (American),penny,nouns,household,penny,penny,26.32129034179081,4.83,575 +English (American),pick,predicates,action_words,pick,pick,28.350959500123274,3.82,4884 +English (American),pickle,nouns,food_drink,pickle,pickle,25.496142784956177,4.64,191 +English (American),picture,nouns,household,picture,picture,24.228451852721275,4.52,6309 +English (American),pig,nouns,animals,pig,pig,21.04970478014681,5,2423 +English (American),pillow,nouns,household,pillow,pillow,22.179110907856753,5,683 +English (American),pizza,nouns,food_drink,pizza,pizza,21.33010054090465,5,2650 +English (American),plant,nouns,household,plant,plant,26.161845259337724,4.76,519 +English (American),plate,nouns,household,plate,plate,24.38848139227323,4.77,2600 +English (American),play,predicates,action_words,play,play,22.50225294745721,3.24,23148 +English (American),play dough,nouns,toys,play dough,play dough,27.200019866322975,4.68,124 +English (American),play pen,nouns,furniture_rooms,play pen,play pen,30.93414000000259,4.76,13 +English (American),pony,nouns,animals,pony,pony,27.149124759679015,4.9,253 +English (American),pool,nouns,outside,pool,pool,23.288489299463958,4.77,860 +English (American),poor,predicates,descriptive_words,poor,poor,32.66767129734731,2.7,2640 +English (American),popcorn,nouns,food_drink,popcorn,popcorn,23.68702745961217,5,195 +English (American),popsicle,nouns,food_drink,popsicle,popsicle,25.14044583723922,4.93,120 +English (American),porch,nouns,furniture_rooms,porch,porch,29.70353523171374,4.92,84 +English (American),potato,nouns,food_drink,potato,potato,25.045299416217585,4.85,594 +English (American),potty,nouns,furniture_rooms,potty,potty,21.163868967261262,4.12,991 +English (American),pour,predicates,action_words,pour,pour,28.27835611871483,4.14,1410 +English (American),present,nouns,toys,present,present,24.99147354255208,3.39,1400 +English (American),pretend,predicates,action_words,pretend,pretend,31.275328228201392,2.11,3413 +English (American),pretty,predicates,descriptive_words,pretty,pretty,23.70523132395096,2.4,6516 +English (American),pretzel,nouns,food_drink,pretzel,pretzel,27.086772724245172,4.74,85 +English (American),pudding,nouns,food_drink,pudding,pudding,28.336874661648956,4.9,306 +English (American),pull,predicates,action_words,pull,pull,26.41351492122096,3.97,4797 +English (American),pumpkin,nouns,food_drink,pumpkin,pumpkin,25.596056826431344,4.9,815 +English (American),puppy,nouns,animals,puppy,puppy,20.642644342886626,4.78,1308 +English (American),purse,nouns,household,purse,purse,24.680534115242512,4.9,487 +English (American),push,predicates,action_words,push,push,24.884039995017684,4.21,5386 +English (American),put,predicates,action_words,put,put,27.180432184212417,2.5,68322 +English (American),puzzle,nouns,toys,puzzle,puzzle,24.603177970245078,4.75,2062 +English (American),quiet,predicates,descriptive_words,quiet,quiet,27.05139131191479,2.76,1380 +English (American),radio,nouns,household,radio,radio,26.613053034699238,4.74,355 +English (American),rain,nouns,outside,rain,rain,22.05272569057193,4.97,1240 +English (American),raisin,nouns,food_drink,raisin,raisin,23.922862069518597,4.97,169 +English (American),read,predicates,action_words,read,read,22.864582930368915,3.56,9185 +English (American),red,predicates,descriptive_words,red,red,24.476663524110176,4.24,9917 +English (American),refrigerator,nouns,furniture_rooms,refrigerator,refrigerator,25.673848500166034,5,562 +English (American),ride,predicates,action_words,ride,ride,24.3290998464415,3.75,2928 +English (American),rip,predicates,action_words,rip,rip,30.898483534092605,3.79,375 +English (American),rock (object),nouns,outside,rock,rock,22.092980833031184,4.91,875 +English (American),rocking chair,nouns,furniture_rooms,rocking chair,rocking chair,26.286720661876167,4.93,184 +English (American),roof,nouns,outside,roof,roof,29.380455204401983,4.79,941 +English (American),room,nouns,furniture_rooms,room,room,24.7800417765674,4.79,5211 +English (American),rooster,nouns,animals,rooster,rooster,27.785104097477404,4.75,422 +English (American),run,predicates,action_words,run,run,23.547368180082476,4.31,2505 +English (American),sad,predicates,descriptive_words,sad,sad,26.975730544989975,3.07,1405 +English (American),salt,nouns,food_drink,salt,salt,28.441896714165853,4.89,431 +English (American),same,function_words,quantifiers,same,same,30.94396519191967,2.64,3923 +English (American),sandbox,nouns,outside,sandbox,sandbox,26.956125375322088,4.86,172 +English (American),sandwich,nouns,food_drink,sandwich,sandwich,24.33836806143716,4.9,1683 +English (American),sauce,nouns,food_drink,sauce,sauce,27.988140408941376,4.75,487 +English (American),say,predicates,action_words,say,say,27.350714856102098,2.58,29942 +English (American),scared,predicates,descriptive_words,scared,scared,27.284769726224756,2.5,1001 +English (American),scarf,nouns,clothing,scarf,scarf,31.0680540145124,4.97,254 +English (American),school,nouns,outside,school,school,23.424986892471757,4.79,6710 +English (American),scissors,nouns,household,scissors,scissors,25.34069479638193,4.85,742 +English (American),see,predicates,action_words,see,see,22.628651743930725,3.21,71442 +English (American),shake,predicates,action_words,shake,shake,27.417577200447113,4.07,2018 +English (American),share,predicates,action_words,share,share,26.777735575736017,2.96,1061 +English (American),she,function_words,pronouns,she,she,29.293434421039155,3.36,53246 +English (American),sheep,nouns,animals,sheep,sheep,23.925866583400786,4.9,2362 +English (American),shirt,nouns,clothing,shirt,shirt,21.792127463554,4.94,2380 +English (American),shoe,nouns,clothing,shoe,shoe,16.401616618833103,4.97,2169 +English (American),shorts,nouns,clothing,shorts,shorts,24.909759184344438,4.82,425 +English (American),shoulder,nouns,body_parts,shoulder,shoulder,27.357714981859722,4.93,225 +English (American),shovel,nouns,outside,shovel,shovel,25.93264008363907,4.97,291 +English (American),show,predicates,action_words,show,show,27.85277401420022,3.97,8568 +English (American),shower,nouns,furniture_rooms,shower,shower,23.85617473782963,4.89,513 +English (American),sick,predicates,descriptive_words,sick,sick,27.025423107263375,2.97,1207 +English (American),sidewalk,nouns,outside,sidewalk,sidewalk,27.982232325403388,4.96,104 +English (American),sing,predicates,action_words,sing,sing,24.699537103250435,4.34,3851 +English (American),sink,nouns,furniture_rooms,sink,sink,25.65056395838028,4.74,560 +English (American),sit,predicates,action_words,sit,sit,21.750618675138146,4.8,14108 +English (American),skate,predicates,action_words,skate,skate,30.980023419231532,4.56,55 +English (American),sky,nouns,outside,sky,sky,24.179550783288896,4.45,1303 +English (American),sled,nouns,vehicles,sled,sled,29.4219079215242,5,90 +English (American),sleep,predicates,action_words,sleep,sleep,22.7674333859663,4.44,5010 +English (American),sleepy,predicates,descriptive_words,sleepy,sleepy,25.765911658239688,2.77,808 +English (American),slide (action),predicates,action_words,slide (action),slide,24.394104047595913,4.48,3556 +English (American),slide (object),nouns,outside,slide (object),slide,23.27441979107034,4.48,3556 +English (American),slipper,nouns,clothing,slipper,slipper,25.90935922531163,4.86,266 +English (American),slow,predicates,descriptive_words,slow,slow,29.462982166623352,3.28,515 +English (American),smile,predicates,action_words,smile,smile,27.022198912378336,4.5,498 +English (American),sneaker,nouns,clothing,sneaker,sneaker,28.458789287265894,4.69,22 +English (American),snow,nouns,outside,snow,snow,25.263272056571832,4.85,1575 +English (American),snowman,nouns,outside,snowman,snowman,27.67210714348651,4.64,577 +English (American),snowsuit,nouns,clothing,snowsuit,snowsuit,32.10541910633961,5,14 +English (American),so,function_words,connecting_words,so,so,31.12075549333711,1.42,44980 +English (American),soap,nouns,household,soap,soap,22.217921774180518,4.93,709 +English (American),sock,nouns,clothing,sock,sock,19.632046621854386,4.91,831 +English (American),soda,nouns,food_drink,soda/pop,soda,24.87205183714649,4.97,226 +English (American),sofa,nouns,furniture_rooms,sofa,sofa,29.394839148936164,4.9,450 +English (American),soft,predicates,descriptive_words,soft,soft,26.4496476384763,3.88,1358 +English (American),some,function_words,quantifiers,some,some,26.15437450894456,2.48,45706 +English (American),soup,nouns,food_drink,soup,soup,24.537539621938286,4.72,1288 +English (American),spill,predicates,action_words,spill,spill,26.85160156093012,4.07,741 +English (American),splash,predicates,action_words,splash,splash,25.60191719014938,4.19,709 +English (American),spoon,nouns,household,spoon,spoon,20.22002234723444,4.96,3524 +English (American),sprinkler,nouns,outside,sprinkler,sprinkler,28.75113806495684,4.89,52 +English (American),squirrel,nouns,animals,squirrel,squirrel,25.05065234245397,4.89,507 +English (American),stairs,nouns,furniture_rooms,stairs,stairs,24.47753335085588,5,1063 +English (American),stand,predicates,action_words,stand,stand,26.47937683978296,4.16,3558 +English (American),star,nouns,outside,star,star,22.957358462903812,4.69,1068 +English (American),stay,predicates,action_words,stay,stay,26.586593702088443,2.15,4403 +English (American),stick,nouns,outside,stick,stick,24.363668955777232,4.59,2642 +English (American),sticky,predicates,descriptive_words,sticky,sticky,26.569871206088944,3.59,1177 +English (American),stone,nouns,outside,stone,stone,29.654573348257873,4.72,306 +English (American),stop,predicates,action_words,stop,stop,22.813711423562257,3.68,5527 +English (American),store,nouns,outside,store,store,23.347629882628436,4.5,1651 +English (American),story,nouns,toys,story,story,24.92454125516168,3.3,5318 +English (American),stove,nouns,furniture_rooms,stove,stove,26.381537023983125,4.96,520 +English (American),strawberry,nouns,food_drink,strawberry,strawberry,24.304122215327734,5,840 +English (American),street,nouns,outside,street,street,25.6074840807896,4.75,987 +English (American),stroller,nouns,vehicles,stroller,stroller,24.603444785777125,4.96,227 +English (American),stuck,predicates,descriptive_words,stuck,stuck,25.790527319514812,3.55,3519 +English (American),sun,nouns,outside,sun,sun,23.386638497101277,4.83,1983 +English (American),sweater,nouns,clothing,sweater,sweater,24.620590384363602,4.78,310 +English (American),sweep,predicates,action_words,sweep,sweep,26.53419291818447,3.72,266 +English (American),swim,predicates,action_words,swim,swim,24.83257594574256,4.43,986 +English (American),swing (action),predicates,action_words,swing (action),swing,23.099517041407093,4.54,2158 +English (American),swing (object),nouns,outside,swing (object),swing,21.97332053501421,4.54,2158 +English (American),table,nouns,furniture_rooms,table,table,23.395064163035677,4.9,5798 +English (American),take,predicates,action_words,take,take,27.354093799677152,3.06,20398 +English (American),talk,predicates,action_words,talk,talk,26.172898281486297,4.07,4771 +English (American),tape,nouns,household,tape,tape,25.98869069431121,4.9,2967 +English (American),taste,predicates,action_words,taste,taste,28.306250651871636,4.07,1493 +English (American),tear,predicates,action_words,tear,tear,30.110713846716816,4.56,352 +English (American),teddybear,nouns,animals,teddybear,teddybear,22.457673785720814,4.87,513 +English (American),telephone,nouns,household,telephone,telephone,20.80553350305347,4.96,1280 +English (American),that,function_words,pronouns,that,that,24.03974924284265,1.54,219719 +English (American),the,function_words,quantifiers,the,the,27.71532064680694,1.43,431031 +English (American),their,function_words,pronouns,their,their,33.050453300967234,3.34,5083 +English (American),them,function_words,pronouns,them,them,31.425327261028436,3.04,41587 +English (American),then,function_words,connecting_words,then,then,32.94497320291506,1.44,43062 +English (American),there,function_words,locations,there,there,25.554374013411266,2.2,106697 +English (American),these,function_words,pronouns,these,these,28.91030410540727,2.03,20843 +English (American),they,function_words,pronouns,they,they,31.558011791688703,2.93,53290 +English (American),think,predicates,action_words,think,think,31.060977381002104,2.41,57446 +English (American),thirsty,predicates,descriptive_words,thirsty,thirsty,25.9814458268787,3.86,776 +English (American),this,function_words,pronouns,this,this,25.53668411833393,2.14,112409 +English (American),those,function_words,pronouns,those,those,30.71305685632802,2.32,19370 +English (American),throw,predicates,action_words,throw,throw,24.82027667799749,4.04,5585 +English (American),tickle,predicates,action_words,tickle,tickle,23.030865141786716,3.85,2094 +English (American),tiger,nouns,animals,tiger,tiger,23.97601642667219,5,2349 +English (American),tights,nouns,clothing,tights,tights,30.49526782372822,4.62,89 +English (American),tiny,predicates,descriptive_words,tiny,tiny,30.75118196119715,3.11,1454 +English (American),tired,predicates,descriptive_words,tired,tired,26.126063847560406,3,3628 +English (American),tissue,nouns,household,tissue/kleenex,tissue,24.881517554334437,4.93,915 +English (American),to,function_words,locations,to,to,27.829074756482076,1.55,183409 +English (American),toast,nouns,food_drink,toast,toast,22.957745051917446,4.93,2117 +English (American),toe,nouns,body_parts,toe,toe,21.063402869488534,4.93,596 +English (American),tongue,nouns,body_parts,tongue,tongue,23.578714641827492,4.93,776 +English (American),too,function_words,quantifiers,too,too,25.97451456969294,1.7,21317 +English (American),tooth,nouns,body_parts,tooth,tooth,21.390186244463926,4.89,351 +English (American),toothbrush,nouns,household,toothbrush,toothbrush,21.327972861784076,5,330 +English (American),touch,predicates,action_words,touch,touch,26.140740016873888,3.86,3096 +English (American),towel,nouns,household,towel,towel,23.309359030136616,4.86,942 +English (American),toy (object),nouns,toys,toy (object),toy,21.250379422750402,4.93,2764 +English (American),tractor,nouns,vehicles,tractor,tractor,26.293460903708173,5,1903 +English (American),train,nouns,vehicles,train,train,21.050773507323704,4.79,8225 +English (American),trash,nouns,household,trash,trash,24.623003619992428,4.7,297 +English (American),tray,nouns,household,tray,tray,31.297608014031766,4.74,570 +English (American),tree,nouns,outside,tree,tree,20.395692984405432,5,4310 +English (American),tricycle,nouns,vehicles,tricycle,tricycle,28.84180554601145,4.68,163 +English (American),truck,nouns,vehicles,truck,truck,19.04437063778738,4.84,4906 +English (American),try,function_words,helping_predicates,try/try to,try,28.751119300000134,2.22,8718 +English (American),tummy,nouns,body_parts,tummy,tummy,21.281768482103764,4.68,1772 +English (American),tuna,nouns,food_drink,tuna,tuna,28.649288768887633,4.89,208 +English (American),turkey,nouns,animals,turkey,turkey,26.30455622910081,4.89,611 +English (American),turtle,nouns,animals,turtle,turtle,23.014453310177558,5,792 +English (American),tv,nouns,furniture_rooms,TV,tv,21.36643906838051,5,1285 +English (American),under,function_words,locations,under,under,27.467319313886602,3.45,4364 +English (American),underpants,nouns,clothing,underpants,underpants,27.702339180277328,4.89,111 +English (American),up,function_words,locations,up,up,19.536229246282858,3.83,54966 +English (American),us,function_words,pronouns,us,us,32.68418082412553,3.59,7432 +English (American),vacuum,nouns,household,vacuum,vacuum,24.31715488103081,4.22,241 +English (American),vagina,nouns,body_parts,vagina*,vagina,30.74482822160873,4.82,8 +English (American),vanilla,nouns,food_drink,vanilla,vanilla,31.134282822644582,4.68,196 +English (American),vitamins,nouns,food_drink,vitamins,vitamins,26.849949095931386,4.5,85 +English (American),wait,predicates,action_words,wait,wait,26.620371092775404,2.68,7959 +English (American),wake,predicates,action_words,wake,wake,27.154377902728353,3.11,1167 +English (American),walk,predicates,action_words,walk,walk,22.817811861035416,4.07,3506 +English (American),walker,nouns,household,walker,walker,31.785543166543064,4.42,60 +English (American),wanna,function_words,helping_predicates,wanna/want to,wanna,25.18163215638973,1.93,45282 +English (American),was,function_words,helping_predicates,was,was,31.797860743565355,1.69,47639 +English (American),wash,predicates,action_words,wash,wash,23.987564663405482,4.35,3451 +English (American),washing machine,nouns,furniture_rooms,washing machine,washing machine,27.543044986998385,4.89,239 +English (American),watch (action),predicates,action_words,watch (action),watch,25.924613964341088,4.61,15632 +English (American),watch (object),nouns,household,watch (object),watch,24.00259582021144,4.61,15632 +English (American),water (beverage),nouns,food_drink,water (beverage),water,19.123463892055845,5,17388 +English (American),water (not beverage),nouns,outside,water (not beverage),water,19.45099861331479,5,17388 +English (American),we,function_words,pronouns,we,we,30.182434631151455,3.08,111064 +English (American),were,function_words,helping_predicates,were,were,33.709463033181024,1.48,30396 +English (American),wet (description),predicates,descriptive_words,wet,wet,22.298452425741512,4.46,2542 +English (American),what,function_words,question_words,what,what,23.658782166516907,2,205010 +English (American),when (question),function_words,question_words,when,when,31.814907045715227,1.6,27792 +English (American),where (question),function_words,question_words,where,where,25.7164381182961,1.66,38793 +English (American),which (question),function_words,question_words,which,which,33.63776900674318,1.54,10214 +English (American),white,predicates,descriptive_words,white,white,27.648764293432254,3.89,3496 +English (American),who,function_words,question_words,who,who,27.953549755744795,1.74,20283 +English (American),why,function_words,question_words,why,why,27.262555741780712,1.86,22267 +English (American),will,function_words,helping_predicates,will,will,30.909905381379183,2.64,14927 +English (American),wind,nouns,outside,wind,wind,26.19931053900361,3.93,1298 +English (American),window,nouns,furniture_rooms,window,window,24.48492597146611,4.86,2662 +English (American),windy,predicates,descriptive_words,windy,windy,27.431143183092633,3.86,342 +English (American),wipe,predicates,action_words,wipe,wipe,25.95723852087949,4,1809 +English (American),wish,predicates,action_words,wish,wish,32.96609018388594,1.77,669 +English (American),with,function_words,locations,with,with,28.56625914577001,2,64176 +English (American),wolf,nouns,animals,wolf,wolf,28.251389709841927,4.79,431 +English (American),work (action),predicates,action_words,work (action),work,24.762232083800356,3.48,12134 +English (American),work (place),nouns,outside,work (place),work,24.024006606906344,3.48,12134 +English (American),would,function_words,helping_predicates,would,would,34.17580948877974,1.12,22070 +English (American),write,predicates,action_words,write,write,26.86079876248854,4.22,2971 +English (American),yard,nouns,outside,backyard,backyard,26.10807276380005,4.82,105 +English (American),yellow,predicates,descriptive_words,yellow,yellow,24.5595931762692,4.3,6782 +English (American),yogurt,nouns,food_drink,yogurt,yogurt,23.91372922424347,4.9,589 +English (American),you,function_words,pronouns,you,you,24.120602892947364,4.11,589501 +English (American),your,function_words,pronouns,your,your,29.787082244817054,2.37,105998 +English (American),yourself,function_words,pronouns,yourself,yourself,34.28546272279592,4.39,3279 +English (American),yucky,predicates,descriptive_words,yucky,yucky,22.051017838612175,1.86,724 +English (American),zebra,nouns,animals,zebra,zebra,25.31795862018206,4.86,902 +English (American),zipper,nouns,clothing,zipper,zipper,24.84585073879806,4.83,421 +English (American),zoo,nouns,outside,zoo,zoo,25.777390810890576,4.81,1804 +English (American),butt,nouns,body_parts,buttocks/bottom*,bottom,21.98056796070707,4.75,2253 +English (American),did,function_words,helping_predicates,did/did ya,did ya,28.236905169713484,2.45,195 +English (American),in,function_words,locations,inside/in,in,24.07962150691501,3,276036 +English (American),need,function_words,helping_predicates,need/need to,need to,28.517326473959855,1.69,829 +English (American),soda,nouns,food_drink,soda/pop,pop,24.87205183714649,4.97,2512 +English (American),tissue,nouns,household,tissue/kleenex,kleenex,24.881517554334437,4.93,91 +English (American),try,function_words,helping_predicates,try/try to,try to,28.751119300000134,2.22,1273 +English (American),wanna,function_words,helping_predicates,wanna/want to,want to,25.18163215638973,1.93,1415 diff --git a/evaluation-pipeline/assets/babylm.png b/evaluation-pipeline/assets/babylm.png new file mode 100644 index 0000000000000000000000000000000000000000..10233d121ba8b216db02be4e3be1a6d635712811 --- /dev/null +++ b/evaluation-pipeline/assets/babylm.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7785304abdfd889ca8f480e00d4b9630602d4926ba5cdd7aa190e15626581398 +size 254671 diff --git a/evaluation-pipeline/babylm_eval.py b/evaluation-pipeline/babylm_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..8beb2d2f2ea56181f37bc2ea9bc0186dd223a91c --- /dev/null +++ b/evaluation-pipeline/babylm_eval.py @@ -0,0 +1,97 @@ +import argparse +import lm_eval +import os +import json + +TASKS = { + "blimp": ["anaphor_agreement.json", "argument_structure.json", "binding.json", + "control_raising.json", "determiner_noun_agreement.json", "ellipsis.json", + "filler_gap.json", "irregular_forms.json", "island_effects.json", + "npi_licensing.json", "quantifiers.json", "subject_verb_agreement.json"], + "supplement": ["hypernym.json", "qa_congruence_easy.json", "qa_congruence_tricky.json", + "subject_aux_inversion.json", "turn_taking.json"] +} + + +def accuracy_on_task(task_name, eval_model, template_name, num_fewshot): + predictions_path = os.path.join(args.model_path, "zeroshot", task_title, "predictions.txt") + predictions_dir = os.path.dirname(predictions_path) + if not os.path.exists(predictions_dir): + os.makedirs(predictions_dir) + + eval_task = lm_eval.get_task_list(task_name, template_names=[template_name]) + results = lm_eval.evaluate(model=eval_model, tasks=eval_task, seed=12, + num_fewshot=num_fewshot, predictions_path=predictions_path) + accuracy = results['results'][0]['acc'] + return accuracy + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("model_path", type=str, + help="Path to huggingface model and tokenizer.") + parser.add_argument("model_type", type=str, choices=["decoder only", "decoder", "encoder only", "encoder", "encoder-decoder",], + help="Language model architecture.") + parser.add_argument("--tasks", "-t", type=str, choices=["blimp", "supplement", "all"], default="all", + help="Tasks on which we evaluate.") + parser.add_argument("--run_aoa", "-a", action="store_true", + help="Will run the additional AoA prediction task.") + parser.add_argument("--trust_remote_code", "-r", action="store_true", + help="Trust remote code (e.g. from huggingface) when loading model.") + parser.add_argument("--num_fewshot", "-n", type=int, default=0, + help="Number of few-shot examples to show the model for each test example.") + args = parser.parse_args() + + MODEL_TYPE_REMAP = {"decoder only": "hf-causal", "decoder": "hf-causal", + "encoder only": "hf-mlm", "encoder": "hf-mlm", + "encoder-decoder": "hf-seq2seq",} + eval_model = lm_eval.get_model(MODEL_TYPE_REMAP[args.model_type], + pretrained=args.model_path, + trust_remote_code=args.trust_remote_code, + device="cuda") + tasks = [] + if args.tasks == "all": + for task_type in TASKS.keys(): + tasks.extend(TASKS[task_type]) + else: + tasks = TASKS[args.tasks] + + accuracies = {} + # Iterate through tasks, get accuracies + for task in tasks: + if task in TASKS["blimp"]: + template = None + task_title = task.split(".json")[0] + task = f"blimp_from_file:filter-data/blimp_filtered/{task}" + elif task in TASKS["supplement"]: + template = None + task_title = task.split(".json")[0] + task = f"blimp_from_file:filter-data/supplement_filtered/{task}" + else: + raise ValueError("Unrecognized task!") + accuracies[task_title] = accuracy_on_task(task, eval_model, template, + args.num_fewshot) + print(f"{task_title}:\t{accuracies[task_title] * 100:.2f}%") + # Write scores to file + out_path = os.path.join(args.model_path, "zeroshot", task_title, "eval_results.json") + out_dir = os.path.dirname(out_path) + if not os.path.exists(out_dir): + os.makedirs(out_dir) + with open(out_path, 'w') as out_file: + json.dump({"eval_accuracy": accuracies[task_title]}, out_file) + + # Print scores + print("\nScores:") + for task in accuracies.keys(): + print(f"{task}:\t{accuracies[task] * 100:.2f}%") + + if args.run_aoa: + # Run AoA prediction evaluation + word_surprisals_n, mad_results = lm_eval.aoa_pred_eval(eval_model.model, eval_model.tokenizer, MODEL_TYPE_REMAP[args.model_type], batch_size = 32) + out_dir = os.path.join(args.model_path, "aoa_prediction") + if not os.path.exists(out_dir): + os.makedirs(out_dir) + with open(os.path.join(out_dir, "extracted_average_surprisals.json") , 'w') as out_file: + json.dump(word_surprisals_n, out_file) + with open(os.path.join(out_dir, "mean_absolute_deviation_results.json"), 'w') as out_file: + json.dump(mad_results, out_file) diff --git a/evaluation-pipeline/collect_results.py b/evaluation-pipeline/collect_results.py new file mode 100644 index 0000000000000000000000000000000000000000..59f302ce07e39f2ab8245c419430f6a0fbcfd99f --- /dev/null +++ b/evaluation-pipeline/collect_results.py @@ -0,0 +1,85 @@ +import argparse +import os +import json + +TASKS = { + "glue": ["cola", "sst2", "mrpc", "qqp", "mnli", "mnli-mm", "qnli", "rte", + "boolq", "multirc", "wsc"], + "blimp": ["anaphor_agreement", "argument_structure", "binding", "control_raising", + "determiner_noun_agreement", "ellipsis", "filler_gap", "irregular_forms", + "island_effects", "npi_licensing", "quantifiers", "subject_verb_agreement"], + "supplement": ["hypernym", "qa_congruence_easy", "qa_congruence_tricky", + "subject_aux_inversion", "turn_taking"], + "msgs": ["main_verb_control", "control_raising_control", "syntactic_category_control", + "relative_position_control", "lexical_content_the_control", + "main_verb_lexical_content_the", "main_verb_relative_token_position", + "control_raising_lexical_content_the", "control_raising_relative_token_position", + "syntactic_category_lexical_content_the", "syntactic_category_relative_position"] +} + +def make_task_dict(task_name, preds_path): + def _add_to_dict(index, prediction, task_dict): + example_id = f"{task_name}_{index}" + prediction = prediction.replace("\\n", "\n") + task_dict["predictions"].append({"id": example_id, "pred": prediction}) + + if task_name in TASKS["glue"]: + task_type = "glue" + elif task_name in TASKS["blimp"]: + task_type = "blimp" + elif task_name in TASKS["supplement"]: + task_type = "supplement" + elif task_name in TASKS["msgs"]: + task_type = "msgs" + else: + raise ValueError(f"Invalid task: {task_name}!") + + if not os.path.exists(preds_path): + raise FileNotFoundError(f"Warning: no predictions found for the \"{task_name}\" ({task_type}) task!") + + task_dict = {"task": task_type, "sub_task": task_name, "predictions": []} + with open(preds_path, 'r') as predictions_file: + # skip header + next(predictions_file) + # collect predictions with ids + index = None + prediction = None + for line in predictions_file: + if "\t" in line: + # add to prediction list + if prediction: + _add_to_dict(index, prediction, task_dict) + # start new prediction + index, prediction = line.strip().split("\t") + else: + prediction += "\n" + line.strip() + # handle final prediction + _add_to_dict(index, prediction, task_dict) + + return task_dict + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("model_path", type=str, + help="Path to huggingface model and tokenizer.") + args = parser.parse_args() + + task_dicts = {} + for task in TASKS["glue"]: + preds_path = os.path.join(args.model_path, "finetune", task, "predict_results.txt") + task_dicts[task] = make_task_dict(task, preds_path) + for task in TASKS["msgs"]: + preds_path = os.path.join(args.model_path, "finetune", task, "predict_results.txt") + task_dicts[task] = make_task_dict(task, preds_path) + for task in TASKS["blimp"]: + preds_path = os.path.join(args.model_path, "zeroshot", task, "predictions.txt") + task_dicts[task] = make_task_dict(task, preds_path) + for task in TASKS["supplement"]: + preds_path = os.path.join(args.model_path, "zeroshot", task, "predictions.txt") + task_dicts[task] = make_task_dict(task, preds_path) + + with open("all_predictions.json", "w") as predictions_out: + for task in task_dicts: + predictions_out.write(json.dumps(task_dicts[task]) + "\n") + print("Predictions output at `all_predictions.json`.") diff --git a/evaluation-pipeline/docs/img/fewshot_example_gpt3.png b/evaluation-pipeline/docs/img/fewshot_example_gpt3.png new file mode 100644 index 0000000000000000000000000000000000000000..aee409d156468b2324b84b8f77034d54cf9e554b --- /dev/null +++ b/evaluation-pipeline/docs/img/fewshot_example_gpt3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6af5dc2196248b29260ba443e882725dd6cfc51ef17ad5a4dbab4f8ce6850c75 +size 315681 diff --git a/evaluation-pipeline/docs/task_guide.md b/evaluation-pipeline/docs/task_guide.md new file mode 100644 index 0000000000000000000000000000000000000000..56592188478c06f9c7eefa6068da975568aa9d2e --- /dev/null +++ b/evaluation-pipeline/docs/task_guide.md @@ -0,0 +1,300 @@ +# `Task` Guide + +The `Task` class is the foundation of all natural language tasks in the `lm-evaluation-harness` (harness). It encompasses everything you’d need to perform few-shot evaluation of an autoregressive language model. Here we’ll provide a step-by-step guide on how to subclass `Task` to create your very own task/s. + +## Setup + +If you haven't already, go ahead and fork the main repo, clone it, create a branch with the name of your task, and install the project requirements in your environment: + +```sh +# After forking... +git clone https://github.com//lm-evaluation-harness.git +cd lm-evaluation-harness +git checkout -b +pip install -e ".[dev]" +``` + +## Creating Your Task File + +From the `lm-evaluation-harness` project root, copy over the `new_task.py` template to `lm_eval/datasets`. + +```sh +cp templates/new_task.py lm_eval/tasks/.py +``` + +or if your task is **multiple-choice**, the `new_multiple_choice_task.py`: + +```sh +cp templates/new_multiple_choice_task.py lm_eval/tasks/.py +``` + +This will set you up with a few `TODO`s to fill-in which we'll now go over in detail. + +## Task Heading + +Open the file you've just created and add a multiline docstring on the first line with the following contents: + +```python +""" + + + + + +Homepage: +""" +``` + +For example, take the QuAC dataset. We have: + +```python +""" +QuAC: Question Answering in Context +https://arxiv.org/abs/1808.07036 + +Question Answering in Context (QuAC) is a dataset for modeling, understanding, and +participating in information seeking dialog. Data instances consist of an interactive +dialog between two crowd workers: (1) a student who poses a sequence of freeform +questions to learn as much as possible about a hidden Wikipedia text, and (2) +a teacher who answers the questions by providing short excerpts (spans) from the text. + +Homepage: https://quac.ai/ +""" +``` + +Next, at the module-level, create a constant variable named +`_CITATION` that contains the citation information for your task in BibTeX format. + +Now let's walk through the actual implementation - from data handling to evaluation. + +## Data Handling + +### Downloading your Data + +All data downloading and management is handled through the HuggingFace (**HF**) [`datasets`](https://github.com/huggingface/datasets) API. So, the first thing you should do is check to see if your task's dataset is already provided in their catalog [here](https://huggingface.co/datasets). If it's not in there, please consider adding it to their Hub to make it accessible to a wider user base by following their [new dataset guide](https://github.com/huggingface/datasets/blob/master/ADD_NEW_DATASET.md) +. +Now, that you have your HF dataset, you need to assign its path and name to your `Task` in the following fields: + +```python +class TaskName(...): + DATASET_PATH = "..." + DATASET_NAME = "..." +``` + +where `DATASET_PATH` is the name of the dataset as listed by HF in the `datasets` Hub and `DATASET_NAME` is the name of, what HF calls, a “data instance” or sub-task of the benchmark. If your task does not contain any data instances, just set `DATASET_NAME = None`. +(If you're familiar with the HF `datasets.load_dataset` function, these are just the first 2 arguments to it.) + +Next up, we have to set some “flags”: + +```python + def has_training_docs(self): + return # True/False + + def has_validation_docs(self): + return # True/False + + def has_test_docs(self): + return # True/False +``` + +These methods return `True`/`False` whether or not your task dataset provides documents for each split type. __Note__: if the test set does not have publicly available answer labels, please do not put it down as having a test set - return False. + +Lastly, we need to load the documents. In our terminology, a document (`doc`) is a single natural language data example stored in a Python `dict`. E.g.: `{“question”: “What is the capital of France?”, “answer”: “Paris”}`. Override the following methods to load your data splits from their storage location in `DATASET_PATH`: + +```python + def training_docs(self): + return #... + + def validation_docs(self): + return #... + + def test_docs(self): + return #... +``` + +These should return a Python iterable (`list` or `generator`) of `dict`s that can be queried for individual `doc` examples. + +#### Processing Documents + +At this point, you can also process each individual document to, for example, strip whitespace or "detokenize" its fields. Put the processing logic into `_process_doc` and map the functions across training/validation/test docs inside of the respective functions. +🔠 If your task is **multiple-choice**, we require you to format your documents such that they contain `gold` and `choices` fields. They can also have other fields, but those will be ignored by `MultipleChoiceTask`. `choices` should be a list of possible continuations, and `gold` should be an integer specifying the index of the correct completion. +See [this task](https://github.com/EleutherAI/lm-evaluation-harness/blob/6caa0afd96a7a7efb2ec4c1f24ad1756e48f3aa7/lm_eval/tasks/sat.py#L60) for an example. 🔠 + +### Formatting your Few-Shot Examples + +The harness is designed to facilitate task evaluations under the few-shot setting. Here we’ll format such examples. + +Format your document into a single query prompt __without the answer__ here. This method takes a single `doc` example of type `dict` with `str` key-value members. You should concatenate these `doc` item values together into a neatly formatted prompt. + +```python +def doc_to_text(self, doc): + return "" +``` + +
+ +️🔠 **Multiple-Choice Formatting** + +If your task is multiple-choice, you can now skip ahead to registering your task. + +️️🔠 **End Multiple-Choice Formatting** + +
+ +Format the target answer from the contents of `doc`. Note that the prepended `" "` is required to space out the `doc_to_text` and `doc_to_target` strings. + +```python +def doc_to_target(self, doc): + target = "" + return " " + target +``` + +Finally, be aware that the strings from `doc_to_text` and `doc_to_target` will be concatenated together to build up labeled examples in the k-shot setting where k > 0. Design with that in mind 👍. + +### Registering Your Task + +Now's a good time to register your task to expose it for usage. All you'll need to do is import your task module in `lm_eval/tasks/__init__.py` and provide an entry in the `TASK_REGISTRY` dictionary with the key as the name of your benchmark task (in the form it'll be referred to in the command line) and the value as the task class. See how it's done for other tasks in the [file](https://github.com/EleutherAI/lm-evaluation-harness/blob/master/lm_eval/tasks/__init__.py). + +### Checking the Data + +After registering your task, you can now check on your data downloading and verify that the few-shot samples look as intended. Run the following command with your desired args: + +```bash +python -m scripts.write_out \ + --output_base_path \ + --tasks \ + --sets \ + --num_fewshot K \ + --num_examples N \ +``` + +Open the file specified at the `--output_base_path ` and ensure it passes +a simple eye test. + +## Evaluation + +**🛑** If your task is a single-true multiple-choice task and you've correctly inherited from `MultipleChoiceTask` then your job here is done; go ‘head and check on the task performance! 🛑 + +Now comes evaluation. The methods you'll need to implement are: + +```python +def construct_requests(self, doc, ctx): + """Uses RequestFactory to construct Requests and returns an iterable of + Requests which will be sent to the LM. + + Args: + doc (dict): + The document as returned from training_docs, validation_docs, or + test_docs. + ctx (str): + The context string, generated by fewshot_context. This includes + the natural language description, as well as the few shot examples, + and the question part of the document for `doc`. + args (dict): + The specifics of the context, including number of few shots. + + Returns: + An iterable of `Request` objects. + """ + return ... +``` +If your task requires generating text you'll need to return a `rf.greedy_until` request otherwise an `rf.loglikelihood` across all labels in a classification tasks will do. + +```python +def process_results(self, doc, results): + """Take a single document and the LM results and evaluates, returning a + dict where keys are the names of sub-metrics and values are the values of + the metric for that one document. + + NOTE: This function automates processing by using the `promptsource` + metadata to determine the metric. + + Args: + doc (dict): + The document as returned from training_docs, validation_docs, or + test_docs. + results (list): + The results of the requests created in construct_requests. + + Returns: + A dict of metric results. + """ + return {} +``` + +```python +def aggregation(self): + """ + Returns: + A dictionary where keys are the names of sub-metrics and values are + functions that aggregate a list of metric scores. + {str: [metric_score] -> float} + """ + return {} +``` + +See `lm_eval/metrics.py` for a few "built-in" aggregate metrics you can easily import. + +```python +def higher_is_better(self): + """ + Returns: + A dictionary where keys are the names of sub-metrics and values are + whether a higher value of the sub-metric is better. + {str: bool} + """ + return {} +``` + +Some tasks that are good examples of various ways evaluation can be implemented can be found here: [LAMBADA](https://github.com/EleutherAI/lm-evaluation-harness/blob/master/lm_eval/tasks/lambada.py), [TriviaQA](https://github.com/EleutherAI/lm-evaluation-harness/blob/master/lm_eval/tasks/triviaqa.py), [SQuAD](https://github.com/EleutherAI/lm-evaluation-harness/blob/master/lm_eval/tasks/squad.py). + +Tip: Feel free to create your own helper-methods for your task! + +### Checking the Task Performance + +```sh +python main.py \ + --model gpt2 \ + --model_args device= \ + --tasks \ + --num_fewshot K +``` + +Set the limit size, `N`, to a smallish number (e.g. 10) and try out the task under different `K`-shot settings. If you have an Nvidia GPU at your disposal, add the argument +`--model_args device=cuda:0`. If you have access to an OpenAI API key, you can also evaluate GPT-3 on various tasks with the following command: + +```sh +export OPENAI_API_SECRET_KEY=YOUR_KEY_HERE +python main.py \ + --model gpt3 \ + --tasks \ + --num_fewshot K +``` + +### Running Unit Tests + +To run the entire test suite, use: + +```sh +pytest +``` + +This is usually overkill; to run only the tests for your task, do: +```sh +pytest -k +``` + +## Versioning + +Lastly, we need to "version control". Tasks in the harness can always evolve. Metrics get updated, data sources change, etc. It’s important to mark each task with a version attribute so users can document which implementation version was used to obtain their results. Add a `VERSION` attribute to your task right below the class name and set it to `0` (this is the first version/implementation of your task): + +```python +class TaskName(...): + VERSION = 0 +``` + +## Submitting your Task + +Although we currently do not work behind a specific style guide, we'd appreciate if you tidy up your file/s with the `black` formatter (which should've been install through the `requirements.txt`). Keep things clean…ish 🙂. + +Now push your work and make a pull request! Thanks for the contribution 👍. If there are any questions, leave a message in the `#lm-thunderdome` channel on the EAI discord. diff --git a/evaluation-pipeline/filter_data.zip b/evaluation-pipeline/filter_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..139c691b3396b422451d08c9a157085b66cc2b80 --- /dev/null +++ b/evaluation-pipeline/filter_data.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8536705c9bb432cc84a9746fe19dacea457b7e2280602fab01431167123dd0b8 +size 61762326 diff --git a/evaluation-pipeline/finetune_all_tasks.sh b/evaluation-pipeline/finetune_all_tasks.sh new file mode 100644 index 0000000000000000000000000000000000000000..40fc114967924565bbae3e1f460dc9bd33e1d0ca --- /dev/null +++ b/evaluation-pipeline/finetune_all_tasks.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +MODEL_PATH=$1 +LR=${2:-5e-5} +PATIENCE=${3:-10} +BSZ=${4:-64} +EVAL_EVERY=${5:-200} +MAX_EPOCHS=${6:-10} +SEED=${7:-12} + +# Fine-tune and evaluate on (Super)GLUE tasks +# If your system uses sbatch or qsub, consider using that to parallelize calls to finetune_model.sh +for subtask in {"cola","sst2","mrpc","qqp","mnli","mnli-mm","qnli","rte","boolq","multirc","wsc"}; do + ./finetune_model.sh $MODEL_PATH glue $subtask $LR $PATIENCE $BSZ $EVAL_EVERY $MAX_EPOCHS $SEED +done + +# Fine-tune and evaluate on MSGS tasks +for subtask in {"main_verb_control","control_raising_control","syntactic_category_control","lexical_content_the_control","relative_position_control","main_verb_lexical_content_the","main_verb_relative_token_position","syntactic_category_lexical_content_the","syntactic_category_relative_position","control_raising_lexical_content_the","control_raising_relative_token_position"}; do + ./finetune_model.sh $MODEL_PATH msgs $subtask $LR $PATIENCE $BSZ $EVAL_EVERY $MAX_EPOCHS $SEED +done diff --git a/evaluation-pipeline/finetune_classification.py b/evaluation-pipeline/finetune_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..5e2ad345e7911e696dcde2b5ed7c4b1963c566bd --- /dev/null +++ b/evaluation-pipeline/finetune_classification.py @@ -0,0 +1,728 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2020 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Finetuning the library models for sequence classification on GLUE.""" +# You can also adapt this script on your own text classification task. Pointers for this are left as comments. + +import logging +import os +import random +import sys +from dataclasses import dataclass, field +from typing import Optional +import evaluate + +import datasets +import numpy as np +from datasets import load_dataset +from sklearn.metrics import f1_score + +import transformers +from transformers import ( + AutoConfig, + AutoModelForSequenceClassification, + AutoTokenizer, + DataCollatorWithPadding, + EarlyStoppingCallback, + EvalPrediction, + HfArgumentParser, + IntervalStrategy, + PretrainedConfig, + Trainer, + TrainingArguments, + default_data_collator, + set_seed, +) +from transformers.trainer_utils import get_last_checkpoint +from transformers.utils import check_min_version, send_example_telemetry +from transformers.utils.versions import require_version + + +# Will error if the minimal version of Transformers is not installed. Remove at your own risks. +# check_min_version("4.27.0.dev0") + +require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/text-classification/requirements.txt") + +task_to_keys = { + "cola": ("sentence", None), + "mnli": ("premise", "hypothesis"), + "mrpc": ("sentence1", "sentence2"), + "qnli": ("question", "sentence"), + "qqp": ("question1", "question2"), + "rte": ("sentence1", "sentence2"), + "sst2": ("sentence", None), + "stsb": ("sentence1", "sentence2"), + "wnli": ("sentence1", "sentence2"), +} + +logger = logging.getLogger(__name__) + + +@dataclass +class DataTrainingArguments: + """ + Arguments pertaining to what data we are going to input our model for training and eval. + + Using `HfArgumentParser` we can turn this class + into argparse arguments to be able to specify them on + the command line. + """ + + task_name: Optional[str] = field( + default=None, + metadata={"help": "The name of the task to train on: " + ", ".join(task_to_keys.keys())}, + ) + dataset_name: Optional[str] = field( + default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."} + ) + dataset_config_name: Optional[str] = field( + default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."} + ) + max_seq_length: int = field( + default=128, + metadata={ + "help": ( + "The maximum total input sequence length after tokenization. Sequences longer " + "than this will be truncated, sequences shorter will be padded." + ) + }, + ) + overwrite_cache: bool = field( + default=False, metadata={"help": "Overwrite the cached preprocessed datasets or not."} + ) + pad_to_max_length: bool = field( + default=True, + metadata={ + "help": ( + "Whether to pad all samples to `max_seq_length`. " + "If False, will pad the samples dynamically when batching to the maximum length in the batch." + ) + }, + ) + max_train_samples: Optional[int] = field( + default=None, + metadata={ + "help": ( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ) + }, + ) + max_eval_samples: Optional[int] = field( + default=None, + metadata={ + "help": ( + "For debugging purposes or quicker training, truncate the number of evaluation examples to this " + "value if set." + ) + }, + ) + max_predict_samples: Optional[int] = field( + default=None, + metadata={ + "help": ( + "For debugging purposes or quicker training, truncate the number of prediction examples to this " + "value if set." + ) + }, + ) + patience: Optional[int] = field( + default=None, + metadata={ + "help": ( + "Number of evaluation steps without improvement > epsilon before stopping fine-tuning. " + "Requires the use of the --eval_every argument." + ) + }, + ) + eval_every: Optional[int] = field( + default=None, + metadata = { + "help": ( + "Number of steps between evaluations (MUST be set if patience is set)." + ) + } + ) + train_file: Optional[str] = field( + default=None, metadata={"help": "A csv or a json file containing the training data."} + ) + validation_file: Optional[str] = field( + default=None, metadata={"help": "A csv or a json file containing the validation data."} + ) + test_file: Optional[str] = field(default=None, metadata={"help": "A csv or a json file containing the test data."}) + + def __post_init__(self): + if self.task_name is not None: + self.task_name = self.task_name.lower() + if self.task_name not in task_to_keys.keys(): + raise ValueError("Unknown task, you should pick one in " + ",".join(task_to_keys.keys())) + elif self.dataset_name is not None: + pass + elif self.train_file is None or self.validation_file is None: + raise ValueError("Need either a GLUE task, a training/validation file or a dataset name.") + else: + train_extension = self.train_file.split(".")[-1] + assert train_extension in ["csv", "json"], "`train_file` should be a csv or a json file." + validation_extension = self.validation_file.split(".")[-1] + assert ( + validation_extension == train_extension + ), "`validation_file` should have the same extension (csv or json) as `train_file`." + + +@dataclass +class ModelArguments: + """ + Arguments pertaining to which model/config/tokenizer we are going to fine-tune from. + """ + + model_name_or_path: str = field( + metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"} + ) + freeze_model: bool = field( + default=False, + metadata={"help": "Whether to freeze the parameters of the base model."} + ) + config_name: Optional[str] = field( + default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"} + ) + tokenizer_name: Optional[str] = field( + default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"} + ) + cache_dir: Optional[str] = field( + default=None, + metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}, + ) + use_fast_tokenizer: bool = field( + default=True, + metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."}, + ) + model_revision: str = field( + default="main", + metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."}, + ) + use_auth_token: bool = field( + default=False, + metadata={ + "help": ( + "Will use the token generated when running `huggingface-cli login` (necessary to use this script " + "with private models)." + ) + }, + ) + ignore_mismatched_sizes: bool = field( + default=False, + metadata={"help": "Will enable to load a pretrained model whose head dimensions are different."}, + ) + + +def main(): + # See all possible arguments in src/transformers/training_args.py + # or by passing the --help flag to this script. + # We now keep distinct sets of args, for a cleaner separation of concerns. + + parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)) + if len(sys.argv) == 2 and sys.argv[1].endswith(".json"): + # If we pass only one argument to the script and it's the path to a json file, + # let's parse it to get our arguments. + model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1])) + else: + model_args, data_args, training_args = parser.parse_args_into_dataclasses() + + # Check for the use of early stopping + if data_args.patience: + training_args.eval_steps = data_args.eval_every + training_args.save_total_limit = 1 + training_args.load_best_model_at_end = True + training_args.evaluation_strategy = "steps" + callbacks = [EarlyStoppingCallback(early_stopping_patience=data_args.patience, + early_stopping_threshold=0.001)] + else: + callbacks = None + + # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The + # information sent is the one passed as arguments along with your Python/PyTorch versions. + send_example_telemetry("run_glue", model_args, data_args) + + # Setup logging + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + handlers=[logging.StreamHandler(sys.stdout)], + ) + + log_level = training_args.get_process_log_level() + logger.setLevel(log_level) + datasets.utils.logging.set_verbosity(log_level) + transformers.utils.logging.set_verbosity(log_level) + transformers.utils.logging.enable_default_handler() + transformers.utils.logging.enable_explicit_format() + + # Log on each process the small summary: + logger.warning( + f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}" + + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}" + ) + logger.info(f"Training/evaluation parameters {training_args}") + + # Detecting last checkpoint. + last_checkpoint = None + if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir: + last_checkpoint = get_last_checkpoint(training_args.output_dir) + if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0: + raise ValueError( + f"Output directory ({training_args.output_dir}) already exists and is not empty. " + "Use --overwrite_output_dir to overcome." + ) + elif last_checkpoint is not None and training_args.resume_from_checkpoint is None: + logger.info( + f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change " + "the `--output_dir` or add `--overwrite_output_dir` to train from scratch." + ) + + # Set seed before initializing model. + set_seed(training_args.seed) + + # Get the datasets: you can either provide your own CSV/JSON training and evaluation files (see below) + # or specify a GLUE benchmark task (the dataset will be downloaded automatically from the datasets Hub). + # + # For CSV/JSON files, this script will use as labels the column called 'label' and as pair of sentences the + # sentences in columns called 'sentence1' and 'sentence2' if such column exists or the first two columns not named + # label if at least two columns are provided. + # + # If the CSVs/JSONs contain only one non-label column, the script does single sentence classification on this + # single column. You can easily tweak this behavior (see below) + # + # In distributed training, the load_dataset function guarantee that only one local process can concurrently + # download the dataset. + if data_args.task_name is not None: + # Downloading and loading a dataset from the hub. + raw_datasets = load_dataset( + "glue", + data_args.task_name, + cache_dir=model_args.cache_dir, + use_auth_token=True if model_args.use_auth_token else None, + ) + elif data_args.dataset_name is not None: + # Downloading and loading a dataset from the hub. + raw_datasets = load_dataset( + data_args.dataset_name, + data_args.dataset_config_name, + cache_dir=model_args.cache_dir, + use_auth_token=True if model_args.use_auth_token else None, + ) + else: + # Loading a dataset from your local files. + # CSV/JSON training and evaluation files are needed. + data_files = {"train": data_args.train_file, "validation": data_args.validation_file} + + # Get the test dataset: you can provide your own CSV/JSON test file (see below) + # when you use `do_predict` without specifying a GLUE benchmark task. + if training_args.do_predict: + if data_args.test_file is not None: + train_extension = data_args.train_file.split(".")[-1] + test_extension = data_args.test_file.split(".")[-1] + assert ( + test_extension == train_extension + ), "`test_file` should have the same extension (csv or json) as `train_file`." + data_files["test"] = data_args.test_file + else: + train_extension = data_args.train_file.split(".")[-1] + validation_extension = data_args.validation_file.split(".")[-1] + assert ( + validation_extension == train_extension + ), "`validation_file` should have the same extension (csv or json) as `train_file`." + data_files["test"] = data_args.validation_file + + for key in data_files.keys(): + logger.info(f"load a local file for {key}: {data_files[key]}") + + if data_args.train_file.endswith(".csv"): + # Loading a dataset from local csv files + raw_datasets = load_dataset( + "csv", + data_files=data_files, + cache_dir=model_args.cache_dir, + use_auth_token=True if model_args.use_auth_token else None, + ) + else: + # Loading a dataset from local json files + raw_datasets = load_dataset( + "json", + data_files=data_files, + cache_dir=model_args.cache_dir, + use_auth_token=True if model_args.use_auth_token else None, + ) + # See more about loading any type of standard or custom dataset at + # https://huggingface.co/docs/datasets/loading_datasets.html. + + # Labels + if data_args.task_name is not None: + is_regression = data_args.task_name == "stsb" + if not is_regression: + label_list = raw_datasets["train"].features["label"].names + num_labels = len(label_list) + else: + num_labels = 1 + else: + # Trying to have good defaults here, don't hesitate to tweak to your needs. + is_regression = raw_datasets["train"].features["label"].dtype in ["float32", "float64"] + if is_regression: + num_labels = 1 + else: + # A useful fast method: + # https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.unique + label_list = raw_datasets["train"].unique("label") + label_list.sort() # Let's sort it for determinism + num_labels = len(label_list) + + is_binary = (num_labels == 2) + + # Load pretrained model and tokenizer + # + # In distributed training, the .from_pretrained methods guarantee that only one local process can concurrently + # download model & vocab. + config = AutoConfig.from_pretrained( + model_args.config_name if model_args.config_name else model_args.model_name_or_path, + num_labels=num_labels, + finetuning_task=data_args.task_name, + cache_dir=model_args.cache_dir, + revision=model_args.model_revision, + use_auth_token=True if model_args.use_auth_token else None, + ) + tokenizer = AutoTokenizer.from_pretrained( + model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path, + cache_dir=model_args.cache_dir, + use_fast=model_args.use_fast_tokenizer, + revision=model_args.model_revision, + use_auth_token=True if model_args.use_auth_token else None, + ) + try: + model = AutoModelForSequenceClassification.from_pretrained( + model_args.model_name_or_path, + from_tf=bool(".ckpt" in model_args.model_name_or_path), + config=config, + cache_dir=model_args.cache_dir, + revision=model_args.model_revision, + use_auth_token=True if model_args.use_auth_token else None, + ignore_mismatched_sizes=model_args.ignore_mismatched_sizes, + ) + except ValueError as e: + from transformers import T5Config + if isinstance(config, T5Config): + from transformers_modified.t5 import T5ForSequenceClassification + model = T5ForSequenceClassification.from_pretrained( + model_args.model_name_or_path, + from_tf=bool(".ckpt" in model_args.model_name_or_path), + config=config, + cache_dir=model_args.cache_dir, + revision=model_args.model_revision, + use_auth_token=True if model_args.use_auth_token else None, + ignore_mismatched_sizes=model_args.ignore_mismatched_sizes, + ) + else: + raise e + + # Freeze all parameters (including embeddings) except the classifier head + if model_args.freeze_model: + for name, param in model.named_parameters(): + if "classifier" not in name and not name.startswith("score"): # classifier layer + param.requires_grad = False + + # Preprocessing the raw_datasets + template = None + if data_args.task_name is not None: + sentence1_key, sentence2_key = task_to_keys[data_args.task_name] + else: + # Again, we try to have some nice defaults but don't hesitate to tweak to your use case. + non_label_column_names = [name for name in raw_datasets["train"].column_names if name not in ("label", "idx")] + if "sentence1" in non_label_column_names and "sentence2" in non_label_column_names: + sentence1_key, sentence2_key = "sentence1", "sentence2" + elif "question1" in non_label_column_names and "question2" in non_label_column_names: + sentence1_key, sentence2_key = "question1", "question2" + elif "premise" in non_label_column_names and "hypothesis" in non_label_column_names: + sentence1_key, sentence2_key = "premise", "hypothesis" + elif "question" in non_label_column_names and "passage" in non_label_column_names: + sentence1_key, sentence2_key = "question", "passage" + elif "question" in non_label_column_names and "sentence" in non_label_column_names: + sentence1_key, sentence2_key = "question", "sentence" + # special cases + elif "paragraph" in non_label_column_names and \ + "question" in non_label_column_names and \ + "answer" in non_label_column_names: # MultiRC + sentence1_key, sentence2_key = ["question", "answer"], "paragraph" + template = "Question: {} Answer: {}" + elif "text" in non_label_column_names and \ + "span1_text" in non_label_column_names and \ + "span2_text" in non_label_column_names: # WSC + sentence1_key, sentence2_key = ["span2_text", "span1_text"], "text" + template = "Does \"{}\" refer to \"{}\" in this passage?" + elif "sentence" in non_label_column_names and "linguistic_feature_type" in non_label_column_names: + sentence1_key, sentence2_key = "sentence", None + else: + if len(non_label_column_names) >= 2: + sentence1_key, sentence2_key = non_label_column_names[:2] + else: + sentence1_key, sentence2_key = non_label_column_names[0], None + + # Padding strategy + if data_args.pad_to_max_length: + padding = "max_length" + else: + # We will pad later, dynamically at batch creation, to the max sequence length in each batch + padding = False + + # Some models have set the order of the labels to use, so let's make sure we do use it. + label_to_id = None + if ( + model.config.label2id != PretrainedConfig(num_labels=num_labels).label2id + and data_args.task_name is not None + and not is_regression + ): + # Some have all caps in their config, some don't. + label_name_to_id = {k.lower(): v for k, v in model.config.label2id.items()} + if list(sorted(label_name_to_id.keys())) == list(sorted(label_list)): + label_to_id = {i: int(label_name_to_id[label_list[i]]) for i in range(num_labels)} + else: + logger.warning( + "Your model seems to have been trained with labels, but they don't match the dataset: ", + f"model labels: {list(sorted(label_name_to_id.keys()))}, dataset labels: {list(sorted(label_list))}." + "\nIgnoring the model labels as a result.", + ) + elif data_args.task_name is None and not is_regression: + label_to_id = {v: i for i, v in enumerate(label_list)} + + if label_to_id is not None: + model.config.label2id = label_to_id + model.config.id2label = {id: label for label, id in config.label2id.items()} + elif data_args.task_name is not None and not is_regression: + model.config.label2id = {l: i for i, l in enumerate(label_list)} + model.config.id2label = {id: label for label, id in config.label2id.items()} + + if data_args.max_seq_length > tokenizer.model_max_length: + logger.warning( + f"The max_seq_length passed ({data_args.max_seq_length}) is larger than the maximum length for the" + f"model ({tokenizer.model_max_length}). Using max_seq_length={tokenizer.model_max_length}." + ) + max_seq_length = min(data_args.max_seq_length, tokenizer.model_max_length) + + def preprocess_function(examples): + # Tokenize the texts + if isinstance(sentence1_key, list): + keya, keyb = examples[sentence1_key[0]], examples[sentence1_key[1]] + keys1 = [template.format(ka, kb) for ka, kb in zip(keya, keyb)] + args = ( + (keys1, examples[sentence2_key]) + ) + else: + args = ( + (examples[sentence1_key],) if sentence2_key is None else (examples[sentence1_key], examples[sentence2_key]) + ) + result = tokenizer(*args, padding=padding, max_length=max_seq_length, truncation=True) + + # Map labels to IDs (not necessary for GLUE tasks) + if label_to_id is not None and "label" in examples: + result["label"] = [(label_to_id[l] if l != -1 else -1) for l in examples["label"]] + return result + + with training_args.main_process_first(desc="dataset map pre-processing"): + raw_datasets = raw_datasets.map( + preprocess_function, + batched=True, + load_from_cache_file=not data_args.overwrite_cache, + desc="Running tokenizer on dataset", + ) + if training_args.do_train: + if "train" not in raw_datasets: + raise ValueError("--do_train requires a train dataset") + train_dataset = raw_datasets["train"] + if data_args.max_train_samples is not None: + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) + + if training_args.do_eval: + if "validation" not in raw_datasets and "validation_matched" not in raw_datasets: + raise ValueError("--do_eval requires a validation dataset") + eval_dataset = raw_datasets["validation_matched" if data_args.task_name == "mnli" else "validation"] + if data_args.max_eval_samples is not None: + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) + + if training_args.do_predict or data_args.task_name is not None or data_args.test_file is not None: + if "test" not in raw_datasets and "test_matched" not in raw_datasets: + raise ValueError("--do_predict requires a test dataset") + predict_dataset = raw_datasets["test_matched" if data_args.task_name == "mnli" else "test"] + if data_args.max_predict_samples is not None: + max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples) + predict_dataset = predict_dataset.select(range(max_predict_samples)) + + # Log a few random samples from the training set: + if training_args.do_train: + for index in random.sample(range(len(train_dataset)), 3): + logger.info(f"Sample {index} of the training set: {train_dataset[index]}.") + + # Get the metric function + if data_args.task_name is not None: + metric = evaluate.load("glue", data_args.task_name) + else: + metric = evaluate.load("accuracy") + + # You can define your custom compute_metrics function. It takes an `EvalPrediction` object (a namedtuple with a + # predictions and label_ids field) and has to return a dictionary string to float. + def compute_metrics(p: EvalPrediction): + preds = p.predictions[0] if isinstance(p.predictions, tuple) else p.predictions + preds = np.squeeze(preds) if is_regression else np.argmax(preds, axis=1) + if data_args.task_name is not None: + result = metric.compute(predictions=preds, references=p.label_ids) + if len(result) > 1: + result["combined_score"] = np.mean(list(result.values())).item() + return result + elif is_regression: + return {"mse": ((preds - p.label_ids) ** 2).mean().item()} + elif is_binary: + return {"accuracy": (preds == p.label_ids).astype(np.float32).mean().item(), + "f1": f1_score(y_true=p.label_ids, y_pred=preds, average="binary")} + else: + return {"accuracy": (preds == p.label_ids).astype(np.float32).mean().item()} + + # Data collator will default to DataCollatorWithPadding when the tokenizer is passed to Trainer, so we change it if + # we already did the padding. + if data_args.pad_to_max_length: + data_collator = default_data_collator + elif training_args.fp16: + data_collator = DataCollatorWithPadding(tokenizer, pad_to_multiple_of=8) + else: + data_collator = None + + if is_regression: + training_args.metric_for_best_model = "mse" + elif is_binary: + training_args.metric_for_best_model = "f1" + training_args.greater_is_better = True + else: + training_args.metric_for_best_model = "accuracy" + training_args.greater_is_better = True + + # Initialize our Trainer + trainer = Trainer( + model=model, + args=training_args, + train_dataset=train_dataset if training_args.do_train else None, + eval_dataset=eval_dataset if training_args.do_eval else None, + compute_metrics=compute_metrics, + tokenizer=tokenizer, + data_collator=data_collator, + callbacks = callbacks, + ) + + # Training + if training_args.do_train: + checkpoint = None + if training_args.resume_from_checkpoint is not None: + checkpoint = training_args.resume_from_checkpoint + elif last_checkpoint is not None: + checkpoint = last_checkpoint + train_result = trainer.train(resume_from_checkpoint=checkpoint) + metrics = train_result.metrics + max_train_samples = ( + data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset) + ) + metrics["train_samples"] = min(max_train_samples, len(train_dataset)) + + trainer.save_model() # Saves the tokenizer too for easy upload + + trainer.log_metrics("train", metrics) + trainer.save_metrics("train", metrics) + trainer.save_state() + + # Evaluation + if training_args.do_eval: + logger.info("*** Evaluate ***") + + # Loop to handle MNLI double evaluation (matched, mis-matched) + tasks = [data_args.task_name] + eval_datasets = [eval_dataset] + if data_args.task_name == "mnli": + tasks.append("mnli-mm") + valid_mm_dataset = raw_datasets["validation_mismatched"] + if data_args.max_eval_samples is not None: + max_eval_samples = min(len(valid_mm_dataset), data_args.max_eval_samples) + valid_mm_dataset = valid_mm_dataset.select(range(max_eval_samples)) + eval_datasets.append(valid_mm_dataset) + combined = {} + + for eval_dataset, task in zip(eval_datasets, tasks): + metrics = trainer.evaluate(eval_dataset=eval_dataset) + + max_eval_samples = ( + data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset) + ) + metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset)) + + if task == "mnli-mm": + metrics = {k + "_mm": v for k, v in metrics.items()} + if task is not None and "mnli" in task: + combined.update(metrics) + + trainer.log_metrics("eval", metrics) + trainer.save_metrics("eval", combined if task is not None and "mnli" in task else metrics) + + if training_args.do_predict: + logger.info("*** Predict ***") + + # Loop to handle MNLI double evaluation (matched, mis-matched) + # We do not use MNLI test data. + tasks = [data_args.task_name] + if data_args.task_name == "mnli": + tasks.append("mnli-mm") + predict_datasets.append(raw_datasets["test_mismatched"]) + + # Removing the `label` columns because it contains -1 and Trainer won't like that. + predict_dataset = predict_dataset.remove_columns("label") + predictions = trainer.predict(predict_dataset, metric_key_prefix="predict").predictions + predictions = np.squeeze(predictions) if is_regression else np.argmax(predictions, axis=1) + + output_predict_file = os.path.join(training_args.output_dir, f"predict_results.txt") + if trainer.is_world_process_zero(): + with open(output_predict_file, "w") as writer: + logger.info(f"***** Predict results *****") + writer.write("index\tprediction\n") + for index, item in enumerate(predictions): + if is_regression: + writer.write(f"{index}\t{item:3.3f}\n") + else: + item = label_list[item] + writer.write(f"{index}\t{item}\n") + + kwargs = {"finetuned_from": model_args.model_name_or_path, "tasks": "text-classification"} + if data_args.task_name is not None: + kwargs["language"] = "en" + kwargs["dataset_tags"] = "glue" + kwargs["dataset_args"] = data_args.task_name + kwargs["dataset"] = f"GLUE {data_args.task_name.upper()}" + + """ + if training_args.push_to_hub: + trainer.push_to_hub(**kwargs) + else: + trainer.create_model_card(**kwargs) + """ + +def _mp_fn(index): + # For xla_spawn (TPUs) + main() + + +if __name__ == "__main__": + main() diff --git a/evaluation-pipeline/finetune_model.sh b/evaluation-pipeline/finetune_model.sh new file mode 100644 index 0000000000000000000000000000000000000000..0c0c372204612f59a8be6f5a46282b640e3c99d3 --- /dev/null +++ b/evaluation-pipeline/finetune_model.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +MODEL_PATH=$1 +TASK_NAME=$2 +SUBTASK_NAME=$3 +LR=${4:-5e-5} # default: 5e-5 +PATIENCE=${5:-10} # default: 10 +BSZ=${6:-64} # default: 64 +EVAL_EVERY=${7:-200} # default: 200 +MAX_EPOCHS=${8:-10} # default: 10 +SEED=${9:-12} # default: 12 + +if [[ "$SUBTASK_NAME" = "mnli" ]]; then + VALID_NAME="validation_matched" + OUT_DIR="mnli" +elif [[ "$SUBTASK_NAME" = "mnli-mm" ]]; then + VALID_NAME="validation_mismatched" + SUBTASK_NAME="mnli" + OUT_DIR="mnli-mm" +else + VALID_NAME="validation" + OUT_DIR=$SUBTASK_NAME +fi + +mkdir -p $MODEL_PATH/finetune/$OUT_DIR/ + +python finetune_classification.py \ + --model_name_or_path $MODEL_PATH \ + --output_dir $MODEL_PATH/finetune/$OUT_DIR/ \ + --train_file filter-data/${TASK_NAME}_filtered/$SUBTASK_NAME.train.json \ + --validation_file filter-data/${TASK_NAME}_filtered/$SUBTASK_NAME.$VALID_NAME.json \ + --do_train \ + --do_eval \ + --do_predict \ + --use_fast_tokenizer False \ + --max_seq_length 128 \ + --per_device_train_batch_size $BSZ \ + --learning_rate $LR \ + --num_train_epochs $MAX_EPOCHS \ + --evaluation_strategy steps \ + --patience $PATIENCE \ + --eval_every $EVAL_EVERY \ + --eval_steps $EVAL_EVERY \ + --save_steps $EVAL_EVERY \ + --overwrite_output_dir \ + --seed $SEED diff --git a/evaluation-pipeline/ignore.txt b/evaluation-pipeline/ignore.txt new file mode 100644 index 0000000000000000000000000000000000000000..bbdc46935fd884b24b2d4efb51523b10f25a6f8b --- /dev/null +++ b/evaluation-pipeline/ignore.txt @@ -0,0 +1,9 @@ +ROUGE +rouge +nin +ond +som +tha +vie +FPR +fpr diff --git a/evaluation-pipeline/lm_eval/__init__.py b/evaluation-pipeline/lm_eval/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..edc726d4f577eac59aff67c3bde1acb4f306b353 --- /dev/null +++ b/evaluation-pipeline/lm_eval/__init__.py @@ -0,0 +1,3 @@ +from .evaluator import evaluate +from .models import get_model, list_model_apis +from .tasks import get_task, get_task_list, list_tasks, get_templates, list_templates, aoa_pred_eval diff --git a/evaluation-pipeline/lm_eval/__pycache__/__init__.cpython-310.pyc b/evaluation-pipeline/lm_eval/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..48056228a6b0d5ce6552cb4a6b2afdca67a0340e Binary files /dev/null and b/evaluation-pipeline/lm_eval/__pycache__/__init__.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/__pycache__/evaluator.cpython-310.pyc b/evaluation-pipeline/lm_eval/__pycache__/evaluator.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9fbe1852de2dce1ad3b144a470a36d3b740adb98 Binary files /dev/null and b/evaluation-pipeline/lm_eval/__pycache__/evaluator.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/api/__init__.py b/evaluation-pipeline/lm_eval/api/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/lm_eval/api/__pycache__/__init__.cpython-310.pyc b/evaluation-pipeline/lm_eval/api/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d4464c5f92e9a1b7b0fda8f425a4c206e4488282 Binary files /dev/null and b/evaluation-pipeline/lm_eval/api/__pycache__/__init__.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/api/__pycache__/metric.cpython-310.pyc b/evaluation-pipeline/lm_eval/api/__pycache__/metric.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..db2af4ab5fd2607204b2a49f36ce67ec79d02b03 Binary files /dev/null and b/evaluation-pipeline/lm_eval/api/__pycache__/metric.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/api/__pycache__/model.cpython-310.pyc b/evaluation-pipeline/lm_eval/api/__pycache__/model.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a02d56bc093ca7b6482d5eb644d06b4ca4efe64e Binary files /dev/null and b/evaluation-pipeline/lm_eval/api/__pycache__/model.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/api/__pycache__/request.cpython-310.pyc b/evaluation-pipeline/lm_eval/api/__pycache__/request.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..57607793f1832a35fcafa634db8f10b80b3dcf06 Binary files /dev/null and b/evaluation-pipeline/lm_eval/api/__pycache__/request.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/api/__pycache__/task.cpython-310.pyc b/evaluation-pipeline/lm_eval/api/__pycache__/task.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..305bcba902705ffffe8570166b0e4bccea60050e Binary files /dev/null and b/evaluation-pipeline/lm_eval/api/__pycache__/task.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/api/__pycache__/utils.cpython-310.pyc b/evaluation-pipeline/lm_eval/api/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ca8fd18fa57d51d29d22e9adbf57d3c3ba4bc36f Binary files /dev/null and b/evaluation-pipeline/lm_eval/api/__pycache__/utils.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/api/metric.py b/evaluation-pipeline/lm_eval/api/metric.py new file mode 100644 index 0000000000000000000000000000000000000000..4f70dd9f374d321a426e7814e83ac5ef531d8d46 --- /dev/null +++ b/evaluation-pipeline/lm_eval/api/metric.py @@ -0,0 +1,371 @@ +import logging +import math +import random +import numpy as np +import sacrebleu +import sklearn.metrics +from collections.abc import Iterable +from rouge_score import rouge_scorer +from typing import List, Mapping, Optional + +from lm_eval.metrics import sari as sari_impl + + +logger = logging.getLogger(__name__) + + +def mean(arr): + return sum(arr) / len(arr) + + +def pop_stddev(arr): + mu = mean(arr) + return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / len(arr)) + + +def sample_stddev(arr): + mu = mean(arr) + if len(arr) == 1: + return 0 + else: + return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / (len(arr) - 1)) + + +def mean_stderr(arr): + return sample_stddev(arr) / math.sqrt(len(arr)) + + +def median(arr): + return arr[len(arr) // 2] + + +def matthews_corrcoef(items): + unzipped_list = list(zip(*items)) + golds = unzipped_list[0] + preds = unzipped_list[1] + return sklearn.metrics.matthews_corrcoef(golds, preds) + + +def f1_score(items): + unzipped_list = list(zip(*items)) + golds = unzipped_list[0] + preds = unzipped_list[1] + fscore = sklearn.metrics.f1_score(golds, preds) + return np.max(fscore) + + +def acc_all(items): + # Only count as correct if all answers are labeled correctly for each question + question_scoring_dict = {} + preds = list(zip(*items))[0] + docs = list(zip(*items))[1] + + for doc, pred in zip(docs, preds): + paragraph_id = doc["idx"]["paragraph"] + question_id = doc["idx"]["question"] + if (paragraph_id, question_id) not in question_scoring_dict: + question_scoring_dict[(paragraph_id, question_id)] = [] + + gold_label = doc["label"] == 1 + + question_scoring_dict[(paragraph_id, question_id)].append(gold_label == pred) + acc = np.mean([int(all(x)) for x in question_scoring_dict.values()]) + return acc + + +def acc_all_stderr(items): + # Only count as correct if all answers are labeled correctly for each question + question_scoring_dict = {} + preds = list(zip(*items))[0] + docs = list(zip(*items))[1] + + for doc, pred in zip(docs, preds): + question_id = doc["idx"]["question"] + if question_id not in question_scoring_dict: + question_scoring_dict[question_id] = [] + + gold_label = doc["label"] == 1 + question_scoring_dict[question_id].append(gold_label == pred) + + acc = mean_stderr([int(all(x)) for x in question_scoring_dict.values()]) + return acc + + +def compute_parity_scores(items): + # Parity checks whether predictions in subsequent pairs of examples are consistent. + # In WinogenderSchema those examples differ only in the gender of the pronoun in the hypothesis. + indices2predictions = {idx: pred for idx, pred in items} + parity_scores = [] + for idx in indices2predictions.keys(): + if (idx % 2) == 0 and (idx + 1) in indices2predictions: + parity_scores.append( + int(indices2predictions[idx] == indices2predictions[idx + 1]) + ) + return parity_scores + + +def parity(items): + parity_scores = compute_parity_scores(items) + if len(parity_scores) > 0: + acc = mean(parity_scores) + else: + acc = 0.0 + return acc + + +def parity_stderr(items): + parity_scores = compute_parity_scores(items) + if len(parity_scores) > 0: + stderr = mean_stderr(parity_scores) + else: + stderr = 0.0 + return stderr + + +def metric_max_over_ground_truths(metric_fn, prediction, ground_truths): + """Compute max metric between prediction and each ground truth.""" + scores_for_ground_truths = [] + for ground_truth in ground_truths: + score = metric_fn(prediction, ground_truth) + scores_for_ground_truths.append(score) + return max(scores_for_ground_truths) + + +def perplexity(items): + return math.exp(-mean(items)) + + +def weighted_mean(items): + a, b = zip(*items) + return sum(a) / sum(b) + + +def weighted_perplexity(items): + return math.exp(-weighted_mean(items)) + + +def bits_per_byte(items): + return -weighted_mean(items) / math.log(2) + + +def sari(sentence_to_simplifiy, generated_sentence, references): + """Implementation of SARI from the authors'.""" + return sari_impl.SARIsent(sentence_to_simplifiy, generated_sentence, references) + + +def bleu(items): + """The Bilingual Evaluation Understudy Score, or BLEU for short, is a metric + for evaluating a generated sentence to a reference sentence. It counts matching + n-grams in the candidate translation to n-grams in the reference text, where + 1-gram or uni-gram would be each token and a bi-gram comparison would be each + word pair. The comparison is made regardless of word order + Source: https://machinelearningmastery.com/calculate-bleu-score-for-text-python/ + Paper: https://www.aclweb.org/anthology/P02-1040/ + + Higher is better + """ + refs = list(zip(*items))[0] + preds = list(zip(*items))[1] + refs, preds = _sacreformat(refs, preds) + return sacrebleu.corpus_bleu(preds, refs).score + + +def chrf(items): + """chrF++ is a tool for automatic evaluation of machine translation output + based on character n-gram precision and recall enhanced with word n-grams. + Source: https://github.com/m-popovic/chrF + Paper: https://www.aclweb.org/anthology/W15-3049.pdf + + Higher is better + """ + refs = list(zip(*items))[0] + preds = list(zip(*items))[1] + refs, preds = _sacreformat(refs, preds) + return sacrebleu.corpus_chrf(preds, refs).score + + +def ter(items): + """Translation Error Rate is an error metric for machine translation that + measures the number of edits required to change a system output into one + of the references + Source: http://www.cs.umd.edu/~snover/tercom/ + Paper: http://mt-archive.info/AMTA-2006-Snover.pdf + + Lower is better + """ + refs = list(zip(*items))[0] + preds = list(zip(*items))[1] + refs, preds = _sacreformat(refs, preds) + return sacrebleu.corpus_ter(preds, refs).score + + +def is_non_str_iterable(obj): + return isinstance(obj, Iterable) and not isinstance(obj, str) + + +def _sacreformat(refs, preds): + """Format refs and preds for sacrebleu corpus calculation. It is very particular""" + # Sacrebleu expects (List[str], List[List[str]) + # e.g. sacrebleu.corpus_bleu([pred_t], [[ref1_stream], [ref2_stream], ...]) + + # Note [ref1_stream] is the first reference for each pred. + # So lists are size N and (M, N) for N preds and M possible refs for each pred + # This is a different order of dimensions that I would expect + + # We expect refs to be List[str] or List[List[str]], the outer list corresponding to preds + # Must become List[List[str]] with the inner list corresponding to preds + if not is_non_str_iterable(refs): + refs = list(refs) + if not is_non_str_iterable(refs[0]): + refs = [[ref] for ref in refs] + refs = list(zip(*refs)) + # Note the number of refs in each ref list much match the number of preds + + # We expect preds to be List[str] or List[List[str]]. Must become List[str] + if not is_non_str_iterable(preds): + preds = list(preds) + if is_non_str_iterable(preds[0]): + assert len(preds[0]) == 1, f"Pred must be a str, was {preds[0]}" + preds = [pred[0] for pred in preds] + + return refs, preds + + +def rouge( + refs: List[str], + pred: str, + rouge_types: Optional[List[str]] = ["rouge1", "rouge2", "rougeL", "rougeLsum"], +) -> Mapping[str, float]: + """ROUGE with multi-reference support + + Implementation based on GEM-metrics: + https://github.com/GEM-benchmark/GEM-metrics/blob/431a8174bd6b3637e8d6118bfad2983e39e99733/gem_metrics/rouge.py + + Args: + refs (List[str]): + A `list` of reference `str`s. + pred (str): + A single prediction `str`s. + rouge_types (Optional[List[str]]): + An optional list of ROUGE types from the set: + ["rouge1", "rouge2", "rougeL", "rougeLsum"] + + Returns: + A `dict` of ROUGE scores. + """ + + # Add newlines between sentences to correctly compute `rougeLsum`. + if "rougeLsum" in rouge_types: + # TODO: Adapt this to handle languages that do not support sentence endings by `.`. + # See GEM-metrics implementation with lang specific `nltk` tokenizers to + # split sentences. + pred = pred.replace(".", ".\n") + refs = [ref.replace(".", ".\n") for ref in refs] + + scorer = rouge_scorer.RougeScorer(rouge_types=rouge_types, use_stemmer=True) + # ROUGE multi-ref jackknifing + if len(refs) > 1: + cur_scores = [scorer.score(ref, pred) for ref in refs] + + # get best score for all leave-one-out sets + best_scores = [] + for leave in range(len(refs)): + cur_scores_leave_one = [ + cur_scores[s] for s in range(len(refs)) if s != leave + ] + best_scores.append( + { + rouge_type: max( + [s[rouge_type] for s in cur_scores_leave_one], + key=lambda s: s.fmeasure, + ) + for rouge_type in rouge_types + } + ) + # average the leave-one-out bests to produce the final score + score = { + rouge_type: rouge_scorer.scoring.Score( + np.mean([b[rouge_type].precision for b in best_scores]), + np.mean([b[rouge_type].recall for b in best_scores]), + np.mean([b[rouge_type].fmeasure for b in best_scores]), + ) + for rouge_type in rouge_types + } + else: + score = scorer.score(refs[0], pred) + # convert the named tuples to plain nested dicts + score = { + rouge_type: { + "precision": score[rouge_type].precision, + "recall": score[rouge_type].recall, + "fmeasure": score[rouge_type].fmeasure, + } + for rouge_type in rouge_types + } + return score + + +# Standard Error Utils + + +class _BootstrapInternal: + def __init__(self, f, n): + self.f = f + self.n = n + + def __call__(self, v): + i, xs = v + rnd = random.Random() + rnd.seed(i) + res = [] + for _ in range(self.n): + res.append(self.f(rnd.choices(xs, k=len(xs)))) + return res + + +def bootstrap_stderr(f, xs, iters): + import multiprocessing as mp + + pool = mp.Pool(mp.cpu_count()) + # this gives a biased estimate of the stderr (i.e w/ the mean, it gives something + # equivalent to stderr calculated without Bessel's correction in the stddev. + # Unfortunately, I haven't been able to figure out what the right correction is + # to make the bootstrap unbiased - I considered multiplying by sqrt(n/(n-1)) but + # that would be ad-hoc and I can't prove that that would actually be an unbiased estimator) + # Thankfully, shouldn't matter because our samples are usually pretty big. + res = [] + chunk_size = min(1000, iters) + from tqdm import tqdm + + logger.info("Bootstrapping for stddev:", f.__name__) + for bootstrap in tqdm( + pool.imap( + _BootstrapInternal(f, chunk_size), + [(i, xs) for i in range(iters // chunk_size)], + ), + total=iters // chunk_size, + ): + # sample w replacement + res.extend(bootstrap) + + pool.close() + return sample_stddev(res) + + +def stderr_for_metric(metric, bootstrap_iters): + bootstrappable = [ + median, + matthews_corrcoef, + f1_score, + perplexity, + bleu, + chrf, + ter, + ] + + if metric in bootstrappable: + return lambda x: bootstrap_stderr(metric, x, iters=bootstrap_iters) + + stderr = {mean: mean_stderr, acc_all: acc_all_stderr, parity: parity_stderr} + + return stderr.get(metric, None) diff --git a/evaluation-pipeline/lm_eval/api/model.py b/evaluation-pipeline/lm_eval/api/model.py new file mode 100644 index 0000000000000000000000000000000000000000..81c0ff0df02c6561469e8cd24c0750d2a9c86711 --- /dev/null +++ b/evaluation-pipeline/lm_eval/api/model.py @@ -0,0 +1,454 @@ +import abc +import hashlib +import json +import os +import torch +import torch.nn.functional as F +from tqdm import tqdm +from typing import Iterable, List, Optional, Tuple, Union +from transformers import BatchEncoding + +from lm_eval.api import utils + + +class LM(abc.ABC): + def __init__(self): + self.cache_hook = CacheHook(None) + + @abc.abstractmethod + def loglikelihood( + self, requests: List[Tuple[str, str]] + ) -> List[Tuple[float, bool]]: + """Compute log-likelihood of generating a continuation from a context. + Downstream tasks should attempt to use loglikelihood instead of other + LM calls whenever possible. + + Args: + requests (List[Tuple[str, str]]): + A list of pairs (context, continuation): + context (str): + Context string. Implementations of LM must be able to handle + an empty context string. + continuation (str): + The continuation over which log likelihood will be calculated. + If there is a word boundary, the space should be in the + continuation. For example, context="hello" continuation=" world" + is correct. + + Returns: + A list of pairs (logprob, isgreedy): + logprob (float): + The log probability of `continuation`. + isgreedy (bool): + Whether `continuation` would be generated by greedy + sampling from `context`. + """ + pass + + @abc.abstractmethod + def loglikelihood_rolling(self, requests: List[Tuple[str, str]]) -> List[float]: + """Compute full log-likelihood of a string, with no truncation, for perplexity computation + - We will use the full max context length of the model. + - For inputs that exceed the max context length, we divide the tokenized string into chunks of up to + the max context length. + - IMPORTANT: Each document's loglikelihood/perplexity is computed *separately*, unlike other implementations + which may simply concatenate multiple documents together. + - IMPORTANT: We maximize the amount of context for each prediction. Specifically, for inputs that we break into + multiple chunks, the last input will still a full-sized context. + Example: + Input tokens: [ 0 1 2 3 4 5 6 7 8 9 ] + Prefix: EOT + Max context length: 4 + Resulting input/prediction pairs: + + INPUT: EOT 0 1 2 + PRED: 0 1 2 3 + + INPUT: 3 4 5 6 + PRED: 4 5 6 7 + + INPUT: 5 6 7 8 + PRED: 8 9 + + Observe that: + 1. Each token is predicted exactly once + 2. For the last pair, we provide the full context, but only score the last two tokens + + Args: + requests (List[Tuple[str, str]]): + A list of paired strings. + string (str): + String for which we are computing per-token loglikelihood. + + Returns: + A list of logprobs on the `continuation`. + """ + pass + + @abc.abstractmethod + def greedy_until(self, requests: List[Tuple[str, dict]]) -> List[str]: + """Generate greedily until a stopping sequence or max generation length. + + Args: + requests (List[Tuple[str, dict]]): + A list of pairs (context, args): + context (str): + Context string. + args (dict): + A dictionary of generation arguments in the form: + { + stop_sequences: str, + max_generation_length: int, + num_fewshot: int + } + + Returns: + A list of strings continuation: + continuation: str + The generated continuation. + """ + pass + + def set_cache_hook(self, cache_hook: "CacheHook"): + self.cache_hook = cache_hook + + +TokenSequence = Union[List[int], torch.LongTensor, torch.Tensor, BatchEncoding] + + +class TokenLM(LM): + """A language model that assumes inputs, and possibly outputs, are + tokenized text as opposed to language model APIs that only support + string-based input and output systems. + """ + + @abc.abstractmethod + def tok_encode(self, string: str): + pass + + @abc.abstractmethod + def tok_decode(self, tokens: Iterable[int]) -> List[str]: + pass + + @property + @abc.abstractmethod + def eot_token(self) -> str: + pass + + @property + @abc.abstractmethod + def eot_token_id(self) -> int: + pass + + @property + @abc.abstractmethod + def max_gen_toks(self) -> int: + """The maximum number of tokens to generate - not including context.""" + pass + + @property + @abc.abstractmethod + def max_length(self) -> int: + """The maximum sequence length of the model.""" + pass + + @property + @abc.abstractmethod + def batch_size(self) -> int: + pass + + @property + @abc.abstractmethod + def device(self) -> Union[int, str, torch.device]: + pass + + def loglikelihood( + self, requests: List[Tuple[str, str]] + ) -> List[Tuple[float, bool]]: + new_requests = [] + for context, continuation in requests: + if context == "": + # End of text as context + context_enc = [self.eot_token_id] + else: + context_enc = self.tok_encode(context) + continuation_enc = self.tok_encode(continuation) + new_requests.append( + ((context, continuation), context_enc, continuation_enc) + ) + return self._loglikelihood_tokens(new_requests) + + def loglikelihood_rolling(self, requests: List[Tuple[str, str]]) -> List[float]: + # TODO: Implement caching once we've confirmed the perplexity implementation + # TODO: Automatic batch size detection for vectorization + loglikelihoods = [] + for (string,) in tqdm(requests): + rolling_token_windows = list( + map( + utils.make_disjoint_window, + utils.get_rolling_token_windows( + token_list=self.tok_encode(string), + prefix_token=self.eot_token_id, + max_seq_len=self.max_length, + context_len=1, + ), + ) + ) + rolling_token_windows = [(None,) + x for x in rolling_token_windows] + # TODO: Extract out this call so it only gets called once and + # also somehow figure out partial caching for that. + string_nll = self._loglikelihood_tokens( + rolling_token_windows, disable_tqdm=True + ) + # Discard `is_greedy` + string_nll = [x[0] for x in string_nll] + string_nll = sum(string_nll) + loglikelihoods.append(string_nll) + return loglikelihoods + + def _loglikelihood_tokens( + self, + requests: List[Tuple[Tuple[str, str], TokenSequence, TokenSequence]], + disable_tqdm: Optional[bool] = False, + ) -> List[Tuple[float, bool]]: + """Helper method for computing log-likelihood of generating a + continuation from a context that have both been tokenized/encoded. + + Args: + requests (List[Tuple[Tuple[str, str], TokenSequence, TokenSequence]]): + A list of pairs ((context, continuation), context_enc, continuation_enc): + context (str): + Context string. Implementations of LM must be able to handle + an empty context string. + continuation (str): + The continuation over which log likelihood will be calculated. + If there is a word boundary, the space should be in the + continuation. For example, context="hello" continuation=" world" + is correct. + context_enc (TokenSequence): + The tokenized/encoded context. + continuation_enc (TokenSequence): + The tokenized/encoded continuation. + disable_tqdm (bool, optional, defaults to False): + Whether to disable `tqdm` progress bar. + + Returns: + A list of pairs (logprob, isgreedy): + logprob (float): + The log probability of `continuation`. + isgreedy (float): + Whether `continuation` would be generated by greedy sampling from `context`. + """ + + def _collate(x): + # The negative sign on len(tokens) sorts descending - this has a few advantages: + # - Time estimates will always be over not underestimates, which is more useful for planning + # - To know the size of a batch when going through the list, you know the first one is always the batch + # padded context length. this is useful to simplify the batching logic and more importantly to make + # automatic adaptive batches much easier to implement + # - Any OOMs will happen right away rather than near the end + tokens = x[1] + x[2] + return -len(tokens), tuple(tokens) + + # TODO: Automatic (variable) batch size detection for vectorization + # TODO: Implement some kind of efficient-request-middleware that lumps together requests with the same context + results = [] + reorder = utils.Reorderer(requests, _collate) + for chunk in utils.chunks( + tqdm(reorder.get_reordered(), disable=disable_tqdm), self.batch_size + ): + inputs = [] + input_lens = [] + cont_tokens_list = [] + padding_length = None + + # Because vectorizing is annoying, we first convert each (context, continuation) pair to padded + # tensors, then we pack them together into a batch, call the model, and then pick it all apart + # again because vectorizing is annoying + for _, context_enc, continuation_enc in chunk: + # sanity check + assert len(context_enc) > 0 + assert len(continuation_enc) > 0 + assert len(continuation_enc) <= self.max_length + + # How this all works: + # CTX CONT + # inp 0 1 2 3|4 5 6 7 8 9 <- last token is deleted by inp[:, :-1] + # gpt2 \ \ + # logits 1 2 3|4 5 6 7 8 9 <- the ctx half gets tossed out by the + # cont_tokens 4 5 6 7 8 9 [:, -len(continuation_enc):, :self.vocab_size] slice + + # When too long to fit in context, truncate from the left + _full_enc = context_enc + continuation_enc + input = torch.tensor( + _full_enc[-(self.max_length + 1) :][:-1], + dtype=torch.long, + ).to(self.device) + (input_len,) = input.shape + + # Since in _collate we make sure length is descending, the longest is always the first one. + padding_length = ( + padding_length if padding_length is not None else input_len + ) + + # Pad length from seq to padding_length + input = torch.cat( + [ + input, # [seq] + torch.zeros(padding_length - input_len, dtype=torch.long).to( + input.device + ), # [padding_length - seq] + ], + dim=0, + ) + inputs.append(input.unsqueeze(0)) # [1, padding_length] + cont_tokens_list.append(continuation_enc) + input_lens.append(input_len) + + batched_inputs = torch.cat(inputs, dim=0) # [batch, padding_length] + multi_logits = F.log_softmax( + self._model_call(batched_inputs), dim=-1 + ).cpu() # [batch, padding_length, vocab] + + for (cache_key, _, _), logits, input, input_len, cont_tokens in zip( + chunk, multi_logits, inputs, input_lens, cont_tokens_list + ): + # Slice to original seq length + cont_len = len(cont_tokens) + # [1, seq, vocab] + logits = logits[input_len - cont_len : input_len].unsqueeze(0) + # Check if per-token argmax is exactly equal to continuation + greedy_tokens = logits.argmax(dim=-1) + # [1, seq] + cont_tokens = torch.tensor(cont_tokens, dtype=torch.long).unsqueeze(0) + max_equal = (greedy_tokens == cont_tokens).all() + + # Obtain logprobs at the corresponding continuation token indices + # last_token_slice = logits[:, -1, :].squeeze(0).tolist() + # [1, seq] + logits = torch.gather(logits, 2, cont_tokens.unsqueeze(-1)).squeeze(-1) + # Answer: (log prob, is-exact-match) + answer = (float(logits.sum()), bool(max_equal)) + # Partial caching + if cache_key is not None: + self.cache_hook.add_partial("loglikelihood", cache_key, answer) + results.append(answer) + return reorder.get_original(results) + + @abc.abstractmethod + def _model_call( + self, inputs: TokenSequence, labels: Optional[TokenSequence] = None + ) -> TokenSequence: + """ + Args: + inputs (TokenSequence): + A list of strings or torch tensor of shape [batch, sequence] + the size of sequence may vary from call to call. + labels (TokenSequence, optional, defaults to None): + A list of strings or torch tensor of shape [batch, sequence] + useful for sequence-to-sequence language models. + + Returns: + A list of ints or torch tensor of shape [batch, sequence, vocab] + with the logits returned from the model. + """ + pass + + @abc.abstractmethod + def _model_generate( + self, inputs: TokenSequence, max_tokens: int, stop: Optional[List[str]] = None + ) -> Union[TokenSequence, List[str]]: + """ + Args: + inputs (TokenSequence): + A list of strings/ints or torch tensor of shape [batch, sequence] + the size of sequence may vary from call to call. + max_tokens (int): + The maximum number of tokens to generate. + stop (List[str], optional, defaults to None): + A list of stopping sequences. If provided, the generation will + stop when any string sequence in the list is encountered. + + Returns: + A list of ints/strings or a torch tensor of shape [batch, sequence, vocab] + with continuation tokens/string of the inputs. + """ + pass + + +def hash_args(attr, args): + data = json.dumps([attr] + list(args)) + return hashlib.sha256(data.encode("utf-8")).hexdigest() + + +class CachingLM: + def __init__(self, lm: LM, cache_db: str): + """LM wrapper that returns cached results if they exist, and uses the underlying LM if not. + + Args: + lm (LM): + The underlying LM to use. + cache_db (str): + Path to the `cache` database. + """ + from sqlitedict import SqliteDict + + self.lm = lm + if os.path.dirname(cache_db): + os.makedirs(os.path.dirname(cache_db), exist_ok=True) + self.cache_db = cache_db + self.dbdict = SqliteDict(cache_db, autocommit=True) + # Add hook to lm + lm.set_cache_hook(self.get_cache_hook()) + + def __getattr__(self, attr): + def fn(requests): + res = [] + remaining_reqs = [] + + # Figure out which ones are cached and which ones are new + for req in requests: + hsh = hash_args(attr, req) + if hsh in self.dbdict: + ob = self.dbdict[hsh] + + assert ob is not None + res.append(ob) + else: + res.append(None) + remaining_reqs.append(req) + + # Actually run the LM on the requests that do not have cached results + rem_res = getattr(self.lm, attr)(remaining_reqs) + + # Stick the new ones back into the list and also cache any of the new ones + resptr = 0 + for req, r in zip(remaining_reqs, rem_res): + while res[resptr] is not None: + resptr += 1 + + res[resptr] = r + # Caching + hsh = hash_args(attr, req) + self.dbdict[hsh] = r + self.dbdict.commit() + return res + + return fn + + def get_cache_hook(self): + return CacheHook(self) + + +class CacheHook: + def __init__(self, cachinglm: CachingLM): + if cachinglm is None: + self.dbdict = None + return + self.dbdict = cachinglm.dbdict + + def add_partial(self, attr, req, res): + if self.dbdict is None: + return + hsh = hash_args(attr, req) + self.dbdict[hsh] = res diff --git a/evaluation-pipeline/lm_eval/api/request.py b/evaluation-pipeline/lm_eval/api/request.py new file mode 100644 index 0000000000000000000000000000000000000000..da938ecf09c9e4dfe9f31f5d87227cd099a010aa --- /dev/null +++ b/evaluation-pipeline/lm_eval/api/request.py @@ -0,0 +1,53 @@ +from typing import Any, Optional + + +REQUEST_RETURN_LENGTHS = { + "loglikelihood": 2, + "greedy_until": None, + "loglikelihood_rolling": None, +} + + +class Request: + def __init__( + self, request_type: str, args: Optional[Any] = None, index: Optional[int] = None + ): + if request_type not in REQUEST_RETURN_LENGTHS.keys(): + raise NotImplementedError( + "The request type {} is not implemented!".format(request_type) + ) + self.request_type = request_type + self.args = args + self.index = index + + def __iter__(self): + if REQUEST_RETURN_LENGTHS[self.request_type] is None: + raise IndexError("This request type does not return multiple arguments!") + for i in range(REQUEST_RETURN_LENGTHS[self.request_type]): + yield Request(self.request_type, self.args, i) + + def __getitem__(self, i: int): + if REQUEST_RETURN_LENGTHS[self.request_type] is None: + raise IndexError("This request type does not return multiple arguments!") + return Request(self.request_type, self.args, i) + + def __eq__(self, other: "Request"): + return ( + self.request_type == other.request_type + and self.args == other.args + and self.index == other.index + ) + + def __repr__(self): + return f"Req_{self.request_type}{self.args}[{self.index}]\n" + + +class RequestFactory: + def __getattr__(self, attr): + def fn(*args): + return Request(attr, args) + + return fn + + +rf = RequestFactory() diff --git a/evaluation-pipeline/lm_eval/api/task.py b/evaluation-pipeline/lm_eval/api/task.py new file mode 100644 index 0000000000000000000000000000000000000000..e92999a9ab522b73cf3eb3be40da885b151bf79e --- /dev/null +++ b/evaluation-pipeline/lm_eval/api/task.py @@ -0,0 +1,874 @@ +import abc +import logging +import re +import datasets +import os +import numpy as np +import promptsource.templates +from abc import abstractmethod +from typing import Callable, List, Mapping, Optional, Tuple, Union + +from lm_eval.api import utils +from lm_eval.api.metric import ( + bits_per_byte, + bleu, + mean, + rouge, + sari, + weighted_perplexity, +) +from lm_eval.api.request import Request, rf + + +logger = logging.getLogger(__name__) + + +class Task(abc.ABC): + """A task represents an entire benchmark including its dataset, problems, + answers, and evaluation methods. See BoolQ for a simple example implementation + + A `doc` can be any python object which represents one instance of evaluation. + This is usually a dictionary e.g. + {"question": ..., "answer": ...} or + {"question": ..., question, answer) + """ + + VERSION = 0 + + # The name of the `Task` benchmark as denoted in the HuggingFace datasets Hub + # or a path to a custom `datasets` loading script. + DATASET_PATH: str = None + + # The name of a subset within `DATASET_PATH`. + DATASET_NAME: str = None + + def __init__( + self, + data_dir: Optional[str] = None, + cache_dir: Optional[str] = None, + download_mode: Optional[str] = None, + file_path: Optional[str] = None, + ): + """ + Args: + data_dir (str, optional, defaults to None): + Stores the path to a local folder containing the `Task`'s data + files. Use this to specify the path to manually downloaded data + (usually when the dataset is not publicly accessible). + cache_dir (str, optional, defaults to None): + The directory to read/write the `Task` dataset. This follows the + HuggingFace `datasets` API with the default cache directory located + at: + `~/.cache/huggingface/datasets` + NOTE: You can change the cache location globally for a given + process by setting the shell environment variable, + `HF_DATASETS_CACHE`, to another directory: + `export HF_DATASETS_CACHE="/path/to/another/directory"` + download_mode (datasets.DownloadMode, optional, defaults to None): + How to treat pre-existing `Task` downloads and data. + - `datasets.DownloadMode.REUSE_DATASET_IF_EXISTS` + Reuse download and reuse dataset. + - `datasets.DownloadMode.REUSE_CACHE_IF_EXISTS` + Reuse download with fresh dataset. + - `datasets.DownloadMode.FORCE_REDOWNLOAD` + Fresh download and fresh dataset. + """ + if file_path: + self.load_from_file(file_path, cache_dir, + download_mode=datasets.DownloadMode.FORCE_REDOWNLOAD) + else: + self.download(data_dir, cache_dir, download_mode) + self._training_docs = None + self._fewshot_docs = None + + def download( + self, + data_dir: Optional[str] = None, + cache_dir: Optional[str] = None, + download_mode: Optional[str] = None, + ): + """Downloads and returns the task dataset. + + NOTE: Override this method to download the dataset from a custom API. + """ + self.dataset = datasets.load_dataset( + path=self.DATASET_PATH, + name=self.DATASET_NAME, + data_dir=data_dir, + cache_dir=cache_dir, + download_mode=download_mode, + ) + + def load_from_file( + self, + file_path, + cache_dir: Optional[str] = None, + download_mode: Optional[str] = None, + ): + # get split names + splits = {} + dirname = os.path.dirname(file_path) + for filename in os.listdir(dirname): + if not filename.startswith(os.path.basename(file_path)): + continue + if filename.count(".") == 2: + splitname = filename.split(".")[1] + splits[splitname] = os.path.join(dirname, filename) + else: + splits["train"] = os.path.join(dirname, filename) + + self.dataset = datasets.load_dataset( + "json", + data_files=splits, + cache_dir=cache_dir, + download_mode=download_mode, + ) + + @abstractmethod + def has_training_docs(self): + """Whether the task has a training set""" + pass + + @abstractmethod + def has_validation_docs(self): + """Whether the task has a validation set""" + pass + + @abstractmethod + def has_test_docs(self): + """Whether the task has a test set""" + pass + + def training_docs(self) -> datasets.Dataset: + """ + Returns: + A dataset of training documents. + """ + return datasets.Dataset.from_dict({}) + + def validation_docs(self) -> datasets.Dataset: + """ + Returns: + A dataset of validation documents. + """ + return datasets.Dataset.from_dict({}) + + def test_docs(self) -> datasets.Dataset: + """ + Returns: + A dataset of test documents. + """ + return datasets.Dataset.from_dict({}) + + def _process_doc(self, doc): + """Override this to process (detokenize, strip, replace, etc.) individual + documents. This can be used in a map over documents of a data split. + E.g. `map(self._process_doc, self.dataset["validation"])` + + Returns: + The processed version of the specified `doc`. + """ + return doc + + @abstractmethod + def doc_to_text(self, doc: dict) -> str: + pass + + @abstractmethod + def doc_to_target(self, doc: dict) -> str: + pass + + @abstractmethod + def construct_requests(self, doc: dict, ctx: str, args: dict) -> List[Request]: + """Uses RequestFactory to construct Requests and returns an iterable of + Requests which will be sent to the LM. + + Args: + doc (dict): + The document as returned from training_docs, validation_docs, or + test_docs. + ctx (str): + The context string, generated by fewshot_context. This includes + the natural language description, as well as the few shot examples, + and the question part of the document for `doc`. + args (dict): + The specifics of the context, including number of few shots. + + Returns: + An iterable of `Request` objects. + """ + pass + + @abstractmethod + def process_results( + self, doc: dict, results: list + ) -> Union[dict, Tuple[dict, dict]]: + """Take a single document and the LM results and evaluates, returning a + dict where keys are the names of sub-metrics and values are the values of + the metric for that one document. + + Args: + doc (dict): + The document as returned from training_docs, validation_docs, or + test_docs. + results (list): + The results of the requests created in construct_requests. + + Returns: + A dict of metric results. + """ + pass + + @abstractmethod + def aggregation(self) -> Mapping[str, Callable]: + """ + Returns: + A dictionary where keys are the names of sub-metrics and values are + functions that aggregate a list of metric scores. + {str: [metric_score] -> float} + """ + pass + + @abstractmethod + def higher_is_better(self) -> Mapping[str, bool]: + """ + Returns: + A dictionary where keys are the names of sub-metrics and values are + whether a higher value of the sub-metric is better. + {str: bool} + """ + pass + + +class PromptSourceTask(Task): + """These are the metrics from promptsource that we have + added default behavior for. If you want to add default behavior for a new metric, + update the functions below. If you want to use one of the following metrics, + *and* add additional custom processing, override `process_results`, `higher_is_better`, and `aggregation`. + """ + + CONFIGURED_RANKED_CHOICE_PS_METRICS = {"Accuracy"} + CONFIGURED_GENERATION_PS_METRICS = {"BLEU", "ROUGE", "SARI"} + SPLIT = None + + def __init__( + self, + data_dir: Optional[str] = None, + cache_dir: Optional[str] = None, + download_mode: Optional[str] = None, + prompt_template: Optional[promptsource.templates.Template] = None, + example_separator: Optional[str] = "\n###\n", + text_target_separator: Optional[str] = " ", + save_examples: Optional[bool] = True, + file_path: Optional[str] = None, + ): + """ + Args: + save_examples (bool, optional, defaults to True): + Whether to save each example and corresponding model predictions + to an output `dict`. + + > Few-shot prompting args + + example_separator (str, optional, defaults to '\n###\n'): + The string that will be used to separate the few-shot examples + from the prompt example. + Default: '\n###\n' + See Webson & Pavlick (2022) https://arxiv.org/pdf/2109.01247.pdf + for justification of this separator. + text_target_separator (str, optional, defaults to ' '): + The string that will be used to separate the prompt example + from the target text. + NOTE: This is assumed to be some form of whitespace-only separation, + e.g. "\n\n", "\t", " ", etc. Otherwise, you should update + the Task's `promptsource` template with the appropriate + separator(s). + Example: + Q: Where is the Eiffel Tower located? A:{text_target_separator}Paris + """ + assert isinstance(save_examples, bool), "`save_examples` must be a bool." + assert isinstance(example_separator, str) and isinstance( + text_target_separator, str + ), "Separator args must be strings." + assert ( + text_target_separator.isspace() + ), f"`text_target_separator` must be whitespace only. Got: `{text_target_separator}`" + + if file_path: + super().__init__(cache_dir=cache_dir, file_path=file_path, + download_mode=download_mode) + else: + super().__init__(data_dir, cache_dir, download_mode) + self.prompt_template = prompt_template + self.save_examples = save_examples + self.example_separator = example_separator + self.text_target_separator = text_target_separator + + def stop_sequences(self) -> List[str]: + """Denote where the generation should end based on the few-shot example + separator. + + NOTE: Override this if you want to use a sequence other than just the + task's few-shot example separator. + """ + return [self.example_separator] + + def max_generation_length(self) -> Optional[int]: + """Denote where the max length of the generation if it is obvious from the task.""" + return None + + def evaluation_docs(self) -> datasets.Dataset: + """Returns the `dataset` split to be used for evaluation.""" + if self.has_test_docs(): + return self.test_docs() + elif self.has_validation_docs(): + return self.validation_docs() + else: + raise RuntimeError("Task has neither test_docs nor validation_docs") + + def fewshot_docs(self) -> datasets.Dataset: + """Returns the `dataset` split that the few-shot examples should be sample + from. This prioritizes the `train_docs` split as the few-shot example + source, then `validation_docs`, and lastly `test_docs`. + """ + if self.has_training_docs(): + return self.training_docs() + elif self.has_validation_docs(): + return self.validation_docs() + else: + return self.test_docs() + + def doc_to_text(self, doc: dict) -> str: + """Returns the input string for a particular example, given the hf dict.""" + if self.prompt_template is None: + return self.null_prompt_doc_to_text(doc) + # is just a string + text, _ = self.prompt_template.apply(doc) + return text + + def null_prompt_doc_to_text(self, doc: dict) -> str: + return NotImplementedError("Override this method in your task!") + + def doc_to_target(self, doc: dict) -> List[str]: + """Returns the target string for a particular example, given the hf dict.""" + if self.prompt_template is None: + return self.null_prompt_doc_to_target(doc) + # is a list of strings where it usually only has one element: the correct answer + _, target = self.prompt_template.apply(doc) + return target + + def null_prompt_doc_to_target(self, doc: dict) -> List[str]: + return NotImplementedError("Override this method in your task!") + + def doc_to_rawtext(self, doc: dict) -> str: + """This should be used for selecting the raw text of the document. + + The current use case is for computing SARI which requires the text + without the prompt. The `text` field is not standardized across tasks + so this is task specific. + """ + raise NotImplementedError("This is task specific.") + + def invalid_doc_for_prompt(self, doc) -> bool: + """Some prompts may not work for some documents. + Default: False + """ + return False + + def format_example(self, text: str, target: str, separator: str) -> str: + """Returns the text and target combined by the specified `separator`""" + return text + separator + target + + def null_prompt_answer_choices(self, doc: dict) -> List[str]: + return NotImplementedError("Override this method in your task!") + + def fewshot_examples( + self, + docs: datasets.Dataset, + k: int, + rng: np.random.Generator, + prompt: dict = None, + ) -> Tuple[List[dict], List[int]]: + """Returns `k` random examples from the set of documents in `docs`. + + Args: + docs (datasets.Dataset): + The dataset of documents to sample few-shot examples from. + k (int): + The number of few-shot examples. + rng (np.random.Generator): + The pseudo-random number generator used to randomly sample examples. + prompt (Optional[dict]): + The prompt document. Specify this to ensure the prompt is not in + the set of few-shot examples. + + Returns: + A tuple of two lists. The first list contains the few-shot examples + """ + random_indices = np.arange(len(docs)).tolist() + rng.shuffle(random_indices) + + i = 0 + fewshot_examples, fewshot_idx = [], [] + for idx in random_indices: + if i >= k: # Break when we have enough examples. + break + is_same_prompt = prompt is not None and all( + # Skips the `doc_id` key assigned to `prompt`s during eval pre-processing. + docs[idx][k] == prompt[k] + for k in docs[idx].keys() + ) + if self.invalid_doc_for_prompt(docs[idx]) or is_same_prompt: + continue + fewshot_examples.append(docs[idx]) + fewshot_idx.append(int(idx)) + i += 1 + return fewshot_examples, fewshot_idx + + def fewshot_context( + self, doc: dict, num_fewshot: int, rng: Optional[np.random.Generator] + ) -> Tuple[str, dict]: + """Returns a few-shot context string made up of `num_fewshot` number of + labeled examples, and an appended prompt example without labeling. + + Args: + doc (dict): + The document as returned from training_docs, validation_docs, or test_docs. + num_fewshot (int): + The number of fewshot examples to provide in the returned context string. + rng (numpy.random.Generator): + The pseudo-random number generator used to randomly sample few-shot examples. + + Returns: + A few-shot context string and a dictionary containing few-shot context + logging information. + ctx (str): + The fewshot context. + logging_info (dict): + A `dict` of logging info that can be used to identify few-shot + sources. + """ + assert ( + rng is not None + ), "A `numpy.random.Generator` argument must be provided to `rng`" + + if num_fewshot == 0: + labeled_examples = "" + fewshot_idx, fewshot_target_idx, fewshot_src = ([], [], None) + else: + # Construct few-shot labeled examples. + fewshot_docs = self.fewshot_docs() + fewshot_src = str(fewshot_docs.split) + fewshot_examples, fewshot_idx = self.fewshot_examples( + fewshot_docs, k=num_fewshot, rng=rng, prompt=doc + ) + labeled_examples_list = [] + fewshot_target_idx = [] + for fewshot_example in fewshot_examples: + text = self.doc_to_text(fewshot_example) + targets = self.doc_to_target(fewshot_example) + # Choose 1 random target from multi-reference targets. + target_idx = int(rng.integers(0, len(targets))) + target = targets[target_idx].strip() + labeled_examples_list.append( + self.format_example(text, target, self.text_target_separator) + ) + fewshot_target_idx.append(target_idx) + labeled_examples = self.example_separator.join(labeled_examples_list) + # Leave an extra `example_separator` right before the prompt. + labeled_examples += self.example_separator + + prompt = self.doc_to_text(doc) + ctx = labeled_examples + prompt + logging_info = { + "fewshot_idx": fewshot_idx, + "fewshot_target_idx": fewshot_target_idx, + "fewshot_source": fewshot_src, + "fewshot_num": num_fewshot, + "ctx": ctx, + } + return ctx, logging_info + + def construct_requests(self, doc: dict, ctx: str, args: dict) -> List[Request]: + """Uses RequestFactory to construct Requests and returns an iterable of + Requests which will be sent to the LM. + + Args: + doc (dict): + The document as returned from training_docs, validation_docs, or + test_docs. + ctx (str): + The context string, generated by fewshot_context. This includes + the natural language description, as well as the few shot examples, + and the question part of the document for `doc`. + args (dict): + The specifics of the context, including number of few shots. + + Returns: + An iterable of `Request` objects. + """ + requests = [] + if self.prompt_template is None: + answer_choices_list = self.null_prompt_answer_choices(doc) + else: + answer_choices_list = self.prompt_template.get_answer_choices_list(doc) + if answer_choices_list: + # If answer_choices_list, then this is a ranked choice prompt. + for answer_choice in answer_choices_list: + ll_answer_choice, _ = rf.loglikelihood( + ctx, self.text_target_separator + answer_choice + ) + requests.append(ll_answer_choice) + else: + # If not, then this is a generation prompt. + request_args = { + "stop_sequences": self.stop_sequences(), + "max_generation_length": self.max_generation_length(), + "num_fewshot": args["num_fewshot"], + } + cont_request = rf.greedy_until(ctx, request_args) + requests.append(cont_request) + return requests + + def process_results( + self, doc: dict, results: list + ) -> Union[dict, Tuple[dict, dict]]: + """Take a single document and the LM results and evaluates, returning a + dict where keys are the names of sub-metrics and values are the values of + the metric for that one document. + + NOTE: This function automates processing by using the `promptsource` + metadata to determine the metric. + + Args: + doc (dict): + The document as returned from training_docs, validation_docs, or + test_docs. + results (list): + The results of the requests created in construct_requests. + + Returns: + A dict of metric results. + """ + if self.prompt_template is None: + answer_choices_list = self.null_prompt_answer_choices(doc) + else: + answer_choices_list = self.prompt_template.get_answer_choices_list(doc) + target = self.doc_to_target(doc) + if answer_choices_list: + # If answer_choices_list, then this is a ranked choice prompt. + # NOTE: In the future, target could be a list of strings. + assert isinstance(target, list) and len(target) == 1 + target = target[0].strip() + try: + target_idx = answer_choices_list.index(target) + except ValueError as e: + print("answer_choices_list:", answer_choices_list) + print("target:", target) + raise ValueError(e) + + pred = answer_choices_list[np.argmax(results)] + out = {} + metric_list = ["Accuracy"] # TODO: CLI framework for specifying metrics + + if self.prompt_template: + metric_list = self.prompt_template.metadata.metrics + for metric in metric_list: + if metric not in self.CONFIGURED_RANKED_CHOICE_PS_METRICS: + logger.warning( + f"Unexpected metric: `{metric}`. Add it, or use a task-specific solution." + ) + if metric == "Accuracy": + out["acc"] = pred == target + # Byte-length normalization. + completion_len = np.array( + [float(len(i)) for i in answer_choices_list] + ) + out["acc_norm"] = ( + 1.0 + if np.argmax(results / completion_len) == target_idx + else 0.0 + ) + # TODO: Add metrics here. + else: + # If not, then this is a generation prompt. + # NOTE: In the future, target will be a list of strings. + assert isinstance(target, list) + pred = results[0].strip() + out = {} + for metric in self.prompt_template.metadata.metrics: + if metric not in self.CONFIGURED_GENERATION_PS_METRICS: + logger.warning( + f"Unexpected metric: `{metric}`. Add it, or use a task-specific solution." + ) + if metric == "BLEU": + out["bleu"] = (target, pred) + elif metric == "ROUGE": + # TODO: This computes all rouge sub-metrics. Find a generic + # way to handle user specified rouge sub-metrics to avoid extra + # compute. + rouge_scores = rouge(target, pred) + # Flatten rouge score dict. + rouge_scores = utils.flatten(rouge_scores) + # Merge all the rouge-type scores into the `out` dict. + out = {**out, **rouge_scores} + elif metric == "SARI": + out["sari"] = sari(self.doc_to_rawtext(doc), pred, target) + + # TODO: Wrap process results s.t. override impl do not + # override the save examples. + if self.save_examples: + example = { + "pred": pred, + "target": target, + "answer_choices_list": answer_choices_list, + } + return out, example + return out + + def aggregation(self) -> Mapping[str, Callable]: + out = {} + metric_list = ["Accuracy"] + if self.prompt_template: + metric_list = self.prompt_template.metadata.metrics + for metric in metric_list: + if metric == "Accuracy": + out["acc"] = mean + out["acc_norm"] = mean + elif metric == "BLEU": + out["bleu"] = bleu + elif metric == "ROUGE": + # TODO: Find a generic way to handle user specified rouge metrics. + out["rouge1_precision"] = mean + out["rouge1_recall"] = mean + out["rouge1_fmeasure"] = mean + + out["rouge2_precision"] = mean + out["rouge2_recall"] = mean + out["rouge2_fmeasure"] = mean + + out["rougeL_precision"] = mean + out["rougeL_recall"] = mean + out["rougeL_fmeasure"] = mean + + out["rougeLsum_precision"] = mean + out["rougeLsum_recall"] = mean + out["rougeLsum_fmeasure"] = mean + elif metric == "SARI": + out["sari"] = mean + return out + + def higher_is_better(self) -> Mapping[str, bool]: + out = {} + for metric in self.prompt_template.metadata.metrics: + if metric == "Accuracy": + out["acc"] = True + out["acc_norm"] = True + elif metric == "BLEU": + out["bleu"] = True + elif metric == "ROUGE": + # TODO: Find a generic way to handle user specified rouge metrics. + out["rouge1_precision"] = True + out["rouge1_recall"] = True + out["rouge1_fmeasure"] = True + + out["rouge2_precision"] = True + out["rouge2_recall"] = True + out["rouge2_fmeasure"] = True + + out["rougeL_precision"] = True + out["rougeL_recall"] = True + out["rougeL_fmeasure"] = True + + out["rougeLsum_precision"] = True + out["rougeLsum_recall"] = True + out["rougeLsum_fmeasure"] = True + elif metric == "SARI": + out["sari"] = True + return out + + def get_logging_info(self): + if self.prompt_template is None: + return self.null_prompt_get_logging_info() + return { + "fixed_answer_choice_list": self.prompt_template.get_fixed_answer_choices_list(), + "dataset_path": self.DATASET_PATH, + "dataset_name": self.DATASET_NAME, + "subset": self.SPLIT, + "prompt_name": self.prompt_template.get_name(), + "prompt_id": self.prompt_template.get_id(), + "prompt_jinja": self.prompt_template.jinja, + "prompt_original_task": self.prompt_template.metadata.original_task, + # Placeholder for comment in post-processing. + "comment": "", + } + + +class TranslationTask(PromptSourceTask): + + # Language specific functions. + @classmethod + def zh_split(cls, zh_text: str) -> List[str]: + """Chinese splitting""" + import jieba + + return [" ".join(jieba.cut(txt.strip())) for txt in zh_text] + + @classmethod + def ja_split(cls, ja_text: str) -> List[str]: + """Japanese splitting""" + import nagisa + + return [" ".join(nagisa.tagging(txt.strip()).words) for txt in ja_text] + + NO_SPACE_LANG = {"zh": zh_split, "ja": ja_split} + + def invalid_doc_for_prompt(self, doc) -> bool: + # Skip docs with empty references. + if self.doc_to_target(doc) == [""]: + return True + return False + + def _get_src_ref_codes(self, template_name: str) -> Tuple[str, str]: + """Returns a 2-tuple of (src_lang, ref_lang) codes from the prompt template name.""" + # Get the lang codes from the dataset name. + lang_pairs = self.DATASET_NAME.split("-") + # Template name ordering defines the src and ref lang codes. + if self.DATASET_NAME in template_name: + return lang_pairs[0], lang_pairs[1] + # Flip the lang pairs following the prompt source. + return lang_pairs[1], lang_pairs[0] + + def process_results( + self, doc: dict, results: list + ) -> Union[dict, Tuple[dict, dict]]: + answer_choices_list = self.prompt_template.get_answer_choices_list(doc) + target = self.doc_to_target(doc) + + # Add spaces between words for BLEU score calculation of target languages like Chinese + _, tar_lang_code = self._get_src_ref_codes(self.prompt_template.name) + if tar_lang_code in self.NO_SPACE_LANG: + target = [self.NO_SPACE_LANG[tar_lang_code]([t])[0] for t in target] + results = self.NO_SPACE_LANG[tar_lang_code](results) + pred = results[0].strip() + + out = {} + for metric in self.prompt_template.metadata.metrics: + assert ( + metric in self.CONFIGURED_GENERATION_PS_METRICS + ), "Unexpected metric. Add it, or use a task-specific solution." + if metric == "BLEU": + out["bleu"] = (target, pred) + elif metric == "ROUGE": + # TODO: This computes all rouge sub-metrics. Find a generic + # way to handle user specified rouge sub-metrics to avoid extra + # compute. + rouge_scores = rouge(target, pred) + # Flatten rouge score dict. + rouge_scores = utils.flatten(rouge_scores) + # Merge all the rouge-type scores into the `out` dict. + out = {**out, **rouge_scores} + + # TODO: Wrap process results s.t. override impl do not + # override the save examples. + if self.save_examples: + example = { + "pred": pred, + "target": target, + "answer_choices_list": answer_choices_list, + } + return out, example + return out + + +class PerplexityTask(PromptSourceTask): + """NOTE: Prompts are ignored for perplexity tasks.""" + + def doc_to_text(self, doc: dict) -> str: + return "" + + def doc_to_target(self, doc: dict) -> List[str]: + """Because prompts are ignored, return the relevant text from doc.""" + raise NotImplementedError() + + def fewshot_context( + self, + doc: dict, + num_fewshot: int, + rng: Optional[np.random.Generator], + ) -> Tuple[str, dict]: + assert ( + num_fewshot == 0 + ), "The number of fewshot examples must be 0 for perplexity tasks." + assert ( + rng is not None + ), "A `numpy.random.Generator` argument must be provided to `rng`" + return ( + "", + { + "fewshot_idx": [], + "fewshot_target_idx": [], + "fewshot_source": None, + "fewshot_num": 0, + "ctx": "", + }, + ) + + def construct_requests(self, doc: dict, ctx: str, args: dict) -> List[Request]: + assert not ctx + string = self.doc_to_target(doc)[0] + req = rf.loglikelihood_rolling(string) + return req + + def process_results( + self, doc: dict, results: list + ) -> Union[dict, Tuple[dict, dict]]: + (loglikelihood,) = results + target = self.doc_to_target(doc)[0] + words = self.count_words(target) + bytes_ = self.count_bytes(target) + + out = { + "word_perplexity": (loglikelihood, words), + "byte_perplexity": (loglikelihood, bytes_), + "bits_per_byte": (loglikelihood, bytes_), + } + if self.save_examples: + return out, { + "word_perplexity_instance": weighted_perplexity( + [(loglikelihood, words)] + ), + "byte_perplexity_instance": weighted_perplexity( + [(loglikelihood, bytes_)] + ), + "bits_per_byte_instance": bits_per_byte([(loglikelihood, bytes_)]), + } + return out + + def aggregation(self) -> Mapping[str, Callable]: + return { + "word_perplexity": weighted_perplexity, + "byte_perplexity": weighted_perplexity, + "bits_per_byte": bits_per_byte, + } + + def higher_is_better(self) -> Mapping[str, bool]: + return { + "word_perplexity": False, + "byte_perplexity": False, + "bits_per_byte": False, + } + + @classmethod + def count_bytes(cls, doc): + return len(doc.encode("utf-8")) + + @classmethod + def count_words(cls, doc): + """Downstream tasks with custom word boundaries should override this!""" + return len(re.split(r"\s+", doc)) + + def get_logging_info(self): + return { + "prompt_name": None, + } diff --git a/evaluation-pipeline/lm_eval/api/utils.py b/evaluation-pipeline/lm_eval/api/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..73d8f8d6cf837efec9d2b147e9cec50f49dde341 --- /dev/null +++ b/evaluation-pipeline/lm_eval/api/utils.py @@ -0,0 +1,357 @@ +import collections +import pathlib +import re +import sys +import torch +from typing import Callable, Final, Iterable, List, Optional, Tuple, Union +from collections.abc import MutableMapping +from transformers import set_seed as transformers_set_seed + + +# General Utils + + +class ExitCodeError(Exception): + pass + + +# Reproducibility utils + + +DEFAULT_SEED: Final[int] = 1234 + + +def set_seed(seed: Optional[int] = DEFAULT_SEED): + transformers_set_seed(seed) + + +# Token Utils + + +def general_detokenize(s: str) -> str: + s = s.replace(" n't", "n't") + s = s.replace(" )", ")") + s = s.replace("( ", "(") + s = s.replace('" ', '"') + s = s.replace(' "', '"') + s = re.sub(r" (['.,])", r"\1", s) + return s + + +def get_rolling_token_windows( + token_list: List[int], prefix_token: int, max_seq_len: int, context_len: int +) -> Iterable[Tuple[List[int], List[int]]]: + """Returns a generator of rolling windows of length `max_seq_len` from a list of tokens. + + Args: + token_list (List[int]): + List of tokens to be predicted. + prefix_token (int): + Dummy token like so the first token has something to condition + on. + max_seq_len (int): + The maximum sequence length of the model or a length we want to use. + context_len (int): + Amount of desired token context for prediction. Needs to be at least 1. + This allows for a rolling window context, letting each prediction + window to potentially condition on some context. + + Returns: + Generator of tuples: (input_tokens, pred_tokens) + NOTE: Score only the last len(pred_tokens) logits of the LM. + """ + assert 1 <= context_len <= max_seq_len + if not token_list: + return + # +1 offset, going from input->preds + pred_len = max_seq_len - context_len + 1 + predicted = 0 + + # Special handling for first window: predict all tokens + first_seq_len = min(max_seq_len, len(token_list)) + yield [prefix_token] + token_list[: first_seq_len - 1], token_list[:first_seq_len] + predicted += first_seq_len + + while predicted < len(token_list): + window_pred_len = min(len(token_list) - predicted, pred_len) + window_end = predicted + window_pred_len + + yield ( + token_list[window_end - max_seq_len - 1 : window_end - 1], + token_list[window_end - window_pred_len : window_end], + ) + predicted += window_pred_len + + +def split_and_pad_windows( + windows: List[Tuple[str, str]], pad_token_id: int, max_seq_len: int +) -> Tuple[List[int], List[int]]: + """Splits and pads a sequence of rolling context and continuation windows + from `get_rolling_token_windows`. + + Example: + [ + ([1] , [23, 19, 3]), # (context, continuation) + ([43], [2, 4]]) + ] + + Output: + [ + [[1],[43]], # Split & padded contexts. + [[23, 19, 3], [2, 4, 1]]` # Split & padded continuations. + ] + where `1` = `pad_token` id. + + Args: + windows (List[Tuple[str, str]]): + A generator of rolling `(context, continuation)` token windows + (tuples). + pad_token_id (int): + The token id to pad with. + max_seq_len (int): + The maximum sequence length of the model or a length we want to use. + + Returns: + A tuple of (context, continuation) padding windows. + """ + contexts, continuations = zip(*windows) + contexts, continuations = list(contexts), list(continuations) + + # Pad contexts: + rollover_context = contexts[-1] + rollover_context_size = len(rollover_context) + # Handle empty final context token list - just add 1 token. + if rollover_context_size == 0: + contexts[-1] += [pad_token_id] + elif rollover_context_size > 1: + for i in range(len(contexts[:-1])): + contexts[i] += [pad_token_id] * (rollover_context_size - len(contexts[i])) + + # Pad continuations: + rollover_continuation = continuations[-1] + rollover_continuation_size = len(rollover_continuation) + is_multiple_windows = len(continuations) > 1 + if rollover_continuation_size < max_seq_len and is_multiple_windows: + continuations[-1] = rollover_continuation + [pad_token_id] * ( + max_seq_len - rollover_continuation_size + ) + return contexts, continuations + + +def make_disjoint_window(pair): + """Takes output from get_rolling_token_windows and makes the context not + overlap with the continuation. + """ + a, b = pair + return a[: -(len(b) - 1)], b + + +def select_continuation_from_batch_left_padding( + generations: Union[List[List[int]], torch.Tensor], max_context_size: int +): + """Select the continuation from the batch, removing prompts of different lengths. + + Args: + generations (Union[List[List[int]], torch.Tensor]): + A tensor or list-of-lists of shape [batch_size, sequence length]. + max_context_size (int): + The size of the biggest context; generations will proceed from that + index. + + Example: + PAD PAD Continue : The dog chased the cat [every day of the week] + Riddle me this : The dog chased the cat [yesterday] PAD PAD PAD PAD + + Output: + [every day of the week] + [yesterday] PAD PAD PAD PAD + """ + return generations[:, max_context_size:] + + +# Container Utils + + +class Reorderer: + def __init__(self, arr, fn): + self.size = len(arr) + arr = list(enumerate(arr)) + arr = group(arr, lambda x: fn(x[1])) + arr = [([y[0] for y in x], x[0][1]) for x in arr] + arr.sort(key=lambda x: fn(x[1])) + self.arr = arr + + def get_reordered(self): + return [x[1] for x in self.arr] + + def get_original(self, newarr): + res = [None] * self.size + cov = [False] * self.size + for (inds, _), v in zip(self.arr, newarr): + for ind in inds: + res[ind] = v + cov[ind] = True + assert all(cov) + return res + + +def flatten( + d: Union[dict, MutableMapping], + parent_key: str = "", + sep: str = "_", +) -> dict: + # From: https://stackoverflow.com/a/6027615 + items = [] + for k, v in d.items(): + new_key = parent_key + sep + k if parent_key else k + if isinstance(v, MutableMapping): + items.extend(flatten(v, new_key, sep=sep).items()) + else: + items.append((new_key, v)) + return dict(items) + + +def join_iters(iterables: Iterable) -> List: + for iterable in iterables: + yield from iterable + + +def chunks(iterable: Iterable, n: int) -> List: + arr = [] + for x in iterable: + arr.append(x) + if len(arr) == n: + yield arr + arr = [] + if arr: + yield arr + + +def group(arr: Iterable, fn: Callable) -> List: + res = collections.defaultdict(list) + for ob in arr: + res[fn(ob)].append(ob) + return list(res.values()) + + +# CLI utils + + +def cli_template_names( + task_name: str, template_names: str, template_idx: int = None +) -> List[str]: + """Returns a selection of template names for a given task and comma- + separated string of template names. + + Example: + cli_template_names("task", "A,B,C") -> ["A", "B", "C"] + + Args: + task_name (str): + Name of the task from which to retrieve template names. + template_names (str): + A string of template names separated by a comma if multiple names + are given. + General Selectors: + "all_templates": + Returns all templates for the task. + "original_templates": + Returns all templates with formatting that matches the + original task design. + template_idx (int, optional, defaults to None): + If given, returns only the template at the given index. + + Returns: + A list of template names. + """ + import lm_eval.tasks + + if template_names == "all_templates": + selections = lm_eval.tasks.list_templates(task_name) + elif template_names == "original_templates": + templates = lm_eval.tasks.get_templates(task_name) + selections = [] + for name in templates.all_template_names: + if templates[name].metadata.original_task is True: + selections.append(name) + if not selections: + raise ValueError(f"No original task templates found for {task_name}") + else: + selections = template_names.split(",") + if template_idx is not None: + selections = [selections[template_idx]] + return selections + + +def parse_cli_args_string(args: str) -> dict: + """Parses a string in the following format to a kwargs dictionary. + "args1=val1,arg2=val2" + """ + # Remove leading whitespace but not trailing in case a `val` contains necessary whitespace. + args = args.lstrip() + if not args: + return {} + arg_list = args.split(",") + args_dict = {} + for arg in arg_list: + # Split on the first `=` to allow for `=`s in `val`. + k, v = arg.split("=", 1) + args_dict[k] = str_to_builtin_type(v) + return args_dict + + +def str_to_builtin_type(s: str) -> str: + for fn in (to_bool, int, float): + try: + return fn(s) + except ValueError: + pass + return s + + +# https://stackoverflow.com/questions/7019283/automatically-type-cast-parameters-in-python +def to_bool(s: str): + if s == "True" or s == "true": + return True + if s == "False" or s == "false": + return False + raise ValueError(f"The input `{s}` is not of boolean form.") + + +# Test utils + + +def find_test_root(*, start_path: pathlib.Path) -> pathlib.Path: + """Search upward in the directory tree to a maximum of three layers + to find and return the package root (containing the 'tests' folder) + """ + cur_path = start_path.resolve() + max_layers = 3 + for _ in range(max_layers): + if (cur_path / "tests" / "test_version_stable.py").exists(): + return cur_path + else: + cur_path = cur_path.parent.resolve() + raise FileNotFoundError( + f"Unable to find package root within {max_layers} upwards" + f"of {start_path}" + ) + + +def run_task_tests(*, task_list: List[str]): + """Find the package root and run the tests for the given tasks.""" + import pytest + + package_root = find_test_root(start_path=pathlib.Path(__file__)) + task_string = " or ".join(task_list) + args = [ + f"{package_root}/tests/test_version_stable.py", + f"--rootdir={package_root}", + "-k", + f"{task_string}", + ] + sys.path.append(str(package_root)) + pytest_return_val = pytest.main(args) + if pytest_return_val: + raise ValueError( + f"Not all tests for the specified tasks ({task_list}) ran successfully! Error code: {pytest_return_val}" + ) diff --git a/evaluation-pipeline/lm_eval/datasets/README.md b/evaluation-pipeline/lm_eval/datasets/README.md new file mode 100644 index 0000000000000000000000000000000000000000..61515a31b9a7d85fbcbb4782f03cc9cf2f44c98a --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/README.md @@ -0,0 +1,6 @@ +# datasets + +This directory contains custom EleutherAI datasets not available in the HuggingFace `datasets` hub. + +In the rare case that you need to add a custom dataset to this collection, follow the +HuggingFace `datasets` guide found [here](https://huggingface.co/docs/datasets/dataset_script). diff --git a/evaluation-pipeline/lm_eval/datasets/__init__.py b/evaluation-pipeline/lm_eval/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/lm_eval/datasets/__pycache__/__init__.cpython-310.pyc b/evaluation-pipeline/lm_eval/datasets/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..96d81210f767fd67cd1662478ab1457a87bbbe3b Binary files /dev/null and b/evaluation-pipeline/lm_eval/datasets/__pycache__/__init__.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/datasets/arithmetic/__init__.py b/evaluation-pipeline/lm_eval/datasets/arithmetic/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/lm_eval/datasets/arithmetic/arithmetic.py b/evaluation-pipeline/lm_eval/datasets/arithmetic/arithmetic.py new file mode 100644 index 0000000000000000000000000000000000000000..649577fcb973d2e8a36197f34f5142be36b9eda7 --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/arithmetic/arithmetic.py @@ -0,0 +1,216 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""GPT-3 Arithmetic Test Dataset.""" + + +import json + +import datasets + + +_CITATION = """\ +@inproceedings{NEURIPS2020_1457c0d6, + author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario}, + booktitle = {Advances in Neural Information Processing Systems}, + editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin}, + pages = {1877--1901}, + publisher = {Curran Associates, Inc.}, + title = {Language Models are Few-Shot Learners}, + url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf}, + volume = {33}, + year = {2020} +} +""" + +_DESCRIPTION = """\ +A small battery of 10 tests that involve asking language models a simple arithmetic +problem in natural language. +""" + +_HOMEPAGE = "https://github.com/openai/gpt-3/tree/master/data" + +# TODO: Add the licence for the dataset here if you can find it +_LICENSE = "" + + +class ArithmeticConfig(datasets.BuilderConfig): + """BuilderConfig for GPT3 Arithmetic Test Dataset.""" + + def __init__(self, url, features, **kwargs): + """BuilderConfig for GPT3 Arithmetic dataset. + + Args: + url: *string*, the url to the specific subset of the GPT3 Arithmetic dataset. + features: *list[string]*, list of the features that will appear in the + feature dict. + """ + # Version history: + super().__init__(version=datasets.Version("0.0.1"), **kwargs) + self.url = url + self.features = features + + +class Arithmetic(datasets.GeneratorBasedBuilder): + """A small battery of 10 tests involving simple arithmetic problems.""" + + BUILDER_CONFIGS = [ + ArithmeticConfig( + name="arithmetic_2da", + url="https://raw.githubusercontent.com/openai/gpt-3/master/data/two_digit_addition.jsonl", + features=datasets.Features( + { + "context": datasets.Value("string"), + "completion": datasets.Value("string"), + } + ), + description="2-digit addition", + ), + ArithmeticConfig( + name="arithmetic_2ds", + url="https://raw.githubusercontent.com/openai/gpt-3/master/data/two_digit_subtraction.jsonl", + features=datasets.Features( + { + "context": datasets.Value("string"), + "completion": datasets.Value("string"), + } + ), + description="2-digit subtraction", + ), + ArithmeticConfig( + name="arithmetic_3da", + url="https://raw.githubusercontent.com/openai/gpt-3/master/data/three_digit_addition.jsonl", + features=datasets.Features( + { + "context": datasets.Value("string"), + "completion": datasets.Value("string"), + } + ), + description="3-digit addition", + ), + ArithmeticConfig( + name="arithmetic_3ds", + url="https://raw.githubusercontent.com/openai/gpt-3/master/data/three_digit_subtraction.jsonl", + features=datasets.Features( + { + "context": datasets.Value("string"), + "completion": datasets.Value("string"), + } + ), + description="3-digit subtraction", + ), + ArithmeticConfig( + name="arithmetic_4da", + url="https://raw.githubusercontent.com/openai/gpt-3/master/data/four_digit_addition.jsonl", + features=datasets.Features( + { + "context": datasets.Value("string"), + "completion": datasets.Value("string"), + } + ), + description="4-digit addition", + ), + ArithmeticConfig( + name="arithmetic_4ds", + url="https://raw.githubusercontent.com/openai/gpt-3/master/data/four_digit_subtraction.jsonl", + features=datasets.Features( + { + "context": datasets.Value("string"), + "completion": datasets.Value("string"), + } + ), + description="4-digit subtraction", + ), + ArithmeticConfig( + name="arithmetic_5da", + url="https://raw.githubusercontent.com/openai/gpt-3/master/data/five_digit_addition.jsonl", + features=datasets.Features( + { + "context": datasets.Value("string"), + "completion": datasets.Value("string"), + } + ), + description="5-digit addition", + ), + ArithmeticConfig( + name="arithmetic_5ds", + url="https://raw.githubusercontent.com/openai/gpt-3/master/data/five_digit_subtraction.jsonl", + features=datasets.Features( + { + "context": datasets.Value("string"), + "completion": datasets.Value("string"), + } + ), + description="5-digit subtraction", + ), + ArithmeticConfig( + name="arithmetic_2dm", + url="https://raw.githubusercontent.com/openai/gpt-3/master/data/two_digit_multiplication.jsonl", + features=datasets.Features( + { + "context": datasets.Value("string"), + "completion": datasets.Value("string"), + } + ), + description="2-digit multiplication", + ), + ArithmeticConfig( + name="arithmetic_1dc", + url="https://raw.githubusercontent.com/openai/gpt-3/master/data/single_digit_three_ops.jsonl", + features=datasets.Features( + { + "context": datasets.Value("string"), + "completion": datasets.Value("string"), + } + ), + description="Single digit 3 operations", + ), + ] + + def _info(self): + return datasets.DatasetInfo( + description=f"{_DESCRIPTION}\n{self.config.description}", + features=self.config.features, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager): + urls = self.config.url + data_dir = dl_manager.download_and_extract(urls) + return [ + datasets.SplitGenerator( + name=datasets.Split.VALIDATION, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "filepath": data_dir, + "split": datasets.Split.VALIDATION, + }, + ), + ] + + # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` + def _generate_examples(self, filepath, split): + with open(filepath, encoding="utf-8") as f: + for key, row in enumerate(f): + data = json.loads(row) + context = ( + data["context"] + .strip() + .replace("\n\n", "\n") + .replace("Q:", "Question:") + .replace("A:", "Answer:") + ) + completion = data["completion"] + yield key, {"context": context, "completion": completion} diff --git a/evaluation-pipeline/lm_eval/datasets/arithmetic/dataset_infos.json b/evaluation-pipeline/lm_eval/datasets/arithmetic/dataset_infos.json new file mode 100644 index 0000000000000000000000000000000000000000..fedef8de2846f1cfbd9afd0aed8ce3509fd7df32 --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/arithmetic/dataset_infos.json @@ -0,0 +1 @@ +{"arithmetic_2da": {"description": "A small battery of 10 tests that involve asking language models a simple arithmetic\nproblem in natural language.\n\n2-digit addition", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n booktitle = {Advances in Neural Information Processing Systems},\n editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n pages = {1877--1901},\n publisher = {Curran Associates, Inc.},\n title = {Language Models are Few-Shot Learners},\n url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n volume = {33},\n year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "arithmetic", "config_name": "arithmetic_2da", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 96624, "num_examples": 2000, "dataset_name": "arithmetic"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/two_digit_addition.jsonl": {"num_bytes": 138624, "checksum": "75a54b7a3db3b23369df74fe440c23025f3d3c51f664300bd3d56632b2617b3d"}}, "download_size": 138624, "post_processing_size": null, "dataset_size": 96624, "size_in_bytes": 235248}, "arithmetic_2ds": {"description": "A small battery of 10 tests that involve asking language models a simple arithmetic\nproblem in natural language.\n\n2-digit subtraction", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n booktitle = {Advances in Neural Information Processing Systems},\n editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n pages = {1877--1901},\n publisher = {Curran Associates, Inc.},\n title = {Language Models are Few-Shot Learners},\n url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n volume = {33},\n year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "arithmetic", "config_name": "arithmetic_2ds", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 98216, "num_examples": 2000, "dataset_name": "arithmetic"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/two_digit_subtraction.jsonl": {"num_bytes": 140216, "checksum": "da956066ff108c00b341d360567472784f5fd872d6465071b44a14291205bc03"}}, "download_size": 140216, "post_processing_size": null, "dataset_size": 98216, "size_in_bytes": 238432}, "arithmetic_3da": {"description": "A small battery of 10 tests that involve asking language models a simple arithmetic\nproblem in natural language.\n\n3-digit addition", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n booktitle = {Advances in Neural Information Processing Systems},\n editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n pages = {1877--1901},\n publisher = {Curran Associates, Inc.},\n title = {Language Models are Few-Shot Learners},\n url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n volume = {33},\n year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "arithmetic", "config_name": "arithmetic_3da", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 102612, "num_examples": 2000, "dataset_name": "arithmetic"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/three_digit_addition.jsonl": {"num_bytes": 144612, "checksum": "124865e30efd2abfbc1855dd34c218fc02d32d780ace970ab9b4ea3fa74c798b"}}, "download_size": 144612, "post_processing_size": null, "dataset_size": 102612, "size_in_bytes": 247224}, "arithmetic_3ds": {"description": "A small battery of 10 tests that involve asking language models a simple arithmetic\nproblem in natural language.\n\n3-digit subtraction", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n booktitle = {Advances in Neural Information Processing Systems},\n editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n pages = {1877--1901},\n publisher = {Curran Associates, Inc.},\n title = {Language Models are Few-Shot Learners},\n url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n volume = {33},\n year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "arithmetic", "config_name": "arithmetic_3ds", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 104150, "num_examples": 2000, "dataset_name": "arithmetic"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/three_digit_subtraction.jsonl": {"num_bytes": 146150, "checksum": "7fc6aaedcb0e2bd17c398dd4147c5585b1e608278a8e98b914e69656707d6a29"}}, "download_size": 146150, "post_processing_size": null, "dataset_size": 104150, "size_in_bytes": 250300}, "arithmetic_4da": {"description": "A small battery of 10 tests that involve asking language models a simple arithmetic\nproblem in natural language.\n\n4-digit addition", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n booktitle = {Advances in Neural Information Processing Systems},\n editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n pages = {1877--1901},\n publisher = {Curran Associates, Inc.},\n title = {Language Models are Few-Shot Learners},\n url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n volume = {33},\n year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "arithmetic", "config_name": "arithmetic_4da", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 108570, "num_examples": 2000, "dataset_name": "arithmetic"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/four_digit_addition.jsonl": {"num_bytes": 150570, "checksum": "459c6f75baa2e8d7cf50bdd07db6d0ca9133a6b137d95d09267db85b6e07f391"}}, "download_size": 150570, "post_processing_size": null, "dataset_size": 108570, "size_in_bytes": 259140}, "arithmetic_4ds": {"description": "A small battery of 10 tests that involve asking language models a simple arithmetic\nproblem in natural language.\n\n4-digit subtraction", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n booktitle = {Advances in Neural Information Processing Systems},\n editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n pages = {1877--1901},\n publisher = {Curran Associates, Inc.},\n title = {Language Models are Few-Shot Learners},\n url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n volume = {33},\n year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "arithmetic", "config_name": "arithmetic_4ds", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 110150, "num_examples": 2000, "dataset_name": "arithmetic"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/four_digit_subtraction.jsonl": {"num_bytes": 152150, "checksum": "0c47db40a10c052ef0cf732a9ef2edaa53d66377d43eb47a9c382d33a8af7102"}}, "download_size": 152150, "post_processing_size": null, "dataset_size": 110150, "size_in_bytes": 262300}, "arithmetic_5da": {"description": "A small battery of 10 tests that involve asking language models a simple arithmetic\nproblem in natural language.\n\n5-digit addition", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n booktitle = {Advances in Neural Information Processing Systems},\n editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n pages = {1877--1901},\n publisher = {Curran Associates, Inc.},\n title = {Language Models are Few-Shot Learners},\n url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n volume = {33},\n year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "arithmetic", "config_name": "arithmetic_5da", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 114476, "num_examples": 2000, "dataset_name": "arithmetic"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/five_digit_addition.jsonl": {"num_bytes": 156476, "checksum": "30ada42efe315b958c6e9649274005d3b720e50298e92c3a2d321f8996e58f54"}}, "download_size": 156476, "post_processing_size": null, "dataset_size": 114476, "size_in_bytes": 270952}, "arithmetic_5ds": {"description": "A small battery of 10 tests that involve asking language models a simple arithmetic\nproblem in natural language.\n\n5-digit subtraction", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n booktitle = {Advances in Neural Information Processing Systems},\n editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n pages = {1877--1901},\n publisher = {Curran Associates, Inc.},\n title = {Language Models are Few-Shot Learners},\n url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n volume = {33},\n year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "arithmetic", "config_name": "arithmetic_5ds", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 116119, "num_examples": 2000, "dataset_name": "arithmetic"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/five_digit_subtraction.jsonl": {"num_bytes": 158119, "checksum": "8b98ccfc943cbf9193bcf1984954aa0b1a4527016072d972a2b055cc1482ca3c"}}, "download_size": 158119, "post_processing_size": null, "dataset_size": 116119, "size_in_bytes": 274238}, "arithmetic_2dm": {"description": "A small battery of 10 tests that involve asking language models a simple arithmetic\nproblem in natural language.\n\n2-digit multiplication", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n booktitle = {Advances in Neural Information Processing Systems},\n editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n pages = {1877--1901},\n publisher = {Curran Associates, Inc.},\n title = {Language Models are Few-Shot Learners},\n url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n volume = {33},\n year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "arithmetic", "config_name": "arithmetic_2dm", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 100685, "num_examples": 2000, "dataset_name": "arithmetic"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/two_digit_multiplication.jsonl": {"num_bytes": 142685, "checksum": "5613d1d1cc3b2c03edc1990252247d34c10ec82944b2cdeb19e71b00f237f431"}}, "download_size": 142685, "post_processing_size": null, "dataset_size": 100685, "size_in_bytes": 243370}, "arithmetic_1dc": {"description": "A small battery of 10 tests that involve asking language models a simple arithmetic\nproblem in natural language.\n\nSingle digit 3 operations", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n booktitle = {Advances in Neural Information Processing Systems},\n editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n pages = {1877--1901},\n publisher = {Curran Associates, Inc.},\n title = {Language Models are Few-Shot Learners},\n url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n volume = {33},\n year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "arithmetic", "config_name": "arithmetic_1dc", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 97651, "num_examples": 2000, "dataset_name": "arithmetic"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/single_digit_three_ops.jsonl": {"num_bytes": 139651, "checksum": "08b34e3272a8ff1d4932d63f251519d14c485c38d582366e1e323d0b859c3925"}}, "download_size": 139651, "post_processing_size": null, "dataset_size": 97651, "size_in_bytes": 237302}} diff --git a/evaluation-pipeline/lm_eval/datasets/asdiv/__init__.py b/evaluation-pipeline/lm_eval/datasets/asdiv/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/lm_eval/datasets/asdiv/asdiv.py b/evaluation-pipeline/lm_eval/datasets/asdiv/asdiv.py new file mode 100644 index 0000000000000000000000000000000000000000..927de50c2b4db05db38e22487cdf8e1c87f6f1ee --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/asdiv/asdiv.py @@ -0,0 +1,111 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""ASDIV dataset.""" + + +import os +import xml.etree.ElementTree as ET + +import datasets + + +_CITATION = """\ +@misc{miao2021diverse, + title={A Diverse Corpus for Evaluating and Developing English Math Word Problem Solvers}, + author={Shen-Yun Miao and Chao-Chun Liang and Keh-Yih Su}, + year={2021}, + eprint={2106.15772}, + archivePrefix={arXiv}, + primaryClass={cs.AI} +} +""" + +_DESCRIPTION = """\ +ASDiv (Academia Sinica Diverse MWP Dataset) is a diverse (in terms of both language +patterns and problem types) English math word problem (MWP) corpus for evaluating +the capability of various MWP solvers. Existing MWP corpora for studying AI progress +remain limited either in language usage patterns or in problem types. We thus present +a new English MWP corpus with 2,305 MWPs that cover more text patterns and most problem +types taught in elementary school. Each MWP is annotated with its problem type and grade +level (for indicating the level of difficulty). +""" + +_HOMEPAGE = "https://github.com/chaochun/nlu-asdiv-dataset" + +# TODO: Add the licence for the dataset here if you can find it +_LICENSE = "" + +_URLS = "https://github.com/chaochun/nlu-asdiv-dataset/archive/55790e5270bb91ccfa5053194b25732534696b50.zip" + + +class ASDiv(datasets.GeneratorBasedBuilder): + """ASDiv: A Diverse Corpus for Evaluating and Developing English Math Word Problem Solvers""" + + VERSION = datasets.Version("0.0.1") + + BUILDER_CONFIGS = [ + datasets.BuilderConfig( + name="asdiv", + version=VERSION, + description="A diverse corpus for evaluating and developing english math word problem solvers", + ) + ] + + def _info(self): + features = datasets.Features( + { + "body": datasets.Value("string"), + "question": datasets.Value("string"), + "solution_type": datasets.Value("string"), + "answer": datasets.Value("string"), + "formula": datasets.Value("string"), + } + ) + return datasets.DatasetInfo( + description=_DESCRIPTION, + features=features, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager): + urls = _URLS + data_dir = dl_manager.download_and_extract(urls) + base_filepath = "nlu-asdiv-dataset-55790e5270bb91ccfa5053194b25732534696b50" + return [ + datasets.SplitGenerator( + name=datasets.Split.VALIDATION, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "filepath": os.path.join( + data_dir, base_filepath, "dataset", "ASDiv.xml" + ), + "split": datasets.Split.VALIDATION, + }, + ), + ] + + # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` + def _generate_examples(self, filepath, split): + tree = ET.parse(filepath) + root = tree.getroot() + for key, problem in enumerate(root.iter("Problem")): + yield key, { + "body": problem.find("Body").text, + "question": problem.find("Question").text, + "solution_type": problem.find("Solution-Type").text, + "answer": problem.find("Answer").text, + "formula": problem.find("Formula").text, + } diff --git a/evaluation-pipeline/lm_eval/datasets/asdiv/dataset_infos.json b/evaluation-pipeline/lm_eval/datasets/asdiv/dataset_infos.json new file mode 100644 index 0000000000000000000000000000000000000000..cfeea0d389b1c0e0150b6b4d16a71ac9ce0dcfbb --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/asdiv/dataset_infos.json @@ -0,0 +1 @@ +{"asdiv": {"description": "ASDiv (Academia Sinica Diverse MWP Dataset) is a diverse (in terms of both language\npatterns and problem types) English math word problem (MWP) corpus for evaluating\nthe capability of various MWP solvers. Existing MWP corpora for studying AI progress\nremain limited either in language usage patterns or in problem types. We thus present\na new English MWP corpus with 2,305 MWPs that cover more text patterns and most problem\ntypes taught in elementary school. Each MWP is annotated with its problem type and grade\nlevel (for indicating the level of difficulty).\n", "citation": "@misc{miao2021diverse,\n title={A Diverse Corpus for Evaluating and Developing English Math Word Problem Solvers},\n author={Shen-Yun Miao and Chao-Chun Liang and Keh-Yih Su},\n year={2021},\n eprint={2106.15772},\n archivePrefix={arXiv},\n primaryClass={cs.AI}\n}\n", "homepage": "https://github.com/chaochun/nlu-asdiv-dataset", "license": "", "features": {"body": {"dtype": "string", "id": null, "_type": "Value"}, "question": {"dtype": "string", "id": null, "_type": "Value"}, "solution_type": {"dtype": "string", "id": null, "_type": "Value"}, "answer": {"dtype": "string", "id": null, "_type": "Value"}, "formula": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "as_div", "config_name": "asdiv", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 501489, "num_examples": 2305, "dataset_name": "as_div"}}, "download_checksums": {"https://github.com/chaochun/nlu-asdiv-dataset/archive/55790e5270bb91ccfa5053194b25732534696b50.zip": {"num_bytes": 440966, "checksum": "8f1fe4f6d5f170ec1e24ab78c244153c14c568b1bb2b1dad0324e71f37939a2d"}}, "download_size": 440966, "post_processing_size": null, "dataset_size": 501489, "size_in_bytes": 942455}} diff --git a/evaluation-pipeline/lm_eval/datasets/coqa/__init__.py b/evaluation-pipeline/lm_eval/datasets/coqa/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/lm_eval/datasets/coqa/coqa.py b/evaluation-pipeline/lm_eval/datasets/coqa/coqa.py new file mode 100644 index 0000000000000000000000000000000000000000..0f0983e17263225739416805602ae6b2725a1e3d --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/coqa/coqa.py @@ -0,0 +1,245 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""CoQA dataset. + +This `CoQA` adds the "additional_answers" feature that's missing in the original +datasets version: +https://github.com/huggingface/datasets/blob/master/datasets/coqa/coqa.py +""" + + +import json + +import datasets + + +_CITATION = """\ +@misc{reddy2018coqa, + title={CoQA: A Conversational Question Answering Challenge}, + author={Siva Reddy and Danqi Chen and Christopher D. Manning}, + year={2018}, + eprint={1808.07042}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} +""" + +_DESCRIPTION = """\ +CoQA is a large-scale dataset for building Conversational Question Answering +systems. The goal of the CoQA challenge is to measure the ability of machines to +understand a text passage and answer a series of interconnected questions that +appear in a conversation. +""" + +_HOMEPAGE = "https://stanfordnlp.github.io/coqa/" + +# TODO: Add the licence for the dataset here if you can find it +_LICENSE = "" + +_URLS = { + "train": "https://nlp.stanford.edu/data/coqa/coqa-train-v1.0.json", + "validation": "https://nlp.stanford.edu/data/coqa/coqa-dev-v1.0.json", +} + +# `additional_answers` are not available in the train set so we fill them with +# empty dicts of the same form. +_EMPTY_ADDITIONAL_ANSWER = { + "0": [ + { + "span_start": -1, + "span_end": -1, + "span_text": "", + "input_text": "", + "turn_id": -1, + } + ], + "1": [ + { + "span_start": -1, + "span_end": -1, + "span_text": "", + "input_text": "", + "turn_id": -1, + } + ], + "2": [ + { + "span_start": -1, + "span_end": -1, + "span_text": "", + "input_text": "", + "turn_id": -1, + } + ], +} + + +class Coqa(datasets.GeneratorBasedBuilder): + """CoQA is a large-scale dataset for building Conversational Question Answering systems.""" + + VERSION = datasets.Version("0.0.1") + + BUILDER_CONFIGS = [ + datasets.BuilderConfig( + name="coqa", version=VERSION, description="The CoQA dataset." + ), + ] + + def _info(self): + features = datasets.Features( + { + "id": datasets.Value("string"), + "source": datasets.Value("string"), + "story": datasets.Value("string"), + "questions": datasets.features.Sequence( + { + "input_text": datasets.Value("string"), + "turn_id": datasets.Value("int32"), + } + ), + "answers": datasets.features.Sequence( + { + "span_start": datasets.Value("int32"), + "span_end": datasets.Value("int32"), + "span_text": datasets.Value("string"), + "input_text": datasets.Value("string"), + "turn_id": datasets.Value("int32"), + } + ), + "additional_answers": { + "0": datasets.features.Sequence( + { + "span_start": datasets.Value("int32"), + "span_end": datasets.Value("int32"), + "span_text": datasets.Value("string"), + "input_text": datasets.Value("string"), + "turn_id": datasets.Value("int32"), + } + ), + "1": datasets.features.Sequence( + { + "span_start": datasets.Value("int32"), + "span_end": datasets.Value("int32"), + "span_text": datasets.Value("string"), + "input_text": datasets.Value("string"), + "turn_id": datasets.Value("int32"), + } + ), + "2": datasets.features.Sequence( + { + "span_start": datasets.Value("int32"), + "span_end": datasets.Value("int32"), + "span_text": datasets.Value("string"), + "input_text": datasets.Value("string"), + "turn_id": datasets.Value("int32"), + } + ), + }, + } + ) + return datasets.DatasetInfo( + description=_DESCRIPTION, + features=features, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager): + urls = {"train": _URLS["train"], "validation": _URLS["validation"]} + data_dirs = dl_manager.download_and_extract(urls) + return [ + datasets.SplitGenerator( + name=datasets.Split.TRAIN, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "filepath": data_dirs["train"], + "split": datasets.Split.TRAIN, + }, + ), + datasets.SplitGenerator( + name=datasets.Split.VALIDATION, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "filepath": data_dirs["validation"], + "split": datasets.Split.VALIDATION, + }, + ), + ] + + # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` + def _generate_examples(self, filepath, split): + with open(filepath, encoding="utf-8") as f: + data = json.load(f) + for row in data["data"]: + id = row["id"] + source = row["source"] + story = row["story"] + questions = [ + {"input_text": q["input_text"], "turn_id": q["turn_id"]} + for q in row["questions"] + ] + answers = [ + { + "span_start": a["span_start"], + "span_end": a["span_end"], + "span_text": a["span_text"], + "input_text": a["input_text"], + "turn_id": a["turn_id"], + } + for a in row["answers"] + ] + if split == datasets.Split.TRAIN: + additional_answers = _EMPTY_ADDITIONAL_ANSWER + else: + additional_answers = { + "0": [ + { + "span_start": a0["span_start"], + "span_end": a0["span_end"], + "span_text": a0["span_text"], + "input_text": a0["input_text"], + "turn_id": a0["turn_id"], + } + for a0 in row["additional_answers"]["0"] + ], + "1": [ + { + "span_start": a1["span_start"], + "span_end": a1["span_end"], + "span_text": a1["span_text"], + "input_text": a1["input_text"], + "turn_id": a1["turn_id"], + } + for a1 in row["additional_answers"]["1"] + ], + "2": [ + { + "span_start": a2["span_start"], + "span_end": a2["span_end"], + "span_text": a2["span_text"], + "input_text": a2["input_text"], + "turn_id": a2["turn_id"], + } + for a2 in row["additional_answers"]["2"] + ], + } + yield row["id"], { + "id": id, + "story": story, + "source": source, + "questions": questions, + "answers": answers, + "additional_answers": additional_answers, + } diff --git a/evaluation-pipeline/lm_eval/datasets/coqa/dataset_infos.json b/evaluation-pipeline/lm_eval/datasets/coqa/dataset_infos.json new file mode 100644 index 0000000000000000000000000000000000000000..ff8ab4a73199e65d8ce4f4e25e0a31f2e492cf22 --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/coqa/dataset_infos.json @@ -0,0 +1 @@ +{"coqa": {"description": "CoQA is a large-scale dataset for building Conversational Question Answering\nsystems. The goal of the CoQA challenge is to measure the ability of machines to\nunderstand a text passage and answer a series of interconnected questions that\nappear in a conversation.\n", "citation": "@misc{reddy2018coqa,\n title={CoQA: A Conversational Question Answering Challenge},\n author={Siva Reddy and Danqi Chen and Christopher D. Manning},\n year={2018},\n eprint={1808.07042},\n archivePrefix={arXiv},\n primaryClass={cs.CL}\n}\n", "homepage": "https://stanfordnlp.github.io/coqa/", "license": "", "features": {"id": {"dtype": "string", "id": null, "_type": "Value"}, "source": {"dtype": "string", "id": null, "_type": "Value"}, "story": {"dtype": "string", "id": null, "_type": "Value"}, "questions": {"feature": {"input_text": {"dtype": "string", "id": null, "_type": "Value"}, "turn_id": {"dtype": "int32", "id": null, "_type": "Value"}}, "length": -1, "id": null, "_type": "Sequence"}, "answers": {"feature": {"span_start": {"dtype": "int32", "id": null, "_type": "Value"}, "span_end": {"dtype": "int32", "id": null, "_type": "Value"}, "span_text": {"dtype": "string", "id": null, "_type": "Value"}, "input_text": {"dtype": "string", "id": null, "_type": "Value"}, "turn_id": {"dtype": "int32", "id": null, "_type": "Value"}}, "length": -1, "id": null, "_type": "Sequence"}, "additional_answers": {"0": {"feature": {"span_start": {"dtype": "int32", "id": null, "_type": "Value"}, "span_end": {"dtype": "int32", "id": null, "_type": "Value"}, "span_text": {"dtype": "string", "id": null, "_type": "Value"}, "input_text": {"dtype": "string", "id": null, "_type": "Value"}, "turn_id": {"dtype": "int32", "id": null, "_type": "Value"}}, "length": -1, "id": null, "_type": "Sequence"}, "1": {"feature": {"span_start": {"dtype": "int32", "id": null, "_type": "Value"}, "span_end": {"dtype": "int32", "id": null, "_type": "Value"}, "span_text": {"dtype": "string", "id": null, "_type": "Value"}, "input_text": {"dtype": "string", "id": null, "_type": "Value"}, "turn_id": {"dtype": "int32", "id": null, "_type": "Value"}}, "length": -1, "id": null, "_type": "Sequence"}, "2": {"feature": {"span_start": {"dtype": "int32", "id": null, "_type": "Value"}, "span_end": {"dtype": "int32", "id": null, "_type": "Value"}, "span_text": {"dtype": "string", "id": null, "_type": "Value"}, "input_text": {"dtype": "string", "id": null, "_type": "Value"}, "turn_id": {"dtype": "int32", "id": null, "_type": "Value"}}, "length": -1, "id": null, "_type": "Sequence"}}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "coqa", "config_name": "coqa", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"train": {"name": "train", "num_bytes": 26250528, "num_examples": 7199, "dataset_name": "coqa"}, "validation": {"name": "validation", "num_bytes": 3765933, "num_examples": 500, "dataset_name": "coqa"}}, "download_checksums": {"https://nlp.stanford.edu/data/coqa/coqa-train-v1.0.json": {"num_bytes": 49001836, "checksum": "b0fdb2bc1bd38dd3ca2ce5fa2ac3e02c6288ac914f241ac409a655ffb6619fa6"}, "https://nlp.stanford.edu/data/coqa/coqa-dev-v1.0.json": {"num_bytes": 9090845, "checksum": "dfa367a9733ce53222918d0231d9b3bedc2b8ee831a2845f62dfc70701f2540a"}}, "download_size": 58092681, "post_processing_size": null, "dataset_size": 30016461, "size_in_bytes": 88109142}} diff --git a/evaluation-pipeline/lm_eval/datasets/drop/__init__.py b/evaluation-pipeline/lm_eval/datasets/drop/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/lm_eval/datasets/drop/dataset_infos.json b/evaluation-pipeline/lm_eval/datasets/drop/dataset_infos.json new file mode 100644 index 0000000000000000000000000000000000000000..f155e7720d0aa5e330496c4f945f7c047424cc61 --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/drop/dataset_infos.json @@ -0,0 +1 @@ +{"drop": {"description": "DROP is a QA dataset which tests comprehensive understanding of paragraphs. In \nthis crowdsourced, adversarially-created, 96k question-answering benchmark, a \nsystem must resolve multiple references in a question, map them onto a paragraph,\nand perform discrete operations over them (such as addition, counting, or sorting).\n", "citation": "@misc{dua2019drop,\n title={DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs}, \n author={Dheeru Dua and Yizhong Wang and Pradeep Dasigi and Gabriel Stanovsky and Sameer Singh and Matt Gardner},\n year={2019},\n eprint={1903.00161},\n archivePrefix={arXiv},\n primaryClass={cs.CL}\n}\n", "homepage": "https://allenai.org/data/drop", "license": "", "features": {"section_id": {"dtype": "string", "id": null, "_type": "Value"}, "passage": {"dtype": "string", "id": null, "_type": "Value"}, "question": {"dtype": "string", "id": null, "_type": "Value"}, "query_id": {"dtype": "string", "id": null, "_type": "Value"}, "answer": {"number": {"dtype": "string", "id": null, "_type": "Value"}, "date": {"day": {"dtype": "string", "id": null, "_type": "Value"}, "month": {"dtype": "string", "id": null, "_type": "Value"}, "year": {"dtype": "string", "id": null, "_type": "Value"}}, "spans": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "worker_id": {"dtype": "string", "id": null, "_type": "Value"}, "hit_id": {"dtype": "string", "id": null, "_type": "Value"}}, "validated_answers": {"feature": {"number": {"dtype": "string", "id": null, "_type": "Value"}, "date": {"day": {"dtype": "string", "id": null, "_type": "Value"}, "month": {"dtype": "string", "id": null, "_type": "Value"}, "year": {"dtype": "string", "id": null, "_type": "Value"}}, "spans": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "worker_id": {"dtype": "string", "id": null, "_type": "Value"}, "hit_id": {"dtype": "string", "id": null, "_type": "Value"}}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "drop", "config_name": "drop", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"train": {"name": "train", "num_bytes": 108858121, "num_examples": 77409, "dataset_name": "drop"}, "validation": {"name": "validation", "num_bytes": 12560739, "num_examples": 9536, "dataset_name": "drop"}}, "download_checksums": {"https://s3-us-west-2.amazonaws.com/allennlp/datasets/drop/drop_dataset.zip": {"num_bytes": 8308692, "checksum": "39d2278a29fd729de301b111a45f434c24834f40df8f4ff116d864589e3249d6"}}, "download_size": 8308692, "post_processing_size": null, "dataset_size": 121418860, "size_in_bytes": 129727552}} diff --git a/evaluation-pipeline/lm_eval/datasets/drop/drop.py b/evaluation-pipeline/lm_eval/datasets/drop/drop.py new file mode 100644 index 0000000000000000000000000000000000000000..d892427b48f82dfb59b33f6844a7a5519ab09cf2 --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/drop/drop.py @@ -0,0 +1,192 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Custom DROP dataset that, unlike HF, keeps all question-answer pairs +# even if there are multiple types of answers for the same question. +"""DROP dataset.""" + + +import json +import os + +import datasets + + +_CITATION = """\ +@misc{dua2019drop, + title={DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs}, + author={Dheeru Dua and Yizhong Wang and Pradeep Dasigi and Gabriel Stanovsky and Sameer Singh and Matt Gardner}, + year={2019}, + eprint={1903.00161}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} +""" + +_DESCRIPTION = """\ +DROP is a QA dataset which tests comprehensive understanding of paragraphs. In +this crowdsourced, adversarially-created, 96k question-answering benchmark, a +system must resolve multiple references in a question, map them onto a paragraph, +and perform discrete operations over them (such as addition, counting, or sorting). +""" + +_HOMEPAGE = "https://allenai.org/data/drop" + +# TODO: Add the licence for the dataset here if you can find it +_LICENSE = "" + +_URLS = { + "drop": "https://s3-us-west-2.amazonaws.com/allennlp/datasets/drop/drop_dataset.zip", +} + +_EMPTY_VALIDATED_ANSWER = [ + { + "number": "", + "date": { + "day": "", + "month": "", + "year": "", + }, + "spans": [], + "worker_id": "", + "hit_id": "", + } +] + + +class Drop(datasets.GeneratorBasedBuilder): + """DROP is a QA dataset which tests comprehensive understanding of paragraphs.""" + + VERSION = datasets.Version("0.0.1") + + BUILDER_CONFIGS = [ + datasets.BuilderConfig( + name="drop", version=VERSION, description="The DROP dataset." + ), + ] + + def _info(self): + features = datasets.Features( + { + "section_id": datasets.Value("string"), + "passage": datasets.Value("string"), + "question": datasets.Value("string"), + "query_id": datasets.Value("string"), + "answer": { + "number": datasets.Value("string"), + "date": { + "day": datasets.Value("string"), + "month": datasets.Value("string"), + "year": datasets.Value("string"), + }, + "spans": datasets.features.Sequence(datasets.Value("string")), + "worker_id": datasets.Value("string"), + "hit_id": datasets.Value("string"), + }, + "validated_answers": datasets.features.Sequence( + { + "number": datasets.Value("string"), + "date": { + "day": datasets.Value("string"), + "month": datasets.Value("string"), + "year": datasets.Value("string"), + }, + "spans": datasets.features.Sequence(datasets.Value("string")), + "worker_id": datasets.Value("string"), + "hit_id": datasets.Value("string"), + } + ), + } + ) + return datasets.DatasetInfo( + description=_DESCRIPTION, + features=features, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager): + urls = _URLS[self.config.name] + data_dir = dl_manager.download_and_extract(urls) + return [ + datasets.SplitGenerator( + name=datasets.Split.TRAIN, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "filepath": os.path.join( + data_dir, "drop_dataset", "drop_dataset_train.json" + ), + "split": "train", + }, + ), + datasets.SplitGenerator( + name=datasets.Split.VALIDATION, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "filepath": os.path.join( + data_dir, "drop_dataset", "drop_dataset_dev.json" + ), + "split": "validation", + }, + ), + ] + + # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` + def _generate_examples(self, filepath, split): + with open(filepath, encoding="utf-8") as f: + data = json.load(f) + key = 0 + for section_id, example in data.items(): + # Each example (passage) has multiple sub-question-answer pairs. + for qa in example["qa_pairs"]: + # Build answer. + answer = qa["answer"] + answer = { + "number": answer["number"], + "date": { + "day": answer["date"].get("day", ""), + "month": answer["date"].get("month", ""), + "year": answer["date"].get("year", ""), + }, + "spans": answer["spans"], + "worker_id": answer.get("worker_id", ""), + "hit_id": answer.get("hit_id", ""), + } + validated_answers = [] + if "validated_answers" in qa: + for validated_answer in qa["validated_answers"]: + va = { + "number": validated_answer.get("number", ""), + "date": { + "day": validated_answer["date"].get("day", ""), + "month": validated_answer["date"].get("month", ""), + "year": validated_answer["date"].get("year", ""), + }, + "spans": validated_answer.get("spans", ""), + "worker_id": validated_answer.get("worker_id", ""), + "hit_id": validated_answer.get("hit_id", ""), + } + validated_answers.append(va) + else: + validated_answers = _EMPTY_VALIDATED_ANSWER + yield key, { + "section_id": section_id, + "passage": example["passage"], + "question": qa["question"], + "query_id": qa["query_id"], + "answer": answer, + "validated_answers": validated_answers, + } + key += 1 diff --git a/evaluation-pipeline/lm_eval/datasets/gsm8k/__init__.py b/evaluation-pipeline/lm_eval/datasets/gsm8k/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/lm_eval/datasets/gsm8k/dataset_infos.json b/evaluation-pipeline/lm_eval/datasets/gsm8k/dataset_infos.json new file mode 100644 index 0000000000000000000000000000000000000000..8984ea99b203fc4e812395c6bcc006c7caa16369 --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/gsm8k/dataset_infos.json @@ -0,0 +1 @@ +{"gsm8k": {"description": "State-of-the-art language models can match human performance on many tasks, but \nthey still struggle to robustly perform multi-step mathematical reasoning. To \ndiagnose the failures of current models and support research, we introduce GSM8K,\na dataset of 8.5K high quality linguistically diverse grade school math word problems.\nWe find that even the largest transformer models fail to achieve high test performance, \ndespite the conceptual simplicity of this problem distribution.\n", "citation": "@misc{cobbe2021training,\n title={Training Verifiers to Solve Math Word Problems},\n author={Karl Cobbe and Vineet Kosaraju and Mohammad Bavarian and Jacob Hilton and Reiichiro Nakano and Christopher Hesse and John Schulman},\n year={2021},\n eprint={2110.14168},\n archivePrefix={arXiv},\n primaryClass={cs.LG}\n}\n", "homepage": "https://github.com/openai/grade-school-math", "license": "", "features": {"question": {"dtype": "string", "id": null, "_type": "Value"}, "answer": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "gsm8_k", "config_name": "gsm8k", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"train": {"name": "train", "num_bytes": 3963202, "num_examples": 7473, "dataset_name": "gsm8_k"}, "test": {"name": "test", "num_bytes": 713732, "num_examples": 1319, "dataset_name": "gsm8_k"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/train.jsonl": {"num_bytes": 4166206, "checksum": "17f347dc51477c50d4efb83959dbb7c56297aba886e5544ee2aaed3024813465"}, "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl": {"num_bytes": 749738, "checksum": "3730d312f6e3440559ace48831e51066acaca737f6eabec99bccb9e4b3c39d14"}}, "download_size": 4915944, "post_processing_size": null, "dataset_size": 4676934, "size_in_bytes": 9592878}} diff --git a/evaluation-pipeline/lm_eval/datasets/gsm8k/gsm8k.py b/evaluation-pipeline/lm_eval/datasets/gsm8k/gsm8k.py new file mode 100644 index 0000000000000000000000000000000000000000..02c981e65482abeedeb0e3bc230910cbc4ff9a03 --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/gsm8k/gsm8k.py @@ -0,0 +1,108 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Grade School Math 8k dataset.""" + + +import json + +import datasets + + +_CITATION = """\ +@misc{cobbe2021training, + title={Training Verifiers to Solve Math Word Problems}, + author={Karl Cobbe and Vineet Kosaraju and Mohammad Bavarian and Jacob Hilton and Reiichiro Nakano and Christopher Hesse and John Schulman}, + year={2021}, + eprint={2110.14168}, + archivePrefix={arXiv}, + primaryClass={cs.LG} +} +""" + +_DESCRIPTION = """\ +State-of-the-art language models can match human performance on many tasks, but +they still struggle to robustly perform multi-step mathematical reasoning. To +diagnose the failures of current models and support research, we introduce GSM8K, +a dataset of 8.5K high quality linguistically diverse grade school math word problems. +We find that even the largest transformer models fail to achieve high test performance, +despite the conceptual simplicity of this problem distribution. +""" + +_HOMEPAGE = "https://github.com/openai/grade-school-math" + +# TODO: Add the licence for the dataset here if you can find it +_LICENSE = "" + +_URLS = { + "train": "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/train.jsonl", + "test": "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl", +} + + +class GSM8K(datasets.GeneratorBasedBuilder): + """Grade School Math 8k""" + + VERSION = datasets.Version("0.0.1") + + BUILDER_CONFIGS = [ + datasets.BuilderConfig( + name="gsm8k", + version=VERSION, + description="The Grade School Math 8k dataset.", + ), + ] + + def _info(self): + features = datasets.Features( + { + "question": datasets.Value("string"), + "answer": datasets.Value("string"), + } + ) + return datasets.DatasetInfo( + description=_DESCRIPTION, + features=features, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager): + urls = {"train": _URLS["train"], "test": _URLS["test"]} + data_dir = dl_manager.download_and_extract(urls) + return [ + datasets.SplitGenerator( + name=datasets.Split.TRAIN, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "filepath": data_dir["train"], + "split": "train", + }, + ), + datasets.SplitGenerator( + name=datasets.Split.TEST, + # These kwargs will be passed to _generate_examples + gen_kwargs={"filepath": data_dir["test"], "split": "test"}, + ), + ] + + # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` + def _generate_examples(self, filepath, split): + with open(filepath, encoding="utf-8") as f: + for key, row in enumerate(f): + data = json.loads(row) + yield key, { + "question": data["question"], + "answer": data["answer"], + } diff --git a/evaluation-pipeline/lm_eval/datasets/headqa/__init__.py b/evaluation-pipeline/lm_eval/datasets/headqa/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/lm_eval/datasets/headqa/dataset_infos.json b/evaluation-pipeline/lm_eval/datasets/headqa/dataset_infos.json new file mode 100644 index 0000000000000000000000000000000000000000..47d6707dbd120aa42cdb37e4822594c5350d91fd --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/headqa/dataset_infos.json @@ -0,0 +1 @@ +{"es": {"description": "HEAD-QA is a multi-choice HEAlthcare Dataset. The questions come from exams to access a specialized position in the\nSpanish healthcare system, and are challenging even for highly specialized humans. They are designed by the Ministerio\nde Sanidad, Consumo y Bienestar Social.\nThe dataset contains questions about the following topics: medicine, nursing, psychology, chemistry, pharmacology and biology.\n", "citation": "@inproceedings{vilares-gomez-rodriguez-2019-head,\n title = \"{HEAD}-{QA}: A Healthcare Dataset for Complex Reasoning\",\n author = \"Vilares, David and\n G{'o}mez-Rodr{'i}guez, Carlos\",\n booktitle = \"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics\",\n month = jul,\n year = \"2019\",\n address = \"Florence, Italy\",\n publisher = \"Association for Computational Linguistics\",\n url = \"https://www.aclweb.org/anthology/P19-1092\",\n doi = \"10.18653/v1/P19-1092\",\n pages = \"960--966\",\n abstract = \"We present HEAD-QA, a multi-choice question answering testbed to encourage research on complex reasoning. The questions come from exams to access a specialized position in the Spanish healthcare system, and are challenging even for highly specialized humans. We then consider monolingual (Spanish) and cross-lingual (to English) experiments with information retrieval and neural techniques. We show that: (i) HEAD-QA challenges current methods, and (ii) the results lag well behind human performance, demonstrating its usefulness as a benchmark for future work.\",\n}\n", "homepage": "https://aghie.github.io/head-qa/", "license": "MIT License", "features": {"name": {"dtype": "string", "id": null, "_type": "Value"}, "year": {"dtype": "string", "id": null, "_type": "Value"}, "category": {"dtype": "string", "id": null, "_type": "Value"}, "qid": {"dtype": "int32", "id": null, "_type": "Value"}, "qtext": {"dtype": "string", "id": null, "_type": "Value"}, "ra": {"dtype": "int32", "id": null, "_type": "Value"}, "answers": [{"aid": {"dtype": "int32", "id": null, "_type": "Value"}, "atext": {"dtype": "string", "id": null, "_type": "Value"}}]}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "head_qa", "config_name": "es", "version": {"version_str": "1.1.0", "description": null, "major": 1, "minor": 1, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 1196021, "num_examples": 2657, "dataset_name": "head_qa"}, "test": {"name": "test", "num_bytes": 1169819, "num_examples": 2742, "dataset_name": "head_qa"}, "validation": {"name": "validation", "num_bytes": 556924, "num_examples": 1366, "dataset_name": "head_qa"}}, "download_checksums": {"https://drive.google.com/uc?export=download&confirm=t&id=1a_95N5zQQoUCq8IBNVZgziHbeM-QxG2t": {"num_bytes": 79365502, "checksum": "6ec29a3f55153d167f0bdf05395558919ba0b1df9c63e79ffceda2a09884ad8b"}}, "download_size": 79365502, "post_processing_size": null, "dataset_size": 2922764, "size_in_bytes": 82288266}, "en": {"description": "HEAD-QA is a multi-choice HEAlthcare Dataset. The questions come from exams to access a specialized position in the\nSpanish healthcare system, and are challenging even for highly specialized humans. They are designed by the Ministerio\nde Sanidad, Consumo y Bienestar Social.\nThe dataset contains questions about the following topics: medicine, nursing, psychology, chemistry, pharmacology and biology.\n", "citation": "@inproceedings{vilares-gomez-rodriguez-2019-head,\n title = \"{HEAD}-{QA}: A Healthcare Dataset for Complex Reasoning\",\n author = \"Vilares, David and\n G{'o}mez-Rodr{'i}guez, Carlos\",\n booktitle = \"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics\",\n month = jul,\n year = \"2019\",\n address = \"Florence, Italy\",\n publisher = \"Association for Computational Linguistics\",\n url = \"https://www.aclweb.org/anthology/P19-1092\",\n doi = \"10.18653/v1/P19-1092\",\n pages = \"960--966\",\n abstract = \"We present HEAD-QA, a multi-choice question answering testbed to encourage research on complex reasoning. The questions come from exams to access a specialized position in the Spanish healthcare system, and are challenging even for highly specialized humans. We then consider monolingual (Spanish) and cross-lingual (to English) experiments with information retrieval and neural techniques. We show that: (i) HEAD-QA challenges current methods, and (ii) the results lag well behind human performance, demonstrating its usefulness as a benchmark for future work.\",\n}\n", "homepage": "https://aghie.github.io/head-qa/", "license": "MIT License", "features": {"name": {"dtype": "string", "id": null, "_type": "Value"}, "year": {"dtype": "string", "id": null, "_type": "Value"}, "category": {"dtype": "string", "id": null, "_type": "Value"}, "qid": {"dtype": "int32", "id": null, "_type": "Value"}, "qtext": {"dtype": "string", "id": null, "_type": "Value"}, "ra": {"dtype": "int32", "id": null, "_type": "Value"}, "answers": [{"aid": {"dtype": "int32", "id": null, "_type": "Value"}, "atext": {"dtype": "string", "id": null, "_type": "Value"}}]}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "head_qa", "config_name": "en", "version": {"version_str": "1.1.0", "description": null, "major": 1, "minor": 1, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 1123151, "num_examples": 2657, "dataset_name": "head_qa"}, "test": {"name": "test", "num_bytes": 1097349, "num_examples": 2742, "dataset_name": "head_qa"}, "validation": {"name": "validation", "num_bytes": 523462, "num_examples": 1366, "dataset_name": "head_qa"}}, "download_checksums": {"https://drive.google.com/uc?export=download&confirm=t&id=1a_95N5zQQoUCq8IBNVZgziHbeM-QxG2t": {"num_bytes": 79365502, "checksum": "6ec29a3f55153d167f0bdf05395558919ba0b1df9c63e79ffceda2a09884ad8b"}}, "download_size": 79365502, "post_processing_size": null, "dataset_size": 2743962, "size_in_bytes": 82109464}} diff --git a/evaluation-pipeline/lm_eval/datasets/headqa/headqa.py b/evaluation-pipeline/lm_eval/datasets/headqa/headqa.py new file mode 100644 index 0000000000000000000000000000000000000000..73be342b98253edfe2e119d5ee2d06d6d7d7af4e --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/headqa/headqa.py @@ -0,0 +1,162 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# NOTE: This is an exact copy of +# https://github.com/huggingface/datasets/blob/3804442bb7cfcb9d52044d92688115cfdc69c2da/datasets/head_qa/head_qa.py +# with the exception of the `image` feature. This is to avoid adding `Pillow` +# as a dependency. +"""HEAD-QA: A Healthcare Dataset for Complex Reasoning.""" + + +import json +import os + +import datasets + + +_CITATION = """\ +@inproceedings{vilares-gomez-rodriguez-2019-head, + title = "{HEAD}-{QA}: A Healthcare Dataset for Complex Reasoning", + author = "Vilares, David and + G{\'o}mez-Rodr{\'i}guez, Carlos", + booktitle = "Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics", + month = jul, + year = "2019", + address = "Florence, Italy", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/P19-1092", + doi = "10.18653/v1/P19-1092", + pages = "960--966", + abstract = "We present HEAD-QA, a multi-choice question answering testbed to encourage research on complex reasoning. The questions come from exams to access a specialized position in the Spanish healthcare system, and are challenging even for highly specialized humans. We then consider monolingual (Spanish) and cross-lingual (to English) experiments with information retrieval and neural techniques. We show that: (i) HEAD-QA challenges current methods, and (ii) the results lag well behind human performance, demonstrating its usefulness as a benchmark for future work.", +} +""" + +_DESCRIPTION = """\ +HEAD-QA is a multi-choice HEAlthcare Dataset. The questions come from exams to access a specialized position in the +Spanish healthcare system, and are challenging even for highly specialized humans. They are designed by the Ministerio +de Sanidad, Consumo y Bienestar Social. +The dataset contains questions about the following topics: medicine, nursing, psychology, chemistry, pharmacology and biology. +""" + +_HOMEPAGE = "https://aghie.github.io/head-qa/" + +_LICENSE = "MIT License" + +_URL = "https://drive.google.com/uc?export=download&confirm=t&id=1a_95N5zQQoUCq8IBNVZgziHbeM-QxG2t" + +_DIRS = {"es": "HEAD", "en": "HEAD_EN"} + + +class HeadQA(datasets.GeneratorBasedBuilder): + """HEAD-QA: A Healthcare Dataset for Complex Reasoning""" + + VERSION = datasets.Version("1.1.0") + + BUILDER_CONFIGS = [ + datasets.BuilderConfig( + name="es", version=VERSION, description="Spanish HEAD dataset" + ), + datasets.BuilderConfig( + name="en", version=VERSION, description="English HEAD dataset" + ), + ] + + DEFAULT_CONFIG_NAME = "es" + + def _info(self): + return datasets.DatasetInfo( + description=_DESCRIPTION, + features=datasets.Features( + { + "name": datasets.Value("string"), + "year": datasets.Value("string"), + "category": datasets.Value("string"), + "qid": datasets.Value("int32"), + "qtext": datasets.Value("string"), + "ra": datasets.Value("int32"), + "answers": [ + { + "aid": datasets.Value("int32"), + "atext": datasets.Value("string"), + } + ], + } + ), + supervised_keys=None, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager): + """Returns SplitGenerators.""" + data_dir = dl_manager.download_and_extract(_URL) + + dir = _DIRS[self.config.name] + data_lang_dir = os.path.join(data_dir, dir) + + return [ + datasets.SplitGenerator( + name=datasets.Split.TRAIN, + gen_kwargs={ + "data_dir": data_dir, + "filepath": os.path.join(data_lang_dir, f"train_{dir}.json"), + }, + ), + datasets.SplitGenerator( + name=datasets.Split.TEST, + gen_kwargs={ + "data_dir": data_dir, + "filepath": os.path.join(data_lang_dir, f"test_{dir}.json"), + }, + ), + datasets.SplitGenerator( + name=datasets.Split.VALIDATION, + gen_kwargs={ + "data_dir": data_dir, + "filepath": os.path.join(data_lang_dir, f"dev_{dir}.json"), + }, + ), + ] + + def _generate_examples(self, data_dir, filepath): + """Yields examples.""" + with open(filepath, encoding="utf-8") as f: + head_qa = json.load(f) + for exam_id, exam in enumerate(head_qa["exams"]): + content = head_qa["exams"][exam] + name = content["name"].strip() + year = content["year"].strip() + category = content["category"].strip() + for question in content["data"]: + qid = int(question["qid"].strip()) + qtext = question["qtext"].strip() + ra = int(question["ra"].strip()) + + aids = [answer["aid"] for answer in question["answers"]] + atexts = [answer["atext"].strip() for answer in question["answers"]] + answers = [ + {"aid": aid, "atext": atext} for aid, atext in zip(aids, atexts) + ] + + id_ = f"{exam_id}_{qid}" + yield id_, { + "name": name, + "year": year, + "category": category, + "qid": qid, + "qtext": qtext, + "ra": ra, + "answers": answers, + } diff --git a/evaluation-pipeline/lm_eval/datasets/hendrycks_ethics/__init__.py b/evaluation-pipeline/lm_eval/datasets/hendrycks_ethics/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/lm_eval/datasets/hendrycks_ethics/dataset_infos.json b/evaluation-pipeline/lm_eval/datasets/hendrycks_ethics/dataset_infos.json new file mode 100644 index 0000000000000000000000000000000000000000..54aecc3bed829a951c32ec612db33252097c6ba6 --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/hendrycks_ethics/dataset_infos.json @@ -0,0 +1 @@ +{"commonsense": {"description": "The ETHICS dataset is a benchmark that spans concepts in justice, well-being,\nduties, virtues, and commonsense morality. Models predict widespread moral\njudgments about diverse text scenarios. This requires connecting physical and\nsocial world knowledge to value judgements, a capability that may enable us\nto steer chatbot outputs or eventually regularize open-ended reinforcement\nlearning agents.\n\nThe Commonsense subset contains examples focusing on moral standards and principles that most people intuitively accept.", "citation": "@article{hendrycks2021ethics\n title={Aligning AI With Shared Human Values},\n author={Dan Hendrycks and Collin Burns and Steven Basart and Andrew Critch and Jerry Li and Dawn Song and Jacob Steinhardt},\n journal={Proceedings of the International Conference on Learning Representations (ICLR)},\n year={2021}\n}\n", "homepage": "https://github.com/hendrycks/ethics", "license": "", "features": {"label": {"dtype": "int32", "id": null, "_type": "Value"}, "input": {"dtype": "string", "id": null, "_type": "Value"}, "is_short": {"dtype": "bool", "id": null, "_type": "Value"}, "edited": {"dtype": "bool", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "hendrycks_ethics", "config_name": "commonsense", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"train": {"name": "train", "num_bytes": 14435215, "num_examples": 13910, "dataset_name": "hendrycks_ethics"}, "test": {"name": "test", "num_bytes": 3150094, "num_examples": 3885, "dataset_name": "hendrycks_ethics"}}, "download_checksums": {"https://people.eecs.berkeley.edu/~hendrycks/ethics.tar": {"num_bytes": 35585024, "checksum": "40acbf1ac0da79a2aabef394d58889136b8d38b05be09482006de2453fb06333"}}, "download_size": 35585024, "post_processing_size": null, "dataset_size": 17585309, "size_in_bytes": 53170333}, "deontology": {"description": "The ETHICS dataset is a benchmark that spans concepts in justice, well-being,\nduties, virtues, and commonsense morality. Models predict widespread moral\njudgments about diverse text scenarios. This requires connecting physical and\nsocial world knowledge to value judgements, a capability that may enable us\nto steer chatbot outputs or eventually regularize open-ended reinforcement\nlearning agents.\n\nThe Deontology subset contains examples focusing on whether an act is required, permitted, or forbidden according to a set of rules or constraints", "citation": "@article{hendrycks2021ethics\n title={Aligning AI With Shared Human Values},\n author={Dan Hendrycks and Collin Burns and Steven Basart and Andrew Critch and Jerry Li and Dawn Song and Jacob Steinhardt},\n journal={Proceedings of the International Conference on Learning Representations (ICLR)},\n year={2021}\n}\n", "homepage": "https://github.com/hendrycks/ethics", "license": "", "features": {"group_id": {"dtype": "int32", "id": null, "_type": "Value"}, "label": {"dtype": "int32", "id": null, "_type": "Value"}, "scenario": {"dtype": "string", "id": null, "_type": "Value"}, "excuse": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "hendrycks_ethics", "config_name": "deontology", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"train": {"name": "train", "num_bytes": 1931475, "num_examples": 18164, "dataset_name": "hendrycks_ethics"}, "test": {"name": "test", "num_bytes": 384602, "num_examples": 3596, "dataset_name": "hendrycks_ethics"}}, "download_checksums": {"https://people.eecs.berkeley.edu/~hendrycks/ethics.tar": {"num_bytes": 35585024, "checksum": "40acbf1ac0da79a2aabef394d58889136b8d38b05be09482006de2453fb06333"}}, "download_size": 35585024, "post_processing_size": null, "dataset_size": 2316077, "size_in_bytes": 37901101}, "justice": {"description": "The ETHICS dataset is a benchmark that spans concepts in justice, well-being,\nduties, virtues, and commonsense morality. Models predict widespread moral\njudgments about diverse text scenarios. This requires connecting physical and\nsocial world knowledge to value judgements, a capability that may enable us\nto steer chatbot outputs or eventually regularize open-ended reinforcement\nlearning agents.\n\nThe Justice subset contains examples focusing on how a character treats another person", "citation": "@article{hendrycks2021ethics\n title={Aligning AI With Shared Human Values},\n author={Dan Hendrycks and Collin Burns and Steven Basart and Andrew Critch and Jerry Li and Dawn Song and Jacob Steinhardt},\n journal={Proceedings of the International Conference on Learning Representations (ICLR)},\n year={2021}\n}\n", "homepage": "https://github.com/hendrycks/ethics", "license": "", "features": {"group_id": {"dtype": "int32", "id": null, "_type": "Value"}, "label": {"dtype": "int32", "id": null, "_type": "Value"}, "scenario": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "hendrycks_ethics", "config_name": "justice", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"train": {"name": "train", "num_bytes": 2516501, "num_examples": 21791, "dataset_name": "hendrycks_ethics"}, "test": {"name": "test", "num_bytes": 309427, "num_examples": 2704, "dataset_name": "hendrycks_ethics"}}, "download_checksums": {"https://people.eecs.berkeley.edu/~hendrycks/ethics.tar": {"num_bytes": 35585024, "checksum": "40acbf1ac0da79a2aabef394d58889136b8d38b05be09482006de2453fb06333"}}, "download_size": 35585024, "post_processing_size": null, "dataset_size": 2825928, "size_in_bytes": 38410952}, "utilitarianism": {"description": "The ETHICS dataset is a benchmark that spans concepts in justice, well-being,\nduties, virtues, and commonsense morality. Models predict widespread moral\njudgments about diverse text scenarios. This requires connecting physical and\nsocial world knowledge to value judgements, a capability that may enable us\nto steer chatbot outputs or eventually regularize open-ended reinforcement\nlearning agents.\n\nThe Utilitarianism subset contains scenarios that should be ranked from most pleasant to least pleasant for the person in the scenario", "citation": "@article{hendrycks2021ethics\n title={Aligning AI With Shared Human Values},\n author={Dan Hendrycks and Collin Burns and Steven Basart and Andrew Critch and Jerry Li and Dawn Song and Jacob Steinhardt},\n journal={Proceedings of the International Conference on Learning Representations (ICLR)},\n year={2021}\n}\n", "homepage": "https://github.com/hendrycks/ethics", "license": "", "features": {"activity": {"dtype": "string", "id": null, "_type": "Value"}, "baseline": {"dtype": "string", "id": null, "_type": "Value"}, "rating": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "hendrycks_ethics", "config_name": "utilitarianism", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"train": {"name": "train", "num_bytes": 2241770, "num_examples": 13738, "dataset_name": "hendrycks_ethics"}, "test": {"name": "test", "num_bytes": 749768, "num_examples": 4808, "dataset_name": "hendrycks_ethics"}}, "download_checksums": {"https://people.eecs.berkeley.edu/~hendrycks/ethics.tar": {"num_bytes": 35585024, "checksum": "40acbf1ac0da79a2aabef394d58889136b8d38b05be09482006de2453fb06333"}}, "download_size": 35585024, "post_processing_size": null, "dataset_size": 2991538, "size_in_bytes": 38576562}, "virtue": {"description": "The ETHICS dataset is a benchmark that spans concepts in justice, well-being,\nduties, virtues, and commonsense morality. Models predict widespread moral\njudgments about diverse text scenarios. This requires connecting physical and\nsocial world knowledge to value judgements, a capability that may enable us\nto steer chatbot outputs or eventually regularize open-ended reinforcement\nlearning agents.\n\nThe Virtue subset contains scenarios focusing on whether virtues or vices are being exemplified", "citation": "@article{hendrycks2021ethics\n title={Aligning AI With Shared Human Values},\n author={Dan Hendrycks and Collin Burns and Steven Basart and Andrew Critch and Jerry Li and Dawn Song and Jacob Steinhardt},\n journal={Proceedings of the International Conference on Learning Representations (ICLR)},\n year={2021}\n}\n", "homepage": "https://github.com/hendrycks/ethics", "license": "", "features": {"group_id": {"dtype": "int32", "id": null, "_type": "Value"}, "label": {"dtype": "int32", "id": null, "_type": "Value"}, "scenario": {"dtype": "string", "id": null, "_type": "Value"}, "trait": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "hendrycks_ethics", "config_name": "virtue", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"train": {"name": "train", "num_bytes": 2640328, "num_examples": 28245, "dataset_name": "hendrycks_ethics"}, "test": {"name": "test", "num_bytes": 473473, "num_examples": 4975, "dataset_name": "hendrycks_ethics"}}, "download_checksums": {"https://people.eecs.berkeley.edu/~hendrycks/ethics.tar": {"num_bytes": 35585024, "checksum": "40acbf1ac0da79a2aabef394d58889136b8d38b05be09482006de2453fb06333"}}, "download_size": 35585024, "post_processing_size": null, "dataset_size": 3113801, "size_in_bytes": 38698825}} diff --git a/evaluation-pipeline/lm_eval/datasets/hendrycks_ethics/hendrycks_ethics.py b/evaluation-pipeline/lm_eval/datasets/hendrycks_ethics/hendrycks_ethics.py new file mode 100644 index 0000000000000000000000000000000000000000..520f912e27e669db6fd71d7b88e68836645b83f4 --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/hendrycks_ethics/hendrycks_ethics.py @@ -0,0 +1,229 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""ETHICS dataset.""" +# TODO: Add the `hard` dataset splits. + + +import csv +import os + +import datasets + + +_CITATION = """\ +@article{hendrycks2021ethics + title={Aligning AI With Shared Human Values}, + author={Dan Hendrycks and Collin Burns and Steven Basart and Andrew Critch and Jerry Li and Dawn Song and Jacob Steinhardt}, + journal={Proceedings of the International Conference on Learning Representations (ICLR)}, + year={2021} +} +""" + +_DESCRIPTION = """\ +The ETHICS dataset is a benchmark that spans concepts in justice, well-being, +duties, virtues, and commonsense morality. Models predict widespread moral +judgments about diverse text scenarios. This requires connecting physical and +social world knowledge to value judgements, a capability that may enable us +to steer chatbot outputs or eventually regularize open-ended reinforcement +learning agents. +""" + +_HOMEPAGE = "https://github.com/hendrycks/ethics" + +# TODO: Add the licence for the dataset here if you can find it +_LICENSE = "" + +_URLS = "https://people.eecs.berkeley.edu/~hendrycks/ethics.tar" + + +class EthicsConfig(datasets.BuilderConfig): + """BuilderConfig for Hendrycks ETHICS.""" + + def __init__(self, prefix, features, **kwargs): + """BuilderConfig for Hendrycks ETHICS. + + Args: + prefix: *string*, prefix to add to the dataset name for path location. + features: *list[string]*, list of the features that will appear in the + feature dict. + """ + # Version history: + super().__init__(version=datasets.Version("0.0.1"), **kwargs) + self.prefix = prefix + self.features = features + + +class HendrycksEthics(datasets.GeneratorBasedBuilder): + """The ETHICS dataset is a benchmark that spans concepts in justice, well-being, duties, virtues, and commonsense morality.""" + + BUILDER_CONFIGS = [ + EthicsConfig( + name="commonsense", + prefix="cm", + features=datasets.Features( + { + "label": datasets.Value("int32"), + "input": datasets.Value("string"), + "is_short": datasets.Value("bool"), + "edited": datasets.Value("bool"), + } + ), + description="The Commonsense subset contains examples focusing on moral standards and principles that most people intuitively accept.", + ), + EthicsConfig( + name="deontology", + prefix="deontology", + features=datasets.Features( + { + "group_id": datasets.Value("int32"), + "label": datasets.Value("int32"), + "scenario": datasets.Value("string"), + "excuse": datasets.Value("string"), + } + ), + description="The Deontology subset contains examples focusing on whether an act is required, permitted, or forbidden according to a set of rules or constraints", + ), + EthicsConfig( + name="justice", + prefix="justice", + features=datasets.Features( + { + "group_id": datasets.Value("int32"), + "label": datasets.Value("int32"), + "scenario": datasets.Value("string"), + } + ), + description="The Justice subset contains examples focusing on how a character treats another person", + ), + EthicsConfig( + name="utilitarianism", + prefix="util", + features=datasets.Features( + { + "activity": datasets.Value("string"), + "baseline": datasets.Value("string"), + "rating": datasets.Value("string"), # Empty rating. + } + ), + description="The Utilitarianism subset contains scenarios that should be ranked from most pleasant to least pleasant for the person in the scenario", + ), + EthicsConfig( + name="virtue", + prefix="virtue", + features=datasets.Features( + { + "group_id": datasets.Value("int32"), + "label": datasets.Value("int32"), + "scenario": datasets.Value("string"), + "trait": datasets.Value("string"), + } + ), + description="The Virtue subset contains scenarios focusing on whether virtues or vices are being exemplified", + ), + ] + + def _info(self): + return datasets.DatasetInfo( + description=f"{_DESCRIPTION}\n{self.config.description}", + features=self.config.features, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager): + urls = _URLS + data_dir = dl_manager.download_and_extract(urls) + return [ + datasets.SplitGenerator( + name=datasets.Split.TRAIN, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "filepath": os.path.join( + data_dir, + "ethics", + self.config.name, + f"{self.config.prefix}_train.csv", + ), + "split": "train", + }, + ), + datasets.SplitGenerator( + name=datasets.Split.TEST, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "filepath": os.path.join( + data_dir, + "ethics", + self.config.name, + f"{self.config.prefix}_test.csv", + ), + "split": "test", + }, + ), + ] + + # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` + def _generate_examples(self, filepath, split): + with open(filepath, newline="") as f: + if self.config.name == "utilitarianism": + contents = csv.DictReader(f, fieldnames=["activity", "baseline"]) + else: + contents = csv.DictReader(f) + # For subsets with grouped scenarios, tag them with an id. + group_id = 0 + for key, row in enumerate(contents): + if self.config.name == "deontology": + # Scenarios come in groups of 4. + if key % 4 == 0 and key != 0: + group_id += 1 + yield key, { + "group_id": group_id, + "label": row["label"], + "scenario": row["scenario"], + "excuse": row["excuse"], + } + elif self.config.name == "justice": + # Scenarios come in groups of 4. + if key % 4 == 0 and key != 0: + group_id += 1 + yield key, { + "group_id": group_id, + "label": row["label"], + "scenario": row["scenario"], + } + elif self.config.name == "commonsense": + yield key, { + "label": row["label"], + "input": row["input"], + "is_short": row["is_short"], + "edited": row["edited"], + } + elif self.config.name == "virtue": + # Scenarios come in groups of 5. + if key % 5 == 0 and key != 0: + group_id += 1 + scenario, trait = row["scenario"].split(" [SEP] ") + yield key, { + "group_id": group_id, + "label": row["label"], + "scenario": scenario, + "trait": trait, + } + elif self.config.name == "utilitarianism": + yield key, { + "activity": row["activity"], + "baseline": row["baseline"], + "rating": "", + } diff --git a/evaluation-pipeline/lm_eval/datasets/hendrycks_math/__init__.py b/evaluation-pipeline/lm_eval/datasets/hendrycks_math/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/lm_eval/datasets/hendrycks_math/dataset_infos.json b/evaluation-pipeline/lm_eval/datasets/hendrycks_math/dataset_infos.json new file mode 100644 index 0000000000000000000000000000000000000000..27d154efa50fd68aa23d3de656c9ce6449faed61 --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/hendrycks_math/dataset_infos.json @@ -0,0 +1 @@ +{"algebra": {"description": "MATH is a dataset of 12,500 challenging competition mathematics problems. Each\nproblem in Math has a full step-by-step solution which can be used to teach\nmodels to generate answer derivations and explanations.\n", "citation": "@article{hendrycksmath2021,\n title={Measuring Mathematical Problem Solving With the Math Dataset},\n author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},\n journal={NeurIPS},\n year={2021}\n}\n", "homepage": "https://github.com/hendrycks/math", "license": "", "features": {"problem": {"dtype": "string", "id": null, "_type": "Value"}, "level": {"dtype": "string", "id": null, "_type": "Value"}, "type": {"dtype": "string", "id": null, "_type": "Value"}, "solution": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "hendrycks_math", "config_name": "algebra", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"train": {"name": "train", "num_bytes": 955021, "num_examples": 1744, "dataset_name": "hendrycks_math"}, "test": {"name": "test", "num_bytes": 648291, "num_examples": 1187, "dataset_name": "hendrycks_math"}}, "download_checksums": {"https://people.eecs.berkeley.edu/~hendrycks/MATH.tar": {"num_bytes": 20327936, "checksum": "0fbe4fad0df66942db6c221cdcc95b298cc7f4595a2f0f518360cce84e90d9ac"}}, "download_size": 20327936, "post_processing_size": null, "dataset_size": 1603312, "size_in_bytes": 21931248}, "counting_and_probability": {"description": "MATH is a dataset of 12,500 challenging competition mathematics problems. Each\nproblem in Math has a full step-by-step solution which can be used to teach\nmodels to generate answer derivations and explanations.\n", "citation": "@article{hendrycksmath2021,\n title={Measuring Mathematical Problem Solving With the Math Dataset},\n author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},\n journal={NeurIPS},\n year={2021}\n}\n", "homepage": "https://github.com/hendrycks/math", "license": "", "features": {"problem": {"dtype": "string", "id": null, "_type": "Value"}, "level": {"dtype": "string", "id": null, "_type": "Value"}, "type": {"dtype": "string", "id": null, "_type": "Value"}, "solution": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "hendrycks_math", "config_name": "counting_and_probability", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"train": {"name": "train", "num_bytes": 667385, "num_examples": 771, "dataset_name": "hendrycks_math"}, "test": {"name": "test", "num_bytes": 353803, "num_examples": 474, "dataset_name": "hendrycks_math"}}, "download_checksums": {"https://people.eecs.berkeley.edu/~hendrycks/MATH.tar": {"num_bytes": 20327936, "checksum": "0fbe4fad0df66942db6c221cdcc95b298cc7f4595a2f0f518360cce84e90d9ac"}}, "download_size": 20327936, "post_processing_size": null, "dataset_size": 1021188, "size_in_bytes": 21349124}, "geometry": {"description": "MATH is a dataset of 12,500 challenging competition mathematics problems. Each\nproblem in Math has a full step-by-step solution which can be used to teach\nmodels to generate answer derivations and explanations.\n", "citation": "@article{hendrycksmath2021,\n title={Measuring Mathematical Problem Solving With the Math Dataset},\n author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},\n journal={NeurIPS},\n year={2021}\n}\n", "homepage": "https://github.com/hendrycks/math", "license": "", "features": {"problem": {"dtype": "string", "id": null, "_type": "Value"}, "level": {"dtype": "string", "id": null, "_type": "Value"}, "type": {"dtype": "string", "id": null, "_type": "Value"}, "solution": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "hendrycks_math", "config_name": "geometry", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"train": {"name": "train", "num_bytes": 1077241, "num_examples": 870, "dataset_name": "hendrycks_math"}, "test": {"name": "test", "num_bytes": 523126, "num_examples": 479, "dataset_name": "hendrycks_math"}}, "download_checksums": {"https://people.eecs.berkeley.edu/~hendrycks/MATH.tar": {"num_bytes": 20327936, "checksum": "0fbe4fad0df66942db6c221cdcc95b298cc7f4595a2f0f518360cce84e90d9ac"}}, "download_size": 20327936, "post_processing_size": null, "dataset_size": 1600367, "size_in_bytes": 21928303}, "intermediate_algebra": {"description": "MATH is a dataset of 12,500 challenging competition mathematics problems. Each\nproblem in Math has a full step-by-step solution which can be used to teach\nmodels to generate answer derivations and explanations.\n", "citation": "@article{hendrycksmath2021,\n title={Measuring Mathematical Problem Solving With the Math Dataset},\n author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},\n journal={NeurIPS},\n year={2021}\n}\n", "homepage": "https://github.com/hendrycks/math", "license": "", "features": {"problem": {"dtype": "string", "id": null, "_type": "Value"}, "level": {"dtype": "string", "id": null, "_type": "Value"}, "type": {"dtype": "string", "id": null, "_type": "Value"}, "solution": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "hendrycks_math", "config_name": "intermediate_algebra", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"train": {"name": "train", "num_bytes": 1157476, "num_examples": 1295, "dataset_name": "hendrycks_math"}, "test": {"name": "test", "num_bytes": 795070, "num_examples": 903, "dataset_name": "hendrycks_math"}}, "download_checksums": {"https://people.eecs.berkeley.edu/~hendrycks/MATH.tar": {"num_bytes": 20327936, "checksum": "0fbe4fad0df66942db6c221cdcc95b298cc7f4595a2f0f518360cce84e90d9ac"}}, "download_size": 20327936, "post_processing_size": null, "dataset_size": 1952546, "size_in_bytes": 22280482}, "number_theory": {"description": "MATH is a dataset of 12,500 challenging competition mathematics problems. Each\nproblem in Math has a full step-by-step solution which can be used to teach\nmodels to generate answer derivations and explanations.\n", "citation": "@article{hendrycksmath2021,\n title={Measuring Mathematical Problem Solving With the Math Dataset},\n author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},\n journal={NeurIPS},\n year={2021}\n}\n", "homepage": "https://github.com/hendrycks/math", "license": "", "features": {"problem": {"dtype": "string", "id": null, "_type": "Value"}, "level": {"dtype": "string", "id": null, "_type": "Value"}, "type": {"dtype": "string", "id": null, "_type": "Value"}, "solution": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "hendrycks_math", "config_name": "number_theory", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"train": {"name": "train", "num_bytes": 595793, "num_examples": 869, "dataset_name": "hendrycks_math"}, "test": {"name": "test", "num_bytes": 349455, "num_examples": 540, "dataset_name": "hendrycks_math"}}, "download_checksums": {"https://people.eecs.berkeley.edu/~hendrycks/MATH.tar": {"num_bytes": 20327936, "checksum": "0fbe4fad0df66942db6c221cdcc95b298cc7f4595a2f0f518360cce84e90d9ac"}}, "download_size": 20327936, "post_processing_size": null, "dataset_size": 945248, "size_in_bytes": 21273184}, "prealgebra": {"description": "MATH is a dataset of 12,500 challenging competition mathematics problems. Each\nproblem in Math has a full step-by-step solution which can be used to teach\nmodels to generate answer derivations and explanations.\n", "citation": "@article{hendrycksmath2021,\n title={Measuring Mathematical Problem Solving With the Math Dataset},\n author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},\n journal={NeurIPS},\n year={2021}\n}\n", "homepage": "https://github.com/hendrycks/math", "license": "", "features": {"problem": {"dtype": "string", "id": null, "_type": "Value"}, "level": {"dtype": "string", "id": null, "_type": "Value"}, "type": {"dtype": "string", "id": null, "_type": "Value"}, "solution": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "hendrycks_math", "config_name": "prealgebra", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"train": {"name": "train", "num_bytes": 715611, "num_examples": 1205, "dataset_name": "hendrycks_math"}, "test": {"name": "test", "num_bytes": 510195, "num_examples": 871, "dataset_name": "hendrycks_math"}}, "download_checksums": {"https://people.eecs.berkeley.edu/~hendrycks/MATH.tar": {"num_bytes": 20327936, "checksum": "0fbe4fad0df66942db6c221cdcc95b298cc7f4595a2f0f518360cce84e90d9ac"}}, "download_size": 20327936, "post_processing_size": null, "dataset_size": 1225806, "size_in_bytes": 21553742}, "precalculus": {"description": "MATH is a dataset of 12,500 challenging competition mathematics problems. Each\nproblem in Math has a full step-by-step solution which can be used to teach\nmodels to generate answer derivations and explanations.\n", "citation": "@article{hendrycksmath2021,\n title={Measuring Mathematical Problem Solving With the Math Dataset},\n author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},\n journal={NeurIPS},\n year={2021}\n}\n", "homepage": "https://github.com/hendrycks/math", "license": "", "features": {"problem": {"dtype": "string", "id": null, "_type": "Value"}, "level": {"dtype": "string", "id": null, "_type": "Value"}, "type": {"dtype": "string", "id": null, "_type": "Value"}, "solution": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "hendrycks_math", "config_name": "precalculus", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"train": {"name": "train", "num_bytes": 816245, "num_examples": 746, "dataset_name": "hendrycks_math"}, "test": {"name": "test", "num_bytes": 552893, "num_examples": 546, "dataset_name": "hendrycks_math"}}, "download_checksums": {"https://people.eecs.berkeley.edu/~hendrycks/MATH.tar": {"num_bytes": 20327936, "checksum": "0fbe4fad0df66942db6c221cdcc95b298cc7f4595a2f0f518360cce84e90d9ac"}}, "download_size": 20327936, "post_processing_size": null, "dataset_size": 1369138, "size_in_bytes": 21697074}} diff --git a/evaluation-pipeline/lm_eval/datasets/hendrycks_math/hendrycks_math.py b/evaluation-pipeline/lm_eval/datasets/hendrycks_math/hendrycks_math.py new file mode 100644 index 0000000000000000000000000000000000000000..043adeeed6648ce04e3209b6359cb119699eddd1 --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/hendrycks_math/hendrycks_math.py @@ -0,0 +1,122 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MATH dataset.""" + + +import json +import os +import pathlib + +import datasets + + +_CITATION = """\ +@article{hendrycksmath2021, + title={Measuring Mathematical Problem Solving With the Math Dataset}, + author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt}, + journal={NeurIPS}, + year={2021} +} +""" + +_DESCRIPTION = """\ +MATH is a dataset of 12,500 challenging competition mathematics problems. Each +problem in Math has a full step-by-step solution which can be used to teach +models to generate answer derivations and explanations. +""" + +_HOMEPAGE = "https://github.com/hendrycks/math" + +# TODO: Add the licence for the dataset here if you can find it +_LICENSE = "" + +_URLS = "https://people.eecs.berkeley.edu/~hendrycks/MATH.tar" + +_NAMES = [ + "algebra", + "counting_and_probability", + "geometry", + "intermediate_algebra", + "number_theory", + "prealgebra", + "precalculus", +] + + +class HendrycksMath(datasets.GeneratorBasedBuilder): + """MATH is a dataset of 12,500 challenging competition mathematics problems.""" + + VERSION = datasets.Version("0.0.1") + + BUILDER_CONFIGS = [ + datasets.BuilderConfig(name=name, version=version, description=name) + for name, version in zip(_NAMES, [VERSION] * len(_NAMES)) + ] + + def _info(self): + features = datasets.Features( + { + "problem": datasets.Value("string"), + "level": datasets.Value("string"), + "type": datasets.Value("string"), + "solution": datasets.Value("string"), + } + ) + return datasets.DatasetInfo( + description=_DESCRIPTION, + features=features, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager): + urls = _URLS + data_dir = dl_manager.download_and_extract(urls) + return [ + datasets.SplitGenerator( + name=datasets.Split.TRAIN, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "basepath": os.path.join( + data_dir, "MATH", "train", self.config.name + ), + "split": "train", + }, + ), + datasets.SplitGenerator( + name=datasets.Split.TEST, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "basepath": os.path.join( + data_dir, "MATH", "test", self.config.name + ), + "split": "test", + }, + ), + ] + + # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` + def _generate_examples(self, basepath, split): + key = 0 + for file in sorted(pathlib.Path(basepath).iterdir()): + with open(file, "r", encoding="utf-8") as f: + data = json.load(f) + yield key, { + "problem": data["problem"], + "level": data["level"], + "type": data["type"], + "solution": data["solution"], + } + key += 1 diff --git a/evaluation-pipeline/lm_eval/datasets/jigsaw_unintended_bias/__init__.py b/evaluation-pipeline/lm_eval/datasets/jigsaw_unintended_bias/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/lm_eval/datasets/jigsaw_unintended_bias/__pycache__/__init__.cpython-310.pyc b/evaluation-pipeline/lm_eval/datasets/jigsaw_unintended_bias/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..319a62011b126ffd1e053d976bfa5c3b5a461ac6 Binary files /dev/null and b/evaluation-pipeline/lm_eval/datasets/jigsaw_unintended_bias/__pycache__/__init__.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/datasets/jigsaw_unintended_bias/__pycache__/jigsaw_unintended_bias.cpython-310.pyc b/evaluation-pipeline/lm_eval/datasets/jigsaw_unintended_bias/__pycache__/jigsaw_unintended_bias.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fc9f454944c2a41bef3970be46bad6a5cc499d83 Binary files /dev/null and b/evaluation-pipeline/lm_eval/datasets/jigsaw_unintended_bias/__pycache__/jigsaw_unintended_bias.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/datasets/jigsaw_unintended_bias/dataset_infos.json b/evaluation-pipeline/lm_eval/datasets/jigsaw_unintended_bias/dataset_infos.json new file mode 100644 index 0000000000000000000000000000000000000000..03110859da0e61fee1a49b9c1bbd5ea0d04cd678 --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/jigsaw_unintended_bias/dataset_infos.json @@ -0,0 +1 @@ +{"default": {"description": "A collection of comments from the defunct Civil Comments platform that have been annotated for their toxicity.\n", "citation": "", "homepage": "https://www.kaggle.com/c/jigsaw-unintended-bias-in-toxicity-classification/", "license": "CC0 (both the dataset and underlying text)", "features": {"target": {"dtype": "float32", "id": null, "_type": "Value"}, "comment_text": {"dtype": "string", "id": null, "_type": "Value"}, "severe_toxicity": {"dtype": "float32", "id": null, "_type": "Value"}, "obscene": {"dtype": "float32", "id": null, "_type": "Value"}, "identity_attack": {"dtype": "float32", "id": null, "_type": "Value"}, "insult": {"dtype": "float32", "id": null, "_type": "Value"}, "threat": {"dtype": "float32", "id": null, "_type": "Value"}, "asian": {"dtype": "float32", "id": null, "_type": "Value"}, "atheist": {"dtype": "float32", "id": null, "_type": "Value"}, "bisexual": {"dtype": "float32", "id": null, "_type": "Value"}, "black": {"dtype": "float32", "id": null, "_type": "Value"}, "buddhist": {"dtype": "float32", "id": null, "_type": "Value"}, "christian": {"dtype": "float32", "id": null, "_type": "Value"}, "female": {"dtype": "float32", "id": null, "_type": "Value"}, "heterosexual": {"dtype": "float32", "id": null, "_type": "Value"}, "hindu": {"dtype": "float32", "id": null, "_type": "Value"}, "homosexual_gay_or_lesbian": {"dtype": "float32", "id": null, "_type": "Value"}, "intellectual_or_learning_disability": {"dtype": "float32", "id": null, "_type": "Value"}, "jewish": {"dtype": "float32", "id": null, "_type": "Value"}, "latino": {"dtype": "float32", "id": null, "_type": "Value"}, "male": {"dtype": "float32", "id": null, "_type": "Value"}, "muslim": {"dtype": "float32", "id": null, "_type": "Value"}, "other_disability": {"dtype": "float32", "id": null, "_type": "Value"}, "other_gender": {"dtype": "float32", "id": null, "_type": "Value"}, "other_race_or_ethnicity": {"dtype": "float32", "id": null, "_type": "Value"}, "other_religion": {"dtype": "float32", "id": null, "_type": "Value"}, "other_sexual_orientation": {"dtype": "float32", "id": null, "_type": "Value"}, "physical_disability": {"dtype": "float32", "id": null, "_type": "Value"}, "psychiatric_or_mental_illness": {"dtype": "float32", "id": null, "_type": "Value"}, "transgender": {"dtype": "float32", "id": null, "_type": "Value"}, "white": {"dtype": "float32", "id": null, "_type": "Value"}, "created_date": {"dtype": "string", "id": null, "_type": "Value"}, "publication_id": {"dtype": "int32", "id": null, "_type": "Value"}, "parent_id": {"dtype": "float32", "id": null, "_type": "Value"}, "article_id": {"dtype": "int32", "id": null, "_type": "Value"}, "rating": {"num_classes": 2, "names": ["rejected", "approved"], "names_file": null, "id": null, "_type": "ClassLabel"}, "funny": {"dtype": "int32", "id": null, "_type": "Value"}, "wow": {"dtype": "int32", "id": null, "_type": "Value"}, "sad": {"dtype": "int32", "id": null, "_type": "Value"}, "likes": {"dtype": "int32", "id": null, "_type": "Value"}, "disagree": {"dtype": "int32", "id": null, "_type": "Value"}, "sexual_explicit": {"dtype": "float32", "id": null, "_type": "Value"}, "identity_annotator_count": {"dtype": "int32", "id": null, "_type": "Value"}, "toxicity_annotator_count": {"dtype": "int32", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "jigsaw_unintended_bias", "config_name": "default", "version": {"version_str": "1.1.0", "description": null, "major": 1, "minor": 1, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 914264058, "num_examples": 1804874, "dataset_name": "jigsaw_unintended_bias"}, "test_private_leaderboard": {"name": "test_private_leaderboard", "num_bytes": 49188921, "num_examples": 97320, "dataset_name": "jigsaw_unintended_bias"}, "test_public_leaderboard": {"name": "test_public_leaderboard", "num_bytes": 49442360, "num_examples": 97320, "dataset_name": "jigsaw_unintended_bias"}}, "download_checksums": {}, "download_size": 0, "post_processing_size": null, "dataset_size": 1012895339, "size_in_bytes": 1012895339}} diff --git a/evaluation-pipeline/lm_eval/datasets/jigsaw_unintended_bias/jigsaw_unintended_bias.py b/evaluation-pipeline/lm_eval/datasets/jigsaw_unintended_bias/jigsaw_unintended_bias.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/lm_eval/datasets/lambada/__init__.py b/evaluation-pipeline/lm_eval/datasets/lambada/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/lm_eval/datasets/lambada/dataset_infos.json b/evaluation-pipeline/lm_eval/datasets/lambada/dataset_infos.json new file mode 100644 index 0000000000000000000000000000000000000000..855c6aef53176900285ef5a4dae05e2011a5df97 --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/lambada/dataset_infos.json @@ -0,0 +1 @@ +{"original": {"description": "LAMBADA is a dataset to evaluate the capabilities of computational models for text\nunderstanding by means of a word prediction task. LAMBADA is a collection of narrative\ntexts sharing the characteristic that human subjects are able to guess their last\nword if they are exposed to the whole text, but not if they only see the last\nsentence preceding the target word. To succeed on LAMBADA, computational models\ncannot simply rely on local context, but must be able to keep track of information\nin the broader discourse.\n\nThe LAMBADA dataset", "citation": "@misc{\n author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel}, \n title={The LAMBADA dataset},\n DOI={10.5281/zenodo.2630551},\n publisher={Zenodo},\n year={2016},\n month={Aug}\n}\n", "homepage": "https://zenodo.org/record/2630551#.X4Xzn5NKjUI", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "lambada", "config_name": "original", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 1709449, "num_examples": 5153, "dataset_name": "lambada"}}, "download_checksums": {"http://eaidata.bmk.sh/data/lambada_test.jsonl": {"num_bytes": 1819752, "checksum": "4aa8d02cd17c719165fc8a7887fddd641f43fcafa4b1c806ca8abc31fabdb226"}}, "download_size": 1819752, "post_processing_size": null, "dataset_size": 1709449, "size_in_bytes": 3529201}, "en": {"description": "LAMBADA is a dataset to evaluate the capabilities of computational models for text\nunderstanding by means of a word prediction task. LAMBADA is a collection of narrative\ntexts sharing the characteristic that human subjects are able to guess their last\nword if they are exposed to the whole text, but not if they only see the last\nsentence preceding the target word. To succeed on LAMBADA, computational models\ncannot simply rely on local context, but must be able to keep track of information\nin the broader discourse.\n\nThe English translated LAMBADA dataset", "citation": "@misc{\n author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel}, \n title={The LAMBADA dataset},\n DOI={10.5281/zenodo.2630551},\n publisher={Zenodo},\n year={2016},\n month={Aug}\n}\n", "homepage": "https://zenodo.org/record/2630551#.X4Xzn5NKjUI", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "lambada", "config_name": "en", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 1709449, "num_examples": 5153, "dataset_name": "lambada"}}, "download_checksums": {"http://eaidata.bmk.sh/data/lambada_test_en.jsonl": {"num_bytes": 1819752, "checksum": "4aa8d02cd17c719165fc8a7887fddd641f43fcafa4b1c806ca8abc31fabdb226"}}, "download_size": 1819752, "post_processing_size": null, "dataset_size": 1709449, "size_in_bytes": 3529201}, "fr": {"description": "LAMBADA is a dataset to evaluate the capabilities of computational models for text\nunderstanding by means of a word prediction task. LAMBADA is a collection of narrative\ntexts sharing the characteristic that human subjects are able to guess their last\nword if they are exposed to the whole text, but not if they only see the last\nsentence preceding the target word. To succeed on LAMBADA, computational models\ncannot simply rely on local context, but must be able to keep track of information\nin the broader discourse.\n\nThe French translated LAMBADA dataset", "citation": "@misc{\n author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel}, \n title={The LAMBADA dataset},\n DOI={10.5281/zenodo.2630551},\n publisher={Zenodo},\n year={2016},\n month={Aug}\n}\n", "homepage": "https://zenodo.org/record/2630551#.X4Xzn5NKjUI", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "lambada", "config_name": "fr", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 1948795, "num_examples": 5153, "dataset_name": "lambada"}}, "download_checksums": {"http://eaidata.bmk.sh/data/lambada_test_fr.jsonl": {"num_bytes": 2028703, "checksum": "941ec6a73dba7dc91c860bf493eb66a527cd430148827a4753a4535a046bf362"}}, "download_size": 2028703, "post_processing_size": null, "dataset_size": 1948795, "size_in_bytes": 3977498}, "de": {"description": "LAMBADA is a dataset to evaluate the capabilities of computational models for text\nunderstanding by means of a word prediction task. LAMBADA is a collection of narrative\ntexts sharing the characteristic that human subjects are able to guess their last\nword if they are exposed to the whole text, but not if they only see the last\nsentence preceding the target word. To succeed on LAMBADA, computational models\ncannot simply rely on local context, but must be able to keep track of information\nin the broader discourse.\n\nThe German translated LAMBADA dataset", "citation": "@misc{\n author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel}, \n title={The LAMBADA dataset},\n DOI={10.5281/zenodo.2630551},\n publisher={Zenodo},\n year={2016},\n month={Aug}\n}\n", "homepage": "https://zenodo.org/record/2630551#.X4Xzn5NKjUI", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "lambada", "config_name": "de", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 1904576, "num_examples": 5153, "dataset_name": "lambada"}}, "download_checksums": {"http://eaidata.bmk.sh/data/lambada_test_de.jsonl": {"num_bytes": 1985231, "checksum": "51c6c1795894c46e88e4c104b5667f488efe79081fb34d746b82b8caa663865e"}}, "download_size": 1985231, "post_processing_size": null, "dataset_size": 1904576, "size_in_bytes": 3889807}, "it": {"description": "LAMBADA is a dataset to evaluate the capabilities of computational models for text\nunderstanding by means of a word prediction task. LAMBADA is a collection of narrative\ntexts sharing the characteristic that human subjects are able to guess their last\nword if they are exposed to the whole text, but not if they only see the last\nsentence preceding the target word. To succeed on LAMBADA, computational models\ncannot simply rely on local context, but must be able to keep track of information\nin the broader discourse.\n\nThe Italian translated LAMBADA dataset", "citation": "@misc{\n author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel}, \n title={The LAMBADA dataset},\n DOI={10.5281/zenodo.2630551},\n publisher={Zenodo},\n year={2016},\n month={Aug}\n}\n", "homepage": "https://zenodo.org/record/2630551#.X4Xzn5NKjUI", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "lambada", "config_name": "it", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 1813420, "num_examples": 5153, "dataset_name": "lambada"}}, "download_checksums": {"http://eaidata.bmk.sh/data/lambada_test_it.jsonl": {"num_bytes": 1894613, "checksum": "86654237716702ab74f42855ae5a78455c1b0e50054a4593fb9c6fcf7fad0850"}}, "download_size": 1894613, "post_processing_size": null, "dataset_size": 1813420, "size_in_bytes": 3708033}, "es": {"description": "LAMBADA is a dataset to evaluate the capabilities of computational models for text\nunderstanding by means of a word prediction task. LAMBADA is a collection of narrative\ntexts sharing the characteristic that human subjects are able to guess their last\nword if they are exposed to the whole text, but not if they only see the last\nsentence preceding the target word. To succeed on LAMBADA, computational models\ncannot simply rely on local context, but must be able to keep track of information\nin the broader discourse.\n\nThe Spanish translated LAMBADA dataset", "citation": "@misc{\n author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel}, \n title={The LAMBADA dataset},\n DOI={10.5281/zenodo.2630551},\n publisher={Zenodo},\n year={2016},\n month={Aug}\n}\n", "homepage": "https://zenodo.org/record/2630551#.X4Xzn5NKjUI", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "lambada", "config_name": "es", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 1821735, "num_examples": 5153, "dataset_name": "lambada"}}, "download_checksums": {"http://eaidata.bmk.sh/data/lambada_test_es.jsonl": {"num_bytes": 1902349, "checksum": "ffd760026c647fb43c67ce1bc56fd527937304b348712dce33190ea6caba6f9c"}}, "download_size": 1902349, "post_processing_size": null, "dataset_size": 1821735, "size_in_bytes": 3724084}} diff --git a/evaluation-pipeline/lm_eval/datasets/lambada/lambada.py b/evaluation-pipeline/lm_eval/datasets/lambada/lambada.py new file mode 100644 index 0000000000000000000000000000000000000000..506bef8279c4cea9836b62021543e40855c81e8e --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/lambada/lambada.py @@ -0,0 +1,130 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# TODO: Address all TODOs and remove all explanatory comments +"""LAMBADA dataset.""" + + +import json + +import datasets + + +_CITATION = """\ +@misc{ + author={Paperno, Denis and Kruszewski, Germán and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fernández, Raquel}, + title={The LAMBADA dataset}, + DOI={10.5281/zenodo.2630551}, + publisher={Zenodo}, + year={2016}, + month={Aug} +} +""" + +_DESCRIPTION = """\ +LAMBADA is a dataset to evaluate the capabilities of computational models for text +understanding by means of a word prediction task. LAMBADA is a collection of narrative +texts sharing the characteristic that human subjects are able to guess their last +word if they are exposed to the whole text, but not if they only see the last +sentence preceding the target word. To succeed on LAMBADA, computational models +cannot simply rely on local context, but must be able to keep track of information +in the broader discourse. +""" + +_HOMEPAGE = "https://zenodo.org/record/2630551#.X4Xzn5NKjUI" + +# TODO: Add the licence for the dataset here if you can find it +_LICENSE = "" + +_URLS = { + "original": "http://eaidata.bmk.sh/data/lambada_test.jsonl", + "en": "http://eaidata.bmk.sh/data/lambada_test_en.jsonl", + "fr": "http://eaidata.bmk.sh/data/lambada_test_fr.jsonl", + "de": "http://eaidata.bmk.sh/data/lambada_test_de.jsonl", + "it": "http://eaidata.bmk.sh/data/lambada_test_it.jsonl", + "es": "http://eaidata.bmk.sh/data/lambada_test_es.jsonl", +} + + +class Lambada(datasets.GeneratorBasedBuilder): + """LAMBADA is a dataset to evaluate the capabilities of computational models for text understanding by means of a word prediction task.""" + + VERSION = datasets.Version("0.0.1") + + BUILDER_CONFIGS = [ + datasets.BuilderConfig( + name="original", version=VERSION, description="The LAMBADA dataset" + ), + datasets.BuilderConfig( + name="en", + version=VERSION, + description="The English translated LAMBADA dataset", + ), + datasets.BuilderConfig( + name="fr", + version=VERSION, + description="The French translated LAMBADA dataset", + ), + datasets.BuilderConfig( + name="de", + version=VERSION, + description="The German translated LAMBADA dataset", + ), + datasets.BuilderConfig( + name="it", + version=VERSION, + description="The Italian translated LAMBADA dataset", + ), + datasets.BuilderConfig( + name="es", + version=VERSION, + description="The Spanish translated LAMBADA dataset", + ), + ] + + DEFAULT_CONFIG_NAME = "original" + + def _info(self): + features = datasets.Features( + { + "text": datasets.Value("string"), + } + ) + return datasets.DatasetInfo( + description=f"{_DESCRIPTION}\n{self.config.description}", + features=features, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager): + urls = _URLS[self.config.name] + data_dir = dl_manager.download_and_extract(urls) + return [ + datasets.SplitGenerator( + name=datasets.Split.VALIDATION, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "filepath": data_dir, + "split": "validation", + }, + ), + ] + + # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` + def _generate_examples(self, filepath, split): + with open(filepath, encoding="utf-8") as f: + for key, row in enumerate(f): + data = json.loads(row) + yield key, {"text": data["text"]} diff --git a/evaluation-pipeline/lm_eval/datasets/logiqa/__init__.py b/evaluation-pipeline/lm_eval/datasets/logiqa/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/lm_eval/datasets/logiqa/dataset_infos.json b/evaluation-pipeline/lm_eval/datasets/logiqa/dataset_infos.json new file mode 100644 index 0000000000000000000000000000000000000000..12a203cb05f9c71c0a135077ec9cae5957592ec7 --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/logiqa/dataset_infos.json @@ -0,0 +1 @@ +{"logiqa": {"description": "LogiQA is a dataset for testing human logical reasoning. It consists of 8,678 QA\ninstances, covering multiple types of deductive reasoning. Results show that state-\nof-the-art neural models perform by far worse than human ceiling. The dataset can\nalso serve as a benchmark for reinvestigating logical AI under the deep learning\nNLP setting.\n", "citation": "@misc{liu2020logiqa,\n title={LogiQA: A Challenge Dataset for Machine Reading Comprehension with Logical Reasoning}, \n author={Jian Liu and Leyang Cui and Hanmeng Liu and Dandan Huang and Yile Wang and Yue Zhang},\n year={2020},\n eprint={2007.08124},\n archivePrefix={arXiv},\n primaryClass={cs.CL}\n}\n", "homepage": "https://github.com/lgw863/LogiQA-dataset", "license": "", "features": {"label": {"dtype": "string", "id": null, "_type": "Value"}, "context": {"dtype": "string", "id": null, "_type": "Value"}, "question": {"dtype": "string", "id": null, "_type": "Value"}, "options": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "logiqa", "config_name": "logiqa", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"train": {"name": "train", "num_bytes": 6419852, "num_examples": 7376, "dataset_name": "logiqa"}, "test": {"name": "test", "num_bytes": 571705, "num_examples": 651, "dataset_name": "logiqa"}, "validation": {"name": "validation", "num_bytes": 562437, "num_examples": 651, "dataset_name": "logiqa"}}, "download_checksums": {"https://raw.githubusercontent.com/lgw863/LogiQA-dataset/master/Train.txt": {"num_bytes": 6281272, "checksum": "7d5bb1f58278e33b395744cd2ad8d7600faa0b3c4d615c659a44ec1181d759fa"}, "https://raw.githubusercontent.com/lgw863/LogiQA-dataset/master/Test.txt": {"num_bytes": 559060, "checksum": "359acb78c37802208f7fde9e2f6574b8526527c63d6a336f90a53f1932cb4701"}, "https://raw.githubusercontent.com/lgw863/LogiQA-dataset/master/Eval.txt": {"num_bytes": 550021, "checksum": "4c49e6753b7262c001506b9151135abf722247035ab075dad93acdea5789c01f"}}, "download_size": 7390353, "post_processing_size": null, "dataset_size": 7553994, "size_in_bytes": 14944347}} diff --git a/evaluation-pipeline/lm_eval/datasets/logiqa/logiqa.py b/evaluation-pipeline/lm_eval/datasets/logiqa/logiqa.py new file mode 100644 index 0000000000000000000000000000000000000000..b1f5521596e502578e95fa9240ead23aea8850ec --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/logiqa/logiqa.py @@ -0,0 +1,124 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""LogiQA dataset.""" + + +import datasets + + +_CITATION = """\ +@misc{liu2020logiqa, + title={LogiQA: A Challenge Dataset for Machine Reading Comprehension with Logical Reasoning}, + author={Jian Liu and Leyang Cui and Hanmeng Liu and Dandan Huang and Yile Wang and Yue Zhang}, + year={2020}, + eprint={2007.08124}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} +""" + +_DESCRIPTION = """\ +LogiQA is a dataset for testing human logical reasoning. It consists of 8,678 QA +instances, covering multiple types of deductive reasoning. Results show that state- +of-the-art neural models perform by far worse than human ceiling. The dataset can +also serve as a benchmark for reinvestigating logical AI under the deep learning +NLP setting. +""" + +_HOMEPAGE = "https://github.com/lgw863/LogiQA-dataset" + +# TODO: Add the licence for the dataset here if you can find it +_LICENSE = "" + +_URLS = { + "train": "https://raw.githubusercontent.com/lgw863/LogiQA-dataset/master/Train.txt", + "validation": "https://raw.githubusercontent.com/lgw863/LogiQA-dataset/master/Eval.txt", + "test": "https://raw.githubusercontent.com/lgw863/LogiQA-dataset/master/Test.txt", +} + + +class Logiqa(datasets.GeneratorBasedBuilder): + """LogiQA: A Challenge Dataset for Machine Reading Comprehension with Logical Reasoning""" + + VERSION = datasets.Version("0.0.1") + + BUILDER_CONFIGS = [ + datasets.BuilderConfig( + name="logiqa", version=VERSION, description="The LogiQA dataset." + ), + ] + + def _info(self): + features = datasets.Features( + { + "label": datasets.Value("string"), + "context": datasets.Value("string"), + "question": datasets.Value("string"), + "options": datasets.features.Sequence(datasets.Value("string")), + } + ) + return datasets.DatasetInfo( + description=_DESCRIPTION, + features=features, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager): + urls = { + "train": _URLS["train"], + "test": _URLS["test"], + "validation": _URLS["validation"], + } + data_dir = dl_manager.download_and_extract(urls) + return [ + datasets.SplitGenerator( + name=datasets.Split.TRAIN, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "filepath": data_dir["train"], + "split": "train", + }, + ), + datasets.SplitGenerator( + name=datasets.Split.TEST, + # These kwargs will be passed to _generate_examples + gen_kwargs={"filepath": data_dir["test"], "split": "test"}, + ), + datasets.SplitGenerator( + name=datasets.Split.VALIDATION, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "filepath": data_dir["validation"], + "split": "validation", + }, + ), + ] + + # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` + def _generate_examples(self, filepath, split): + def normalize(text): + return text.replace(".", ". ").strip() + + with open(filepath, encoding="utf-8") as f: + data = f.read().strip().split("\n\n") + for key, row in enumerate(data): + example = row.split("\n") + yield key, { + "label": example[0].strip(), + "context": normalize(example[1]), + "question": normalize(example[2]), + "options": [normalize(option[2:]) for option in example[3:]], + } diff --git a/evaluation-pipeline/lm_eval/datasets/mutual/__init__.py b/evaluation-pipeline/lm_eval/datasets/mutual/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/lm_eval/datasets/mutual/dataset_infos.json b/evaluation-pipeline/lm_eval/datasets/mutual/dataset_infos.json new file mode 100644 index 0000000000000000000000000000000000000000..f8c438b3f85d96a670fad128cbc555a2ab22c7fd --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/mutual/dataset_infos.json @@ -0,0 +1 @@ +{"mutual": {"description": "MuTual is a retrieval-based dataset for multi-turn dialogue reasoning, which is\nmodified from Chinese high school English listening comprehension test data.\n\nThe MuTual dataset.", "citation": "@inproceedings{mutual,\n title = \"MuTual: A Dataset for Multi-Turn Dialogue Reasoning\",\n author = \"Cui, Leyang and Wu, Yu and Liu, Shujie and Zhang, Yue and Zhou, Ming\" ,\n booktitle = \"Proceedings of the 58th Conference of the Association for Computational Linguistics\",\n year = \"2020\",\n publisher = \"Association for Computational Linguistics\",\n}\n", "homepage": "https://github.com/Nealcly/MuTual", "license": "", "features": {"answers": {"dtype": "string", "id": null, "_type": "Value"}, "options": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "article": {"dtype": "string", "id": null, "_type": "Value"}, "id": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "mutual", "config_name": "mutual", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"train": {"name": "train", "num_bytes": 5141602, "num_examples": 7088, "dataset_name": "mutual"}, "test": {"name": "test", "num_bytes": 634396, "num_examples": 886, "dataset_name": "mutual"}, "validation": {"name": "validation", "num_bytes": 624271, "num_examples": 886, "dataset_name": "mutual"}}, "download_checksums": {"https://github.com/Nealcly/MuTual/archive/master.zip": {"num_bytes": 10997878, "checksum": "bb325cf6c672f0f02699993a37138b0fa0af6fcfc77ec81dfbe46add4d7b29f9"}}, "download_size": 10997878, "post_processing_size": null, "dataset_size": 6400269, "size_in_bytes": 17398147}, "mutual_plus": {"description": "MuTual is a retrieval-based dataset for multi-turn dialogue reasoning, which is\nmodified from Chinese high school English listening comprehension test data.\n\nMuTualPlus is a more difficult MuTual that replaces positive responses with a safe responses.", "citation": "@inproceedings{mutual,\n title = \"MuTual: A Dataset for Multi-Turn Dialogue Reasoning\",\n author = \"Cui, Leyang and Wu, Yu and Liu, Shujie and Zhang, Yue and Zhou, Ming\" ,\n booktitle = \"Proceedings of the 58th Conference of the Association for Computational Linguistics\",\n year = \"2020\",\n publisher = \"Association for Computational Linguistics\",\n}\n", "homepage": "https://github.com/Nealcly/MuTual", "license": "", "features": {"answers": {"dtype": "string", "id": null, "_type": "Value"}, "options": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "article": {"dtype": "string", "id": null, "_type": "Value"}, "id": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "mutual", "config_name": "mutual_plus", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"train": {"name": "train", "num_bytes": 4921179, "num_examples": 7088, "dataset_name": "mutual"}, "test": {"name": "test", "num_bytes": 606620, "num_examples": 886, "dataset_name": "mutual"}, "validation": {"name": "validation", "num_bytes": 597340, "num_examples": 886, "dataset_name": "mutual"}}, "download_checksums": {"https://github.com/Nealcly/MuTual/archive/master.zip": {"num_bytes": 10997878, "checksum": "bb325cf6c672f0f02699993a37138b0fa0af6fcfc77ec81dfbe46add4d7b29f9"}}, "download_size": 10997878, "post_processing_size": null, "dataset_size": 6125139, "size_in_bytes": 17123017}} diff --git a/evaluation-pipeline/lm_eval/datasets/mutual/mutual.py b/evaluation-pipeline/lm_eval/datasets/mutual/mutual.py new file mode 100644 index 0000000000000000000000000000000000000000..c519e663bea10d4097bcaad4299148ce152f25a9 --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/mutual/mutual.py @@ -0,0 +1,136 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MuTual dataset.""" + + +import json +import os +from pathlib import Path + +import datasets + + +_CITATION = """\ +@inproceedings{mutual, + title = "MuTual: A Dataset for Multi-Turn Dialogue Reasoning", + author = "Cui, Leyang and Wu, Yu and Liu, Shujie and Zhang, Yue and Zhou, Ming" , + booktitle = "Proceedings of the 58th Conference of the Association for Computational Linguistics", + year = "2020", + publisher = "Association for Computational Linguistics", +} +""" + +_DESCRIPTION = """\ +MuTual is a retrieval-based dataset for multi-turn dialogue reasoning, which is +modified from Chinese high school English listening comprehension test data. +""" + +_HOMEPAGE = "https://github.com/Nealcly/MuTual" + +# TODO: Add the licence for the dataset here if you can find it +_LICENSE = "" + +_URLS = "https://github.com/Nealcly/MuTual/archive/master.zip" + + +class Mutual(datasets.GeneratorBasedBuilder): + """MuTual: A Dataset for Multi-Turn Dialogue Reasoning""" + + VERSION = datasets.Version("0.0.1") + + BUILDER_CONFIGS = [ + datasets.BuilderConfig( + name="mutual", version=VERSION, description="The MuTual dataset." + ), + datasets.BuilderConfig( + name="mutual_plus", + version=VERSION, + description="MuTualPlus is a more difficult MuTual that replaces positive responses with a safe responses.", + ), + ] + + def _info(self): + features = datasets.Features( + { + "answers": datasets.Value("string"), + "options": datasets.features.Sequence(datasets.Value("string")), + "article": datasets.Value("string"), + "id": datasets.Value("string"), + } + ) + return datasets.DatasetInfo( + description=f"{_DESCRIPTION}\n{self.config.description}", + features=features, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager): + urls = _URLS + data_dir = dl_manager.download_and_extract(urls) + return [ + datasets.SplitGenerator( + name=datasets.Split.TRAIN, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "basepath": os.path.join( + data_dir, "MuTual-master", "data", self.config.name, "train" + ), + "split": "train", + }, + ), + datasets.SplitGenerator( + name=datasets.Split.TEST, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "basepath": os.path.join( + data_dir, "MuTual-master", "data", self.config.name, "test" + ), + "split": "test", + }, + ), + datasets.SplitGenerator( + name=datasets.Split.VALIDATION, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "basepath": os.path.join( + data_dir, "MuTual-master", "data", self.config.name, "dev" + ), + "split": "dev", + }, + ), + ] + + # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` + def _generate_examples(self, basepath, split): + # TODO: This method handles input defined in _split_generators to yield (key, example) tuples from the dataset. + # The `key` is for legacy reasons (tfds) and is not important in itself, but must be unique for each example. + key = 0 + for file in sorted(Path(basepath).iterdir()): + if file.suffix != ".txt": + continue + with open(file, "r", encoding="utf-8") as f: + data_str = f.read() + # Ignore the occasional empty file. + if not data_str: + continue + data = json.loads(data_str) + yield key, { + "answers": data["answers"], + "options": data["options"], + "article": data["article"], + "id": data["id"], + } + key += 1 diff --git a/evaluation-pipeline/lm_eval/datasets/pile/__init__.py b/evaluation-pipeline/lm_eval/datasets/pile/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/lm_eval/datasets/pile/dataset_infos.json b/evaluation-pipeline/lm_eval/datasets/pile/dataset_infos.json new file mode 100644 index 0000000000000000000000000000000000000000..d91e9b5704f390fc5764cbe5e50daaaf54a1fe12 --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/pile/dataset_infos.json @@ -0,0 +1 @@ +{"pile_arxiv": {"description": "The Pile is a 825 GiB diverse, open source language modeling data set that consists\nof 22 smaller, high-quality datasets combined together. To score well on Pile\nBPB (bits per byte), a model must be able to understand many disparate domains\nincluding books, github repositories, webpages, chat logs, and medical, physics,\nmath, computer science, and philosophy papers.\n\nArXiv", "citation": "@article{pile,\n title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},\n author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},\n journal={arXiv preprint arXiv:2101.00027},\n year={2020}\n}\n", "homepage": "https://pile.eleuther.ai/", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "pile", "config_name": "pile_arxiv", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"test": {"name": "test", "num_bytes": 113218251, "num_examples": 2407, "dataset_name": "pile"}, "validation": {"name": "validation", "num_bytes": 115653720, "num_examples": 2434, "dataset_name": "pile"}}, "download_checksums": {"http://eaidata.bmk.sh/data/pile/val.jsonl.zst": {"num_bytes": 470907480, "checksum": "264c875d8bbd355d8daa9d032b75fd8fb91606218bb84dd1155b203fcd5fab92"}, "http://eaidata.bmk.sh/data/pile/test.jsonl.zst": {"num_bytes": 460250856, "checksum": "0bb28c52d0b5596d389bf179ce2d43bf7f7ffae76b0d2d20b180c97f62e0975e"}}, "download_size": 931158336, "post_processing_size": null, "dataset_size": 228871971, "size_in_bytes": 1160030307}, "pile_books3": {"description": "The Pile is a 825 GiB diverse, open source language modeling data set that consists\nof 22 smaller, high-quality datasets combined together. To score well on Pile\nBPB (bits per byte), a model must be able to understand many disparate domains\nincluding books, github repositories, webpages, chat logs, and medical, physics,\nmath, computer science, and philosophy papers.\n\nBooks3", "citation": "@article{pile,\n title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},\n author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},\n journal={arXiv preprint arXiv:2101.00027},\n year={2020}\n}\n", "homepage": "https://pile.eleuther.ai/", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "pile", "config_name": "pile_books3", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"test": {"name": "test", "num_bytes": 150095743, "num_examples": 269, "dataset_name": "pile"}, "validation": {"name": "validation", "num_bytes": 177359876, "num_examples": 301, "dataset_name": "pile"}}, "download_checksums": {"http://eaidata.bmk.sh/data/pile/val.jsonl.zst": {"num_bytes": 470907480, "checksum": "264c875d8bbd355d8daa9d032b75fd8fb91606218bb84dd1155b203fcd5fab92"}, "http://eaidata.bmk.sh/data/pile/test.jsonl.zst": {"num_bytes": 460250856, "checksum": "0bb28c52d0b5596d389bf179ce2d43bf7f7ffae76b0d2d20b180c97f62e0975e"}}, "download_size": 931158336, "post_processing_size": null, "dataset_size": 327455619, "size_in_bytes": 1258613955}, "pile_bookcorpus2": {"description": "The Pile is a 825 GiB diverse, open source language modeling data set that consists\nof 22 smaller, high-quality datasets combined together. To score well on Pile\nBPB (bits per byte), a model must be able to understand many disparate domains\nincluding books, github repositories, webpages, chat logs, and medical, physics,\nmath, computer science, and philosophy papers.\n\nBookCorpus2", "citation": "@article{pile,\n title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},\n author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},\n journal={arXiv preprint arXiv:2101.00027},\n year={2020}\n}\n", "homepage": "https://pile.eleuther.ai/", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "pile", "config_name": "pile_bookcorpus2", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"test": {"name": "test", "num_bytes": 9680652, "num_examples": 28, "dataset_name": "pile"}, "validation": {"name": "validation", "num_bytes": 9776271, "num_examples": 26, "dataset_name": "pile"}}, "download_checksums": {"http://eaidata.bmk.sh/data/pile/val.jsonl.zst": {"num_bytes": 470907480, "checksum": "264c875d8bbd355d8daa9d032b75fd8fb91606218bb84dd1155b203fcd5fab92"}, "http://eaidata.bmk.sh/data/pile/test.jsonl.zst": {"num_bytes": 460250856, "checksum": "0bb28c52d0b5596d389bf179ce2d43bf7f7ffae76b0d2d20b180c97f62e0975e"}}, "download_size": 931158336, "post_processing_size": null, "dataset_size": 19456923, "size_in_bytes": 950615259}, "pile_dm-mathematics": {"description": "The Pile is a 825 GiB diverse, open source language modeling data set that consists\nof 22 smaller, high-quality datasets combined together. To score well on Pile\nBPB (bits per byte), a model must be able to understand many disparate domains\nincluding books, github repositories, webpages, chat logs, and medical, physics,\nmath, computer science, and philosophy papers.\n\nDM Mathematics", "citation": "@article{pile,\n title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},\n author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},\n journal={arXiv preprint arXiv:2101.00027},\n year={2020}\n}\n", "homepage": "https://pile.eleuther.ai/", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "pile", "config_name": "pile_dm-mathematics", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"test": {"name": "test", "num_bytes": 15756556, "num_examples": 1922, "dataset_name": "pile"}, "validation": {"name": "validation", "num_bytes": 16453386, "num_examples": 2007, "dataset_name": "pile"}}, "download_checksums": {"http://eaidata.bmk.sh/data/pile/val.jsonl.zst": {"num_bytes": 470907480, "checksum": "264c875d8bbd355d8daa9d032b75fd8fb91606218bb84dd1155b203fcd5fab92"}, "http://eaidata.bmk.sh/data/pile/test.jsonl.zst": {"num_bytes": 460250856, "checksum": "0bb28c52d0b5596d389bf179ce2d43bf7f7ffae76b0d2d20b180c97f62e0975e"}}, "download_size": 931158336, "post_processing_size": null, "dataset_size": 32209942, "size_in_bytes": 963368278}, "pile_enron": {"description": "The Pile is a 825 GiB diverse, open source language modeling data set that consists\nof 22 smaller, high-quality datasets combined together. To score well on Pile\nBPB (bits per byte), a model must be able to understand many disparate domains\nincluding books, github repositories, webpages, chat logs, and medical, physics,\nmath, computer science, and philosophy papers.\n\nEnron Emails", "citation": "@article{pile,\n title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},\n author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},\n journal={arXiv preprint arXiv:2101.00027},\n year={2020}\n}\n", "homepage": "https://pile.eleuther.ai/", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "pile", "config_name": "pile_enron", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"test": {"name": "test", "num_bytes": 1638859, "num_examples": 1010, "dataset_name": "pile"}, "validation": {"name": "validation", "num_bytes": 1556487, "num_examples": 947, "dataset_name": "pile"}}, "download_checksums": {"http://eaidata.bmk.sh/data/pile/val.jsonl.zst": {"num_bytes": 470907480, "checksum": "264c875d8bbd355d8daa9d032b75fd8fb91606218bb84dd1155b203fcd5fab92"}, "http://eaidata.bmk.sh/data/pile/test.jsonl.zst": {"num_bytes": 460250856, "checksum": "0bb28c52d0b5596d389bf179ce2d43bf7f7ffae76b0d2d20b180c97f62e0975e"}}, "download_size": 931158336, "post_processing_size": null, "dataset_size": 3195346, "size_in_bytes": 934353682}, "pile_europarl": {"description": "The Pile is a 825 GiB diverse, open source language modeling data set that consists\nof 22 smaller, high-quality datasets combined together. To score well on Pile\nBPB (bits per byte), a model must be able to understand many disparate domains\nincluding books, github repositories, webpages, chat logs, and medical, physics,\nmath, computer science, and philosophy papers.\n\nEuroParl", "citation": "@article{pile,\n title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},\n author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},\n journal={arXiv preprint arXiv:2101.00027},\n year={2020}\n}\n", "homepage": "https://pile.eleuther.ai/", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "pile", "config_name": "pile_europarl", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"test": {"name": "test", "num_bytes": 8789652, "num_examples": 157, "dataset_name": "pile"}, "validation": {"name": "validation", "num_bytes": 9111791, "num_examples": 133, "dataset_name": "pile"}}, "download_checksums": {"http://eaidata.bmk.sh/data/pile/val.jsonl.zst": {"num_bytes": 470907480, "checksum": "264c875d8bbd355d8daa9d032b75fd8fb91606218bb84dd1155b203fcd5fab92"}, "http://eaidata.bmk.sh/data/pile/test.jsonl.zst": {"num_bytes": 460250856, "checksum": "0bb28c52d0b5596d389bf179ce2d43bf7f7ffae76b0d2d20b180c97f62e0975e"}}, "download_size": 931158336, "post_processing_size": null, "dataset_size": 17901443, "size_in_bytes": 949059779}, "pile_freelaw": {"description": "The Pile is a 825 GiB diverse, open source language modeling data set that consists\nof 22 smaller, high-quality datasets combined together. To score well on Pile\nBPB (bits per byte), a model must be able to understand many disparate domains\nincluding books, github repositories, webpages, chat logs, and medical, physics,\nmath, computer science, and philosophy papers.\n\nFreeLaw", "citation": "@article{pile,\n title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},\n author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},\n journal={arXiv preprint arXiv:2101.00027},\n year={2020}\n}\n", "homepage": "https://pile.eleuther.ai/", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "pile", "config_name": "pile_freelaw", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"test": {"name": "test", "num_bytes": 80808693, "num_examples": 5101, "dataset_name": "pile"}, "validation": {"name": "validation", "num_bytes": 80363814, "num_examples": 5094, "dataset_name": "pile"}}, "download_checksums": {"http://eaidata.bmk.sh/data/pile/val.jsonl.zst": {"num_bytes": 470907480, "checksum": "264c875d8bbd355d8daa9d032b75fd8fb91606218bb84dd1155b203fcd5fab92"}, "http://eaidata.bmk.sh/data/pile/test.jsonl.zst": {"num_bytes": 460250856, "checksum": "0bb28c52d0b5596d389bf179ce2d43bf7f7ffae76b0d2d20b180c97f62e0975e"}}, "download_size": 931158336, "post_processing_size": null, "dataset_size": 161172507, "size_in_bytes": 1092330843}, "pile_github": {"description": "The Pile is a 825 GiB diverse, open source language modeling data set that consists\nof 22 smaller, high-quality datasets combined together. To score well on Pile\nBPB (bits per byte), a model must be able to understand many disparate domains\nincluding books, github repositories, webpages, chat logs, and medical, physics,\nmath, computer science, and philosophy papers.\n\nGithub", "citation": "@article{pile,\n title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},\n author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},\n journal={arXiv preprint arXiv:2101.00027},\n year={2020}\n}\n", "homepage": "https://pile.eleuther.ai/", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "pile", "config_name": "pile_github", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"test": {"name": "test", "num_bytes": 95654706, "num_examples": 18195, "dataset_name": "pile"}, "validation": {"name": "validation", "num_bytes": 97179576, "num_examples": 18337, "dataset_name": "pile"}}, "download_checksums": {"http://eaidata.bmk.sh/data/pile/val.jsonl.zst": {"num_bytes": 470907480, "checksum": "264c875d8bbd355d8daa9d032b75fd8fb91606218bb84dd1155b203fcd5fab92"}, "http://eaidata.bmk.sh/data/pile/test.jsonl.zst": {"num_bytes": 460250856, "checksum": "0bb28c52d0b5596d389bf179ce2d43bf7f7ffae76b0d2d20b180c97f62e0975e"}}, "download_size": 931158336, "post_processing_size": null, "dataset_size": 192834282, "size_in_bytes": 1123992618}, "pile_gutenberg": {"description": "The Pile is a 825 GiB diverse, open source language modeling data set that consists\nof 22 smaller, high-quality datasets combined together. To score well on Pile\nBPB (bits per byte), a model must be able to understand many disparate domains\nincluding books, github repositories, webpages, chat logs, and medical, physics,\nmath, computer science, and philosophy papers.\n\nGutenberg (PG-19)", "citation": "@article{pile,\n title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},\n author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},\n journal={arXiv preprint arXiv:2101.00027},\n year={2020}\n}\n", "homepage": "https://pile.eleuther.ai/", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "pile", "config_name": "pile_gutenberg", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"test": {"name": "test", "num_bytes": 30243176, "num_examples": 80, "dataset_name": "pile"}, "validation": {"name": "validation", "num_bytes": 24685980, "num_examples": 60, "dataset_name": "pile"}}, "download_checksums": {"http://eaidata.bmk.sh/data/pile/val.jsonl.zst": {"num_bytes": 470907480, "checksum": "264c875d8bbd355d8daa9d032b75fd8fb91606218bb84dd1155b203fcd5fab92"}, "http://eaidata.bmk.sh/data/pile/test.jsonl.zst": {"num_bytes": 460250856, "checksum": "0bb28c52d0b5596d389bf179ce2d43bf7f7ffae76b0d2d20b180c97f62e0975e"}}, "download_size": 931158336, "post_processing_size": null, "dataset_size": 54929156, "size_in_bytes": 986087492}, "pile_hackernews": {"description": "The Pile is a 825 GiB diverse, open source language modeling data set that consists\nof 22 smaller, high-quality datasets combined together. To score well on Pile\nBPB (bits per byte), a model must be able to understand many disparate domains\nincluding books, github repositories, webpages, chat logs, and medical, physics,\nmath, computer science, and philosophy papers.\n\nHackerNews", "citation": "@article{pile,\n title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},\n author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},\n journal={arXiv preprint arXiv:2101.00027},\n year={2020}\n}\n", "homepage": "https://pile.eleuther.ai/", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "pile", "config_name": "pile_hackernews", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"test": {"name": "test", "num_bytes": 8124255, "num_examples": 1632, "dataset_name": "pile"}, "validation": {"name": "validation", "num_bytes": 9803822, "num_examples": 1619, "dataset_name": "pile"}}, "download_checksums": {"http://eaidata.bmk.sh/data/pile/val.jsonl.zst": {"num_bytes": 470907480, "checksum": "264c875d8bbd355d8daa9d032b75fd8fb91606218bb84dd1155b203fcd5fab92"}, "http://eaidata.bmk.sh/data/pile/test.jsonl.zst": {"num_bytes": 460250856, "checksum": "0bb28c52d0b5596d389bf179ce2d43bf7f7ffae76b0d2d20b180c97f62e0975e"}}, "download_size": 931158336, "post_processing_size": null, "dataset_size": 17928077, "size_in_bytes": 949086413}, "pile_nih-exporter": {"description": "The Pile is a 825 GiB diverse, open source language modeling data set that consists\nof 22 smaller, high-quality datasets combined together. To score well on Pile\nBPB (bits per byte), a model must be able to understand many disparate domains\nincluding books, github repositories, webpages, chat logs, and medical, physics,\nmath, computer science, and philosophy papers.\n\nNIH ExPorter", "citation": "@article{pile,\n title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},\n author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},\n journal={arXiv preprint arXiv:2101.00027},\n year={2020}\n}\n", "homepage": "https://pile.eleuther.ai/", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "pile", "config_name": "pile_nih-exporter", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"test": {"name": "test", "num_bytes": 3928804, "num_examples": 1884, "dataset_name": "pile"}, "validation": {"name": "validation", "num_bytes": 3927967, "num_examples": 1825, "dataset_name": "pile"}}, "download_checksums": {"http://eaidata.bmk.sh/data/pile/val.jsonl.zst": {"num_bytes": 470907480, "checksum": "264c875d8bbd355d8daa9d032b75fd8fb91606218bb84dd1155b203fcd5fab92"}, "http://eaidata.bmk.sh/data/pile/test.jsonl.zst": {"num_bytes": 460250856, "checksum": "0bb28c52d0b5596d389bf179ce2d43bf7f7ffae76b0d2d20b180c97f62e0975e"}}, "download_size": 931158336, "post_processing_size": null, "dataset_size": 7856771, "size_in_bytes": 939015107}, "pile_opensubtitles": {"description": "The Pile is a 825 GiB diverse, open source language modeling data set that consists\nof 22 smaller, high-quality datasets combined together. To score well on Pile\nBPB (bits per byte), a model must be able to understand many disparate domains\nincluding books, github repositories, webpages, chat logs, and medical, physics,\nmath, computer science, and philosophy papers.\n\nOpenSubtitles", "citation": "@article{pile,\n title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},\n author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},\n journal={arXiv preprint arXiv:2101.00027},\n year={2020}\n}\n", "homepage": "https://pile.eleuther.ai/", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "pile", "config_name": "pile_opensubtitles", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"test": {"name": "test", "num_bytes": 21008996, "num_examples": 642, "dataset_name": "pile"}, "validation": {"name": "validation", "num_bytes": 19622904, "num_examples": 621, "dataset_name": "pile"}}, "download_checksums": {"http://eaidata.bmk.sh/data/pile/val.jsonl.zst": {"num_bytes": 470907480, "checksum": "264c875d8bbd355d8daa9d032b75fd8fb91606218bb84dd1155b203fcd5fab92"}, "http://eaidata.bmk.sh/data/pile/test.jsonl.zst": {"num_bytes": 460250856, "checksum": "0bb28c52d0b5596d389bf179ce2d43bf7f7ffae76b0d2d20b180c97f62e0975e"}}, "download_size": 931158336, "post_processing_size": null, "dataset_size": 40631900, "size_in_bytes": 971790236}, "pile_openwebtext2": {"description": "The Pile is a 825 GiB diverse, open source language modeling data set that consists\nof 22 smaller, high-quality datasets combined together. To score well on Pile\nBPB (bits per byte), a model must be able to understand many disparate domains\nincluding books, github repositories, webpages, chat logs, and medical, physics,\nmath, computer science, and philosophy papers.\n\nOpenWebText2", "citation": "@article{pile,\n title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},\n author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},\n journal={arXiv preprint arXiv:2101.00027},\n year={2020}\n}\n", "homepage": "https://pile.eleuther.ai/", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "pile", "config_name": "pile_openwebtext2", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"test": {"name": "test", "num_bytes": 128624303, "num_examples": 32925, "dataset_name": "pile"}, "validation": {"name": "validation", "num_bytes": 131554302, "num_examples": 33400, "dataset_name": "pile"}}, "download_checksums": {"http://eaidata.bmk.sh/data/pile/val.jsonl.zst": {"num_bytes": 470907480, "checksum": "264c875d8bbd355d8daa9d032b75fd8fb91606218bb84dd1155b203fcd5fab92"}, "http://eaidata.bmk.sh/data/pile/test.jsonl.zst": {"num_bytes": 460250856, "checksum": "0bb28c52d0b5596d389bf179ce2d43bf7f7ffae76b0d2d20b180c97f62e0975e"}}, "download_size": 931158336, "post_processing_size": null, "dataset_size": 260178605, "size_in_bytes": 1191336941}, "pile_philpapers": {"description": "The Pile is a 825 GiB diverse, open source language modeling data set that consists\nof 22 smaller, high-quality datasets combined together. To score well on Pile\nBPB (bits per byte), a model must be able to understand many disparate domains\nincluding books, github repositories, webpages, chat logs, and medical, physics,\nmath, computer science, and philosophy papers.\n\nPhilPapers", "citation": "@article{pile,\n title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},\n author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},\n journal={arXiv preprint arXiv:2101.00027},\n year={2020}\n}\n", "homepage": "https://pile.eleuther.ai/", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "pile", "config_name": "pile_philpapers", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"test": {"name": "test", "num_bytes": 5090158, "num_examples": 68, "dataset_name": "pile"}, "validation": {"name": "validation", "num_bytes": 6499078, "num_examples": 64, "dataset_name": "pile"}}, "download_checksums": {"http://eaidata.bmk.sh/data/pile/val.jsonl.zst": {"num_bytes": 470907480, "checksum": "264c875d8bbd355d8daa9d032b75fd8fb91606218bb84dd1155b203fcd5fab92"}, "http://eaidata.bmk.sh/data/pile/test.jsonl.zst": {"num_bytes": 460250856, "checksum": "0bb28c52d0b5596d389bf179ce2d43bf7f7ffae76b0d2d20b180c97f62e0975e"}}, "download_size": 931158336, "post_processing_size": null, "dataset_size": 11589236, "size_in_bytes": 942747572}, "pile_pile-cc": {"description": "The Pile is a 825 GiB diverse, open source language modeling data set that consists\nof 22 smaller, high-quality datasets combined together. To score well on Pile\nBPB (bits per byte), a model must be able to understand many disparate domains\nincluding books, github repositories, webpages, chat logs, and medical, physics,\nmath, computer science, and philosophy papers.\n\nPile-CC", "citation": "@article{pile,\n title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},\n author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},\n journal={arXiv preprint arXiv:2101.00027},\n year={2020}\n}\n", "homepage": "https://pile.eleuther.ai/", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "pile", "config_name": "pile_pile-cc", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"test": {"name": "test", "num_bytes": 235004043, "num_examples": 52790, "dataset_name": "pile"}, "validation": {"name": "validation", "num_bytes": 233535650, "num_examples": 52792, "dataset_name": "pile"}}, "download_checksums": {"http://eaidata.bmk.sh/data/pile/val.jsonl.zst": {"num_bytes": 470907480, "checksum": "264c875d8bbd355d8daa9d032b75fd8fb91606218bb84dd1155b203fcd5fab92"}, "http://eaidata.bmk.sh/data/pile/test.jsonl.zst": {"num_bytes": 460250856, "checksum": "0bb28c52d0b5596d389bf179ce2d43bf7f7ffae76b0d2d20b180c97f62e0975e"}}, "download_size": 931158336, "post_processing_size": null, "dataset_size": 468539693, "size_in_bytes": 1399698029}, "pile_pubmed-abstracts": {"description": "The Pile is a 825 GiB diverse, open source language modeling data set that consists\nof 22 smaller, high-quality datasets combined together. To score well on Pile\nBPB (bits per byte), a model must be able to understand many disparate domains\nincluding books, github repositories, webpages, chat logs, and medical, physics,\nmath, computer science, and philosophy papers.\n\nPubMed Abstracts", "citation": "@article{pile,\n title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},\n author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},\n journal={arXiv preprint arXiv:2101.00027},\n year={2020}\n}\n", "homepage": "https://pile.eleuther.ai/", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "pile", "config_name": "pile_pubmed-abstracts", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"test": {"name": "test", "num_bytes": 39908950, "num_examples": 29895, "dataset_name": "pile"}, "validation": {"name": "validation", "num_bytes": 40008336, "num_examples": 29871, "dataset_name": "pile"}}, "download_checksums": {"http://eaidata.bmk.sh/data/pile/val.jsonl.zst": {"num_bytes": 470907480, "checksum": "264c875d8bbd355d8daa9d032b75fd8fb91606218bb84dd1155b203fcd5fab92"}, "http://eaidata.bmk.sh/data/pile/test.jsonl.zst": {"num_bytes": 460250856, "checksum": "0bb28c52d0b5596d389bf179ce2d43bf7f7ffae76b0d2d20b180c97f62e0975e"}}, "download_size": 931158336, "post_processing_size": null, "dataset_size": 79917286, "size_in_bytes": 1011075622}, "pile_pubmed-central": {"description": "The Pile is a 825 GiB diverse, open source language modeling data set that consists\nof 22 smaller, high-quality datasets combined together. To score well on Pile\nBPB (bits per byte), a model must be able to understand many disparate domains\nincluding books, github repositories, webpages, chat logs, and medical, physics,\nmath, computer science, and philosophy papers.\n\nPubMed Central", "citation": "@article{pile,\n title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},\n author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},\n journal={arXiv preprint arXiv:2101.00027},\n year={2020}\n}\n", "homepage": "https://pile.eleuther.ai/", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "pile", "config_name": "pile_pubmed-central", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"test": {"name": "test", "num_bytes": 187251519, "num_examples": 5911, "dataset_name": "pile"}, "validation": {"name": "validation", "num_bytes": 184791818, "num_examples": 5977, "dataset_name": "pile"}}, "download_checksums": {"http://eaidata.bmk.sh/data/pile/val.jsonl.zst": {"num_bytes": 470907480, "checksum": "264c875d8bbd355d8daa9d032b75fd8fb91606218bb84dd1155b203fcd5fab92"}, "http://eaidata.bmk.sh/data/pile/test.jsonl.zst": {"num_bytes": 460250856, "checksum": "0bb28c52d0b5596d389bf179ce2d43bf7f7ffae76b0d2d20b180c97f62e0975e"}}, "download_size": 931158336, "post_processing_size": null, "dataset_size": 372043337, "size_in_bytes": 1303201673}, "pile_stackexchange": {"description": "The Pile is a 825 GiB diverse, open source language modeling data set that consists\nof 22 smaller, high-quality datasets combined together. To score well on Pile\nBPB (bits per byte), a model must be able to understand many disparate domains\nincluding books, github repositories, webpages, chat logs, and medical, physics,\nmath, computer science, and philosophy papers.\n\nStackExchange", "citation": "@article{pile,\n title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},\n author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},\n journal={arXiv preprint arXiv:2101.00027},\n year={2020}\n}\n", "homepage": "https://pile.eleuther.ai/", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "pile", "config_name": "pile_stackexchange", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"test": {"name": "test", "num_bytes": 66441557, "num_examples": 30378, "dataset_name": "pile"}, "validation": {"name": "validation", "num_bytes": 66011397, "num_examples": 29950, "dataset_name": "pile"}}, "download_checksums": {"http://eaidata.bmk.sh/data/pile/val.jsonl.zst": {"num_bytes": 470907480, "checksum": "264c875d8bbd355d8daa9d032b75fd8fb91606218bb84dd1155b203fcd5fab92"}, "http://eaidata.bmk.sh/data/pile/test.jsonl.zst": {"num_bytes": 460250856, "checksum": "0bb28c52d0b5596d389bf179ce2d43bf7f7ffae76b0d2d20b180c97f62e0975e"}}, "download_size": 931158336, "post_processing_size": null, "dataset_size": 132452954, "size_in_bytes": 1063611290}, "pile_upsto": {"description": "The Pile is a 825 GiB diverse, open source language modeling data set that consists\nof 22 smaller, high-quality datasets combined together. To score well on Pile\nBPB (bits per byte), a model must be able to understand many disparate domains\nincluding books, github repositories, webpages, chat logs, and medical, physics,\nmath, computer science, and philosophy papers.\n\nUSPTO Backgrounds", "citation": "@article{pile,\n title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},\n author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},\n journal={arXiv preprint arXiv:2101.00027},\n year={2020}\n}\n", "homepage": "https://pile.eleuther.ai/", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "pile", "config_name": "pile_upsto", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"test": {"name": "test", "num_bytes": 47345405, "num_examples": 11415, "dataset_name": "pile"}, "validation": {"name": "validation", "num_bytes": 48122320, "num_examples": 11387, "dataset_name": "pile"}}, "download_checksums": {"http://eaidata.bmk.sh/data/pile/val.jsonl.zst": {"num_bytes": 470907480, "checksum": "264c875d8bbd355d8daa9d032b75fd8fb91606218bb84dd1155b203fcd5fab92"}, "http://eaidata.bmk.sh/data/pile/test.jsonl.zst": {"num_bytes": 460250856, "checksum": "0bb28c52d0b5596d389bf179ce2d43bf7f7ffae76b0d2d20b180c97f62e0975e"}}, "download_size": 931158336, "post_processing_size": null, "dataset_size": 95467725, "size_in_bytes": 1026626061}, "pile_ubuntu-irc": {"description": "The Pile is a 825 GiB diverse, open source language modeling data set that consists\nof 22 smaller, high-quality datasets combined together. To score well on Pile\nBPB (bits per byte), a model must be able to understand many disparate domains\nincluding books, github repositories, webpages, chat logs, and medical, physics,\nmath, computer science, and philosophy papers.\n\nUbuntu IRC", "citation": "@article{pile,\n title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},\n author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},\n journal={arXiv preprint arXiv:2101.00027},\n year={2020}\n}\n", "homepage": "https://pile.eleuther.ai/", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "pile", "config_name": "pile_ubuntu-irc", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"test": {"name": "test", "num_bytes": 5694218, "num_examples": 22, "dataset_name": "pile"}, "validation": {"name": "validation", "num_bytes": 7410104, "num_examples": 21, "dataset_name": "pile"}}, "download_checksums": {"http://eaidata.bmk.sh/data/pile/val.jsonl.zst": {"num_bytes": 470907480, "checksum": "264c875d8bbd355d8daa9d032b75fd8fb91606218bb84dd1155b203fcd5fab92"}, "http://eaidata.bmk.sh/data/pile/test.jsonl.zst": {"num_bytes": 460250856, "checksum": "0bb28c52d0b5596d389bf179ce2d43bf7f7ffae76b0d2d20b180c97f62e0975e"}}, "download_size": 931158336, "post_processing_size": null, "dataset_size": 13104322, "size_in_bytes": 944262658}, "pile_wikipedia": {"description": "The Pile is a 825 GiB diverse, open source language modeling data set that consists\nof 22 smaller, high-quality datasets combined together. To score well on Pile\nBPB (bits per byte), a model must be able to understand many disparate domains\nincluding books, github repositories, webpages, chat logs, and medical, physics,\nmath, computer science, and philosophy papers.\n\nWikipedia (en)", "citation": "@article{pile,\n title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},\n author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},\n journal={arXiv preprint arXiv:2101.00027},\n year={2020}\n}\n", "homepage": "https://pile.eleuther.ai/", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "pile", "config_name": "pile_wikipedia", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"test": {"name": "test", "num_bytes": 52166968, "num_examples": 17511, "dataset_name": "pile"}, "validation": {"name": "validation", "num_bytes": 53186137, "num_examples": 17478, "dataset_name": "pile"}}, "download_checksums": {"http://eaidata.bmk.sh/data/pile/val.jsonl.zst": {"num_bytes": 470907480, "checksum": "264c875d8bbd355d8daa9d032b75fd8fb91606218bb84dd1155b203fcd5fab92"}, "http://eaidata.bmk.sh/data/pile/test.jsonl.zst": {"num_bytes": 460250856, "checksum": "0bb28c52d0b5596d389bf179ce2d43bf7f7ffae76b0d2d20b180c97f62e0975e"}}, "download_size": 931158336, "post_processing_size": null, "dataset_size": 105353105, "size_in_bytes": 1036511441}, "pile_youtubesubtitles": {"description": "The Pile is a 825 GiB diverse, open source language modeling data set that consists\nof 22 smaller, high-quality datasets combined together. To score well on Pile\nBPB (bits per byte), a model must be able to understand many disparate domains\nincluding books, github repositories, webpages, chat logs, and medical, physics,\nmath, computer science, and philosophy papers.\n\nYoutubeSubtitles", "citation": "@article{pile,\n title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling},\n author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor},\n journal={arXiv preprint arXiv:2101.00027},\n year={2020}\n}\n", "homepage": "https://pile.eleuther.ai/", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "pile", "config_name": "pile_youtubesubtitles", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"test": {"name": "test", "num_bytes": 7377448, "num_examples": 342, "dataset_name": "pile"}, "validation": {"name": "validation", "num_bytes": 8937546, "num_examples": 326, "dataset_name": "pile"}}, "download_checksums": {"http://eaidata.bmk.sh/data/pile/val.jsonl.zst": {"num_bytes": 470907480, "checksum": "264c875d8bbd355d8daa9d032b75fd8fb91606218bb84dd1155b203fcd5fab92"}, "http://eaidata.bmk.sh/data/pile/test.jsonl.zst": {"num_bytes": 460250856, "checksum": "0bb28c52d0b5596d389bf179ce2d43bf7f7ffae76b0d2d20b180c97f62e0975e"}}, "download_size": 931158336, "post_processing_size": null, "dataset_size": 16314994, "size_in_bytes": 947473330}} diff --git a/evaluation-pipeline/lm_eval/datasets/pile/pile.py b/evaluation-pipeline/lm_eval/datasets/pile/pile.py new file mode 100644 index 0000000000000000000000000000000000000000..208f45365607bd83e4a8ee347f06db628841edf3 --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/pile/pile.py @@ -0,0 +1,126 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Pile dataset.""" + + +import json + +import datasets + + +_CITATION = """\ +@article{pile, + title={The {P}ile: An 800GB Dataset of Diverse Text for Language Modeling}, + author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and Presser, Shawn and Leahy, Connor}, + journal={arXiv preprint arXiv:2101.00027}, + year={2020} +} +""" + +_DESCRIPTION = """\ +The Pile is a 825 GiB diverse, open source language modeling data set that consists +of 22 smaller, high-quality datasets combined together. To score well on Pile +BPB (bits per byte), a model must be able to understand many disparate domains +including books, github repositories, webpages, chat logs, and medical, physics, +math, computer science, and philosophy papers. +""" + +_HOMEPAGE = "https://pile.eleuther.ai/" + +# TODO: Add the licence for the dataset here if you can find it +_LICENSE = "" + +_URLS = { + "validation": "http://eaidata.bmk.sh/data/pile/val.jsonl.zst", + "test": "http://eaidata.bmk.sh/data/pile/test.jsonl.zst", +} + +_NAMES = { + "pile_arxiv": "ArXiv", + "pile_books3": "Books3", + "pile_bookcorpus2": "BookCorpus2", + "pile_dm-mathematics": "DM Mathematics", + "pile_enron": "Enron Emails", + "pile_europarl": "EuroParl", + "pile_freelaw": "FreeLaw", + "pile_github": "Github", + "pile_gutenberg": "Gutenberg (PG-19)", + "pile_hackernews": "HackerNews", + "pile_nih-exporter": "NIH ExPorter", + "pile_opensubtitles": "OpenSubtitles", + "pile_openwebtext2": "OpenWebText2", + "pile_philpapers": "PhilPapers", + "pile_pile-cc": "Pile-CC", + "pile_pubmed-abstracts": "PubMed Abstracts", + "pile_pubmed-central": "PubMed Central", + "pile_stackexchange": "StackExchange", + "pile_upsto": "USPTO Backgrounds", + "pile_ubuntu-irc": "Ubuntu IRC", + "pile_wikipedia": "Wikipedia (en)", + "pile_youtubesubtitles": "YoutubeSubtitles", +} + + +class Pile(datasets.GeneratorBasedBuilder): + """The Pile is a 825 GiB diverse, open source language modeling dataset.""" + + VERSION = datasets.Version("0.0.1") + + BUILDER_CONFIGS = [ + datasets.BuilderConfig(name=name, version=version, description=_NAMES[name]) + for name, version in zip(_NAMES.keys(), [VERSION] * len(_NAMES)) + ] + + def _info(self): + features = datasets.Features( + { + "text": datasets.Value("string"), + } + ) + return datasets.DatasetInfo( + description=f"{_DESCRIPTION}\n{self.config.description}", + features=features, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager): + urls = {"validation": _URLS["validation"], "test": _URLS["test"]} + data_dir = dl_manager.download_and_extract(urls) + return [ + datasets.SplitGenerator( + name=datasets.Split.TEST, + # These kwargs will be passed to _generate_examples + gen_kwargs={"filepath": data_dir["test"], "split": "test"}, + ), + datasets.SplitGenerator( + name=datasets.Split.VALIDATION, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "filepath": data_dir["validation"], + "split": "validation", + }, + ), + ] + + # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` + def _generate_examples(self, filepath, split): + with open(filepath, encoding="utf-8") as f: + for key, row in enumerate(f): + data = json.loads(row) + if data["meta"]["pile_set_name"] == _NAMES[self.config.name]: + yield key, { + "text": data["text"], + } diff --git a/evaluation-pipeline/lm_eval/datasets/quac/__init__.py b/evaluation-pipeline/lm_eval/datasets/quac/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/lm_eval/datasets/quac/dataset_infos.json b/evaluation-pipeline/lm_eval/datasets/quac/dataset_infos.json new file mode 100644 index 0000000000000000000000000000000000000000..86fe853167145358addae0443729646dfe9585ec --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/quac/dataset_infos.json @@ -0,0 +1 @@ +{"quac": {"description": "Question Answering in Context (QuAC) is a dataset for modeling, understanding, and \nparticipating in information seeking dialog. Data instances consist of an interactive\ndialog between two crowd workers: (1) a student who poses a sequence of freeform\nquestions to learn as much as possible about a hidden Wikipedia text, and (2)\na teacher who answers the questions by providing short excerpts (spans) from the text.\n", "citation": "@article{choi2018quac,\n title={Quac: Question answering in context},\n author={Choi, Eunsol and He, He and Iyyer, Mohit and Yatskar, Mark and Yih, Wen-tau and Choi, Yejin and Liang, Percy and Zettlemoyer, Luke},\n journal={arXiv preprint arXiv:1808.07036},\n year={2018}\n}\n", "homepage": "https://quac.ai/", "license": "", "features": {"title": {"dtype": "string", "id": null, "_type": "Value"}, "section_title": {"dtype": "string", "id": null, "_type": "Value"}, "paragraph": {"dtype": "string", "id": null, "_type": "Value"}, "question": {"dtype": "string", "id": null, "_type": "Value"}, "answer": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "quac", "config_name": "quac", "version": {"version_str": "1.1.0", "description": null, "major": 1, "minor": 1, "patch": 0}, "splits": {"train": {"name": "train", "num_bytes": 212391958, "num_examples": 83568, "dataset_name": "quac"}, "validation": {"name": "validation", "num_bytes": 20678483, "num_examples": 7354, "dataset_name": "quac"}}, "download_checksums": {"https://s3.amazonaws.com/my89public/quac/train_v0.2.json": {"num_bytes": 68114819, "checksum": "ff5cca5a2e4b4d1cb5b5ced68b9fce88394ef6d93117426d6d4baafbcc05c56a"}, "https://s3.amazonaws.com/my89public/quac/val_v0.2.json": {"num_bytes": 8929167, "checksum": "09e622916280ba04c9352acb1bc5bbe80f11a2598f6f34e934c51d9e6570f378"}}, "download_size": 77043986, "post_processing_size": null, "dataset_size": 233070441, "size_in_bytes": 310114427}} diff --git a/evaluation-pipeline/lm_eval/datasets/quac/quac.py b/evaluation-pipeline/lm_eval/datasets/quac/quac.py new file mode 100644 index 0000000000000000000000000000000000000000..4328ec89eba5f95588f9ddb7c67409e040ce89f6 --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/quac/quac.py @@ -0,0 +1,117 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# TODO: Address all TODOs and remove all explanatory comments +"""QuAC dataset.""" + + +import json + +import datasets + + +_CITATION = """\ +@article{choi2018quac, + title={Quac: Question answering in context}, + author={Choi, Eunsol and He, He and Iyyer, Mohit and Yatskar, Mark and Yih, Wen-tau and Choi, Yejin and Liang, Percy and Zettlemoyer, Luke}, + journal={arXiv preprint arXiv:1808.07036}, + year={2018} +} +""" + +_DESCRIPTION = """\ +Question Answering in Context (QuAC) is a dataset for modeling, understanding, and +participating in information seeking dialog. Data instances consist of an interactive +dialog between two crowd workers: (1) a student who poses a sequence of freeform +questions to learn as much as possible about a hidden Wikipedia text, and (2) +a teacher who answers the questions by providing short excerpts (spans) from the text. +""" + +_HOMEPAGE = "https://quac.ai/" + +# TODO: Add the licence for the dataset here if you can find it +_LICENSE = "" + +_URLS = { + "train": "https://s3.amazonaws.com/my89public/quac/train_v0.2.json", + "validation": "https://s3.amazonaws.com/my89public/quac/val_v0.2.json", +} + + +class Quac(datasets.GeneratorBasedBuilder): + """Question Answering in Context (QuAC) is a dataset for modeling, understanding, and participating in information seeking dialog.""" + + VERSION = datasets.Version("1.1.0") + + BUILDER_CONFIGS = [ + datasets.BuilderConfig( + name="quac", version=VERSION, description="The QuAC dataset" + ), + ] + + def _info(self): + features = datasets.Features( + { + "title": datasets.Value("string"), + "section_title": datasets.Value("string"), + "paragraph": datasets.Value("string"), + "question": datasets.Value("string"), + "answer": datasets.Value("string"), + } + ) + return datasets.DatasetInfo( + description=_DESCRIPTION, + features=features, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager): + urls = {"train": _URLS["train"], "validation": _URLS["validation"]} + data_dir = dl_manager.download_and_extract(urls) + return [ + datasets.SplitGenerator( + name=datasets.Split.TRAIN, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "filepath": data_dir["train"], + "split": "train", + }, + ), + datasets.SplitGenerator( + name=datasets.Split.VALIDATION, + # These kwargs will be passed to _generate_examples + gen_kwargs={"filepath": data_dir["validation"], "split": "validation"}, + ), + ] + + # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` + def _generate_examples(self, filepath, split): + with open(filepath, encoding="utf-8") as f: + data = json.load(f)["data"] + key = 0 + for row in data: + paragraph = row["paragraphs"][0]["context"].replace("CANNOTANSWER", "") + qas = row["paragraphs"][0]["qas"] + qa_pairs = [(qa["question"], qa["answers"][0]["text"]) for qa in qas] + for (question, answer) in qa_pairs: + # Yields examples as (key, example) tuples + yield key, { + "title": row["title"], + "section_title": row["section_title"], + "paragraph": paragraph, + "question": question, + "answer": answer, + } + key += 1 diff --git a/evaluation-pipeline/lm_eval/datasets/sat_analogies/__init__.py b/evaluation-pipeline/lm_eval/datasets/sat_analogies/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/lm_eval/datasets/sat_analogies/sat_analogies.py b/evaluation-pipeline/lm_eval/datasets/sat_analogies/sat_analogies.py new file mode 100644 index 0000000000000000000000000000000000000000..76ae9af713723c08afecc0b6b3b7a4f9befd0ba4 --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/sat_analogies/sat_analogies.py @@ -0,0 +1,128 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""SAT Analogy Questions dataset.""" + + +import os + +import datasets + + +_CITATION = """\ +@article{article, + author = {Turney, Peter}, + year = {2006}, + month = {09}, + pages = {379-416}, + title = {Similarity of Semantic Relations}, + volume = {32}, + journal = {Computational Linguistics}, + doi = {10.1162/coli.2006.32.3.379} +} +""" + +_DESCRIPTION = """\ +SAT (Scholastic Aptitude Test) Analogy Questions is a dataset comprising 374 +multiple-choice analogy questions; 5 choices per question. +""" + +_HOMEPAGE = "https://aclweb.org/aclwiki/SAT_Analogy_Questions_(State_of_the_art)" + +# TODO: Add the licence for the dataset here if you can find it +_LICENSE = "" + + +class SatAnalogies(datasets.GeneratorBasedBuilder): + """SAT (Scholastic Aptitude Test) Analogy Questions is a dataset comprising 374 multiple-choice analogy questions.""" + + VERSION = datasets.Version("0.0.1") + + BUILDER_CONFIGS = [ + datasets.BuilderConfig( + name="sat_analogies", + version=VERSION, + description="The SAT Analogy Questions dataset", + ), + ] + + @property + def manual_download_instructions(self): + return ( + "To use SAT Analogy Questions you have to download it manually. Please " + "email Peter Turney to request the data (https://www.apperceptual.com). " + "Once you receive a download link for the dataset, supply the local path " + "as the `data_dir` arg: " + "`datasets.load_dataset('sat_analogies', data_dir='path/to/folder/folder_name')`" + ) + + def _info(self): + features = datasets.Features( + { + "source": datasets.Value("string"), + "stem": datasets.Value("string"), + "choices": datasets.features.Sequence(datasets.Value("string")), + "solution": datasets.Value("string"), + } + ) + return datasets.DatasetInfo( + description=_DESCRIPTION, + features=features, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager): + data_dir = os.path.abspath(os.path.expanduser(dl_manager.manual_dir)) + if not os.path.exists(data_dir): + raise FileNotFoundError( + f"{data_dir} does not exist. Make sure you insert a manual dir via `datasets.load_dataset('matinf', data_dir=...)` that includes SAT-package-V3.txt. Manual download instructions: {self.manual_download_instructions}" + ) + return [ + datasets.SplitGenerator( + name=datasets.Split.VALIDATION, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "filepath": os.path.join(data_dir, "SAT-package-V3.txt"), + }, + ) + ] + + # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` + def _generate_examples(self, filepath): + data = [] + with open(filepath, "r", encoding="utf-8") as f: + record = [] + for line in f: + line = line.strip() + if len(line) == 0 and record: + data.append(record) + record = [] + elif len(line) > 0 and line[0] == "#": + # Skip comments. + continue + else: + record.append(line) + data.append(record) + for key, record in enumerate(data): + source = record[-8] + stem = record[-7] + choices = record[-6:-1] + solution = record[-1] + yield key, { + "source": source, + "stem": stem, + "choices": choices, + "solution": solution, + } diff --git a/evaluation-pipeline/lm_eval/datasets/triviaqa/__init__.py b/evaluation-pipeline/lm_eval/datasets/triviaqa/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/lm_eval/datasets/triviaqa/dataset_infos.json b/evaluation-pipeline/lm_eval/datasets/triviaqa/dataset_infos.json new file mode 100644 index 0000000000000000000000000000000000000000..87f4e064cfea01354dbc69b1c784553fce783172 --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/triviaqa/dataset_infos.json @@ -0,0 +1 @@ +{"triviaqa": {"description": "TriviaQA is a reading comprehension dataset containing over 650K question-answer-evidence\ntriples. TriviaQA includes 95K question-answer pairs authored by trivia enthusiasts\nand independently gathered evidence documents, six per question on average, that provide\nhigh quality distant supervision for answering the questions.\n", "citation": "@InProceedings{JoshiTriviaQA2017,\n author = {Joshi, Mandar and Choi, Eunsol and Weld, Daniel S. and Zettlemoyer, Luke},\n title = {TriviaQA: A Large Scale Distantly Supervised Challenge Dataset for Reading Comprehension},\n booktitle = {Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics},\n month = {July},\n year = {2017},\n address = {Vancouver, Canada},\n publisher = {Association for Computational Linguistics},\n}\n", "homepage": "https://nlp.cs.washington.edu/triviaqa/", "license": "Apache License 2.0", "features": {"question_id": {"dtype": "string", "id": null, "_type": "Value"}, "question_source": {"dtype": "string", "id": null, "_type": "Value"}, "question": {"dtype": "string", "id": null, "_type": "Value"}, "answer": {"aliases": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "value": {"dtype": "string", "id": null, "_type": "Value"}}, "search_results": {"feature": {"description": {"dtype": "string", "id": null, "_type": "Value"}, "filename": {"dtype": "string", "id": null, "_type": "Value"}, "rank": {"dtype": "int32", "id": null, "_type": "Value"}, "title": {"dtype": "string", "id": null, "_type": "Value"}, "url": {"dtype": "string", "id": null, "_type": "Value"}, "search_context": {"dtype": "string", "id": null, "_type": "Value"}}, "length": -1, "id": null, "_type": "Sequence"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "triviaqa", "config_name": "triviaqa", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"train": {"name": "train", "num_bytes": 1271393601, "num_examples": 87622, "dataset_name": "triviaqa"}, "validation": {"name": "validation", "num_bytes": 163819509, "num_examples": 11313, "dataset_name": "triviaqa"}}, "download_checksums": {"http://eaidata.bmk.sh/data/triviaqa-unfiltered.tar.gz": {"num_bytes": 546481381, "checksum": "adc19b42769062d241a8fbe834c56e58598d9322eb6c614e9f33a68a2cf5523e"}}, "download_size": 546481381, "post_processing_size": null, "dataset_size": 1435213110, "size_in_bytes": 1981694491}} diff --git a/evaluation-pipeline/lm_eval/datasets/triviaqa/triviaqa.py b/evaluation-pipeline/lm_eval/datasets/triviaqa/triviaqa.py new file mode 100644 index 0000000000000000000000000000000000000000..a1c179ca20be4a83ae98dce0e841785e3eef2d91 --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/triviaqa/triviaqa.py @@ -0,0 +1,153 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Custom TriviaQA because HF version sanitizes the dataset differently. +# https://github.com/huggingface/datasets/blob/9977ade72191ff0b6907ec63935448c6269a91a1/datasets/trivia_qa/trivia_qa.py#L285 +"""TriviaQA (Unfiltered Raw) dataset.""" + + +import json +import os + +import datasets + + +_CITATION = """\ +@InProceedings{JoshiTriviaQA2017, + author = {Joshi, Mandar and Choi, Eunsol and Weld, Daniel S. and Zettlemoyer, Luke}, + title = {TriviaQA: A Large Scale Distantly Supervised Challenge Dataset for Reading Comprehension}, + booktitle = {Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics}, + month = {July}, + year = {2017}, + address = {Vancouver, Canada}, + publisher = {Association for Computational Linguistics}, +} +""" + +_DESCRIPTION = """\ +TriviaQA is a reading comprehension dataset containing over 650K question-answer-evidence +triples. TriviaQA includes 95K question-answer pairs authored by trivia enthusiasts +and independently gathered evidence documents, six per question on average, that provide +high quality distant supervision for answering the questions. +""" + +_HOMEPAGE = "https://nlp.cs.washington.edu/triviaqa/" + +_LICENSE = "Apache License 2.0" + +_URLS = "http://eaidata.bmk.sh/data/triviaqa-unfiltered.tar.gz" + + +class Triviaqa(datasets.GeneratorBasedBuilder): + """TriviaQA is a reading comprehension dataset containing over 650K question-answer-evidence triples""" + + VERSION = datasets.Version("0.0.1") + + BUILDER_CONFIGS = [ + datasets.BuilderConfig( + name="triviaqa", version=VERSION, description="The TriviaQA dataset" + ), + ] + + def _info(self): + features = datasets.Features( + { + "question_id": datasets.Value("string"), + "question_source": datasets.Value("string"), + "question": datasets.Value("string"), + "answer": { + "aliases": datasets.features.Sequence( + datasets.Value("string"), + ), + "value": datasets.Value("string"), + }, + "search_results": datasets.features.Sequence( + { + "description": datasets.Value("string"), + "filename": datasets.Value("string"), + "rank": datasets.Value("int32"), + "title": datasets.Value("string"), + "url": datasets.Value("string"), + "search_context": datasets.Value("string"), + } + ), + } + ) + return datasets.DatasetInfo( + description=_DESCRIPTION, + features=features, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager): + urls = _URLS + data_dir = dl_manager.download_and_extract(urls) + return [ + datasets.SplitGenerator( + name=datasets.Split.TRAIN, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "filepath": os.path.join(data_dir, "unfiltered-web-train.jsonl"), + }, + ), + datasets.SplitGenerator( + name=datasets.Split.VALIDATION, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "filepath": os.path.join(data_dir, "unfiltered-web-dev.jsonl"), + }, + ), + ] + + # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` + def _generate_examples(self, filepath): + with open(filepath, encoding="utf-8") as f: + for key, row in enumerate(f): + data = json.loads(row) + search_results = [] + for search_result in data["SearchResults"]: + search_results.append( + { + "description": search_result["Description"] + if "Description" in search_result + else "", + "filename": search_result["Filename"] + if "Filename" in search_result + else "", + "rank": search_result["Rank"] + if "Rank" in search_result + else -1, + "title": search_result["Title"] + if "Title" in search_result + else "", + "url": search_result["Url"] + if "Url" in search_result + else "", + "search_context": search_result["SearchContext"] + if "SearchContext" in search_result + else "", + } + ) + yield key, { + "question_id": data["QuestionId"], + "question_source": data["QuestionSource"], + "question": data["Question"], + "answer": { + "aliases": data["Answer"]["Aliases"], + "value": data["Answer"]["Value"], + }, + "search_results": search_results, + } diff --git a/evaluation-pipeline/lm_eval/datasets/truthfulqa/__init__.py b/evaluation-pipeline/lm_eval/datasets/truthfulqa/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/lm_eval/datasets/truthfulqa/dataset_infos.json b/evaluation-pipeline/lm_eval/datasets/truthfulqa/dataset_infos.json new file mode 100644 index 0000000000000000000000000000000000000000..18d52feaa844d385ee2d5bd34617b3dc9186f79a --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/truthfulqa/dataset_infos.json @@ -0,0 +1 @@ +{"multiple_choice": {"description": "TruthfulQA is a benchmark to measure whether a language model is truthful in\ngenerating answers to questions. The benchmark comprises 817 questions that\nspan 38 categories, including health, law, finance and politics. Questions are\ncrafted so that some humans would answer falsely due to a false belief or\nmisconception. To perform well, models must avoid generating false answers\nlearned from imitating human texts.\n\nThe multiple choice TruthfulQA task", "citation": "@misc{lin2021truthfulqa,\n title={TruthfulQA: Measuring How Models Mimic Human Falsehoods},\n author={Stephanie Lin and Jacob Hilton and Owain Evans},\n year={2021},\n eprint={2109.07958},\n archivePrefix={arXiv},\n primaryClass={cs.CL}\n}\n", "homepage": "https://github.com/sylinrl/TruthfulQA", "license": "", "features": {"question": {"dtype": "string", "id": null, "_type": "Value"}, "mc1_targets": {"choices": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "labels": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}, "mc2_targets": {"choices": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "labels": {"feature": {"dtype": "int32", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "truthfulqa", "config_name": "multiple_choice", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 610333, "num_examples": 817, "dataset_name": "truthfulqa"}}, "download_checksums": {"https://raw.githubusercontent.com/sylinrl/TruthfulQA/013686a06be7a7bde5bf8223943e106c7250123c/data/mc_task.json": {"num_bytes": 710607, "checksum": "6eb4125d25750c0145c4be2dce00440736684ab6f74ce6bff2139571cc758954"}}, "download_size": 710607, "post_processing_size": null, "dataset_size": 610333, "size_in_bytes": 1320940}, "generation": {"description": "TruthfulQA is a benchmark to measure whether a language model is truthful in\ngenerating answers to questions. The benchmark comprises 817 questions that\nspan 38 categories, including health, law, finance and politics. Questions are\ncrafted so that some humans would answer falsely due to a false belief or\nmisconception. To perform well, models must avoid generating false answers\nlearned from imitating human texts.\n\nThe generative TruthfulQA task", "citation": "@misc{lin2021truthfulqa,\n title={TruthfulQA: Measuring How Models Mimic Human Falsehoods},\n author={Stephanie Lin and Jacob Hilton and Owain Evans},\n year={2021},\n eprint={2109.07958},\n archivePrefix={arXiv},\n primaryClass={cs.CL}\n}\n", "homepage": "https://github.com/sylinrl/TruthfulQA", "license": "", "features": {"category": {"dtype": "string", "id": null, "_type": "Value"}, "question": {"dtype": "string", "id": null, "_type": "Value"}, "best_answer": {"dtype": "string", "id": null, "_type": "Value"}, "correct_answers": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "incorrect_answers": {"feature": {"dtype": "string", "id": null, "_type": "Value"}, "length": -1, "id": null, "_type": "Sequence"}, "source": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "truthfulqa", "config_name": "generation", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 463860, "num_examples": 817, "dataset_name": "truthfulqa"}}, "download_checksums": {"https://raw.githubusercontent.com/sylinrl/TruthfulQA/013686a06be7a7bde5bf8223943e106c7250123c/TruthfulQA.csv": {"num_bytes": 443723, "checksum": "8d7dd15f033196140f032d97d30f037da7a7b1192c3f36f9937c1850925335a2"}}, "download_size": 443723, "post_processing_size": null, "dataset_size": 463860, "size_in_bytes": 907583}} diff --git a/evaluation-pipeline/lm_eval/datasets/truthfulqa/truthfulqa.py b/evaluation-pipeline/lm_eval/datasets/truthfulqa/truthfulqa.py new file mode 100644 index 0000000000000000000000000000000000000000..af1eb896f62c1c53d9484f159900367fc46a81cc --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/truthfulqa/truthfulqa.py @@ -0,0 +1,169 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""TruthfulQA dataset.""" + + +import csv +import json + +import datasets + + +_CITATION = """\ +@misc{lin2021truthfulqa, + title={TruthfulQA: Measuring How Models Mimic Human Falsehoods}, + author={Stephanie Lin and Jacob Hilton and Owain Evans}, + year={2021}, + eprint={2109.07958}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} +""" + +_DESCRIPTION = """\ +TruthfulQA is a benchmark to measure whether a language model is truthful in +generating answers to questions. The benchmark comprises 817 questions that +span 38 categories, including health, law, finance and politics. Questions are +crafted so that some humans would answer falsely due to a false belief or +misconception. To perform well, models must avoid generating false answers +learned from imitating human texts. +""" + +_HOMEPAGE = "https://github.com/sylinrl/TruthfulQA" + +# TODO: Add the licence for the dataset here if you can find it +_LICENSE = "" + + +class TruthfulqaConfig(datasets.BuilderConfig): + """BuilderConfig for TruthfulQA.""" + + def __init__(self, url, features, **kwargs): + """BuilderConfig for TruthfulQA. + + Args: + url: *string*, the url to the specific subset of the GPT3 Arithmetic dataset. + features: *list[string]*, list of the features that will appear in the + feature dict. + """ + # Version history: + super().__init__(version=datasets.Version("0.0.1"), **kwargs) + self.url = url + self.features = features + + +class Truthfulqa(datasets.GeneratorBasedBuilder): + """TruthfulQA is a benchmark to measure whether a language model is truthful in + generating answers to questions.""" + + BUILDER_CONFIGS = [ + TruthfulqaConfig( + name="multiple_choice", + url="https://raw.githubusercontent.com/sylinrl/TruthfulQA/013686a06be7a7bde5bf8223943e106c7250123c/data/mc_task.json", + features=datasets.Features( + { + "question": datasets.Value("string"), + "mc1_targets": { + "choices": datasets.features.Sequence(datasets.Value("string")), + "labels": datasets.features.Sequence(datasets.Value("int32")), + }, + "mc2_targets": { + "choices": datasets.features.Sequence(datasets.Value("string")), + "labels": datasets.features.Sequence(datasets.Value("int32")), + }, + } + ), + description="The multiple choice TruthfulQA task", + ), + TruthfulqaConfig( + name="generation", + url="https://raw.githubusercontent.com/sylinrl/TruthfulQA/013686a06be7a7bde5bf8223943e106c7250123c/TruthfulQA.csv", + features=datasets.Features( + { + "category": datasets.Value("string"), + "question": datasets.Value("string"), + "best_answer": datasets.Value("string"), + "correct_answers": datasets.features.Sequence( + datasets.Value("string") + ), + "incorrect_answers": datasets.features.Sequence( + datasets.Value("string") + ), + "source": datasets.Value("string"), + } + ), + description="The generative TruthfulQA task", + ), + ] + + def _info(self): + return datasets.DatasetInfo( + description=f"{_DESCRIPTION}\n{self.config.description}", + features=self.config.features, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager): + urls = self.config.url + data_dir = dl_manager.download_and_extract(urls) + return [ + datasets.SplitGenerator( + name=datasets.Split.VALIDATION, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "filepath": data_dir, + "split": "validation", + }, + ), + ] + + # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` + def _generate_examples(self, filepath, split): + if self.config.name == "multiple_choice": + # Multiple choice data is in a `JSON` file. + with open(filepath, encoding="utf-8") as f: + contents = json.load(f) + for key, row in enumerate(contents): + yield key, { + "question": row["question"], + "mc1_targets": { + "choices": row["mc1_targets"].keys(), + "labels": row["mc1_targets"].values(), + }, + "mc2_targets": { + "choices": row["mc2_targets"].keys(), + "labels": row["mc2_targets"].values(), + }, + } + else: + # Generation data is in a `CSV` file. + with open(filepath, newline="") as f: + contents = csv.DictReader(f) + for key, row in enumerate(contents): + # Ensure that references exist. + if not row["Correct Answers"] or not row["Incorrect Answers"]: + continue + yield key, { + "category": row["Category"], + "question": row["Question"], + "best_answer": row["Best Answer"], + # split on ";" + "correct_answers": row["Correct Answers"].strip().split(";"), + "incorrect_answers": row["Incorrect Answers"] + .strip() + .split(";"), + "source": row["Source"], + } diff --git a/evaluation-pipeline/lm_eval/datasets/unscramble/__init__.py b/evaluation-pipeline/lm_eval/datasets/unscramble/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/lm_eval/datasets/unscramble/dataset_infos.json b/evaluation-pipeline/lm_eval/datasets/unscramble/dataset_infos.json new file mode 100644 index 0000000000000000000000000000000000000000..bae29209daf50ccb5bd6adc0779a2138cddb908b --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/unscramble/dataset_infos.json @@ -0,0 +1 @@ +{"mid_word_1_anagrams": {"description": "Unscramble is a small battery of 5 \u201ccharacter manipulation\u201d tasks. Each task\ninvolves giving the model a word distorted by some combination of scrambling,\naddition, or deletion of characters, and asking it to recover the original word.\n", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n booktitle = {Advances in Neural Information Processing Systems},\n editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n pages = {1877--1901},\n publisher = {Curran Associates, Inc.},\n title = {Language Models are Few-Shot Learners},\n url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n volume = {33},\n year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "unscramble", "config_name": "mid_word_1_anagrams", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 271516, "num_examples": 10000, "dataset_name": "unscramble"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/mid_word_1_anagrams.jsonl.gz": {"num_bytes": 106533, "checksum": "6768a86896083199de4815d4964cb2f6f1046476cfd80c2a562784f182905979"}}, "download_size": 106533, "post_processing_size": null, "dataset_size": 271516, "size_in_bytes": 378049}, "mid_word_2_anagrams": {"description": "Unscramble is a small battery of 5 \u201ccharacter manipulation\u201d tasks. Each task\ninvolves giving the model a word distorted by some combination of scrambling,\naddition, or deletion of characters, and asking it to recover the original word.\n", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n booktitle = {Advances in Neural Information Processing Systems},\n editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n pages = {1877--1901},\n publisher = {Curran Associates, Inc.},\n title = {Language Models are Few-Shot Learners},\n url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n volume = {33},\n year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "unscramble", "config_name": "mid_word_2_anagrams", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 282654, "num_examples": 10000, "dataset_name": "unscramble"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/mid_word_2_anagrams.jsonl.gz": {"num_bytes": 109091, "checksum": "c3d839d09a7954b78a27cd2cd75d4ed0488656c56ef4dbd741a005343826cb01"}}, "download_size": 109091, "post_processing_size": null, "dataset_size": 282654, "size_in_bytes": 391745}, "cycle_letters_in_word": {"description": "Unscramble is a small battery of 5 \u201ccharacter manipulation\u201d tasks. Each task\ninvolves giving the model a word distorted by some combination of scrambling,\naddition, or deletion of characters, and asking it to recover the original word.\n", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n booktitle = {Advances in Neural Information Processing Systems},\n editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n pages = {1877--1901},\n publisher = {Curran Associates, Inc.},\n title = {Language Models are Few-Shot Learners},\n url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n volume = {33},\n year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "unscramble", "config_name": "cycle_letters_in_word", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 282654, "num_examples": 10000, "dataset_name": "unscramble"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/cycle_letters_in_word.jsonl.gz": {"num_bytes": 98451, "checksum": "1689c9002bb8c5988bf5f05e977c9db92f57932c1b5a38998c29ac0dd71e1d42"}}, "download_size": 98451, "post_processing_size": null, "dataset_size": 282654, "size_in_bytes": 381105}, "random_insertion_in_word": {"description": "Unscramble is a small battery of 5 \u201ccharacter manipulation\u201d tasks. Each task\ninvolves giving the model a word distorted by some combination of scrambling,\naddition, or deletion of characters, and asking it to recover the original word.\n", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n booktitle = {Advances in Neural Information Processing Systems},\n editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n pages = {1877--1901},\n publisher = {Curran Associates, Inc.},\n title = {Language Models are Few-Shot Learners},\n url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n volume = {33},\n year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "unscramble", "config_name": "random_insertion_in_word", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 353981, "num_examples": 10000, "dataset_name": "unscramble"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/random_insertion_in_word.jsonl.gz": {"num_bytes": 143626, "checksum": "72e65d83da53d15752ee0c47379509de149ddbad32d61184e5991df29616b78a"}}, "download_size": 143626, "post_processing_size": null, "dataset_size": 353981, "size_in_bytes": 497607}, "reversed_words": {"description": "Unscramble is a small battery of 5 \u201ccharacter manipulation\u201d tasks. Each task\ninvolves giving the model a word distorted by some combination of scrambling,\naddition, or deletion of characters, and asking it to recover the original word.\n", "citation": "@inproceedings{NEURIPS2020_1457c0d6,\n author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},\n booktitle = {Advances in Neural Information Processing Systems},\n editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},\n pages = {1877--1901},\n publisher = {Curran Associates, Inc.},\n title = {Language Models are Few-Shot Learners},\n url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},\n volume = {33},\n year = {2020}\n}\n", "homepage": "https://github.com/openai/gpt-3/tree/master/data", "license": "", "features": {"context": {"dtype": "string", "id": null, "_type": "Value"}, "completion": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "unscramble", "config_name": "reversed_words", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 282654, "num_examples": 10000, "dataset_name": "unscramble"}}, "download_checksums": {"https://raw.githubusercontent.com/openai/gpt-3/master/data/reversed_words.jsonl.gz": {"num_bytes": 91917, "checksum": "133a08f875cd6c1ef8608a3233571a773881cc27b1c707de738cc6543439332a"}}, "download_size": 91917, "post_processing_size": null, "dataset_size": 282654, "size_in_bytes": 374571}} diff --git a/evaluation-pipeline/lm_eval/datasets/unscramble/unscramble.py b/evaluation-pipeline/lm_eval/datasets/unscramble/unscramble.py new file mode 100644 index 0000000000000000000000000000000000000000..86f1e1ef5cbabf4ae57368e0d4fd00b126d4e37e --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/unscramble/unscramble.py @@ -0,0 +1,110 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unscramble dataset.""" + + +import json +import os + +import datasets + + +_CITATION = """\ +@inproceedings{NEURIPS2020_1457c0d6, + author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario}, + booktitle = {Advances in Neural Information Processing Systems}, + editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin}, + pages = {1877--1901}, + publisher = {Curran Associates, Inc.}, + title = {Language Models are Few-Shot Learners}, + url = {https://proceedings.neurips.cc/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf}, + volume = {33}, + year = {2020} +} +""" + +_DESCRIPTION = """\ +Unscramble is a small battery of 5 “character manipulation” tasks. Each task +involves giving the model a word distorted by some combination of scrambling, +addition, or deletion of characters, and asking it to recover the original word. +""" + +_HOMEPAGE = "https://github.com/openai/gpt-3/tree/master/data" + +# TODO: Add the licence for the dataset here if you can find it +_LICENSE = "" + +_BASE_URL = "https://raw.githubusercontent.com/openai/gpt-3/master/data" + + +_DESCRIPTIONS = { + "mid_word_1_anagrams": "Anagrams of all but the first and last letter.", + "mid_word_2_anagrams": "Anagrams of all but the first and last 2 letters.", + "cycle_letters_in_word": "Cycle letters in the word.", + "random_insertion_in_word": "Random insertions in the word that must be removed.", + "reversed_words": "Words spelled backwards that must be reversed.", +} +_NAMES = _DESCRIPTIONS.keys() + + +class Unscramble(datasets.GeneratorBasedBuilder): + """Unscramble is a small battery of 5 “character manipulation” tasks.""" + + VERSION = datasets.Version("0.0.1") + + BUILDER_CONFIGS = [ + datasets.BuilderConfig( + name=name, version=version, description=_DESCRIPTIONS[name] + ) + for name, version in zip(_NAMES, [VERSION] * len(_NAMES)) + ] + + def _info(self): + features = datasets.Features( + { + "context": datasets.Value("string"), + "completion": datasets.Value("string"), + } + ) + return datasets.DatasetInfo( + description=_DESCRIPTION, + features=features, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager): + urls = os.path.join(_BASE_URL, f"{self.config.name}.jsonl.gz") + data_dir = dl_manager.download_and_extract(urls) + return [ + datasets.SplitGenerator( + name=datasets.Split.VALIDATION, + # These kwargs will be passed to _generate_examples + gen_kwargs={ + "filepath": data_dir, + "split": "validation", + }, + ), + ] + + # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` + def _generate_examples(self, filepath, split): + with open(filepath, encoding="utf-8") as f: + for key, row in enumerate(f): + data = json.loads(row) + yield key, { + "context": data["context"], + "completion": data["completion"], + } diff --git a/evaluation-pipeline/lm_eval/datasets/wikitext/__init__.py b/evaluation-pipeline/lm_eval/datasets/wikitext/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/lm_eval/datasets/wikitext/dataset_infos.json b/evaluation-pipeline/lm_eval/datasets/wikitext/dataset_infos.json new file mode 100644 index 0000000000000000000000000000000000000000..db8249c1b172e8ad2457cad3214d15dbab4ae856 --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/wikitext/dataset_infos.json @@ -0,0 +1 @@ +{"wikitext-103-v1": {"description": " The WikiText language modeling dataset is a collection of over 100 million tokens extracted from the set of verified\n Good and Featured articles on Wikipedia. The dataset is available under the Creative Commons Attribution-ShareAlike\n License.\n", "citation": "@misc{merity2016pointer,\n title={Pointer Sentinel Mixture Models},\n author={Stephen Merity and Caiming Xiong and James Bradbury and Richard Socher},\n year={2016},\n eprint={1609.07843},\n archivePrefix={arXiv},\n primaryClass={cs.CL}\n}\n", "homepage": "https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/", "license": "Creative Commons Attribution-ShareAlike 4.0 International (CC BY-SA 4.0)", "features": {"page": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "wikitext", "config_name": "wikitext-103-v1", "version": {"version_str": "1.0.0", "description": null, "major": 1, "minor": 0, "patch": 0}, "splits": {"test": {"name": "test", "num_bytes": 1281262, "num_examples": 62, "dataset_name": "wikitext"}, "train": {"name": "train", "num_bytes": 539297488, "num_examples": 29444, "dataset_name": "wikitext"}, "validation": {"name": "validation", "num_bytes": 1142488, "num_examples": 60, "dataset_name": "wikitext"}}, "download_checksums": {"https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-103-v1.zip": {"num_bytes": 190229076, "checksum": "242ba0f20b329cfdf1ccc61e9e9e5b59becf189db7f7a81cd2a0e2fc31539590"}}, "download_size": 190229076, "post_processing_size": null, "dataset_size": 541721238, "size_in_bytes": 731950314}, "wikitext-2-v1": {"description": " The WikiText language modeling dataset is a collection of over 100 million tokens extracted from the set of verified\n Good and Featured articles on Wikipedia. The dataset is available under the Creative Commons Attribution-ShareAlike\n License.\n", "citation": "@misc{merity2016pointer,\n title={Pointer Sentinel Mixture Models},\n author={Stephen Merity and Caiming Xiong and James Bradbury and Richard Socher},\n year={2016},\n eprint={1609.07843},\n archivePrefix={arXiv},\n primaryClass={cs.CL}\n}\n", "homepage": "https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/", "license": "Creative Commons Attribution-ShareAlike 4.0 International (CC BY-SA 4.0)", "features": {"page": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "wikitext", "config_name": "wikitext-2-v1", "version": {"version_str": "1.0.0", "description": null, "major": 1, "minor": 0, "patch": 0}, "splits": {"test": {"name": "test", "num_bytes": 1256634, "num_examples": 62, "dataset_name": "wikitext"}, "train": {"name": "train", "num_bytes": 10799034, "num_examples": 629, "dataset_name": "wikitext"}, "validation": {"name": "validation", "num_bytes": 1121860, "num_examples": 60, "dataset_name": "wikitext"}}, "download_checksums": {"https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-v1.zip": {"num_bytes": 4475746, "checksum": "92675f1d63015c1c8b51f1656a52d5bdbc33aafa60cc47a218a66e7ee817488c"}}, "download_size": 4475746, "post_processing_size": null, "dataset_size": 13177528, "size_in_bytes": 17653274}, "wikitext-103-raw-v1": {"description": " The WikiText language modeling dataset is a collection of over 100 million tokens extracted from the set of verified\n Good and Featured articles on Wikipedia. The dataset is available under the Creative Commons Attribution-ShareAlike\n License.\n", "citation": "@misc{merity2016pointer,\n title={Pointer Sentinel Mixture Models},\n author={Stephen Merity and Caiming Xiong and James Bradbury and Richard Socher},\n year={2016},\n eprint={1609.07843},\n archivePrefix={arXiv},\n primaryClass={cs.CL}\n}\n", "homepage": "https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/", "license": "Creative Commons Attribution-ShareAlike 4.0 International (CC BY-SA 4.0)", "features": {"page": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "wikitext", "config_name": "wikitext-103-raw-v1", "version": {"version_str": "1.0.0", "description": null, "major": 1, "minor": 0, "patch": 0}, "splits": {"test": {"name": "test", "num_bytes": 1290775, "num_examples": 62, "dataset_name": "wikitext"}, "train": {"name": "train", "num_bytes": 540656522, "num_examples": 29444, "dataset_name": "wikitext"}, "validation": {"name": "validation", "num_bytes": 1147025, "num_examples": 60, "dataset_name": "wikitext"}}, "download_checksums": {"https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-103-raw-v1.zip": {"num_bytes": 191984949, "checksum": "91c00ae287f0d699e18605c84afc9e45c192bc6b7797ff8837e5474655a33794"}}, "download_size": 191984949, "post_processing_size": null, "dataset_size": 543094322, "size_in_bytes": 735079271}, "wikitext-2-raw-v1": {"description": " The WikiText language modeling dataset is a collection of over 100 million tokens extracted from the set of verified\n Good and Featured articles on Wikipedia. The dataset is available under the Creative Commons Attribution-ShareAlike\n License.\n", "citation": "@misc{merity2016pointer,\n title={Pointer Sentinel Mixture Models},\n author={Stephen Merity and Caiming Xiong and James Bradbury and Richard Socher},\n year={2016},\n eprint={1609.07843},\n archivePrefix={arXiv},\n primaryClass={cs.CL}\n}\n", "homepage": "https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/", "license": "Creative Commons Attribution-ShareAlike 4.0 International (CC BY-SA 4.0)", "features": {"page": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "wikitext", "config_name": "wikitext-2-raw-v1", "version": {"version_str": "1.0.0", "description": null, "major": 1, "minor": 0, "patch": 0}, "splits": {"test": {"name": "test", "num_bytes": 1290775, "num_examples": 62, "dataset_name": "wikitext"}, "train": {"name": "train", "num_bytes": 10942633, "num_examples": 629, "dataset_name": "wikitext"}, "validation": {"name": "validation", "num_bytes": 1147025, "num_examples": 60, "dataset_name": "wikitext"}}, "download_checksums": {"https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-raw-v1.zip": {"num_bytes": 4721645, "checksum": "ef7edb566e3e2b2d31b29c1fdb0c89a4cc683597484c3dc2517919c615435a11"}}, "download_size": 4721645, "post_processing_size": null, "dataset_size": 13380433, "size_in_bytes": 18102078}} diff --git a/evaluation-pipeline/lm_eval/datasets/wikitext/wikitext.py b/evaluation-pipeline/lm_eval/datasets/wikitext/wikitext.py new file mode 100644 index 0000000000000000000000000000000000000000..a7d305303bc9e5358bdaf425b8afcf59e386d03e --- /dev/null +++ b/evaluation-pipeline/lm_eval/datasets/wikitext/wikitext.py @@ -0,0 +1,252 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# NOTE: This is a modified version of https://github.com/huggingface/datasets/blob/master/datasets/wikitext/wikitext.py +# that returns Wiki pages instead of Wiki text line-by-line. +"""WikiText Dataset.""" + + +import os + +import datasets + + +_CITATION = """\ +@misc{merity2016pointer, + title={Pointer Sentinel Mixture Models}, + author={Stephen Merity and Caiming Xiong and James Bradbury and Richard Socher}, + year={2016}, + eprint={1609.07843}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} +""" + +_DESCRIPTION = """\ + The WikiText language modeling dataset is a collection of over 100 million tokens extracted from the set of verified + Good and Featured articles on Wikipedia. The dataset is available under the Creative Commons Attribution-ShareAlike + License. +""" +_HOMEPAGE = "https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/" +_LICENSE = "Creative Commons Attribution-ShareAlike 4.0 International (CC BY-SA 4.0)" +_DATA_URL = "https://s3.amazonaws.com/research.metamind.io/wikitext" + + +class WikitextConfig(datasets.BuilderConfig): + """BuilderConfig for GLUE.""" + + def __init__(self, data_url, **kwargs): + """BuilderConfig for Wikitext + Args: + data_url: `string`, url to the dataset (word or raw level) + **kwargs: keyword arguments forwarded to super. + """ + super(WikitextConfig, self).__init__( + version=datasets.Version( + "1.0.0", + ), + **kwargs, + ) + self.data_url = data_url + + +class Wikitext(datasets.GeneratorBasedBuilder): + """TODO(wikitext_103): Short description of my dataset.""" + + # TODO(wikitext_103): Set up version. + VERSION = datasets.Version("0.1.0") + BUILDER_CONFIGS = [ + WikitextConfig( + name="wikitext-103-v1", + data_url=_DATA_URL + "/" + "wikitext-103-v1.zip", + description="Word level dataset. No processing is needed other than replacing newlines with tokens.", + ), + WikitextConfig( + name="wikitext-2-v1", + data_url=_DATA_URL + "/" + "wikitext-2-v1.zip", + description="Word level dataset. No processing is needed other than replacing newlines with tokens.", + ), + WikitextConfig( + name="wikitext-103-raw-v1", + data_url=_DATA_URL + "/" + "wikitext-103-raw-v1.zip", + description="Raw level dataset: the raw tokens before the addition of tokens. " + "They should only be used for character level work or for creating newly derived datasets.", + ), + WikitextConfig( + name="wikitext-2-raw-v1", + data_url=_DATA_URL + "/" + "wikitext-2-raw-v1.zip", + description="Raw level dataset: the raw tokens before the addition of tokens. " + "They should only be used for character level work or for creating newly derived datasets.", + ), + ] + + def _info(self): + # TODO(wikitext): Specifies the datasets.DatasetInfo object + return datasets.DatasetInfo( + # This is the description that will appear on the datasets page. + description=_DESCRIPTION, + # datasets.features.FeatureConnectors + features=datasets.Features( + { + "page": datasets.Value("string") + # These are the features of your dataset like images, labels ... + } + ), + # If there's a common (input, target) tuple from the features, + # specify them here. They'll be used if as_supervised=True in + # builder.as_dataset. + supervised_keys=None, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager): + """Returns SplitGenerators.""" + # TODO(wikitext): Downloads the data and defines the splits + # dl_manager is a datasets.download.DownloadManager that can be used to + # download and extract URLs + if self.config.name == "wikitext-103-v1": + data_file = dl_manager.download_and_extract(self.config.data_url) + data_dir = os.path.join(data_file, "wikitext-103") + return [ + datasets.SplitGenerator( + name=datasets.Split.TEST, + gen_kwargs={ + "data_file": os.path.join(data_dir, "wiki.test.tokens"), + "split": "test", + }, + ), + datasets.SplitGenerator( + name=datasets.Split.TRAIN, + gen_kwargs={ + "data_file": os.path.join(data_dir, "wiki.train.tokens"), + "split": "train", + }, + ), + datasets.SplitGenerator( + name=datasets.Split.VALIDATION, + gen_kwargs={ + "data_file": os.path.join(data_dir, "wiki.valid.tokens"), + "split": "valid", + }, + ), + ] + else: + if self.config.name == "wikitext-103-raw-v1": + data_file = dl_manager.download_and_extract(self.config.data_url) + data_dir = os.path.join(data_file, "wikitext-103-raw") + return [ + datasets.SplitGenerator( + name=datasets.Split.TEST, + gen_kwargs={ + "data_file": os.path.join(data_dir, "wiki.test.raw"), + "split": "test", + }, + ), + datasets.SplitGenerator( + name=datasets.Split.TRAIN, + gen_kwargs={ + "data_file": os.path.join(data_dir, "wiki.train.raw"), + "split": "train", + }, + ), + datasets.SplitGenerator( + name=datasets.Split.VALIDATION, + gen_kwargs={ + "data_file": os.path.join(data_dir, "wiki.valid.raw"), + "split": "valid", + }, + ), + ] + else: + if self.config.name == "wikitext-2-raw-v1": + data_file = dl_manager.download_and_extract(self.config.data_url) + data_dir = os.path.join(data_file, "wikitext-2-raw") + return [ + datasets.SplitGenerator( + name=datasets.Split.TEST, + gen_kwargs={ + "data_file": os.path.join(data_dir, "wiki.test.raw"), + "split": "test", + }, + ), + datasets.SplitGenerator( + name=datasets.Split.TRAIN, + gen_kwargs={ + "data_file": os.path.join(data_dir, "wiki.train.raw"), + "split": "train", + }, + ), + datasets.SplitGenerator( + name=datasets.Split.VALIDATION, + gen_kwargs={ + "data_file": os.path.join(data_dir, "wiki.valid.raw"), + "split": "valid", + }, + ), + ] + else: + if self.config.name == "wikitext-2-v1": + data_file = dl_manager.download_and_extract( + self.config.data_url + ) + data_dir = os.path.join(data_file, "wikitext-2") + return [ + datasets.SplitGenerator( + name=datasets.Split.TEST, + gen_kwargs={ + "data_file": os.path.join( + data_dir, "wiki.test.tokens" + ), + "split": "test", + }, + ), + datasets.SplitGenerator( + name=datasets.Split.TRAIN, + gen_kwargs={ + "data_file": os.path.join( + data_dir, "wiki.train.tokens" + ), + "split": "train", + }, + ), + datasets.SplitGenerator( + name=datasets.Split.VALIDATION, + gen_kwargs={ + "data_file": os.path.join( + data_dir, "wiki.valid.tokens" + ), + "split": "valid", + }, + ), + ] + + def _generate_examples(self, data_file, split): + """Yields examples.""" + with open(data_file, encoding="utf-8") as f: + key = 0 + ret = [] + data = f.read().split("\n") + for line in data: + rline = line.replace("= = =", "===").replace("= =", "==").strip() + if rline.startswith("= ") and rline.strip().endswith(" ="): + page = "\n".join(ret) + if page.strip(): + yield key, {"page": page} + key += 1 + ret = [] + ret.append(line) + page = "\n".join(ret) + yield key, {"page": page} diff --git a/evaluation-pipeline/lm_eval/evaluator.py b/evaluation-pipeline/lm_eval/evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..c0e23b6c60a812f43e9b7be4482a294eb773cb34 --- /dev/null +++ b/evaluation-pipeline/lm_eval/evaluator.py @@ -0,0 +1,362 @@ +import collections +import itertools +import json +import logging +import sys +import numpy as np +from tqdm import tqdm +from typing import List, Optional + +import lm_eval.models +import lm_eval.tasks +import lm_eval.api.metric +import lm_eval.api.model +from lm_eval.api.utils import DEFAULT_SEED, set_seed +from lm_eval.api.task import Task + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +logger.addHandler(logging.StreamHandler(sys.stdout)) + + +def cli_evaluate( + *, + model_api_name: str, + model_args: str, + task_name: str, + task_args: str, + template_names: List[str], + num_fewshot: Optional[int] = 0, + batch_size: Optional[int] = None, + device: Optional[str] = None, + use_cache: Optional[bool] = False, + bootstrap_iters: Optional[int] = 100000, + seed: Optional[int] = DEFAULT_SEED, + limit: Optional[int] = None, +) -> dict: + """Evaluate a model from an api on a given task with multiple possible prompt + formats. This is effectively a wrapper around `evaluate` for command-line + interface (CLI) like usage; only primitive type arguments. + + Args: + model_api_name (str): + Name of the language model api to use. See: + `lm_eval.models.list_model_apis` + model_args (str): + String arguments for the model api. See: + `lm_eval.api.model.get_model_from_args_string` + task_name (str): + The task name of the task to evaluate the model on. + task_args (str): + String arguments for the task. See: + `lm_eval.api.task.get_task_list_from_args_string` + WARNING: To avoid parse errors, separators must not contain commas. + template_names (List[str]): + List of template names for the specified `task_name` to evaluate + under. + num_fewshot (int, optional, defaults to 0): + Number of examples in few-shot context. + batch_size (int, optional, defaults to None): + Batch size to use for model evaluation. + device (str, optional, defaults to None): + PyTorch device (e.g. "cpu" or "cuda:0") for running models. + use_cache (bool, optional, defaults to False): + Whether or not to use a cache for language model results. + bootstrap_iters (int, optional, defaults to 100000): + Number of iterations for bootstrap statistics. + seed (int, optional, defaults to 1234 = `DEFAULT_SEED`): + Seed for pseudo-random number generation. This controls document + shuffling, few-shot prompt selection, and framework seeding. + limit (int, optional, defaults to None): + Limit the number of examples per task (only use this for testing). + + Returns: + Dictionary of results. + """ + tasks = lm_eval.tasks.get_task_list_from_args_string( + task_name, template_names, task_args + ) + model = lm_eval.models.get_model_from_args_string( + model_api_name, model_args, {"batch_size": batch_size, "device": device} + ) + + if use_cache: + cache_args = model_args.replace("=", "-").replace(",", "_").replace("/", "-") + # TODO: Make `cache_location` path configurable thru an environment var. + cache_location = f"lm_cache/{model_api_name}_{cache_args}.db" + model = lm_eval.api.model.CachingLM(model, cache_location) + + results = evaluate( + model=model, + tasks=tasks, + num_fewshot=num_fewshot, + bootstrap_iters=bootstrap_iters, + seed=seed, + limit=limit, + ) + + # Add info about the model and few shot config. + results["config"] = { + "model": model_api_name, + "model_args": model_args, + "task_args": task_args, + "num_fewshot": num_fewshot, + "batch_size": batch_size, + "device": device, + "use_cache": use_cache, + "limit": limit, + "bootstrap_iters": bootstrap_iters, + "seed": seed, + } + return results + + +def evaluate( + *, + model: lm_eval.api.model.LM, + tasks: List[Task], + num_fewshot: Optional[int] = 0, + bootstrap_iters: Optional[int] = 100000, + seed: Optional[int] = DEFAULT_SEED, + limit: Optional[int] = None, + predictions_path: Optional[str] = None, +) -> dict: + """Instantiate and evaluate a model on a list of tasks. + + Args: + model (lm_eval.api.model.LM): + Language model API instance. + tasks (List[Task]): + List of tasks to evaluate `model` on. + num_fewshot (int, optional, defaults to 0): + Number of examples in the few-shot context. + bootstrap_iters (int, optional, defaults to 100000): + Number of iterations for bootstrap statistics. + seed (int, optional, defaults to 1234 = `DEFAULT_SEED`): + Seed for pseudo-random number generation. This controls document + shuffling, few-shot prompt selection, and framework seeding. + limit (int, optional, defaults to None): + Limit the number of examples per task. + WARNING: This is only for testing purposes. + + Returns: + Dictionary of results. + """ + set_seed(seed) + rng = np.random.default_rng(seed) + + # TODO: Completely refactor this entire function to not be a huge mess, ideally breaking it down into smaller pieces + task_dict = {} + for task in tasks: + if task.has_validation_docs() is False and task.has_test_docs() is False: + logger.info( + f"Ignoring Task: {lm_eval.tasks.get_registry_name_from_task(task)} has no validation or test docs" + ) + continue + # Create unique keys for each task-template pair. + task_name = lm_eval.tasks.get_registry_name_from_task(task) + template_name = task.prompt_template.name if task.prompt_template else None + key = lm_eval.tasks._get_task_template_key(task_name, template_name) + task_dict[key] = task + + results = collections.defaultdict(dict) + versions = collections.defaultdict(dict) + requests = collections.defaultdict(list) + requests_origin = collections.defaultdict(list) + + # TODO: We need unit tests & sanity checks or something to ensure that the return of `validation_docs` is stable + docs = {} + + # Build contexts and collect language model requests. + for task_template_key, task in task_dict.items(): + task_docs = task.evaluation_docs() + + logger.info(f"\n» Assigning unique IDs to '{task_template_key}' docs") + task_docs = task_docs.map( + lambda ex, idx: {**ex, "doc_id": idx}, with_indices=True + ) + + logger.info(f"\n» Filtering invalid docs from '{task_template_key}'") + task_docs = task_docs.filter(lambda d: not task.invalid_doc_for_prompt(d)) + # task_docs = task_docs.shuffle(generator=rng) + + logger.info(f"\n» Constructing '{task_template_key}' contexts and requests") + pbar_limit = len(task_docs) if not limit else np.minimum(limit, len(task_docs)) + + for doc_id, doc in enumerate( + tqdm(itertools.islice(task_docs, 0, limit), total=pbar_limit) + ): + docs[(task_template_key, doc_id)] = doc + ctx, fewshotex_logging_info = task.fewshot_context( + doc=doc, + num_fewshot=num_fewshot, + rng=rng, + ) + fewshotex_logging_info["doc_id"] = doc["doc_id"] + args = {"num_fewshot": num_fewshot} + reqs = task.construct_requests(doc, ctx, args) + if not isinstance(reqs, (list, tuple)): + reqs = [reqs] + for i, req in enumerate(reqs): + requests[req.request_type].append(req) + # i: Index in requests for a single task instance + # doc_id: Unique id that we can get back to a doc using `docs` + requests_origin[req.request_type].append( + (i, task_template_key, doc, doc_id, fewshotex_logging_info) + ) + # Store the task version. + versions[task_template_key] = task.VERSION + + # All responses for each (task, doc) + process_response_queue = collections.defaultdict(list) + # Execute each type of request + for reqtype, reqs in requests.items(): + # TODO: Right now, this code runs multiple separate LM requests for + # multiple Requests differing only in index. We could implement some + # kind of caching, but that would be more of a band-aid solution. We + # could also implement some kind of auto-grouping here; they should + # end up next to each other. + logger.info(f"\n» Running all `{reqtype}` requests") + resps = getattr(model, reqtype)([req.args for req in reqs]) + resps = [ + x if req.index is None else x[req.index] for x, req in zip(resps, reqs) + ] + for resp, (i, task_template_key, doc, doc_id, fewshotex_logging_info) in zip( + resps, requests_origin[reqtype] + ): + process_response_queue[(task_template_key, doc_id)].append( + (i, resp, fewshotex_logging_info) + ) + + # Unpack results and sort back in order and return control to Task + if predictions_path: + preds = [] + vals = collections.defaultdict(list) + example_logger = logging.getLogger("examples") + for (task_template_key, doc_id), per_doc_requests in process_response_queue.items(): + per_doc_requests.sort(key=lambda x: x[0]) + per_doc_results = [x[1] for x in per_doc_requests] + fewshot_logging_info = [x[2] for x in per_doc_requests][0] + + task = task_dict[task_template_key] + doc = docs[(task_template_key, doc_id)] + + output = task.process_results(doc, per_doc_results) + + if task.save_examples: + metrics, example = output + preds.append(example["pred"]) + example.update(fewshot_logging_info) + example.update(task.get_logging_info()) + example_logger.info(json.dumps(example)) + else: + metrics = output + example = fewshot_logging_info + example.update(task.get_logging_info()) + example_logger.info(json.dumps(example)) + + for metric, value in metrics.items(): + vals[(task_template_key, metric)].append(value) + + if predictions_path: + with open(predictions_path, 'w') as preds_file: + preds_file.write("index\tprediction\n") + for index, prediction in enumerate(preds): + prediction = prediction.replace("\n", "\\n") + preds_file.write(f"{index}\t{prediction}\n") + + # Aggregate results + metric_results = [] + for (task_template_key, metric), items in vals.items(): + task_name, prompt_name = lm_eval.tasks._split_task_template_key( + task_template_key + ) + + results[task_template_key]["task_name"] = task_name + results[task_template_key]["prompt_name"] = prompt_name + task = task_dict[task_template_key] + results[task_template_key][metric] = task.aggregation()[metric](items) + + _metric_results = { + "task_name": task_name, + "prompt_name": prompt_name, + metric: task.aggregation()[metric](items), + **task.get_logging_info(), + } + # NOTE: bleu, chrf, ter seem to be really expensive to bootstrap + # so we run them less iterations. + # TODO: Find an efficient work around. + stderr = lm_eval.api.metric.stderr_for_metric( + metric=task.aggregation()[metric], + bootstrap_iters=min(bootstrap_iters, 1000) + if metric in ["bleu", "chrf", "ter"] + else bootstrap_iters, + ) + if stderr is not None: + results[task_template_key][metric + "_stderr"] = stderr(items) + _metric_results[metric + "_stderr"] = stderr(items) + metric_results.append(_metric_results) + + return { + # List of results that tracks the averages per model and prompt. + "results": metric_results, + "versions": dict(versions), + # List of all prompt x doc examples with additional information in it. + # Original results used for generating the table when running this file. + "table_results": dict(results), + } + + +def make_table(results: dict) -> str: + """Returns a markdown table from an evaluation results `dict`. + + Args: + results (dict): + A dict of results as found in the `"table_results"` key of the + dictionary returned by `evaluate`. + + Returns: + The markdown table of results as a string. + """ + from pytablewriter import MarkdownTableWriter + + md_writer = MarkdownTableWriter() + md_writer.headers = ["Task", "Prompt", "Version", "Metric", "Value", "", "Stderr"] + + values = [] + for k, result_dict in results["table_results"].items(): + version = results["versions"][k] + for m, v in result_dict.items(): + if m.endswith("_stderr"): + continue + if "_name" in m: + continue + if m + "_stderr" in result_dict: + se = result_dict[m + "_stderr"] + values.append( + [ + result_dict["task_name"], + result_dict["prompt_name"], + version, + m, + "%.4f" % v, + "±", + "%.4f" % se, + ] + ) + else: + values.append( + [ + result_dict["task_name"], + result_dict["prompt_name"], + version, + m, + "%.4f" % v, + "", + "", + ] + ) + version = "" + md_writer.value_matrix = values + return md_writer.dumps() diff --git a/evaluation-pipeline/lm_eval/metrics/__init__.py b/evaluation-pipeline/lm_eval/metrics/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/lm_eval/metrics/__pycache__/__init__.cpython-310.pyc b/evaluation-pipeline/lm_eval/metrics/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..71509320ff5ab7dcb4d29b35ddeb20dc29c6964e Binary files /dev/null and b/evaluation-pipeline/lm_eval/metrics/__pycache__/__init__.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/metrics/__pycache__/sari.cpython-310.pyc b/evaluation-pipeline/lm_eval/metrics/__pycache__/sari.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a74d0d1cf085c3c9ac736a8d1bc6a8692c3a8a46 Binary files /dev/null and b/evaluation-pipeline/lm_eval/metrics/__pycache__/sari.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/metrics/sari.py b/evaluation-pipeline/lm_eval/metrics/sari.py new file mode 100644 index 0000000000000000000000000000000000000000..ead7153e109aa38f4270fc2f6d8c8d7e8a2d7de1 --- /dev/null +++ b/evaluation-pipeline/lm_eval/metrics/sari.py @@ -0,0 +1,231 @@ +# ======================================================= +# SARI -- Text Simplification Tunable Evaluation Metric +# ======================================================= +# +# SOURCE: https://github.com/cocoxu/simplification/blob/master/SARI.py +# This is the implementation provided by the author. +# +# Author: Wei Xu (UPenn xwe@cis.upenn.edu) +# +# A Python implementation of the SARI metric for text simplification +# evaluation in the following paper +# +# "Optimizing Statistical Machine Translation for Text Simplification" +# Wei Xu, Courtney Napoles, Ellie Pavlick, Quanze Chen and Chris Callison-Burch +# In Transactions of the Association for Computational Linguistics (TACL) 2015 +# +# There is also a Java implementation of the SARI metric +# that is integrated into the Joshua MT Decoder. It can +# be used for tuning Joshua models for a real end-to-end +# text simplification model. + + +""""@article{Xu-EtAl:2016:TACL, + author = {Wei Xu and Courtney Napoles and Ellie Pavlick and Quanze Chen and Chris Callison-Burch}, + title = {Optimizing Statistical Machine Translation for Text Simplification}, + journal = {Transactions of the Association for Computational Linguistics}, + volume = {4}, + year = {2016}, + url = {https://cocoxu.github.io/publications/tacl2016-smt-simplification.pdf}, + pages = {401--415} + }""" + +from __future__ import division +from collections import Counter + + +def SARIngram(sgrams, cgrams, rgramslist, numref): + rgramsall = [rgram for rgrams in rgramslist for rgram in rgrams] + rgramcounter = Counter(rgramsall) + + sgramcounter = Counter(sgrams) + sgramcounter_rep = Counter() + for sgram, scount in sgramcounter.items(): + sgramcounter_rep[sgram] = scount * numref + + cgramcounter = Counter(cgrams) + cgramcounter_rep = Counter() + for cgram, ccount in cgramcounter.items(): + cgramcounter_rep[cgram] = ccount * numref + + # KEEP + keepgramcounter_rep = sgramcounter_rep & cgramcounter_rep + keepgramcountergood_rep = keepgramcounter_rep & rgramcounter + keepgramcounterall_rep = sgramcounter_rep & rgramcounter + + keeptmpscore1 = 0 + keeptmpscore2 = 0 + for keepgram in keepgramcountergood_rep: + keeptmpscore1 += ( + keepgramcountergood_rep[keepgram] / keepgramcounter_rep[keepgram] + ) + keeptmpscore2 += ( + keepgramcountergood_rep[keepgram] / keepgramcounterall_rep[keepgram] + ) + # print "KEEP", keepgram, keepscore, cgramcounter[keepgram], sgramcounter[keepgram], rgramcounter[keepgram] + keepscore_precision = 0 + if len(keepgramcounter_rep) > 0: + keepscore_precision = keeptmpscore1 / len(keepgramcounter_rep) + keepscore_recall = 0 + if len(keepgramcounterall_rep) > 0: + keepscore_recall = keeptmpscore2 / len(keepgramcounterall_rep) + keepscore = 0 + if keepscore_precision > 0 or keepscore_recall > 0: + keepscore = ( + 2 + * keepscore_precision + * keepscore_recall + / (keepscore_precision + keepscore_recall) + ) + + # DELETION + delgramcounter_rep = sgramcounter_rep - cgramcounter_rep + delgramcountergood_rep = delgramcounter_rep - rgramcounter + delgramcounterall_rep = sgramcounter_rep - rgramcounter + deltmpscore1 = 0 + deltmpscore2 = 0 + for delgram in delgramcountergood_rep: + deltmpscore1 += delgramcountergood_rep[delgram] / delgramcounter_rep[delgram] + deltmpscore2 += delgramcountergood_rep[delgram] / delgramcounterall_rep[delgram] + delscore_precision = 0 + if len(delgramcounter_rep) > 0: + delscore_precision = deltmpscore1 / len(delgramcounter_rep) + # delscore_recall = 0 + # if len(delgramcounterall_rep) > 0: + # delscore_recall = deltmpscore1 / len(delgramcounterall_rep) + # delscore = 0 + # if delscore_precision > 0 or delscore_recall > 0: + # delscore = ( + # 2 + # * delscore_precision + # * delscore_recall + # / (delscore_precision + delscore_recall) + # ) + + # ADDITION + addgramcounter = set(cgramcounter) - set(sgramcounter) + addgramcountergood = set(addgramcounter) & set(rgramcounter) + addgramcounterall = set(rgramcounter) - set(sgramcounter) + + addtmpscore = 0 + for addgram in addgramcountergood: + addtmpscore += 1 + + addscore_precision = 0 + addscore_recall = 0 + if len(addgramcounter) > 0: + addscore_precision = addtmpscore / len(addgramcounter) + if len(addgramcounterall) > 0: + addscore_recall = addtmpscore / len(addgramcounterall) + addscore = 0 + if addscore_precision > 0 or addscore_recall > 0: + addscore = ( + 2 + * addscore_precision + * addscore_recall + / (addscore_precision + addscore_recall) + ) + + return (keepscore, delscore_precision, addscore) + + +def SARIsent(ssent, csent, rsents): + numref = len(rsents) + + s1grams = ssent.lower().split(" ") + c1grams = csent.lower().split(" ") + s2grams = [] + c2grams = [] + s3grams = [] + c3grams = [] + s4grams = [] + c4grams = [] + + r1gramslist = [] + r2gramslist = [] + r3gramslist = [] + r4gramslist = [] + for rsent in rsents: + r1grams = rsent.lower().split(" ") + r2grams = [] + r3grams = [] + r4grams = [] + r1gramslist.append(r1grams) + for i in range(0, len(r1grams) - 1): + if i < len(r1grams) - 1: + r2gram = r1grams[i] + " " + r1grams[i + 1] + r2grams.append(r2gram) + if i < len(r1grams) - 2: + r3gram = r1grams[i] + " " + r1grams[i + 1] + " " + r1grams[i + 2] + r3grams.append(r3gram) + if i < len(r1grams) - 3: + r4gram = ( + r1grams[i] + + " " + + r1grams[i + 1] + + " " + + r1grams[i + 2] + + " " + + r1grams[i + 3] + ) + r4grams.append(r4gram) + r2gramslist.append(r2grams) + r3gramslist.append(r3grams) + r4gramslist.append(r4grams) + + for i in range(0, len(s1grams) - 1): + if i < len(s1grams) - 1: + s2gram = s1grams[i] + " " + s1grams[i + 1] + s2grams.append(s2gram) + if i < len(s1grams) - 2: + s3gram = s1grams[i] + " " + s1grams[i + 1] + " " + s1grams[i + 2] + s3grams.append(s3gram) + if i < len(s1grams) - 3: + s4gram = ( + s1grams[i] + + " " + + s1grams[i + 1] + + " " + + s1grams[i + 2] + + " " + + s1grams[i + 3] + ) + s4grams.append(s4gram) + + for i in range(0, len(c1grams) - 1): + if i < len(c1grams) - 1: + c2gram = c1grams[i] + " " + c1grams[i + 1] + c2grams.append(c2gram) + if i < len(c1grams) - 2: + c3gram = c1grams[i] + " " + c1grams[i + 1] + " " + c1grams[i + 2] + c3grams.append(c3gram) + if i < len(c1grams) - 3: + c4gram = ( + c1grams[i] + + " " + + c1grams[i + 1] + + " " + + c1grams[i + 2] + + " " + + c1grams[i + 3] + ) + c4grams.append(c4gram) + + (keep1score, del1score, add1score) = SARIngram( + s1grams, c1grams, r1gramslist, numref + ) + (keep2score, del2score, add2score) = SARIngram( + s2grams, c2grams, r2gramslist, numref + ) + (keep3score, del3score, add3score) = SARIngram( + s3grams, c3grams, r3gramslist, numref + ) + (keep4score, del4score, add4score) = SARIngram( + s4grams, c4grams, r4gramslist, numref + ) + avgkeepscore = sum([keep1score, keep2score, keep3score, keep4score]) / 4 + avgdelscore = sum([del1score, del2score, del3score, del4score]) / 4 + avgaddscore = sum([add1score, add2score, add3score, add4score]) / 4 + finalscore = (avgkeepscore + avgdelscore + avgaddscore) / 3 + + return finalscore diff --git a/evaluation-pipeline/lm_eval/models/__init__.py b/evaluation-pipeline/lm_eval/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2cc23c1ae124085cf23981720e4081e618b52390 --- /dev/null +++ b/evaluation-pipeline/lm_eval/models/__init__.py @@ -0,0 +1,75 @@ +import logging +from typing import List, Mapping, Optional, Type + +import lm_eval.api.utils +from lm_eval.api.model import LM + +from . import dummy +from . import openai_completions +from . import huggingface + + +logger = logging.getLogger(__name__) + + +MODEL_API_REGISTRY = { + "hf-causal": huggingface.AutoCausalLM, + "hf-seq2seq": huggingface.AutoSeq2SeqLM, + "hf-mlm": huggingface.AutoMaskedLM, + "openai": openai_completions.OpenAICompletionsLM, + "dummy": dummy.DummyLM, +} + + +def list_model_apis() -> List[str]: + """Returns a list of all the model API names available for language model construction.""" + return sorted(list(MODEL_API_REGISTRY)) + + +def get_model(model_api_name: str, **model_kwargs) -> LM: + """Returns a language model from the specified model API, instantiated + with the specified kwargs. + + Args + model_api_name: Name of the model API to use as found in the model registry. + **model_kwargs: Keyword arguments to pass to the model constructor. See constructor + args for the model API in `lm_eval.models`. + + Returns: + A language model instance. + """ + model_api_class = _get_model_api_from_registry(model_api_name) + return model_api_class(**model_kwargs) + + +def get_model_from_args_string( + model_api_name: str, + model_args: str, + additional_config: Optional[Mapping[str, str]] = None, +) -> LM: + """Returns a language model from the specified model API, instantiated with + the given kwargs. + + Args: + model_api_name: Name of the model API to use as found in the model registry. + model_args: A string of comma-separated key=value pairs that will be passed + to the model constructor. E.g. "pretrained=gpt2,batch_size=32". + additional_config: An additional dictionary of key=value pairs that will be + passed to the model constructor. + + Returns: + A language model instance. + """ + additional_config = {} if additional_config is None else additional_config + additional_args = {k: v for k, v in additional_config.items() if v is not None} + kwargs = lm_eval.api.utils.parse_cli_args_string(model_args) + kwargs.update(additional_args) + return get_model(model_api_name, **kwargs) + + +def _get_model_api_from_registry(model_api_name: str) -> Type[LM]: + try: + return MODEL_API_REGISTRY[model_api_name] + except KeyError: + logger.warning(f"Available model APIs:\n{list_model_apis()}") + raise KeyError(f"Model API `{model_api_name}` is missing.") diff --git a/evaluation-pipeline/lm_eval/models/__pycache__/__init__.cpython-310.pyc b/evaluation-pipeline/lm_eval/models/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7793a97fca65a5e0c8747ad475f634589a716ade Binary files /dev/null and b/evaluation-pipeline/lm_eval/models/__pycache__/__init__.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/models/__pycache__/dummy.cpython-310.pyc b/evaluation-pipeline/lm_eval/models/__pycache__/dummy.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fe366deb93eab9ec328758d0933192fe7ec44767 Binary files /dev/null and b/evaluation-pipeline/lm_eval/models/__pycache__/dummy.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/models/__pycache__/huggingface.cpython-310.pyc b/evaluation-pipeline/lm_eval/models/__pycache__/huggingface.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9491107814adce2d6808d0e6f96e3916f750ebf5 Binary files /dev/null and b/evaluation-pipeline/lm_eval/models/__pycache__/huggingface.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/models/__pycache__/openai_completions.cpython-310.pyc b/evaluation-pipeline/lm_eval/models/__pycache__/openai_completions.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1e10dd6829122203e8e49182845d306b39174734 Binary files /dev/null and b/evaluation-pipeline/lm_eval/models/__pycache__/openai_completions.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/models/dummy.py b/evaluation-pipeline/lm_eval/models/dummy.py new file mode 100644 index 0000000000000000000000000000000000000000..0d9d28a4c2ff213a835933b7239c2801f73a5b7d --- /dev/null +++ b/evaluation-pipeline/lm_eval/models/dummy.py @@ -0,0 +1,30 @@ +import random +from typing import List, Tuple + +from lm_eval.api.model import LM + + +class DummyLM(LM): + def __init__(self): + super().__init__() + + def loglikelihood( + self, requests: List[Tuple[str, str]] + ) -> List[Tuple[float, bool]]: + res = [] + for _ in requests: + res.append((-random.random(), False)) + return res + + def loglikelihood_rolling(self, requests: List[Tuple[str, str]]) -> List[float]: + res = [] + for _ in requests: + res.append(-random.random()) + return res + + def greedy_until(self, requests: List[Tuple[str, dict]]) -> List[str]: + res = [] + for ctx, _ in requests: + res.append("null") + assert ctx.strip() != "" + return res diff --git a/evaluation-pipeline/lm_eval/models/huggingface.py b/evaluation-pipeline/lm_eval/models/huggingface.py new file mode 100644 index 0000000000000000000000000000000000000000..c2271e18d671d5d879da7574e8e41e8fa7620c84 --- /dev/null +++ b/evaluation-pipeline/lm_eval/models/huggingface.py @@ -0,0 +1,825 @@ +import math +import torch +import torch.nn.functional as F +import transformers +from typing import List, Mapping, NewType, Optional, Tuple, Union, Dict +from tqdm import tqdm +from itertools import chain + +from lm_eval.api import utils +from lm_eval.api.model import TokenLM, TokenSequence + + +_DeviceMapping = NewType("DeviceMapping", Mapping[str, Union[int, str, torch.device]]) + + +def _get_accelerate_args( + device_map_option: Optional[str] = "auto", + max_memory_per_gpu: Optional[Union[int, str]] = None, + max_cpu_memory: Optional[Union[int, str]] = None, + offload_folder: Optional[str] = "./offload", +) -> dict: + """Returns the kwargs needed to apply `accelerate` in `AutoModel.from_pretrained`.""" + max_memory = {} + if max_memory_per_gpu is not None: + max_memory_per_gpu_map = { + device_idx: max_memory_per_gpu + for device_idx in range(torch.cuda.device_count()) + } + max_memory.update(max_memory_per_gpu_map) + if max_cpu_memory is not None: + max_memory["cpu"] = max_cpu_memory + + args = {} + if max_memory: + args["max_memory"] = max_memory + args["device_map"] = device_map_option + args["offload_folder"] = offload_folder + return args + + +def _get_dtype( + dtype: Union[str, torch.dtype], config: Optional[transformers.AutoConfig] = None +) -> torch.dtype: + """Converts `dtype` from `str` to torch.dtype when possible.""" + if dtype is None and config is not None: + _torch_dtype = config.torch_dtype + elif isinstance(dtype, str) and dtype != "auto": + # Convert `str` args torch dtype: `float16` -> `torch.float16` + _torch_dtype = getattr(torch, dtype) + else: + _torch_dtype = dtype + return _torch_dtype + + +class HuggingFaceAutoLM(TokenLM): + AUTO_CONFIG_CLASS: transformers.AutoConfig = transformers.AutoConfig + AUTO_TOKENIZER_CLASS: transformers.AutoTokenizer = transformers.AutoTokenizer + AUTO_MODEL_CLASS: transformers.AutoModel = None + + # Default max sequence length setting for when no `max_length` is provided + # or no max length config setting is found in the model or tokenizer. + _DEFAULT_MAX_LENGTH: int = 2048 + + def __init__( + self, + pretrained: str, + tokenizer: Optional[str] = None, + subfolder: Optional[str] = None, + revision: Optional[str] = "main", + batch_size: Optional[int] = 1, + max_gen_toks: Optional[int] = 256, + max_length: Optional[int] = None, + add_special_tokens: Optional[bool] = None, + use_accelerate: Optional[bool] = False, + device_map_option: Optional[str] = "auto", + max_memory_per_gpu: Optional[Union[int, str]] = None, + max_cpu_memory: Optional[Union[int, str]] = None, + offload_folder: Optional[str] = "./offload", + dtype: Optional[Union[str, torch.dtype]] = None, + trust_remote_code: Optional[bool] = False, + device: Optional[Union[int, str]] = "cuda", + ): + """Initializes a HuggingFace `AutoModel` and `AutoTokenizer` for evaluation. + + Args: + pretrained (str): + The HuggingFace Hub model ID name or the path to a pre-trained + model to load. This is effectively the `pretrained_model_name_or_path` + argument of `from_pretrained` in the HuggingFace `transformers` API. + add_special_tokens (bool, optional, defaults to True): + Whether to add special tokens to the input sequences. If `None`, the + default value will be set to `True` for seq2seq models (e.g. T5) and + `False` for causal models. + + WARNING: Evaluating causal models with `add_special_tokens=True` is + currently __not__ supported. + + > Large model loading `accelerate` arguments + + use_accelerate (bool, optional, defaults to False): + If True, uses the `accelerate` library to load a large model across + multiple devices. + device_map_option (str, optional, defaults to "auto"): + The device map option to use when loading the model with + `accelerate`. + Options: + "auto", "balanced", "balanced_low_0", "sequential" + See the `accelerate` docs for more details on these options: + https://huggingface.co/docs/accelerate/v0.12.0/en/usage_guides/big_modeling#designing-a-device-map + max_memory_per_gpu (Union[int, str], optional, defaults to None): + The maximum memory available for each GPU in bytes as `int` or in + the format f"{significand}{unit_symbol}" where {unit_symbol} is + any of ["GB", "MB", "GIB", "MIB"]. Refer to the `max_memory` arg in + the "Parameters for big model inference" section of the following + docs: + https://huggingface.co/docs/transformers/v4.20.1/en/main_classes/model#large-model-loading + max_cpu_memory (Union[int, str], optional, defaults to None): + The maximum available CPU RAM in bytes as `int` or in the format + f"{significand}{unit_symbol}" where {unit_symbol} is any of + ["GB", "MB", "GIB", "MIB"]. Refer to the `max_memory` arg in the + "Parameters for big model inference" section of the following docs: + https://huggingface.co/docs/transformers/v4.20.1/en/main_classes/model#large-model-loading + offload_folder (str, optional, defaults to "./offload"): + The folder to offload weights into if `device_map` contains any + "disk" value. + dtype (Union[str, torch.dtype], optional, defaults to None):): + Converts the model weights to `dtype`, if specified. Strings get + converted to `torch.dtype` objects (e.g. `float16` -> `torch.float16`). + Use `dtype="auto"` to derive the type from the model’s weights. + """ + super().__init__() + assert isinstance(pretrained, str) + assert isinstance(device, str) + assert isinstance(batch_size, int) + if ( + add_special_tokens is not None + and self.AUTO_MODEL_CLASS is transformers.AutoModelForCausalLM + ): + # TODO: Support evaluating causal models with special tokens. Currently, + # this is not possible because the `_loglikelihood_tokens()` method for + # causal LMs makes a no-special-tokens assumption given that contexts + # and labels/continuations are tokenized separately without special + # tokens, concatenated, and then processed as inputs. + assert ( + not add_special_tokens + ), "Evaluating causal models with `add_special_tokens=True` is currently not supported." + + self._batch_size = batch_size # TODO: Adaptive batch size + self._max_gen_toks = max_gen_toks + self._max_length = max_length + self._config = self.AUTO_CONFIG_CLASS.from_pretrained( + pretrained, + revision=revision + ("/" + subfolder if subfolder is not None else ""), + trust_remote_code=trust_remote_code, + ) + + self._add_special_tokens = add_special_tokens + self.tokenizer = self._create_auto_tokenizer( + pretrained=pretrained, + revision=revision, + subfolder=subfolder, + tokenizer=tokenizer, + trust_remote_code=trust_remote_code, + ) + self.tokenizer.model_max_length = self.max_length + + accelerate_kwargs = {} + if use_accelerate: + accelerate_kwargs = _get_accelerate_args( + device_map_option, + max_memory_per_gpu, + max_cpu_memory, + offload_folder, + ) + self.model = self._create_auto_model( + pretrained=pretrained, + revision=revision, + subfolder=subfolder, + torch_dtype=_get_dtype(dtype, self._config), + trust_remote_code=trust_remote_code, + **accelerate_kwargs, + ) + self.model.eval() + torch.set_grad_enabled(False) + + self._device = device + if use_accelerate and "lm_head" in self.model.hf_device_map: + # `accelerate` can place `lm_head` weights on a different device than + # the user specified one so we force `self._device` to be the same as + # `lm_head`'s. + self._device = self.model.hf_device_map["lm_head"] + if not use_accelerate: + self.model.to(self._device) + + def _create_auto_model( + self, + *, + pretrained: str, + revision: str, + subfolder: str, + device_map: Optional[Union[str, _DeviceMapping]] = None, + max_memory: Optional[dict] = None, + offload_folder: Optional[str] = None, + torch_dtype: Optional[Union[str, torch.dtype]] = None, + trust_remote_code: Optional[bool] = False, + ) -> transformers.AutoModel: + """Returns a pre-trained pytorch model from a pre-trained model configuration.""" + model = self.AUTO_MODEL_CLASS.from_pretrained( + pretrained, + revision=revision + ("/" + subfolder if subfolder is not None else ""), + device_map=device_map, + max_memory=max_memory, + offload_folder=offload_folder, + torch_dtype=torch_dtype, + trust_remote_code=trust_remote_code, + ) + return model + + def _create_auto_tokenizer( + self, + *, + pretrained: str, + revision: str, + subfolder: str, + tokenizer: Optional[str] = None, + use_fast: Optional[bool] = True, + trust_remote_code: Optional[bool] = False, + ) -> transformers.PreTrainedTokenizer: + """Returns a pre-trained tokenizer from a pre-trained tokenizer configuration.""" + tokenizer = self.AUTO_TOKENIZER_CLASS.from_pretrained( + pretrained if tokenizer is None else tokenizer, + revision=revision + ("/" + subfolder if subfolder is not None else ""), + use_fast=use_fast, + trust_remote_code=trust_remote_code, + ) + if not tokenizer.eos_token: + tokenizer.add_special_tokens({"eos_token": ""}) + if not tokenizer.pad_token: + tokenizer.pad_token = tokenizer.eos_token + + return tokenizer + + @property + def add_special_tokens(self) -> bool: + """Whether to include special tokens in encoded text. This should be + determined by whether or not the model was trained with special tokens. + + TODO: Remove these conditionals once HuggingFace supports a way to + check whether or not an arbitrary model was trained with special tokens. + """ + if self._add_special_tokens is not None: + return self._add_special_tokens + elif self.AUTO_MODEL_CLASS is transformers.AutoModelForCausalLM: + return False + elif self.AUTO_MODEL_CLASS is transformers.AutoModelForSeq2SeqLM: + return True + elif self.AUTO_MODEL_CLASS is transformers.AutoModelForMaskedLM: + return True + else: + raise ValueError( + "Could not determine `add_special_tokens` value from the model " + "class. Set to `True` or `False` depending on whether the model " + "was pre-trained with special tokens." + ) + + @property + def eot_token(self) -> str: + return self.tokenizer.eos_token + + @property + def eot_token_id(self) -> int: + return self.tokenizer.eos_token_id + + @property + def max_gen_toks(self) -> int: + return self._max_gen_toks + + @property + def max_length(self) -> int: + """Return the maximum sequence length of the model. + NOTE: Different model configurations have different max sequence length + attribute names. + - n_positions: (CTRLConfig) + - max_position_embeddings: (BartConfig, RoFormerConfig) + - n_ctx: (GPT2Config) + NOTE: For relative position encoded models you should specify the max + sequence length of the model in the constructor via `max_length`. + """ + if self._max_length is not None: + return self._max_length + # Try to get the sequence length from the model config. + seqlen_config_attrs = ("n_positions", "max_position_embeddings", "n_ctx") + for attr in seqlen_config_attrs: + if hasattr(self._config, attr): + return getattr(self._config, attr) + if hasattr(self.tokenizer, "model_max_length"): + return self.tokenizer.model_max_length + return self._DEFAULT_MAX_LENGTH + + @property + def batch_size(self) -> int: + # TODO: Add adaptive batch size. + return self._batch_size # * gpus + + @property + def device(self) -> Union[int, str, torch.device]: + return self._device + + def tok_encode(self, string: str) -> TokenSequence: + # TODO: Merge `tok_encode_batch` here. + return self.tokenizer.encode(string, add_special_tokens=self.add_special_tokens) + + def tok_encode_batch(self, strings: List[str]) -> TokenSequence: + return self.tokenizer( + strings, + padding=True, + add_special_tokens=self.add_special_tokens, + return_tensors="pt", + ) + + def tok_decode(self, tokens: torch.LongTensor) -> List[str]: + return self.tokenizer.batch_decode(tokens, skip_special_tokens=True) + + def greedy_until(self, requests: List[Tuple[str, dict]]) -> List[str]: + def _collate(x): + tokens = self.tok_encode(x[0]) + return len(tokens), x[0] + + results = [] + reorder = utils.Reorderer(requests, _collate) + for chunk in utils.chunks( + tqdm(reorder.get_reordered(), disable=False), self.batch_size + ): + context = [c[0] for c in chunk] + request_args = chunk[0][1] + stop_sequences = request_args["stop_sequences"] + max_generation_length = request_args["max_generation_length"] + num_fewshot = request_args["num_fewshot"] + + assert ( + isinstance(max_generation_length, int) or max_generation_length is None + ) + assert isinstance(stop_sequences, list) or stop_sequences is None + assert isinstance(num_fewshot, int) or num_fewshot is None + + # TODO: Find a better way to handle stop sequences for 0-shot. + if stop_sequences is None or num_fewshot == 0: + until = [self.eot_token] + else: + until = stop_sequences + [self.eot_token] + + if max_generation_length is None: + max_tokens = self.max_gen_toks + else: + max_tokens = max_generation_length + + token_context = self.tok_encode_batch(context) + + responses = self._model_generate( + inputs=token_context, + max_tokens=max_tokens, + stop=until, + ) + responses = self.tok_decode(responses.tolist()) + + for response in responses: + # Ensure the generated responses do not contain the stop sequences. + for term in until: + response = response.split(term)[0] + # partial caching + self.cache_hook.add_partial("greedy_until", (context, until), response) + results.append(response) + return reorder.get_original(results) + + +class AutoCausalLM(HuggingFaceAutoLM): + """Causal language modeling. + You can find a set of supported models in the HF documentation: + https://huggingface.co/docs/transformers/main/model_doc/auto#transformers.AutoModelForCausalLM + """ + + AUTO_MODEL_CLASS = transformers.AutoModelForCausalLM + + def _create_auto_tokenizer( + self, + *, + pretrained: str, + revision: str, + subfolder: str, + tokenizer: Optional[str] = None, + trust_remote_code: Optional[bool] = False, + ) -> transformers.PreTrainedTokenizer: + tokenizer = super()._create_auto_tokenizer( + pretrained=pretrained, + revision=revision, + subfolder=subfolder, + tokenizer=tokenizer, + use_fast=False, + trust_remote_code=trust_remote_code, + ) + tokenizer.padding_side = "left" + return tokenizer + + def _model_call( + self, inputs: TokenSequence, labels: Optional[TokenSequence] = None + ) -> TokenSequence: + return self.model(inputs)["logits"] + + def _model_generate( + self, + inputs: transformers.BatchEncoding, + max_tokens: int, + stop: Optional[List[str]] = None, + ) -> TokenSequence: + # Ensure that the context does not encroach into the `space` + # for the generation. + input_ids = inputs["input_ids"][:, self.max_gen_toks - self.max_length :] + attention_mask = inputs["attention_mask"][ + :, self.max_gen_toks - self.max_length : + ] + input_ids = input_ids.to(self.device) + attention_mask = attention_mask.to(self.device) + + stopping_criteria = stop_sequences_criteria( + self.tokenizer, stop, input_ids.shape[1], input_ids.shape[0] + ) + + generations = self.model.generate( + input_ids=input_ids, + attention_mask=attention_mask, + # GPT style models require the `generate` `max_length` arg to include the + # context length, so we instead set `max_new_tokens` which is the number + # of new tokens to generate, excluding the current number of tokens. + max_new_tokens=max_tokens, + stopping_criteria=stopping_criteria, + do_sample=False, + ) + return utils.select_continuation_from_batch_left_padding( + generations, max_context_size=inputs["input_ids"].size(1) + ) + + +class AutoSeq2SeqLM(HuggingFaceAutoLM): + """Seq2Seq language modeling. + You can find a set of supported models in the following documentation: + https://huggingface.co/docs/transformers/main/model_doc/auto#transformers.AutoModelForSeq2SeqLM + """ + + AUTO_MODEL_CLASS = transformers.AutoModelForSeq2SeqLM + + @property + def max_length(self) -> int: + """Return the maximum sequence length of the model. + TODO: Currently only works for relative position encoded Seq2Seq models. + """ + if self._max_length is not None: + return self._max_length + return self._DEFAULT_MAX_LENGTH + + def loglikelihood( + self, requests: List[Tuple[str, str]] + ) -> List[Tuple[float, bool]]: + new_requests = [] + for chunk in utils.chunks(requests, self.batch_size): + context, continuation = zip(*chunk) + + # Fill empty contexts with the EOT token. + context = [ + f"{self.eot_token}" if len(text) == 0 else text for text in context + ] + context_enc = self.tok_encode_batch(context) + for key in context_enc: + context_enc[key] = context_enc[key][:, -self.max_length :] + + # Remove leading whitespace introduced by the default + # `text_target_separator` since the context and continuation + # will not be concatenated as a single (decoder) input. + continuation = [text.lstrip() for text in continuation] + continuation_enc = self.tok_encode_batch(list(continuation)) + for key in continuation_enc: + continuation_enc[key] = continuation_enc[key][:, -self.max_length :] + + new_requests.append( + ((context, continuation), context_enc, continuation_enc) + ) + return self._loglikelihood_tokens(new_requests) + + def loglikelihood_rolling(self, requests: List[Tuple[str, str]]) -> List[float]: + loglikelihoods = [] + for (string,) in tqdm(requests): + rolling_token_windows = list( + map( + utils.make_disjoint_window, + utils.get_rolling_token_windows( + token_list=self.tok_encode(string), + prefix_token=self.eot_token_id, + max_seq_len=self.max_length, + context_len=1, + ), + ) + ) + contexts, conts = utils.split_and_pad_windows( + rolling_token_windows, + pad_token_id=self.eot_token_id, + max_seq_len=self.max_length, + ) + # Manually create BatchEncoding tensors with attention masks as + # expected by `self._model_call` in `self._loglikelihood_tokens`. + contexts_enc = torch.Tensor(contexts).long() + contexts_enc = transformers.tokenization_utils_base.BatchEncoding( + { + "input_ids": contexts_enc, + "attention_mask": (contexts_enc != self.eot_token_id).long(), + } + ) + conts_enc = torch.Tensor(conts).long() + conts_enc = transformers.tokenization_utils_base.BatchEncoding( + { + "input_ids": conts_enc, + "attention_mask": (conts_enc != self.eot_token_id).long(), + } + ) + # TODO: Extract out this call so it only gets called once and also + # somehow figure out partial caching for. + rolling_token_windows_request = [ + ((contexts, conts), contexts_enc, conts_enc) + ] + string_nll = self._loglikelihood_tokens( + rolling_token_windows_request, disable_tqdm=True + ) + string_nll = [x[0] for x in string_nll] # discard is_greedy + string_nll = sum(string_nll) + loglikelihoods.append(string_nll) + return loglikelihoods + + def _loglikelihood_tokens( + self, + requests: List[Tuple[Tuple[str, str], TokenSequence, TokenSequence]], + disable_tqdm: Optional[bool] = False, + ) -> List[Tuple[float, bool]]: + results = [] + for chunk in tqdm( + requests, total=math.ceil(len(requests)), disable=disable_tqdm + ): + cache_keys, inputs_tokens, targets_tokens = chunk + inputs_tokens = inputs_tokens.to(self.device) + targets_tokens = targets_tokens.to(self.device) + outputs = self._model_call(inputs=inputs_tokens, labels=targets_tokens) + log_softmaxes = F.log_softmax(outputs.logits, dim=-1) + + output_iterator = zip( + zip(cache_keys[0], cache_keys[1]), + log_softmaxes, + targets_tokens["input_ids"], + targets_tokens["attention_mask"], + ) + for cache_key, log_softmax, target_tokens, target_mask in output_iterator: + length = target_mask.sum() + log_softmax = log_softmax[:length] + target_tokens = target_tokens[:length] + greedy_tokens = log_softmax.argmax(dim=-1) + max_equal = (greedy_tokens == target_tokens).all() + target_logits = torch.gather( + log_softmax, 1, target_tokens.unsqueeze(-1) + ).squeeze(-1) + answer = (float(target_logits.sum()), bool(max_equal)) + results.append(answer) + if cache_key is not None: + self.cache_hook.add_partial("loglikelihood", cache_key, answer) + return results + + def _model_call( + self, inputs: TokenSequence, labels: Optional[TokenSequence] = None + ) -> TokenSequence: + return self.model(**inputs, labels=labels["input_ids"]) + + def _model_generate( + self, + inputs: transformers.BatchEncoding, + max_tokens: int, + stop: Optional[List[str]] = None, + ) -> TokenSequence: + input_ids = inputs["input_ids"][:, -self.max_length :].to(self.device) + attention_mask = inputs["attention_mask"][:, -self.max_length :].to(self.device) + + # Generate one token to calculate the number of start tokens prepended to decoder_input_ids + # (leaving this here in case the below assumption is violated in the future) + # one_tok_gen = self.model.generate( + # input_ids=torch.zeros((1, 1), dtype=torch.int), + # min_length=2, + # max_new_tokens=1, + # ).squeeze() + # initial_decoder_input_length = len(one_tok_gen) - 1 + + # Assume that there will always only be one token in the decoder inputs, assumption holds for existing HF models + stopping_criteria = stop_sequences_criteria( + self.tokenizer, stop, 1, input_ids.shape[0] + ) + + generations = self.model.generate( + input_ids=input_ids, + attention_mask=attention_mask, + max_new_tokens=max_tokens, + stopping_criteria=stopping_criteria, + do_sample=False, + ) + return generations + + +class AutoMaskedLM(HuggingFaceAutoLM): + """Masked language modeling. + You can find a set of supported models in the following documentation: + https://huggingface.co/docs/transformers/main/model_doc/auto#transformers.AutoModelForMaskedLM + + Much of the code in this class is adapted from minicons, by Kanishka Misra: + https://github.com/kanishkamisra/minicons + which is itself adapted from the code of Salazar et al. (2020): + https://github.com/awslabs/mlm-scoring + """ + + AUTO_MODEL_CLASS = transformers.AutoModelForMaskedLM + + @property + def max_length(self) -> int: + """Return the maximum sequence length of the model. + TODO: Currently only works for relative position encoded Seq2Seq models. + """ + if self._max_length is not None: + return self._max_length + return self._DEFAULT_MAX_LENGTH + + def _model_call( + self, inputs: TokenSequence, labels: Optional[TokenSequence] = None + ) -> TokenSequence: + return self.model(inputs, labels=labels["input_ids"]) + + def _model_generate( + self, + inputs: transformers.BatchEncoding, + max_tokens: int, + stop: Optional[List[str]] = None, + ) -> TokenSequence: + raise NotImplementedError("Masked LMs are not well-suited to generating sequences.") + + def encode(self, text: Union[str, List[str]], manual_special: bool = True, return_tensors: Optional[str] = 'pt') -> Dict: + """ + Encode a batch of sentences using the model's tokenizer. + Equivalent of calling `model.tokenizer(input)` + :param ``Union[str, List[str]]`` text: Input batch/sentence to + be encoded. + :param manual_special: Specification of whether special tokens + will be manually encoded. + :type manual_special: bool + :param return_tensors: returned tensor format. Default `'pt'` + :type manual_special: str + :return: Encoded batch + :rtype: ``Dict`` + """ + sentences = [text] if isinstance(text, str) else text + + if manual_special: + # manually add special tokens + sentences = self.add_special_tokens(sentences) + if return_tensors: + tokens = self.tokenizer.batch_encode_plus(sentences, add_special_tokens = False, padding = 'longest', return_attention_mask = True, return_tensors = return_tensors) + else: + # mostly for masked LMs + tokens = self.tokenizer.batch_encode_plus(sentences, padding = 'longest', return_attention_mask = True) + + return tokens + + def _prepare_text(self, text: Union[str, List[str]]): + sentences = [text] if isinstance(text, str) else list(text) if isinstance(text, tuple) else text + encoded = self.encode(sentences, manual_special = False) + + token_idx = encoded['input_ids'] + attention_masks = encoded['attention_mask'] + + masked_tensors = [] # token ids, attention masks, lengths + + for token_ids, attention_mask in zip(token_idx, attention_masks): + token_ids = torch.tensor(token_ids) + # final_lengths = len(token_ids) - 2 + attention_mask = torch.tensor(attention_mask) + + token_ids_masked_list = [] + attention_masked_list = [] + + effective_token_ids = [token for token in token_ids if token != self.tokenizer.pad_token_id and token != self.tokenizer.cls_token_id and token != self.tokenizer.sep_token_id] + effective_length = len(effective_token_ids) + + mask_indices = [] + mask_indices = [[mask_pos] for mask_pos in range(effective_length+2)] + + # We don't mask the [CLS], [SEP] for now for PLL + mask_indices = mask_indices[1:-1] + + mask_token_id = self.tokenizer.mask_token_id + for mask_set in mask_indices: + token_ids_masked = token_ids.clone() + token_ids_masked[mask_set] = mask_token_id + attention_masked = attention_mask.clone() + + attention_masked_list.append(attention_masked) + token_ids_masked_list.append(token_ids_masked) + masked_tensors.append((torch.stack(token_ids_masked_list), torch.stack(attention_masked_list), effective_token_ids, len(mask_indices), 1)) + + return masked_tensors + + def loglikelihood( + self, requests: List[Tuple[str, str]] + ) -> List[Tuple[float, bool]]: + """ + Returns *pseudo*-loglikelihoods, as described in Salazar et al. (2020). + """ + scores = [] + + for chunk in utils.chunks(tqdm(requests, disable=False), self.batch_size): + context, continuation = zip(*chunk) + context = [ + f"{self.tokenizer.eos_token}" if len(text) == 0 else text for text in context + ] + # context_enc = self._prepare_text(context) + # for key in context_enc: + # context_enc[key] = context_enc[key][:, -self.max_length :] + + tokenized = self._prepare_text(continuation) + + token_ids, attention_masks, effective_token_ids, lengths, offsets = list(zip(*tokenized)) + token_ids = torch.cat(token_ids) + attention_masks = torch.cat(attention_masks) + token_ids = token_ids.to(self.device) + attention_masks = attention_masks.to(self.device) + effective_token_ids = torch.cat([torch.tensor(x) for x in effective_token_ids]) + + indices = list(chain.from_iterable([list(range(o,o+n)) for n, o in zip(lengths, offsets)])) + + with torch.no_grad(): + output = self.model(token_ids, attention_mask = attention_masks) + logits = output.logits.detach()[torch.arange(sum(lengths)), indices] + + logprob_distribution = logits - logits.logsumexp(1).unsqueeze(1) + + # if base_two: + logprob_distribution = logprob_distribution/torch.tensor(2).log() + + # if prob: + # logprob_distribution = logprob_distribution.exp() + + # if rank: + # shape = logprob_distribution.shape + # ''' + # Double argsort trick: + # first argsort returns idxes of values that would return a sorted tensor, + # second argsort returns ranks (0 indexed) + # Proof: https://www.berkayantmen.com/rank.html + # TODO: Try to implement ranking in linear time but across arbitrary dimensions: + # https://stackoverflow.com/a/5284703 + # ''' + # word_ranks = (-1.0 * logprob_distribution).argsort().argsort() + 1 + # word_ranks = word_ranks[torch.arange(shape[0]), effective_token_ids].split(lengths) + # word_ranks = [wr.tolist() for wr in word_ranks] + + batch_scores = logprob_distribution[torch.arange(sum(lengths)), effective_token_ids].type(torch.DoubleTensor).split(lengths) + # print(len(batch_scores[0])) + # print(len(tokenized[0])) + batch_scores = [(float(s.sum()),) for s in batch_scores] + scores.extend(batch_scores) + + # if not return_tensors: + # scores = [s.tolist() for s in scores] + + # if rank: + # return scores, word_ranks + # else: + return scores + + +class MultiTokenEOSCriteria(transformers.StoppingCriteria): + """Criteria to stop on the specified multi-token sequence.""" + + def __init__( + self, + sequence: str, + tokenizer: transformers.PreTrainedTokenizer, + initial_decoder_input_length: int, + batch_size: int, + ): + self.initial_decoder_input_length = initial_decoder_input_length + self.done_tracker = [False] * batch_size + self.sequence = sequence + self.sequence_ids = tokenizer.encode(sequence, add_special_tokens=False) + self.sequence_id_len = len(self.sequence_ids) + self.tokenizer = tokenizer + + def __call__(self, input_ids, scores, **kwargs) -> bool: + # For efficiency, we compare the last n tokens where n is the number of tokens in the stop_sequence + lookback_ids_batch = input_ids[:, self.initial_decoder_input_length :][ + :, -self.sequence_id_len : + ] + + lookback_tokens_batch = self.tokenizer.batch_decode(lookback_ids_batch) + + for i, done in enumerate(self.done_tracker): + if not done: + self.done_tracker[i] = self.sequence in lookback_tokens_batch[i] + return False not in self.done_tracker + + +def stop_sequences_criteria( + tokenizer: transformers.PreTrainedTokenizer, + stop_sequences: List[str], + initial_decoder_input_length: int, + batch_size: int, +) -> transformers.StoppingCriteriaList: + return transformers.StoppingCriteriaList( + [ + *[ + MultiTokenEOSCriteria( + sequence, tokenizer, initial_decoder_input_length, batch_size + ) + for sequence in stop_sequences + ], + ] + ) diff --git a/evaluation-pipeline/lm_eval/models/openai_completions.py b/evaluation-pipeline/lm_eval/models/openai_completions.py new file mode 100644 index 0000000000000000000000000000000000000000..d6ceaf4e5ca5cc67791d9b6acc5b9738d9a6633b --- /dev/null +++ b/evaluation-pipeline/lm_eval/models/openai_completions.py @@ -0,0 +1,270 @@ +import logging +import os +import time +import transformers +from typing import Iterable, List, Optional, Tuple, Union +from tqdm import tqdm + +from lm_eval.api import utils +from lm_eval.api.model import TokenLM, TokenSequence + + +logging.getLogger("openai").setLevel(logging.WARNING) + + +def get_result(response: dict, ctxlen: int) -> Tuple[float, bool]: + """Process results from OpenAI API response. + + Args: + response (dict): + OpenAI API response. + ctxlen (int): + Length of context (so we can slice them away and only keep the + predictions). + + Returns: + continuation_logprobs: float + Log probabilities of continuation tokens. + is_greedy: bool + Whether argmax matches given continuation exactly. + """ + is_greedy = True + logprobs = response["logprobs"]["token_logprobs"] + continuation_logprobs = sum(logprobs[ctxlen:]) + + for i in range(ctxlen, len(response["logprobs"]["tokens"])): + token = response["logprobs"]["tokens"][i] + top_tokens = response["logprobs"]["top_logprobs"][i] + top_token = max(top_tokens.keys(), key=lambda x: top_tokens[x]) + if top_token != token: + is_greedy = False + break + + return continuation_logprobs, is_greedy + + +def oa_completion(**kwargs): + """Query OpenAI API for completion. + + NOTE: Retry with back-off until they respond. + """ + import openai + + backoff_time = 3 + while True: + try: + return openai.Completion.create(**kwargs) + except openai.error.OpenAIError: + import traceback + + traceback.print_exc() + time.sleep(backoff_time) + backoff_time *= 1.5 + + +class OpenAICompletionsLM(TokenLM): + """Implements a language model interface for OpenAI's Completions API. + See: https://beta.openai.com/docs/api-reference/completions + """ + + def __init__( + self, + engine: str, + device: Optional[str] = None, + batch_size: Optional[int] = 20, + max_gen_toks: Optional[int] = 256, + ): + """ + Args: + engine (str): + OpenAI API engine (e.g. `davinci`). + """ + super().__init__() + assert device is None, "Can't specify `device` in the OpenAI API." + + import openai + + self.engine = engine + self.tokenizer = transformers.GPT2TokenizerFast.from_pretrained("gpt2") + # To make the annoying "Using pad_token, but it is not set yet." error go away + self.tokenizer.pad_token = "<|endoftext|>" + self.vocab_size = self.tokenizer.vocab_size + + self._max_gen_toks = max_gen_toks + self._batch_size = batch_size # TODO: adaptive batch size + + openai.api_key = os.environ["OPENAI_API_SECRET_KEY"] + + @property + def eot_token(self) -> int: + return self.tokenizer.eos_token + + @property + def eot_token_id(self) -> int: + # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence* + return self.tokenizer.eos_token_id + + @property + def max_length(self) -> int: + # Note: the OpenAI API supports up to 2049 tokens, with the first token being the first input token + return 2048 + + @property + def max_gen_toks(self) -> int: + return self._max_gen_toks + + @property + def batch_size(self) -> int: + return self._batch_size + + @property + def device(self) -> str: + raise NotImplementedError() + + def tok_encode(self, string: str): + return self.tokenizer.encode(string, add_special_tokens=False) + + def tok_decode(self, tokens: Iterable[int]) -> List[str]: + return self.tokenizer.decode(tokens) + + def _loglikelihood_tokens( + self, + requests: List[Union[Tuple[str, str], TokenSequence, TokenSequence]], + disable_tqdm: Optional[bool] = False, + ) -> List[Tuple[float, bool]]: + def _collate(x): + # this doesn't efficiently handle last-token differences yet, but those are kinda annoying because + # it's not guaranteed that the 100 or so logprobs we get to see actually contain all the continuations + # we care about and so we need some kind of backup for when it isn't + tokens = x[1] + x[2] + return -len(tokens), tuple(tokens) + + results = [] + reorder = utils.Reorderer(requests, _collate) + for chunk in tqdm( + list(utils.chunks(reorder.get_reordered(), self.batch_size)), + disable=disable_tqdm, + ): + inputs = [] + ctxlens = [] + for cache_key, context_enc, continuation_enc in chunk: + # max_gen_toks+1 because the API takes up to 2049 tokens, including the first context token + input = (context_enc + continuation_enc)[-(self.max_length + 1) :] + # TODO: the logic is much simpler if we just look at the length of continuation tokens + ctxlen = len(context_enc) - max( + 0, + len(context_enc) + len(continuation_enc) - (self.max_length + 1), + ) + inputs.append(input) + ctxlens.append(ctxlen) + + responses = self._model_call(inputs) + + for response, ctxlen, (cache_key, context_enc, continuation_enc) in zip( + responses.choices, ctxlens, chunk + ): + answer = get_result(response, ctxlen) + results.append(answer) + # partial caching + if cache_key is not None: + self.cache_hook.add_partial("loglikelihood", cache_key, answer) + return reorder.get_original(results) + + def greedy_until(self, requests: List[Tuple[str, dict]]) -> List[str]: + def _collate(x): + tokens = self.tok_encode(x[0]) + return len(tokens), x[0] + + def sameuntil_chunks(xs, size): + ret = [] + last_until = xs[0][1] + for x in xs: + if len(ret) >= size or x[1] != last_until: + yield ret, last_until + ret = [] + last_until = x[1] + ret.append(x) + + if ret: + yield ret, last_until + + results = [] + reorder = utils.Reorderer(requests, _collate) + # TODO: more intelligent batching for heterogeneous `until` + for chunk, request_args in tqdm( + list(sameuntil_chunks(reorder.get_reordered(), self.batch_size)) + ): + stop_sequences = request_args["stop_sequences"] + max_generation_length = request_args["max_generation_length"] + num_fewshot = request_args["num_fewshot"] + + assert isinstance(stop_sequences, list) or stop_sequences is None + assert ( + isinstance(max_generation_length, int) or max_generation_length is None + ) + assert isinstance(num_fewshot, int) or num_fewshot is None + + # TODO(jon-tow): This is most likely useless b/c `stop_sequences` is + # never `None`; see `PromptSourceTask.construct_requests`. + if stop_sequences is None or num_fewshot == 0: + until = [self.eot_token] + else: + until = stop_sequences + [self.eot_token] + + if max_generation_length is None: + max_tokens = self.max_gen_toks + else: + max_tokens = max_generation_length + + inputs = [] + for context, _ in chunk: + context_enc = self.tok_encode(context) + input = context_enc[-(self.max_length - self.max_gen_toks) :] + inputs.append(input) + + responses = self._model_generate( + inputs=inputs, + max_tokens=max_tokens, + stop=until, + ) + + # Iterate thru the per-request responses. + for response, (context, _request_args) in zip(responses.choices, chunk): + sentence = response["text"] + _stop_sequences = _request_args["stop_sequences"] + _until = ( + [self.eot_token] if _stop_sequences is None else _stop_sequences + ) + for term in _until: + sentence = sentence.split(term)[0] + # partial caching + self.cache_hook.add_partial("greedy_until", (context, _until), sentence) + results.append(sentence) + return reorder.get_original(results) + + def _model_call( + self, inputs: TokenSequence, labels: Optional[TokenSequence] = None + ) -> TokenSequence: + return oa_completion( + engine=self.engine, + prompt=inputs, + echo=True, + max_tokens=0, + temperature=0.0, + logprobs=5, + ) + + def _model_generate( + self, inputs: TokenSequence, max_tokens: int, stop: Optional[List[str]] = None + ) -> Union[TokenSequence, List[str]]: + # NOTE: We don't need to add context size b/c OpenAI completion only + # expects the max generation count portion. + generations = oa_completion( + engine=self.engine, + prompt=inputs, + max_tokens=max_tokens, + temperature=0.0, + logprobs=5, + stop=stop, + ) + return generations diff --git a/evaluation-pipeline/lm_eval/tasks/__init__.py b/evaluation-pipeline/lm_eval/tasks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..22d2537ca3e0e18f760c373e90e68cab1164739b --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/__init__.py @@ -0,0 +1,408 @@ +import logging +from typing import List, Mapping, Tuple, Type, Optional, Union +from promptsource.templates import DatasetTemplates + +import lm_eval.api.utils +from lm_eval.api.task import Task + +from . import anli +from . import bias_shades +from . import blimp +from . import diabla +from . import cnn_dailymail +from . import comps +from . import coqa +from . import crd3 +from . import crows_pairs_multilingual +from . import drop +from . import e2e_nlg_cleaned +from . import flores_101 +from . import gem_asset_turk +from . import gem_mlsum +from . import gem_webnlg +from . import gem_wikilingua +from . import gem_xsum +from . import glue +from . import hans +from . import huff_post +from . import jigsaw_unintended_bias +from . import lama +from . import lince +from . import olmpics +from . import piaf +from . import race +from . import schema_guided_dstc8 +from . import superglue +from . import syllogisms +from . import tydiqa +from . import wino_bias +from . import wmt +from . import xquad +from . import aoa_pred + +aoa_pred_eval = aoa_pred.aoa_pred_eval + + +logger = logging.getLogger(__name__) + + +TASK_REGISTRY = { + # GLUE + "cola": glue.CoLA, + "mnli": glue.MNLI, + "mnli_mismatched": glue.MNLIMismatched, + "mrpc": glue.MRPC, + "rte": glue.RTE, + "qnli": glue.QNLI, + "qqp": glue.QQP, + # "stsb": glue.STSB, # not implemented yet + "sst": glue.SST, + "wnli": glue.WNLI, + # SuperGLUE + "axb": superglue.BroadcoverageDiagnostics, + "axg": superglue.WinogenderSchemaDiagnostics, + "boolq": superglue.BoolQ, + "cb": superglue.CommitmentBank, + "copa": superglue.Copa, + "multirc": superglue.MultiRC, + "record": superglue.ReCoRD, + "superglue_rte": superglue.RTE, + "wic": superglue.WordsInContext, + "wsc": superglue.SGWinogradSchemaChallenge, + # CoQA + "coqa": coqa.CoQA, + # DROP + "drop": drop.DROP, + # E2E NLG + "e2e_nlg_cleaned": e2e_nlg_cleaned.E2E_NLG_Cleaned, + # DSTC8 + "schema_guided_dstc8": schema_guided_dstc8.Schema_Guided_DSTC8, + # RACE + "race": race.RACE, + # ANLI + "anli_r1": anli.ANLIRound1, + "anli_r2": anli.ANLIRound2, + "anli_r3": anli.ANLIRound3, + # HANS + "hans": hans.HANS, + # CNN Daily Mail + "cnn_dailymail": cnn_dailymail.CnnDailyMail, + # GEM/xum + "gem_xsum": gem_xsum.GEMXSUM, + "gem_xsum_challenge_sample": gem_xsum.GEMXSUMChallgeSample, + "gem_xsum_challenge_test_backtranslation": gem_xsum.GEMXSUMChallgeTestBacktranslation, + "gem_xsum_challenge_test_bfp_02": gem_xsum.GEMXSUMChallgeTestBFP02, + "gem_xsum_challenge_test_bfp_05": gem_xsum.GEMXSUMChallgeTestBFP05, + "gem_xsum_challenge_test_nopunc": gem_xsum.GEMXSUMChallgeTestNopunc, + "gem_xsum_challenge_test_covid": gem_xsum.GEMXSUMChallgeTestCovid, + # LAMA + "lama-trex": lama.Trex, + "lama-squad": lama.Squad, + "lama-google_re": lama.google_re, + "lama-conceptnet": lama.Conceptnet, + # WinoBias + "wino_bias_type1_pro": wino_bias.WinoBiasType1Pro, + "wino_bias_type1_anti": wino_bias.WinoBiasType1Anti, + "wino_bias_type2_pro": wino_bias.WinoBiasType2Pro, + "wino_bias_type2_anti": wino_bias.WinoBiasType2Anti, + # Crows-Pairs + "crows_pairs_english": crows_pairs_multilingual.CrowsPairsEnglish, + "crows_pairs_french": crows_pairs_multilingual.CrowsPairsFrench, + # News + "huffpost": huff_post.HuffPost, + # Code-switching + "lince_sa": lince.LinCESentimentAnalysis, + # CRD3 + "crd3": crd3.CRD3, + # DiaBLa + "diabla": diabla.DiaBLa, + "diabla_1_shot_context_orig": diabla.DiaBLa_1_shot_context_orig, + "diabla_1_shot_context_same": diabla.DiaBLa_1_shot_context_same, + "diabla_1_shot_context_opposite": diabla.DiaBLa_1_shot_context_opposite, + # XQuAD + "xquad_en": xquad.XQuADEnglish, + "xquad_ar": xquad.XQuADArabic, + # PIAF + "piaf": piaf.PIAF, + # Flores 101 (MT) + "flores_101_mt": flores_101.Flores101MT, + "flores_101_mt_fewshot_fr2en": flores_101.Flores101MT_fewshot_fr2en, + "flores_101_mt_fewshot_hi2en": flores_101.Flores101MT_fewshot_hi2en, + "flores_101_mt_fewshot_fr2ar": flores_101.Flores101MT_fewshot_fr2ar, + "flores_101_mt_fewshot_en2bn": flores_101.Flores101MT_fewshot_en2bn, + "flores_101_mt_fewshot_wmt_fr2en": flores_101.Flores101MT_fewshot_wmt_fr2en, + "flores_101_mt_fewshot_wmt_hi2en": flores_101.Flores101MT_fewshot_wmt_hi2en, + # Flores101 (Perplexity) + "flores_101_ppl": flores_101.Flores101Perplexity, + # GEM/WebNLG + # Format: `GEM/web_nlg_{webnlg.subset_name}_{split}` + **gem_webnlg.construct_tasks(), + # GEM/WikiAssetTurk + # Format: `GEM/wiki_auto_asset_turk_{split}` + **gem_asset_turk.construct_tasks(), + # GEM WikiLingua + # Format: `GEM/wiki_lingua_{lang}` + **gem_wikilingua.construct_tasks(), + # WMT + # Format: `wmt{year}_{lang1}_{lang2}` + **wmt.construct_tasks(), + # Bias-Shades + # Format: `bias_shades_{lang}` + **bias_shades.construct_tasks(), + # BLiMP + "blimp_adjunct_island": blimp.BlimpAdjunctIsland, + "blimp_anaphor_gender_agreement": blimp.BlimpAnaphorGenderAgreement, + "blimp_anaphor_number_agreement": blimp.BlimpAnaphorNumberAgreement, + "blimp_animate_subject_passive": blimp.BlimpAnimateSubjectPassive, + "blimp_animate_subject_trans": blimp.BlimpAnimateSubjectTrans, + "blimp_causative": blimp.BlimpCausative, + "blimp_complex_NP_island": blimp.BlimpComplex_NPIsland, + "blimp_coordinate_structure_constraint_complex_left_branch": blimp.BlimpCoordinateStructureConstraintComplexLeftBranch, + "blimp_coordinate_structure_constraint_object_extraction": blimp.BlimpCoordinateStructureConstraintObjectExtraction, + "blimp_determiner_noun_agreement_1": blimp.BlimpDeterminerNounAgreement_1, + "blimp_determiner_noun_agreement_2": blimp.BlimpDeterminerNounAgreement_2, + "blimp_determiner_noun_agreement_irregular_1": blimp.BlimpDeterminerNounAgreementIrregular_1, + "blimp_determiner_noun_agreement_irregular_2": blimp.BlimpDeterminerNounAgreementIrregular_2, + "blimp_determiner_noun_agreement_with_adj_2": blimp.BlimpDeterminerNounAgreementWithAdj_2, + "blimp_determiner_noun_agreement_with_adj_irregular_1": blimp.BlimpDeterminerNounAgreementWithAdjIrregular_1, + "blimp_determiner_noun_agreement_with_adj_irregular_2": blimp.BlimpDeterminerNounAgreementWithAdjIrregular_2, + "blimp_determiner_noun_agreement_with_adjective_1": blimp.BlimpDeterminerNounAgreementWithAdjective_1, + "blimp_distractor_agreement_relational_noun": blimp.BlimpDistractorAgreementRelationalNoun, + "blimp_distractor_agreement_relative_clause": blimp.BlimpDistractorAgreementRelativeClause, + "blimp_drop_argument": blimp.BlimpDropArgument, + "blimp_ellipsis_n_bar_1": blimp.BlimpEllipsisNBar_1, + "blimp_ellipsis_n_bar_2": blimp.BlimpEllipsisNBar_2, + "blimp_existential_there_object_raising": blimp.BlimpExistentialThereObjectRaising, + "blimp_existential_there_quantifiers_1": blimp.BlimpExistentialThereQuantifiers_1, + "blimp_existential_there_quantifiers_2": blimp.BlimpExistentialThereQuantifiers_2, + "blimp_existential_there_subject_raising": blimp.BlimpExistentialThereSubjectRaising, + "blimp_expletive_it_object_raising": blimp.BlimpExpletiveItObjectRaising, + "blimp_inchoative": blimp.BlimpInchoative, + "blimp_intransitive": blimp.BlimpIntransitive, + "blimp_irregular_past_participle_adjectives": blimp.BlimpIrregularPastParticipleAdjectives, + "blimp_irregular_past_participle_verbs": blimp.BlimpIrregularPastParticipleVerbs, + "blimp_irregular_plural_subject_verb_agreement_1": blimp.BlimpIrregularPluralSubjectVerbAgreement_1, + "blimp_irregular_plural_subject_verb_agreement_2": blimp.BlimpIrregularPluralSubjectVerbAgreement_2, + "blimp_left_branch_island_echo_question": blimp.BlimpLeftBranchIslandEchoQuestion, + "blimp_left_branch_island_simple_question": blimp.BlimpLeftBranchIslandSimpleQuestion, + "blimp_matrix_question_npi_licensor_present": blimp.BlimpMatrixQuestionNpiLicensorPresent, + "blimp_npi_present_1": blimp.BlimpNpiPresent_1, + "blimp_npi_present_2": blimp.BlimpNpiPresent_2, + "blimp_only_npi_licensor_present": blimp.BlimpOnlyNpiLicensorPresent, + "blimp_only_npi_scope": blimp.BlimpOnlyNpiScope, + "blimp_passive_1": blimp.BlimpPassive_1, + "blimp_passive_2": blimp.BlimpPassive_2, + "blimp_principle_A_c_command": blimp.BlimpPrinciple_ACCommand, + "blimp_principle_A_case_1": blimp.BlimpPrinciple_ACase_1, + "blimp_principle_A_case_2": blimp.BlimpPrinciple_ACase_2, + "blimp_principle_A_domain_1": blimp.BlimpPrinciple_ADomain_1, + "blimp_principle_A_domain_2": blimp.BlimpPrinciple_ADomain_2, + "blimp_principle_A_domain_3": blimp.BlimpPrinciple_ADomain_3, + "blimp_principle_A_reconstruction": blimp.BlimpPrinciple_AReconstruction, + "blimp_regular_plural_subject_verb_agreement_1": blimp.BlimpRegularPluralSubjectVerbAgreement_1, + "blimp_regular_plural_subject_verb_agreement_2": blimp.BlimpRegularPluralSubjectVerbAgreement_2, + "blimp_sentential_negation_npi_licensor_present": blimp.BlimpSententialNegationNpiLicensorPresent, + "blimp_sentential_negation_npi_scope": blimp.BlimpSententialNegationNpiScope, + "blimp_sentential_subject_island": blimp.BlimpSententialSubjectIsland, + "blimp_superlative_quantifiers_1": blimp.BlimpSuperlativeQuantifiers_1, + "blimp_superlative_quantifiers_2": blimp.BlimpSuperlativeQuantifiers_2, + "blimp_tough_vs_raising_1": blimp.BlimpToughVsRaising_1, + "blimp_tough_vs_raising_2": blimp.BlimpToughVsRaising_2, + "blimp_transitive": blimp.BlimpTransitive, + "blimp_wh_island": blimp.BlimpWhIsland, + "blimp_wh_questions_object_gap": blimp.BlimpWhQuestionsObjectGap, + "blimp_wh_questions_subject_gap": blimp.BlimpWhQuestionsSubjectGap, + "blimp_wh_questions_subject_gap_long_distance": blimp.BlimpWhQuestionsSubjectGapLongDistance, + "blimp_wh_vs_that_no_gap": blimp.BlimpWhVsThatNoGap, + "blimp_wh_vs_that_no_gap_long_distance": blimp.BlimpWhVsThatNoGapLongDistance, + "blimp_wh_vs_that_with_gap": blimp.BlimpWhVsThatWithGap, + "blimp_wh_vs_that_with_gap_long_distance": blimp.BlimpWhVsThatWithGapLongDistance, + "blimp_from_file": blimp.BlimpFromFile, + # COMPS + "comps_base": comps.CompsBase, + "comps_wugs": comps.CompsWugs, + # Syllogisms + "syllogisms_nonsense": syllogisms.SyllogismsNonsense, + # oLMpics + "olmpics_age_comparison": olmpics.OlmpicsAgeComparison, + "olmpics_always_never": olmpics.OlmpicsAlwaysNever, + "olmpics_multihop_composition": olmpics.OlmpicsMultihopComposition, + "olmpics_object_comparison": olmpics.OlmpicsObjectComparison, + "olmpics_property_conjunction": olmpics.OlmpicsPropertyConjunction, + "olmpics_taxonomy_conjunction": olmpics.OlmpicsTaxonomyConjunction, + # TyDi QA + "tydiqa_primary": tydiqa.TyDiQAPrimaryClassification, + "tydiqa_secondary": tydiqa.TyDiQAGoldPGeneration, + ####################################################### + # TODO: Not Yet Available in `promptsource/eval-hackathon` + ######################################################## + # GEM/mlsum + # "mlsum_es": gem_mlsum.GEMMLSUMEs, + # "mlsum_de": gem_mlsum.GEMMLSUMDe, + # "mlsum_es_covid_challenge_set": gem_mlsum.GEMMLSUMEsChallgeTestCovid, + # "mlsum_de_covid_challenge_set": gem_mlsum.GEMMLSUMDeChallgeTestCovid, + # LAMA + # "bigscience-lama": lama.BigScienceLAMA, + ######################################################## + # TODO: Tasks That Require Manual Download: + ######################################################## + # JigSaw + # "jigsaw_unintended_bias": jigsaw_unintended_bias.JigsawUnintendedBias, + ######################################################## +} + + +def list_tasks() -> List[str]: + """Returns a list of all the available tasks by name.""" + return sorted(list(TASK_REGISTRY)) + + +def get_task(task_name: str, template_name: Optional[str] = None, **task_kwargs) -> Task: + """Returns a task from the registry and instantiates it with the `promptsource` + template specified by `template_name`. + + Args: + task_name: Name of the task to load from the task registry. + template_name: Name of the prompt template from `promptsource` to use + for this task. + **task_kwargs: Keyword arguments to pass to the task constructor. See constructor + args for `lm_eval.api.task.Task`. + + Returns: + A task instance with formatting specified by `template_name`. + """ + if ":" in task_name: + task_str, file_path = task_name.split(":") + task_name = task_str + task_kwargs.update({"file_path": file_path}) + task_class = _get_task_from_registry(task_name) + + if template_name is None: + return task_class(**task_kwargs) + + template = get_templates(task_name)[template_name] + return task_class(prompt_template=template, **task_kwargs) + + +def get_task_list( + task_name: str, template_names: Optional[List[str]] = None, **task_kwargs +) -> List[Task]: + """Returns a list of the same task but with multiple prompt templates. + + Args: + task_name: Name of the task to load from the task registry. + template_names: Name of the prompt template from `promptsource` to use + for this task. + **task_kwargs: Keyword arguments to pass to the task constructor. See constructor + args for `lm_eval.api.task.Task`. + + Returns: + A list of tasks with the same name but different prompt templates. + """ + # assert template_names, "Must specify at least one template name" + if template_names is None: + return [get_task(task_name, None, **task_kwargs)] + template_names = sorted(set(template_names)) + return [get_task(task_name, t, **task_kwargs) for t in template_names] + + +def list_templates(task_name: str) -> List[str]: + """Returns all template names available in `promptsource` for a given task.""" + templates = get_templates(task_name) + return sorted(templates.all_template_names) + + +def get_templates(task_name: str) -> DatasetTemplates: + """Returns the `promptsource` `DatasetTemplates` for the specified task name.""" + task_class = _get_task_from_registry(task_name) + return _get_templates_from_task(task_class) + + +def get_task_list_from_args_string( + task_name: str, + template_names: List[str], + task_args: str, + additional_config: Optional[Mapping[str, str]] = None, +) -> List[Task]: + """Returns a list of the same task but with multiple prompt templates, each + task instantiated with the given kwargs. + + Args: + task_name: Name of the task to use as found in the task registry. + template_names: Name of the prompt template from `promptsource` to use + for this task. + task_args: A string of comma-separated key=value pairs that will be passed + to the task constructor. E.g. "data_dir=./datasets,example_separator=\n\n" + additional_config: An additional dictionary of key=value pairs that will + be passed to the task constructor. + + Returns: + A list of `Task` instances. + """ + kwargs = lm_eval.api.utils.parse_cli_args_string(task_args) + assert "prompt_template" not in kwargs, ( + "Cannot specify a `prompt_template` object in the `task_args` string. " + "Only primitive type arguments are allowed." + ) + additional_config = {} if additional_config is None else additional_config + additional_args = {k: v for k, v in additional_config.items() if v is not None} + kwargs.update(additional_args) + return get_task_list(task_name, template_names, **kwargs) + + +# Helper functions + + +def _get_task_from_registry(task_name: str) -> Type[Task]: + try: + return TASK_REGISTRY[task_name] + except KeyError: + logger.warning(f"Available tasks:\n{list_tasks()}") + raise KeyError(f"`{task_name}` is missing from the task registry.") + + +def _get_templates_from_task(task: Union[Task, Type[Task]]) -> DatasetTemplates: + dataset_name = ( + task.DATASET_PATH + if task.DATASET_NAME is None + else f"{task.DATASET_PATH}/{task.DATASET_NAME}" + ) + return DatasetTemplates(dataset_name) + + +# TODO(jon-tow): Refactor everything below! These functions are only required +# b/c the task registry is non-uniformly hard-coded. + + +# TODO(jon-tow): Remove this function after refactoring the task registry to use +# `Task` object __str__ representations for task names as opposed to +# hardcoded string keys. +def get_registry_name_from_task(task: Task) -> str: + """Returns the task registry name from a Task instance.""" + for name, class_ in TASK_REGISTRY.items(): + if isinstance(task, class_): + return name + # This gives a mechanism for non-registered tasks to have a custom name anyways when reporting. + return type(task).__name__ + + +_TASK_TEMPLATE_KEY_SEP = "+" + + +def _get_task_template_key(task_name: str, template_name: str) -> str: + """Returns a `str` key for a task with that prompt template name appended. + This should be used to uniquely identify a task by its name AND + its specific prompt template - as a task can have many templates. + """ + if not template_name: + # Add `null` prompt template to the key if no template name is specified. + template_name = "null" + return f"{task_name}{_TASK_TEMPLATE_KEY_SEP}{template_name}" + + +def _split_task_template_key(task_template_key: str) -> Tuple[str, str]: + """Splits a task template key as returned from `_get_task_template_key` + into it's constituent parts: (task name, template name). + """ + task_name, template_name = task_template_key.split(_TASK_TEMPLATE_KEY_SEP, 1) + return task_name, template_name diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/__init__.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0ff794192371fb4558382629de521ce69dff15e5 Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/__init__.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/anli.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/anli.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4af1f0702c9ed1ca96fd73c47432b1dd5dfd33e7 Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/anli.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/aoa_pred.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/aoa_pred.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f6d56bca085f1f236413124736a04ebfd2d89f3f Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/aoa_pred.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/bias_shades.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/bias_shades.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a8926a4abef119a5d7bc455e0b51e6913e8d5bf2 Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/bias_shades.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/blimp.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/blimp.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1eaa3871c7f1e5ae650160b49f422ba13b3d1444 Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/blimp.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/cnn_dailymail.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/cnn_dailymail.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5b32eb7f411b55e888182500b47c16ccde400c9b Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/cnn_dailymail.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/comps.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/comps.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4226392520882aac8564a26a9162d0b05f602702 Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/comps.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/coqa.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/coqa.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b4e665dce7b6d5837f4bd0aee571070f85e4db0d Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/coqa.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/crd3.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/crd3.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0d133f1988d205ca1b380fca43a71f100a0ee519 Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/crd3.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/crows_pairs_multilingual.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/crows_pairs_multilingual.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e347e8c44bdcf416d139e5f921c6fdd2aacc51c0 Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/crows_pairs_multilingual.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/diabla.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/diabla.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..60714cc8579ddab24c6258f7da365906ef25ec4f Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/diabla.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/drop.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/drop.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..30132d01053974ca457cf173cbe7a8037d99a4a3 Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/drop.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/e2e_nlg_cleaned.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/e2e_nlg_cleaned.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c5d9e384192139a3b0734da4fecefa12cbfce1b4 Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/e2e_nlg_cleaned.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/flores_101.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/flores_101.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..250dfcf3f6bb11afcd82b51a307928f42b9e20bc Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/flores_101.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/gem_asset_turk.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/gem_asset_turk.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a331f03e5da4013acf41a99fd5a2338afc15774b Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/gem_asset_turk.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/gem_mlsum.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/gem_mlsum.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8ffe5ed5ec42e997211c55cf96448a7412efe0de Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/gem_mlsum.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/gem_webnlg.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/gem_webnlg.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a3b776829f0722b198850fc076d5355304b4287f Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/gem_webnlg.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/gem_wikilingua.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/gem_wikilingua.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..804be31856186297f0def1070c7d069ae20ddce7 Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/gem_wikilingua.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/gem_xsum.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/gem_xsum.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0cf54d4e05f90db6235206d16d8d3f458636d25c Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/gem_xsum.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/glue.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/glue.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a3cb69845c53aaaf4fda3977ed4c294f477b5671 Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/glue.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/hans.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/hans.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bb21d5b4151e727184ef854a037b306f8886edc0 Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/hans.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/huff_post.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/huff_post.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d9087653598257bb373610243974c2c1012a2f7f Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/huff_post.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/jigsaw_unintended_bias.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/jigsaw_unintended_bias.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..653e18fff58ab3bf7a894404cb47259592d1c441 Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/jigsaw_unintended_bias.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/lama.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/lama.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9aae749352ff088f7a8a13407c0d27423697dad4 Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/lama.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/lince.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/lince.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a655e9ef42f24b5c6916c4c2ba3a539e0e9ebc35 Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/lince.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/olmpics.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/olmpics.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3a9f914e603754148bae909a7c8038916f1ab6b5 Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/olmpics.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/piaf.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/piaf.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ea882e48dea1615b30693300426445e0743c7453 Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/piaf.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/race.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/race.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f38dc0453e668efb32de54c0f1f0891e6ed86cc8 Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/race.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/schema_guided_dstc8.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/schema_guided_dstc8.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..616f0d41b191f0ab362b27a826aea1ef3c650740 Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/schema_guided_dstc8.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/superglue.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/superglue.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e7d9523472e99bba60c364b927a1d965cd8c781c Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/superglue.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/syllogisms.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/syllogisms.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..de79c392d4f56a630146eb6c5ecd4f6ece3fc346 Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/syllogisms.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/tydiqa.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/tydiqa.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a0ce13b2781064b5d592b81433b74a065f0f33b1 Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/tydiqa.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/wino_bias.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/wino_bias.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..472996b0a51cc0baaa589236a6daba7838dbe01b Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/wino_bias.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/wmt.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/wmt.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..853978a56ca273a75d4cff25b2e9787ad3f246c4 Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/wmt.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/__pycache__/xquad.cpython-310.pyc b/evaluation-pipeline/lm_eval/tasks/__pycache__/xquad.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9c35b168ca26db1d0314e9750092689cc37eb577 Binary files /dev/null and b/evaluation-pipeline/lm_eval/tasks/__pycache__/xquad.cpython-310.pyc differ diff --git a/evaluation-pipeline/lm_eval/tasks/anli.py b/evaluation-pipeline/lm_eval/tasks/anli.py new file mode 100644 index 0000000000000000000000000000000000000000..918983448afe6314474835141bce4b886372fac0 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/anli.py @@ -0,0 +1,68 @@ +""" +Adversarial NLI: A New Benchmark for Natural Language Understanding +https://arxiv.org/pdf/1910.14599.pdf + +Adversarial NLI (ANLI) is a dataset collected via an iterative, adversarial +human-and-model-in-the-loop procedure. It consists of three rounds that progressively +increase in difficulty and complexity, and each question-answer includes annotator- +provided explanations. + +Homepage: "https://github.com/facebookresearch/anli" +""" +from lm_eval.api.task import PromptSourceTask + + +_CITATION = """ +@inproceedings{nie-etal-2020-adversarial, + title = "Adversarial {NLI}: A New Benchmark for Natural Language Understanding", + author = "Nie, Yixin and + Williams, Adina and + Dinan, Emily and + Bansal, Mohit and + Weston, Jason and + Kiela, Douwe", + booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics", + year = "2020", + publisher = "Association for Computational Linguistics", +} +""" + + +class ANLIBase(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "anli" + DATASET_NAME = None + SPLIT = None + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return True + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train_r" + str(self.SPLIT)] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["dev_r" + str(self.SPLIT)] + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["test_r" + str(self.SPLIT)] + + +class ANLIRound1(ANLIBase): + SPLIT = 1 + + +class ANLIRound2(ANLIBase): + SPLIT = 2 + + +class ANLIRound3(ANLIBase): + SPLIT = 3 diff --git a/evaluation-pipeline/lm_eval/tasks/aoa_pred.py b/evaluation-pipeline/lm_eval/tasks/aoa_pred.py new file mode 100644 index 0000000000000000000000000000000000000000..dfad1e6beb1fd22cbd923cd6727924ce3edbb397 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/aoa_pred.py @@ -0,0 +1,223 @@ +import torch +import torch.nn.functional as F +import numpy as np +import operator +import functools +import math +import csv +import json +import sys +import logging +from tqdm import tqdm +from torch.utils.data import DataLoader, Dataset +from sklearn.model_selection import LeaveOneOut +from sklearn.linear_model import LinearRegression +from sklearn.preprocessing import OneHotEncoder, StandardScaler, PolynomialFeatures +from typing import Tuple, Dict, AnyStr +from numbers import Number + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +logger.addHandler(logging.StreamHandler(sys.stdout)) + +def open_word_list_csv(): + with open("./aoa_data/word_list.csv", newline='') as csv_file: + word_list = [] + csv_contents = csv.DictReader(csv_file, delimiter=',', quotechar='|') + for row in csv_contents: + word_list.append(row) + return word_list + +## Custom Dataset class for CHILDES utterances +class CHILDESDataset(Dataset): + def __init__(self, file_path): + self.sentences = [] + self.words = [] + with open(file_path, "r") as f: + sent_words = json.load(f) + for item in sent_words: + self.sentences.append(item['sent']) + self.words.append(item['words']) + def __len__(self): + return len(self.sentences) + def __getitem__(self, index): + sent = self.sentences[index] + words = '-'.join(self.words[index]) + return {'sent':sent, 'words': words} + +# create mapping from words to tokens +def make_token_word_mappings(word_list_dict, tokenizer): + word_mappings = dict() + for word in word_list_dict: + w = word['word_clean'] + s_w = " "+w + seq = tokenizer(w, add_special_tokens=False)['input_ids'] + # if tokenizer has different tokens for words at the begining of a string vs followed by a space, get both. + s_seq = tokenizer(s_w, add_special_tokens=False)['input_ids'] + if seq == s_seq: + tokens= [torch.Tensor(seq).long()] + n_tokens= [len(seq)] + else: + tokens= [torch.Tensor(seq).long(), torch.Tensor(s_seq).long()] + n_tokens= [len(seq), len(s_seq)] + word_mappings[w] = (tokens, n_tokens) + return word_mappings + +# find index matches for sequences of indexes i.e. if a word is tokenized as multiple tokens +def indexes_in_sequence(query, base): + batch_id, label = base[0], base[1] + label = label.squeeze() + l = len(query) + locations = [] + for index in range((len(label)-l)): + if torch.all(label[index:index+l] == query): + locations.append([batch_id, index]) + return locations + +# get average surprisal values from model +r""" + Note for Causal LMs, the code currently assumes that labels **are shifted** inside the model, like GPT and OPT models, + i.e. we can set ``labels = input_ids``. If this is not the case, you can either provide the labels using your tokenizer + under the key ['labels'] which the code currently handles, or you will have to adjust it to make sure that labels shift + accordingly here. +""" +def get_batched_surprisal(model, tokenizer, model_type, dataloader, word_mapping, device): + model.eval() + word_surprisals_n = {} + for word in word_mapping.keys(): + word_surprisals_n[word] = [0, 0] + batch_size = dataloader.batch_size + for n, item in tqdm(enumerate(dataloader), total=len(dataloader)): + sentences = item['sent'] + words = item['words'] + batch = tokenizer(sentences, return_tensors='pt', padding=True, truncation=True, max_length=50) + for key in batch.keys(): + batch[key] = batch[key].to(device) + if model_type == "hf-causal": + if 'labels' in batch.keys(): + labels = batch['labels'] + else: + labels = batch['input_ids'] + else: + labels = batch['input_ids'] + outputs = model(**batch, labels=labels) + surprisals = -F.log_softmax(outputs.logits, -1) + labels_split = torch.tensor_split(labels, batch_size) + word_list = list(set(sum([w.split('-') for w in words], []))) + for word in word_list: + tokens, n_tokens = word_mapping[word] + for indexes, n_tokens in zip(tokens, n_tokens): + indexes = indexes.to(device) + # If word is represented by a single token we can optimize run time by searching over whole batch matrix + if n_tokens == 1: + index_matches = (labels == indexes).nonzero(as_tuple=False) + if len(index_matches) > 0: + for i in index_matches: + match = surprisals[tuple(i)] + surprisal = match[indexes].item() + word_surprisals_n[word][0] += surprisal + word_surprisals_n[word][1] += 1 + # If word is represented by sequence of tokens, we have to match the whole sequence + else: + match_list = list(map(lambda x: indexes_in_sequence(indexes, x), enumerate(labels_split))) + index_matches = functools.reduce(operator.iconcat, match_list) + if len(index_matches) > 0: + for i in index_matches: + surprisal = 0.0 + batch_id = int(i[0]) + # we sum the log probs of each sub token in sequence to get sequence surprisal + for j, index in enumerate(indexes): + current_id = int(i[1] + j) + match = surprisals[(batch_id, current_id)] + sub_surprisal = match[index].item() + surprisal += sub_surprisal + word_surprisals_n[word][0] += surprisal + word_surprisals_n[word][1] += 1 + # Average the summed suprisal values for each non zero count word + for word in word_mapping.keys(): + if word_surprisals_n[word][1] > 0: + word_surprisals_n[word][0] = word_surprisals_n[word][0]/ word_surprisals_n[word][1] + return word_surprisals_n + +# fit linear regressions using LOO cross validation to get MAD scores for predicting AoA +def get_loo_mad_results(word_surprisals_n, word_list_dict): + data = [] + # collect all predictors in one table + for w in word_list_dict: + word = w['word_clean'] + lex_cat = w['lexical_category'] + concreteness = w['concreteness'] + frequency = w['frequency'] + aoa = w['aoa'] + avg_surprisal, count = word_surprisals_n[word] + if count != 0: + data.append([word, float(aoa), lex_cat, float(concreteness), avg_surprisal, int(frequency)]) + data = np.array(data) + # turn frequency counts into unigram surprisals + total = np.sum(data[:,5].astype(int)) + unigram = -np.log(data[:,[5]].astype(int) / total) + # get residualized surprisal values + surp_model = LinearRegression().fit(unigram, data[:,4]) + prediction = surp_model.predict(unigram) + residuals = (data[:,4].astype(float) - prediction) + resid_surp = np.expand_dims(residuals, axis=1) + data = np.concatenate((data[:,0:4], resid_surp, unigram), axis=1) + # dummy code lex_category with noun as base and predicate, function_words as separate dummy codes + enc = OneHotEncoder(drop=['nouns'], categories=[['nouns', 'function_words', 'predicates']]) + dummy_coded_categorical_vars = enc.fit_transform(data[:,[2]]).toarray() + # standardize variables so mean=0 and std=1 + scaler = StandardScaler() + scaled_continuous_vars = scaler.fit_transform(data[:,3:]) + scaled_data = np.concatenate((dummy_coded_categorical_vars, scaled_continuous_vars), axis=1) + # get interaction term values between lexical category dummy codes and concreteness/surprisal/frequency add to dataset + inter = PolynomialFeatures(interaction_only=True, include_bias=False) + interactions = inter.fit_transform(scaled_data) + # get LOO data splits + X = np.concatenate((interactions[:,:5], interactions[:,6:12]), axis=1) + y = data[:,1] + loo = LeaveOneOut() + n = loo.get_n_splits(X) + overall_mad = 0 + noun_n = 0 + noun_mad = 0 + pred_n = 0 + pred_mad = 0 + fctword_n = 0 + fctword_mad = 0 + for i, (train_index, test_index) in enumerate(loo.split(X)): + # for each split fit linear regression model and get absolute deviation + model = LinearRegression().fit(X[train_index], y[train_index]) + y_pred = model.predict(X[test_index]) + abs_deviation = abs(float(y[test_index]) - float(y_pred)) + overall_mad += abs_deviation + if bool(X[test_index, 1] == 1): + pred_mad += abs_deviation + pred_n += 1 + elif bool(X[test_index, 0] == 1): + fctword_mad += abs_deviation + fctword_n += 1 + else: + noun_mad += abs_deviation + noun_n += 1 + overall_mad = overall_mad/n + noun_mad = noun_mad/noun_n + pred_mad = pred_mad/pred_n + fctword_mad = fctword_mad/fctword_n + return {'overall_mad':overall_mad, 'n':n, 'noun_mad':noun_mad, 'n_noun':noun_n, 'predicate_mad':pred_mad, 'n_predicate':pred_n, 'functionword_mad':fctword_mad, 'n_functionword':fctword_n} + + +def aoa_pred_eval(model: object, tokenizer: object, model_type: str, batch_size: int = 32) -> Tuple[Dict[AnyStr, Tuple[Number, Number]], Dict[AnyStr, Number]]: + logger.info(f"\n» Evaluating model on predicting the age of acquisition of words.") + device = model.device + if torch.cuda.is_available(): + torch.cuda.empty_cache() + data = CHILDESDataset('./aoa_data/sent_words.json') + dataloader = DataLoader(data, batch_size=batch_size) + word_list_dict = open_word_list_csv() + word_mapping = make_token_word_mappings(word_list_dict, tokenizer) + logger.info(f"\n» Collecting model average surprisal values.") + word_surprisals_n = get_batched_surprisal(model, tokenizer, model_type, dataloader, word_mapping, device) + logger.info(f"\n» Fitting regression models using leave-one-out cross-validation.") + mad_results = get_loo_mad_results(word_surprisals_n, word_list_dict) + return word_surprisals_n, mad_results diff --git a/evaluation-pipeline/lm_eval/tasks/bias_shades.py b/evaluation-pipeline/lm_eval/tasks/bias_shades.py new file mode 100644 index 0000000000000000000000000000000000000000..75e3a33ef72133ef4f19526df17f6d7f840bfed5 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/bias_shades.py @@ -0,0 +1,71 @@ +""" +Multilingual dataset for measuring social biases in language models. +https://huggingface.co/datasets/BigScienceBiasEval/bias-shades/viewer/spanish/test + +TODO: Add `arabic`, `german`, `russian`, and `tamil` subsets when `promptsource` +templates become available. +""" +from lm_eval.api.task import PromptSourceTask + + +_CITATION = """""" + + +class BiasShadesBase(PromptSourceTask): + def has_training_docs(self): + return False + + def has_validation_docs(self): + return False + + def has_test_docs(self): + return True + + def training_docs(self): + pass + + def validation_docs(self): + pass + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["test"] + + +class BiasShadesEnglish(BiasShadesBase): + VERSION = 0 + DATASET_PATH = "BigScienceBiasEval/bias-shades" + DATASET_NAME = "english" + + +class BiasShadesFrench(BiasShadesBase): + VERSION = 0 + DATASET_PATH = "BigScienceBiasEval/bias-shades" + DATASET_NAME = "french" + + +class BiasShadesHindi(BiasShadesBase): + VERSION = 0 + DATASET_PATH = "BigScienceBiasEval/bias-shades" + DATASET_NAME = "hindi" + + +class BiasShadesSpanish(BiasShadesBase): + VERSION = 0 + DATASET_PATH = "BigScienceBiasEval/bias-shades" + DATASET_NAME = "spanish" + + +BIAS_SHADES_CLASSES = [ + BiasShadesEnglish, + BiasShadesFrench, + BiasShadesHindi, + BiasShadesSpanish, +] + + +def construct_tasks(): + tasks = {} + for bias_shades_class in BIAS_SHADES_CLASSES: + tasks[f"bias_shades_{bias_shades_class.DATASET_NAME}"] = bias_shades_class + return tasks diff --git a/evaluation-pipeline/lm_eval/tasks/blimp.py b/evaluation-pipeline/lm_eval/tasks/blimp.py new file mode 100644 index 0000000000000000000000000000000000000000..e9e25b04441f6e27e03b02367a2ca8b32cd96e6e --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/blimp.py @@ -0,0 +1,346 @@ +""" +BLiMP: A Benchmark of Linguistic Minimal Pairs for English +https://arxiv.org/abs/1912.00582 + +BLiMP is a challenge set for evaluating what language models (LMs) know about +major grammatical phenomena in English. BLiMP consists of 67 sub-datasets, each +containing 1000 minimal pairs isolating specific contrasts in syntax, morphology, +or semantics. The data is automatically generated according to expert-crafted +grammars. + +Homepage: https://github.com/alexwarstadt/blimp +""" +from lm_eval.api.task import PromptSourceTask +from typing import Optional, List +from datasets import load_dataset + + +_CITATION = """ +@article{warstadt2019blimp, + author = {Warstadt, Alex and Parrish, Alicia and Liu, Haokun and Mohananey, Anhad and Peng, Wei and Wang, Sheng-Fu and Bowman, Samuel R.}, + title = {BLiMP: The Benchmark of Linguistic Minimal Pairs for English}, + journal = {Transactions of the Association for Computational Linguistics}, + volume = {8}, + number = {}, + pages = {377-392}, + year = {2020}, + doi = {10.1162/tacla00321}, + URL = {https://doi.org/10.1162/tacl_a_00321}, + eprint = {https://doi.org/10.1162/tacl_a_00321}, + abstract = { We introduce The Benchmark of Linguistic Minimal Pairs (BLiMP),1 a challenge set for evaluating the linguistic knowledge of language models (LMs) on major grammatical phenomena in English. BLiMP consists of 67 individual datasets, each containing 1,000 minimal pairs—that is, pairs of minimally different sentences that contrast in grammatical acceptability and isolate specific phenomenon in syntax, morphology, or semantics. We generate the data according to linguist-crafted grammar templates, and human aggregate agreement with the labels is 96.4. We evaluate n-gram, LSTM, and Transformer (GPT-2 and Transformer-XL) LMs by observing whether they assign a higher probability to the acceptable sentence in each minimal pair. We find that state-of-the-art models identify morphological contrasts related to agreement reliably, but they struggle with some subtle semantic and syntactic phenomena, such as negative polarity items and extraction islands. } +} +""" + + +class BlimpTask(PromptSourceTask): + DATASET_PATH = "blimp" + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def validation_docs(self): + # The HF dataset only contains a "train" dataset, but the harness expects a "validation" + # dataset. Let's use the training dataset, on the assumption that the model wasn't actually + # trained on this data. + return self.dataset["train"] + + def null_prompt_doc_to_text(self, doc: dict) -> str: + return "" + + def null_prompt_doc_to_target(self, doc: dict) -> List[str]: + correct_choice = doc["sentence_good"] + return [correct_choice] + + def null_prompt_answer_choices(self, doc: dict) -> List[str]: + choices = [doc["sentence_good"], doc["sentence_bad"]] + return choices + + def null_prompt_get_logging_info(self): + return { + "fixed_answer_choice_list": None, + "dataset_path": self.DATASET_PATH, + "dataset_name": self.DATASET_NAME, + "subset": self.SPLIT, + "prompt_name": None, + "prompt_id": None, + "prompt_jinja": None, + "prompt_original_task": f"{self.DATASET_PATH}/{self.DATASET_NAME}", + # Placeholder for comment in post-processing. + "comment": "", + } + +class BlimpAdjunctIsland(BlimpTask): + DATASET_NAME = "adjunct_island" + + +class BlimpAnaphorGenderAgreement(BlimpTask): + DATASET_NAME = "anaphor_gender_agreement" + + +class BlimpAnaphorNumberAgreement(BlimpTask): + DATASET_NAME = "anaphor_number_agreement" + + +class BlimpAnimateSubjectPassive(BlimpTask): + DATASET_NAME = "animate_subject_passive" + + +class BlimpAnimateSubjectTrans(BlimpTask): + DATASET_NAME = "animate_subject_trans" + + +class BlimpCausative(BlimpTask): + DATASET_NAME = "causative" + + +class BlimpComplex_NPIsland(BlimpTask): + DATASET_NAME = "complex_NP_island" + + +class BlimpCoordinateStructureConstraintComplexLeftBranch(BlimpTask): + DATASET_NAME = "coordinate_structure_constraint_complex_left_branch" + + +class BlimpCoordinateStructureConstraintObjectExtraction(BlimpTask): + DATASET_NAME = "coordinate_structure_constraint_object_extraction" + + +class BlimpDeterminerNounAgreement_1(BlimpTask): + DATASET_NAME = "determiner_noun_agreement_1" + + +class BlimpDeterminerNounAgreement_2(BlimpTask): + DATASET_NAME = "determiner_noun_agreement_2" + + +class BlimpDeterminerNounAgreementIrregular_1(BlimpTask): + DATASET_NAME = "determiner_noun_agreement_irregular_1" + + +class BlimpDeterminerNounAgreementIrregular_2(BlimpTask): + DATASET_NAME = "determiner_noun_agreement_irregular_2" + + +class BlimpDeterminerNounAgreementWithAdj_2(BlimpTask): + DATASET_NAME = "determiner_noun_agreement_with_adj_2" + + +class BlimpDeterminerNounAgreementWithAdjIrregular_1(BlimpTask): + DATASET_NAME = "determiner_noun_agreement_with_adj_irregular_1" + + +class BlimpDeterminerNounAgreementWithAdjIrregular_2(BlimpTask): + DATASET_NAME = "determiner_noun_agreement_with_adj_irregular_2" + + +class BlimpDeterminerNounAgreementWithAdjective_1(BlimpTask): + DATASET_NAME = "determiner_noun_agreement_with_adjective_1" + + +class BlimpDistractorAgreementRelationalNoun(BlimpTask): + DATASET_NAME = "distractor_agreement_relational_noun" + + +class BlimpDistractorAgreementRelativeClause(BlimpTask): + DATASET_NAME = "distractor_agreement_relative_clause" + + +class BlimpDropArgument(BlimpTask): + DATASET_NAME = "drop_argument" + + +class BlimpEllipsisNBar_1(BlimpTask): + DATASET_NAME = "ellipsis_n_bar_1" + + +class BlimpEllipsisNBar_2(BlimpTask): + DATASET_NAME = "ellipsis_n_bar_2" + + +class BlimpExistentialThereObjectRaising(BlimpTask): + DATASET_NAME = "existential_there_object_raising" + + +class BlimpExistentialThereQuantifiers_1(BlimpTask): + DATASET_NAME = "existential_there_quantifiers_1" + + +class BlimpExistentialThereQuantifiers_2(BlimpTask): + DATASET_NAME = "existential_there_quantifiers_2" + + +class BlimpExistentialThereSubjectRaising(BlimpTask): + DATASET_NAME = "existential_there_subject_raising" + + +class BlimpExpletiveItObjectRaising(BlimpTask): + DATASET_NAME = "expletive_it_object_raising" + + +class BlimpInchoative(BlimpTask): + DATASET_NAME = "inchoative" + + +class BlimpIntransitive(BlimpTask): + DATASET_NAME = "intransitive" + + +class BlimpIrregularPastParticipleAdjectives(BlimpTask): + DATASET_NAME = "irregular_past_participle_adjectives" + + +class BlimpIrregularPastParticipleVerbs(BlimpTask): + DATASET_NAME = "irregular_past_participle_verbs" + + +class BlimpIrregularPluralSubjectVerbAgreement_1(BlimpTask): + DATASET_NAME = "irregular_plural_subject_verb_agreement_1" + + +class BlimpIrregularPluralSubjectVerbAgreement_2(BlimpTask): + DATASET_NAME = "irregular_plural_subject_verb_agreement_2" + + +class BlimpLeftBranchIslandEchoQuestion(BlimpTask): + DATASET_NAME = "left_branch_island_echo_question" + + +class BlimpLeftBranchIslandSimpleQuestion(BlimpTask): + DATASET_NAME = "left_branch_island_simple_question" + + +class BlimpMatrixQuestionNpiLicensorPresent(BlimpTask): + DATASET_NAME = "matrix_question_npi_licensor_present" + + +class BlimpNpiPresent_1(BlimpTask): + DATASET_NAME = "npi_present_1" + + +class BlimpNpiPresent_2(BlimpTask): + DATASET_NAME = "npi_present_2" + + +class BlimpOnlyNpiLicensorPresent(BlimpTask): + DATASET_NAME = "only_npi_licensor_present" + + +class BlimpOnlyNpiScope(BlimpTask): + DATASET_NAME = "only_npi_scope" + + +class BlimpPassive_1(BlimpTask): + DATASET_NAME = "passive_1" + + +class BlimpPassive_2(BlimpTask): + DATASET_NAME = "passive_2" + + +class BlimpPrinciple_ACCommand(BlimpTask): + DATASET_NAME = "principle_A_c_command" + + +class BlimpPrinciple_ACase_1(BlimpTask): + DATASET_NAME = "principle_A_case_1" + + +class BlimpPrinciple_ACase_2(BlimpTask): + DATASET_NAME = "principle_A_case_2" + + +class BlimpPrinciple_ADomain_1(BlimpTask): + DATASET_NAME = "principle_A_domain_1" + + +class BlimpPrinciple_ADomain_2(BlimpTask): + DATASET_NAME = "principle_A_domain_2" + + +class BlimpPrinciple_ADomain_3(BlimpTask): + DATASET_NAME = "principle_A_domain_3" + + +class BlimpPrinciple_AReconstruction(BlimpTask): + DATASET_NAME = "principle_A_reconstruction" + + +class BlimpRegularPluralSubjectVerbAgreement_1(BlimpTask): + DATASET_NAME = "regular_plural_subject_verb_agreement_1" + + +class BlimpRegularPluralSubjectVerbAgreement_2(BlimpTask): + DATASET_NAME = "regular_plural_subject_verb_agreement_2" + + +class BlimpSententialNegationNpiLicensorPresent(BlimpTask): + DATASET_NAME = "sentential_negation_npi_licensor_present" + + +class BlimpSententialNegationNpiScope(BlimpTask): + DATASET_NAME = "sentential_negation_npi_scope" + + +class BlimpSententialSubjectIsland(BlimpTask): + DATASET_NAME = "sentential_subject_island" + + +class BlimpSuperlativeQuantifiers_1(BlimpTask): + DATASET_NAME = "superlative_quantifiers_1" + + +class BlimpSuperlativeQuantifiers_2(BlimpTask): + DATASET_NAME = "superlative_quantifiers_2" + + +class BlimpToughVsRaising_1(BlimpTask): + DATASET_NAME = "tough_vs_raising_1" + + +class BlimpToughVsRaising_2(BlimpTask): + DATASET_NAME = "tough_vs_raising_2" + + +class BlimpTransitive(BlimpTask): + DATASET_NAME = "transitive" + + +class BlimpWhIsland(BlimpTask): + DATASET_NAME = "wh_island" + + +class BlimpWhQuestionsObjectGap(BlimpTask): + DATASET_NAME = "wh_questions_object_gap" + + +class BlimpWhQuestionsSubjectGap(BlimpTask): + DATASET_NAME = "wh_questions_subject_gap" + + +class BlimpWhQuestionsSubjectGapLongDistance(BlimpTask): + DATASET_NAME = "wh_questions_subject_gap_long_distance" + + +class BlimpWhVsThatNoGap(BlimpTask): + DATASET_NAME = "wh_vs_that_no_gap" + + +class BlimpWhVsThatNoGapLongDistance(BlimpTask): + DATASET_NAME = "wh_vs_that_no_gap_long_distance" + + +class BlimpWhVsThatWithGap(BlimpTask): + DATASET_NAME = "wh_vs_that_with_gap" + + +class BlimpWhVsThatWithGapLongDistance(BlimpTask): + DATASET_NAME = "wh_vs_that_with_gap_long_distance" + +class BlimpFromFile(BlimpTask): + DATASET_NAME = "causative" diff --git a/evaluation-pipeline/lm_eval/tasks/cnn_dailymail.py b/evaluation-pipeline/lm_eval/tasks/cnn_dailymail.py new file mode 100644 index 0000000000000000000000000000000000000000..643353231bd05645a8656a499e84a681d62461ff --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/cnn_dailymail.py @@ -0,0 +1,62 @@ +""" +CNN/Daily Mail is a dataset for text summarization. Human generated abstractive +summary bullets were generated from news stories in CNN and Daily Mail websites +as questions (with one of the entities hidden), and stories as the corresponding +passages from which the system is expected to answer the fill-in the-blank +question. The authors released the scripts that crawl, extract and generate pairs +of passages and questions from these websites. + +In all, the corpus has 286,817 training pairs, 13,368 validation pairs and 11,487 +test pairs, as defined by their scripts. The source documents in the training set +have 766 words spanning 29.74 sentences on an average while the summaries consist +of 53 words and 3.72 sentences. """ +from lm_eval.api.task import PromptSourceTask + + +_CITATION = """@article{DBLP:journals/corr/NallapatiXZ16, + author = {Ramesh Nallapati and + Bing Xiang and + Bowen Zhou}, + title = {Sequence-to-Sequence RNNs for Text Summarization}, + journal = {CoRR}, + volume = {abs/1602.06023}, + year = {2016}, + url = {http://arxiv.org/abs/1602.06023}, + eprinttype = {arXiv}, + eprint = {1602.06023}, + timestamp = {Mon, 13 Aug 2018 16:46:52 +0200}, + biburl = {https://dblp.org/rec/journals/corr/NallapatiXZ16.bib}, + bibsource = {dblp computer science bibliography, https://dblp.org} +}""" + + +class CnnDailyMail(PromptSourceTask): + + DATASET_PATH = "cnn_dailymail" + DATASET_NAME = "3.0.0" + + def doc_to_rawtext(self, doc): + return doc["article"] + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return True + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["validation"] + + def test_docs(self): + return self.dataset["test"] + + def max_generation_length(self): + return 200 diff --git a/evaluation-pipeline/lm_eval/tasks/comps.py b/evaluation-pipeline/lm_eval/tasks/comps.py new file mode 100644 index 0000000000000000000000000000000000000000..690a078ab46726c1d0e51b49b014b2f8dcb51796 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/comps.py @@ -0,0 +1,87 @@ +""" +BLiMP: A Benchmark of Linguistic Minimal Pairs for English +https://arxiv.org/abs/1912.00582 + +BLiMP is a challenge set for evaluating what language models (LMs) know about +major grammatical phenomena in English. BLiMP consists of 67 sub-datasets, each +containing 1000 minimal pairs isolating specific contrasts in syntax, morphology, +or semantics. The data is automatically generated according to expert-crafted +grammars. + +Homepage: https://github.com/alexwarstadt/blimp +""" +from lm_eval.api.task import PromptSourceTask +from typing import Optional, List +from datasets import load_dataset + + +_CITATION = """ +@article{warstadt2019blimp, + author = {Warstadt, Alex and Parrish, Alicia and Liu, Haokun and Mohananey, Anhad and Peng, Wei and Wang, Sheng-Fu and Bowman, Samuel R.}, + title = {BLiMP: The Benchmark of Linguistic Minimal Pairs for English}, + journal = {Transactions of the Association for Computational Linguistics}, + volume = {8}, + number = {}, + pages = {377-392}, + year = {2020}, + doi = {10.1162/tacla00321}, + URL = {https://doi.org/10.1162/tacl_a_00321}, + eprint = {https://doi.org/10.1162/tacl_a_00321}, + abstract = { We introduce The Benchmark of Linguistic Minimal Pairs (BLiMP),1 a challenge set for evaluating the linguistic knowledge of language models (LMs) on major grammatical phenomena in English. BLiMP consists of 67 individual datasets, each containing 1,000 minimal pairs—that is, pairs of minimally different sentences that contrast in grammatical acceptability and isolate specific phenomenon in syntax, morphology, or semantics. We generate the data according to linguist-crafted grammar templates, and human aggregate agreement with the labels is 96.4. We evaluate n-gram, LSTM, and Transformer (GPT-2 and Transformer-XL) LMs by observing whether they assign a higher probability to the acceptable sentence in each minimal pair. We find that state-of-the-art models identify morphological contrasts related to agreement reliably, but they struggle with some subtle semantic and syntactic phenomena, such as negative polarity items and extraction islands. } +} +""" + +class CompsTask(PromptSourceTask): + DATASET_PATH = "kanishka/comps" + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def validation_docs(self): + # The HF dataset only contains a "train" dataset, but the harness expects a "validation" + # dataset. Let's use the training dataset, on the assumption that the model wasn't actually + # trained on this data. + return self.dataset["train"] + + def null_prompt_doc_to_text(self, doc: dict) -> str: + return "" + + def null_prompt_doc_to_target(self, doc: dict) -> List[str]: + correct_choice = doc["prefix_acceptable"] + sentence_end = doc["property_phrase"] + formatted = f"{correct_choice} {sentence_end}" + return [formatted] + + def null_prompt_answer_choices(self, doc: dict) -> List[str]: + choices = [doc["prefix_acceptable"], doc["prefix_unacceptable"]] + sentence_end = doc["property_phrase"] + formatted = [f"{choice} {sentence_end}" for choice in choices] + return formatted + + def null_prompt_get_logging_info(self): + return { + "fixed_answer_choice_list": None, + "dataset_path": self.DATASET_PATH, + "dataset_name": self.DATASET_NAME, + "subset": self.SPLIT, + "prompt_name": None, + "prompt_id": None, + "prompt_jinja": None, + "prompt_original_task": f"{self.DATASET_PATH}/{self.DATASET_NAME}", + # Placeholder for comment in post-processing. + "comment": "", + } + + +class CompsBase(CompsTask): + DATASET_NAME = "base" + + +class CompsWugs(CompsTask): + DATASET_NAME = "wugs" \ No newline at end of file diff --git a/evaluation-pipeline/lm_eval/tasks/coqa.py b/evaluation-pipeline/lm_eval/tasks/coqa.py new file mode 100644 index 0000000000000000000000000000000000000000..247ba00f2a7b21febf86cd42f6117648542477b2 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/coqa.py @@ -0,0 +1,98 @@ +""" +CoQA: A Conversational Question Answering Challenge +https://arxiv.org/pdf/1808.07042.pdf + +CoQA is a large-scale dataset for building Conversational Question Answering +systems. The goal of the CoQA challenge is to measure the ability of machines to +understand a text passage and answer a series of interconnected questions that +appear in a conversation. + +Homepage: https://stanfordnlp.github.io/coqa/ +""" +import transformers.data.metrics.squad_metrics as squad_metrics + +from lm_eval.api.metric import mean +from lm_eval.api.task import PromptSourceTask + + +_CITATION = """ +@misc{reddy2018coqa, + title={CoQA: A Conversational Question Answering Challenge}, + author={Siva Reddy and Danqi Chen and Christopher D. Manning}, + year={2018}, + eprint={1808.07042}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} +""" + + +class CoQA(PromptSourceTask): + VERSION = 1 + DATASET_PATH = "coqa" + DATASET_NAME = None + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + return self.dataset["validation"] + + @staticmethod + def compute_scores(gold_list, pred): + # tests for exact match and on the normalised answer (compute_exact) + # test for overlap (compute_f1) + f1_sum = 0.0 + em_sum = 0.0 + if len(gold_list) > 1: + for i in range(len(gold_list)): + gold_answers = gold_list[0:i] + gold_list[i + 1 :] + # predictions compared against (n) golds and take maximum + em_sum += max( + squad_metrics.compute_exact(a, pred) for a in gold_answers + ) + f1_sum += max(squad_metrics.compute_f1(a, pred) for a in gold_answers) + else: + em_sum += max(squad_metrics.compute_exact(a, pred) for a in gold_list) + f1_sum += max(squad_metrics.compute_f1(a, pred) for a in gold_list) + + return { + "em": em_sum / max(1, len(gold_list)), + "f1": f1_sum / max(1, len(gold_list)), + } + + def process_results(self, doc, results): + targets = self.doc_to_target(doc) + pred = results[0].strip().split("\n")[0] + scores = self.compute_scores(targets, pred) + + out = { + "f1": scores["f1"], + "em": scores["em"], + } + + if self.save_examples: + example = {"target": targets, "pred": pred} + return out, example + return out + + def higher_is_better(self): + return { + "f1": True, + "em": True, + } + + def aggregation(self): + return { + "f1": mean, + "em": mean, + } diff --git a/evaluation-pipeline/lm_eval/tasks/crd3.py b/evaluation-pipeline/lm_eval/tasks/crd3.py new file mode 100644 index 0000000000000000000000000000000000000000..4b170445fbf3e260f6e6d096f6c4336feb22f171 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/crd3.py @@ -0,0 +1,51 @@ +""" +Storytelling with Dialogue: A Critical Role Dungeons and Dragons Dataset +https://aclanthology.org/2020.acl-main.459.pdf + +Storytelling with Dialogue: A Critical Role Dungeons and Dragons Dataset. Critical Role is an unscripted, live-streamed show where a fixed group of people play Dungeons and Dragons, an open-ended role-playing game. The dataset is collected from 159 Critical Role episodes transcribed to text dialogues, consisting of 398,682 turns. It also includes corresponding abstractive summaries collected from the Fandom wiki. The dataset is linguistically unique in that the narratives are generated entirely through player collaboration and spoken interaction. For each dialogue, there are a large number of turns, multiple abstractive summaries with varying levels of detail, and semantic ties to the previous dialogues. + +Homepage: https://github.com/RevanthRameshkumar/CRD3 +""" +from lm_eval.api.task import PromptSourceTask + + +# TODO: Add the BibTeX citation for the task. +_CITATION = """ +@inproceedings{ +title = {Storytelling with Dialogue: A Critical Role Dungeons and Dragons Dataset}, +author = {Rameshkumar, Revanth and Bailey, Peter}, +year = {2020}, +publisher = {Association for Computational Linguistics}, +conference = {ACL} +} +""" + + +class CRD3(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "shanya/crd3" + DATASET_NAME = None + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return True + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["validation"] + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["test"] + + def max_generation_length(self): + return None diff --git a/evaluation-pipeline/lm_eval/tasks/crows_pairs_multilingual.py b/evaluation-pipeline/lm_eval/tasks/crows_pairs_multilingual.py new file mode 100644 index 0000000000000000000000000000000000000000..de3e0b76bb09288f26e6043e64aebf2832392a7a --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/crows_pairs_multilingual.py @@ -0,0 +1,66 @@ +""" +French CrowS-Pairs: Extending a challenge dataset for measuring social bias in masked language models to a language other than English +https://hal.inria.fr/hal-03629677/file/ACLFinal.pdf +Measuring social biases in masked language models in English and French. +https://gitlab.inria.fr/french-crows-pairs/acl-2022-paper-data-and-code/-/tree/main +""" +from lm_eval.api.task import PromptSourceTask + + +_CITATION = """\ +@inproceedings{neveol2022french, + title={French CrowS-Pairs: Extending a challenge dataset for measuring social bias in masked language models to a language other than English}, + author={N{\'e}v{\'e}ol, Aur{\'e}lie and Dupont, Yoann and Bezancon, Julien and Fort, Kar{\"e}n}, + booktitle={ACL 2022-60th Annual Meeting of the Association for Computational Linguistics}, + year={2022} +""" + + +class CrowsPairsEnglish(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "BigScienceBiasEval/crows_pairs_multilingual" + DATASET_NAME = "english" + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return False + + def has_test_docs(self): + return True + + def training_docs(self): + pass + + def validation_docs(self): + pass + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["test"] + + +class CrowsPairsFrench(PromptSourceTask): + + DATASET_PATH = "BigScienceBiasEval/crows_pairs_multilingual" + DATASET_NAME = "french" + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return False + + def has_test_docs(self): + return True + + def training_docs(self): + pass + + def validation_docs(self): + pass + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["test"] diff --git a/evaluation-pipeline/lm_eval/tasks/diabla.py b/evaluation-pipeline/lm_eval/tasks/diabla.py new file mode 100644 index 0000000000000000000000000000000000000000..bf63a6feb4a0337457dcb76652b4cf00aaa35934 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/diabla.py @@ -0,0 +1,285 @@ +""" +DiaBLa: English-French Bilingual dialogue dataset for Machine Translation +https://link.springer.com/article/10.1007/s10579-020-09514-4 + +Rachel Bawden, Eric Bilinski, Thomas Lavergne and Sophie Rosset +(2021). DiaBLa: A Corpus of Bilingual Spontaneous Written Dialogues +for Machine Translation. Language Resources and Evaluation(55). Pages +635–660. Springer Verlag. 10.1007/s10579-020-09514-4. + +DiaBLa is an English-French dataset for the evaluation of Machine +Translation (MT) for informal, written bilingual dialogue. It +contains 144 spontaneous dialogues (5,700+ sentences) between native +English and French speakers, mediated by one of two neural MT systems +in a range of role-play settings. The dialogues are accompanied by +fine-grained sentence-level judgments of MT quality, produced by the +dialogue participants themselves, as well as by manually normalised +versions and reference translations produced a posteriori + +Homepage: http://almanach.inria.fr/software_and_resources/custom/DiaBLa-en.html +""" +from lm_eval.api.task import PromptSourceTask +from typing import List, Tuple, Optional +import datasets +import copy +import numpy as np + + +_CITATION = """@article{bawden_DiaBLa:-A-Corpus-of_2021, + author = {Bawden, Rachel and Bilinski, Eric and Lavergne, Thomas and Rosset, Sophie}, + doi = {10.1007/s10579-020-09514-4}, + title = {DiaBLa: A Corpus of Bilingual Spontaneous Written Dialogues for Machine Translation}, + year = {2021}, + journal = {Language Resources and Evaluation}, + publisher = {Springer Verlag}, + volume = {55}, + pages = {635--660}, + url = {https://hal.inria.fr/hal-03021633}, + pdf = {https://hal.inria.fr/hal-03021633/file/diabla-lre-personal-formatting.pdf}, +} +""" + + +class DiaBLa(PromptSourceTask): + + DATASET_PATH = "rbawden/DiaBLa" + DATASET_NAME = None + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return False + + def has_test_docs(self): + return True + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["validation"] + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["test"] + + def max_generation_length(self): + return 512 + + def invalid_doc_for_prompt(self, doc) -> bool: + if len(self.doc_to_target(doc)) == 0 or self.doc_to_target(doc)[0] == "": + return True + return False + + +class DiaBLa_1_shot_context_same(PromptSourceTask): + """ + This task is identical to the DiaBLa task, but in the 1-shot setting takes the + 1-shot example from the previous sentence in the dialogue if this is available + (source sentence and MT output, in the same language direction as the direction + of the current example). N.B. this task is not currently designed for more than + 1-shot. + """ + + DATASET_PATH = "rbawden/DiaBLa" + DATASET_NAME = None + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return False + + def has_test_docs(self): + return True + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["validation"] + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["test"] + + def max_generation_length(self): + return 512 + + def invalid_doc_for_prompt(self, doc) -> bool: + if len(self.doc_to_target(doc)) == 0 or self.doc_to_target(doc)[0] == "": + return True + return False + + def doc_to_shot_text(self, doc: dict) -> str: + text, _ = self.shot_prompt_template.apply(doc) + return text + + def doc_to_shot_target(self, doc: dict) -> List[str]: + _, target = self.shot_prompt_template.apply(doc) + return target + + def fewshot_docs(self) -> datasets.Dataset: + """ + Returns the `dataset` split that the few-shot examples should be sample + from. This prioritizes the `train_docs` split as the few-shot example + source, then `validation_docs`, and lastly `test_docs`. + """ + return self.test_docs() + + # heuristically hack the current template to replace the attributes 'orig' and 'ref' by + # the original and reference sentences of the previous sentence (if available) + def get_fewshot_template(self): + self.shot_prompt_template = copy.deepcopy(self.prompt_template) + old_jinja = self.shot_prompt_template.jinja + preamble = '{% set src_sent = ""%}' + preamble += '{% set trg_sent = "" %}' + preamble += "{% if dialogue_history|length > 0 %}{% if utterance_meta.lang == dialogue_history[-1].utterance_meta.lang %}{% set src_sent = dialogue_history[-1].orig %}{% set trg_sent = dialogue_history[-1].ref %}{% else %}{% set src_sent = dialogue_history[-1].ref %}{% set trg_sent = dialogue_history[-1].orig %}{% endif %}{% endif %}" + self.shot_prompt_template.jinja = preamble + old_jinja.replace( + "{{ orig }}", "{{ src_sent }}" + ).replace("{{ ref }}", "{{ trg_sent }}") + return self.shot_prompt_template + + def fewshot_examples( + self, + docs: datasets.Dataset, + k: int, + rng: np.random.Generator, + prompt: dict = None, + ) -> Tuple[List[dict], List[int]]: + """Returns `k` random examples from the set of documents in `docs`. + + :param docs: datasets.Dataset + The dataset of documents to sample few-shot examples from. + :param k: int + The number of few-shot examples. + :param rng: np.random.Generator + The pseudo-random number generator used to randomly sample examples. + :param prompt: Optional[dict] + The prompt document. Specify this to ensure the prompt is not in + the set of few-shot examples. + """ + # hack the hack to the be used so that it uses other attributes + self.get_fewshot_template() + return [prompt], 0 + + def fewshot_context( + self, doc: dict, num_fewshot: int, rng: Optional[np.random.Generator] + ) -> Tuple[str, dict]: + """Returns a few-shot context string made up of `num_fewshot` number of + labeled examples, and an appended prompt example without labeling. + + :param doc: dict + The document as returned from training_docs, validation_docs, or test_docs. + :param num_fewshot: int + The number of fewshot examples to provide in the returned context string. + :param rng: numpy.random.Generator + The pseudo-random number generator used to randomly sample few-shot examples. + :returns: Tuple[str, dict] + ctx: str + The fewshot context. + logging_info: dict + A `dict` of logging info that can be used to identify few-shot sources. + """ + assert ( + rng is not None + ), "A `numpy.random.Generator` argument must be provided to `rng`" + + if num_fewshot == 0: + labeled_examples = "" + fewshot_idx, fewshot_target_idx, fewshot_src = ([], [], None) + else: + # Construct few-shot labeled examples. + fewshot_docs = self.fewshot_docs() + fewshot_src = str(fewshot_docs.split) + fewshot_examples, fewshot_idx = self.fewshot_examples( + fewshot_docs, k=num_fewshot, rng=rng, prompt=doc + ) + labeled_examples_list = [] + fewshot_target_idx = [] + for fewshot_example in fewshot_examples: + # format the example, but use the previous context of the example + text = self.doc_to_shot_text(fewshot_example) + targets = self.doc_to_shot_target(fewshot_example) + # Choose 1 random target from multi-reference targets. + target_idx = int(rng.integers(0, len(targets))) + target = targets[target_idx].strip() + labeled_examples_list.append( + self.format_example(text, target, self.text_target_separator) + ) + fewshot_target_idx.append(target_idx) + labeled_examples = self.example_separator.join(labeled_examples_list) + # Leave an extra `example_separator` right before the prompt. + labeled_examples += self.example_separator + + prompt = self.doc_to_text(doc) + ctx = labeled_examples + prompt + logging_info = { + "fewshot_idx": fewshot_idx, + "fewshot_target_idx": fewshot_target_idx, + "fewshot_source": fewshot_src, + "fewshot_num": num_fewshot, + "ctx": ctx, + } + return ctx, logging_info + + +class DiaBLa_1_shot_context_opposite(DiaBLa_1_shot_context_same): + """ + This task is identical to the DiaBLa task, but in the 1-shot setting takes the + 1-shot example from the previous sentence in the dialogue if this is available + (source sentence and MT output, in the same language direction as the direction + of the current example). N.B. this task is not currently designed for more than + 1-shot. + """ + + DATASET_PATH = "rbawden/DiaBLa" + DATASET_NAME = None + + # heuristically hack the current template to replace the attributes 'orig' and 'ref' by + # the original and reference sentences of the previous sentence (if available) + def get_fewshot_template(self): + self.shot_prompt_template = copy.deepcopy(self.prompt_template) + old_jinja = self.shot_prompt_template.jinja + preamble = '{% set src_sent = ""%}' + preamble += '{% set trg_sent = "" %}' + preamble += "{% if dialogue_history|length > 0 %}{% if utterance_meta.lang != dialogue_history[-1].utterance_meta.lang %}{% set src_sent = dialogue_history[-1].orig %}{% set trg_sent = dialogue_history[-1].ref %}{% else %}{% set src_sent = dialogue_history[-1].ref %}{% set trg_sent = dialogue_history[-1].orig %}{% endif %}{% endif %}" + self.shot_prompt_template.jinja = preamble + old_jinja.replace( + "{{ orig }}", "{{ src_sent }}" + ).replace("{{ ref }}", "{{ trg_sent }}").replace( + '{% if utterance_meta.lang == "french" %}', + '{% if utterance_meta.lang != "french" %}', + ) + return self.shot_prompt_template + + +class DiaBLa_1_shot_context_orig(DiaBLa_1_shot_context_same): + """ + This task is identical to the DiaBLa task, but in the 1-shot setting takes the + 1-shot example from the previous sentence in the dialogue if this is available + (source sentence and MT output, in the same language direction as the direction + of the current example). N.B. this task is not currently designed for more than + 1-shot. + """ + + DATASET_PATH = "rbawden/DiaBLa" + DATASET_NAME = None + + # heuristically hack the current template to replace the attributes 'orig' and 'ref' by + # the original and reference sentences of the previous sentence (if available) + def get_fewshot_template(self): + self.shot_prompt_template = copy.deepcopy(self.prompt_template) + old_jinja = self.shot_prompt_template.jinja + preamble = '{% set src_sent = ""%}{% set trg_sent = "" %}' + preamble += "{% if dialogue_history|length > 0 %}{% set src_sent = dialogue_history[-1].orig %}{% set trg_sent = dialogue_history[-1].ref %}{% endif %}" + self.shot_prompt_template.jinja = preamble + old_jinja.replace( + "{{ orig }}", "{{ src_sent }}" + ).replace("{{ ref }}", "{{ trg_sent }}").replace( + "utterance_meta.lang", "dialogue_history[-1].utterance_meta.lang" + ) + return self.shot_prompt_template diff --git a/evaluation-pipeline/lm_eval/tasks/drop.py b/evaluation-pipeline/lm_eval/tasks/drop.py new file mode 100644 index 0000000000000000000000000000000000000000..744f745460649109cf9d28ddf8b67f81e34075a6 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/drop.py @@ -0,0 +1,204 @@ +""" +DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs +https://aclanthology.org/attachments/N19-1246.Supplementary.pdf + +DROP is a QA dataset which tests comprehensive understanding of paragraphs. In +this crowdsourced, adversarially-created, 96k question-answering benchmark, a +system must resolve multiple references in a question, map them onto a paragraph, +and perform discrete operations over them (such as addition, counting, or sorting). + +Homepage: https://allenai.org/data/drop + +Acknowledgement: This implementation is based on the official evaluation for `DROP`: +https://github.com/allenai/allennlp-reading-comprehension/blob/master/allennlp_rc/eval/drop_eval.py +""" +import numpy as np +import re +import string +from scipy.optimize import linear_sum_assignment + +from lm_eval.api.task import PromptSourceTask +from lm_eval.api.metric import mean + + +_CITATION = """ +@misc{dua2019drop, + title={DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs}, + author={Dheeru Dua and Yizhong Wang and Pradeep Dasigi and Gabriel Stanovsky and Sameer Singh and Matt Gardner}, + year={2019}, + eprint={1903.00161}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} +""" + + +_ARTICLES = re.compile(r"\b(a|an|the)\b", re.UNICODE) + + +class DROP(PromptSourceTask): + VERSION = 1 + DATASET_PATH = "drop" # inspect.getfile(lm_eval.datasets.drop.drop) + DATASET_NAME = None + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + return self.dataset["validation"] + + def process_results(self, doc, results): + golds = self.doc_to_target(doc) + pred = results[0].strip() + preds = [pred] + + max_em = 0 + max_f1 = 0 + for gold_answer in golds: + exact_match, f1_score = self.get_metrics(preds, gold_answer) + if gold_answer[0].strip(): + max_em = max(max_em, exact_match) + max_f1 = max(max_f1, f1_score) + + if self.save_examples: + return {"em": max_em, "f1": max_f1}, {"pred": pred, "target": golds} + return {"em": max_em, "f1": max_f1} + + def get_metrics(self, predicted, gold): + """ + Takes a predicted answer and a gold answer (that are both either a string or a list of + strings), and returns exact match and the DROP F1 metric for the prediction. If you are + writing a script for evaluating objects in memory (say, the output of predictions during + validation, or while training), this is the function you want to call, after using + :func:`answer_json_to_strings` when reading the gold answer from the released data file. + """ + predicted_bags = self._answer_to_bags(predicted) + gold_bags = self._answer_to_bags(gold) + + if set(predicted_bags[0]) == set(gold_bags[0]) and len( + predicted_bags[0] + ) == len(gold_bags[0]): + exact_match = 1.0 + else: + exact_match = 0.0 + + f1_per_bag = self._align_bags(predicted_bags[1], gold_bags[1]) + f1 = np.mean(f1_per_bag) + f1 = round(f1, 2) + return exact_match, f1 + + def _answer_to_bags(self, answer): + if isinstance(answer, (list, tuple)): + raw_spans = answer + else: + raw_spans = [answer] + normalized_spans = [] + token_bags = [] + for raw_span in raw_spans: + normalized_span = self._normalize(raw_span) + normalized_spans.append(normalized_span) + token_bags.append(set(normalized_span.split())) + return normalized_spans, token_bags + + def _align_bags(self, predicted, gold): + """ + Takes gold and predicted answer sets and first finds the optimal 1-1 alignment + between them and gets maximum metric values over all the answers. + """ + scores = np.zeros([len(gold), len(predicted)]) + for gold_index, gold_item in enumerate(gold): + for pred_index, pred_item in enumerate(predicted): + if self._match_numbers_if_present(gold_item, pred_item): + scores[gold_index, pred_index] = self._compute_f1( + pred_item, gold_item + ) + row_ind, col_ind = linear_sum_assignment(-scores) + + max_scores = np.zeros([max(len(gold), len(predicted))]) + for row, column in zip(row_ind, col_ind): + max_scores[row] = max(max_scores[row], scores[row, column]) + return max_scores + + def _compute_f1(self, predicted_bag, gold_bag): + intersection = len(gold_bag.intersection(predicted_bag)) + if not predicted_bag: + precision = 1.0 + else: + precision = intersection / float(len(predicted_bag)) + if not gold_bag: + recall = 1.0 + else: + recall = intersection / float(len(gold_bag)) + f1 = ( + (2 * precision * recall) / (precision + recall) + if not (precision == 0.0 and recall == 0.0) + else 0.0 + ) + return f1 + + def _match_numbers_if_present(self, gold_bag, predicted_bag): + gold_numbers = set() + predicted_numbers = set() + for word in gold_bag: + if self._is_number(word): + gold_numbers.add(word) + for word in predicted_bag: + if self._is_number(word): + predicted_numbers.add(word) + if (not gold_numbers) or gold_numbers.intersection(predicted_numbers): + return True + return False + + def _is_number(self, text): + try: + float(text) + return True + except ValueError: + return False + + def _remove_articles(self, text): + return _ARTICLES.sub(" ", text) + + def _white_space_fix(self, text): + return " ".join(text.split()) + + def _remove_punc(self, text): + exclude = set(string.punctuation) + if not self._is_number(text): + return "".join(ch for ch in text if ch not in exclude) + else: + return text + + def _fix_number(self, text): + return str(float(text)) if self._is_number(text) else text + + def _tokenize(self, text): + return re.split(" |-", text) + + def _normalize(self, answer): + tokens = [ + self._white_space_fix( + self._remove_articles( + self._fix_number(self._remove_punc(token.lower())) + ) + ) + for token in self._tokenize(answer) + ] + tokens = [token for token in tokens if token.strip()] + normalized = " ".join(tokens).strip() + return normalized + + def aggregation(self): + return {"em": mean, "f1": mean} + + def higher_is_better(self): + return {"em": True, "f1": True} diff --git a/evaluation-pipeline/lm_eval/tasks/e2e_nlg_cleaned.py b/evaluation-pipeline/lm_eval/tasks/e2e_nlg_cleaned.py new file mode 100644 index 0000000000000000000000000000000000000000..0e1be5b0256c120bc3aa3d4c355ecaf757dfa530 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/e2e_nlg_cleaned.py @@ -0,0 +1,78 @@ +""" +Semantic Noise Matters for Neural Natural Language Generation +http://arxiv.org/abs/1911.03905 + +A cleaned version of the dataset from the E2E NLG Challenge. +The dataset contains MR with restaurant attributes and corresponding descriptions. + +Homepage: https://github.com/tuetschek/e2e-cleaning +""" +from lm_eval.api.task import PromptSourceTask + + +_CITATION = """ +@inproceedings{dusek-etal-2019-semantic, + title = "Semantic Noise Matters for Neural Natural Language Generation", + author = "Du{\v{s}}ek, Ond{\v{r}}ej and + Howcroft, David M. and + Rieser, Verena", + booktitle = "Proceedings of the 12th International Conference on Natural Language Generation", + year = "2019", + address = "Tokyo, Japan", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/W19-8652", + doi = "10.18653/v1/W19-8652", + pages = "421--426", +} +""" + + +# Work in progress +class E2E_NLG_Cleaned(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "e2e_nlg_cleaned" + DATASET_NAME = None + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return True + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["validation"] + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["test"] + + def max_generation_length(self): + return 64 + + def invalid_doc_for_prompt(self, doc) -> bool: + """The QA prompts are not applicable to all the examples, we want to filter these out.""" + # HACK: Some templates have conditionals that ignore documents + # when the condition is not met, like `{if doc['question'] != \"cause\"}`. + # This means the prompt will never produce an input and target. + # TODO: Remove this when fixed in `promptsource` + try: + text, target = self.prompt_template.apply(doc) + return ( + self.prompt_template.name.endswith("_qa") + or self.prompt_template.name == "family_friendly_yes_no" + ) + except Exception: + return True + + def doc_to_text(self, doc) -> str: + # if the response is not defined in PS, the text will be a single-element list containing an empty string + text = self.prompt_template.apply(doc)[0] + return text diff --git a/evaluation-pipeline/lm_eval/tasks/flores_101.py b/evaluation-pipeline/lm_eval/tasks/flores_101.py new file mode 100644 index 0000000000000000000000000000000000000000..99e1592b19161a23e6a9c8605b5db4fe3ae66834 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/flores_101.py @@ -0,0 +1,536 @@ +""" +The Flores-101 Evaluation Benchmark for Low-Resource and Multilingual Machine Translation +https://aclanthology.org/2022.tacl-1.30/ + +Naman Goyal, Cynthia Gao, Vishrav Chaudhary, Peng-Jen Chen, Guillaume Wenzek, Da Ju, Sanjana Krishnan, +Marc’Aurelio Ranzato, Francisco Guzmán, and Angela Fan. 2022. The Flores-101 Evaluation Benchmark for +Low-Resource and Multilingual Machine Translation. Transactions of the Association for Computational Linguistics, +10:522–538. + +FLORES-101 is a Many-to-Many multilingual translation benchmark dataset for 101 languages. + +Github: https://github.com/facebookresearch/flores +""" +from lm_eval.api.task import PromptSourceTask, PerplexityTask +from typing import List, Tuple, Optional +import datasets +import copy +import re +import numpy as np +import promptsource.templates +from lm_eval import tasks + + +_CITATION = """ +@article{goyal-etal-2022-flores, + title = "The {F}lores-101 Evaluation Benchmark for Low-Resource and Multilingual Machine Translation", + author = "Goyal, Naman and + Gao, Cynthia and + Chaudhary, Vishrav and + Chen, Peng-Jen and + Wenzek, Guillaume and + Ju, Da and + Krishnan, Sanjana and + Ranzato, Marc{'}Aurelio and + Guzm{\'a}n, Francisco and + Fan, Angela", + journal = "Transactions of the Association for Computational Linguistics", + volume = "10", + year = "2022", + address = "Cambridge, MA", + publisher = "MIT Press", + url = "https://aclanthology.org/2022.tacl-1.30", + doi = "10.1162/tacl_a_00474", + pages = "522--538", +}} +""" + + +class Flores101MT(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "gsarti/flores_101" + DATASET_NAME = "all" + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return True + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["dev"] + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["devtest"] + + def max_generation_length(self): + return 512 + + +class Flores101MT_fewshot_wmt_fr2en(Flores101MT): + """ + This task is Identical to the Flores101MT task, except in the few-shot setting + where few-shot examples are created using examples from the WMT14 French-to-English + development set, whatever the language specified in the prompt. + """ + + VERSION = 0 + DATASET_PATH = "gsarti/flores_101" + DATASET_NAME = "all" + + def __init__( + self, + data_dir: Optional[str] = None, + cache_dir: Optional[str] = None, + download_mode: Optional[str] = None, + prompt_template: Optional[promptsource.templates.Template] = None, + example_separator: Optional[str] = "\n###\n", + text_target_separator: Optional[str] = " ", + save_examples: Optional[bool] = True, + ): + """ + Args: + save_examples (bool, optional, defaults to True): + Whether to save each example and corresponding model predictions + to an output `dict`. + > Few-shot prompting args + example_separator (str, optional, defaults to '\n###\n'): + The string that will be used to separate the few-shot examples + from the prompt example. + Default: '\n###\n' + See Webson & Pavlick (2022) https://arxiv.org/pdf/2109.01247.pdf + for justification of this separator. + text_target_separator (str, optional, defaults to ' '): + The string that will be used to separate the prompt example + from the target text. + NOTE: This is assumed to be some form of whitespace-only separation, + e.g. "\n\n", "\t", " ", etc. Otherwise, you should update + the Task's `promptsource` template with the appropriate + separator(s). + Example: + Q: Where is the Eiffel Tower located? A:{text_target_separator}Paris + """ + assert ( + text_target_separator.isspace() + ), f"`text_target_separator` must be whitespace only. Got: `{text_target_separator}`" + super().__init__(data_dir, cache_dir, download_mode) + self.prompt_template = prompt_template + self.save_examples = save_examples + self.example_separator = example_separator + self.text_target_separator = text_target_separator + self.cache_dir = cache_dir + self.download_mode = download_mode + + def fewshot_docs(self) -> datasets.Dataset: + """Returns a wmt dataset split""" + return ( + "valid", + datasets.load_dataset( + "wmt14", + "fr-en", + cache_dir=self.cache_dir, + download_mode=self.download_mode, + )["validation"]["translation"], + ) + + def fewshot_context( + self, doc: dict, num_fewshot: int, rng: Optional[np.random.Generator] + ) -> Tuple[str, dict]: + """Returns a few-shot context string made up of `num_fewshot` number of + labeled examples, and an appended prompt example without labeling. + + :param doc: dict + The document as returned from training_docs, validation_docs, or test_docs. + :param num_fewshot: int + The number of fewshot examples to provide in the returned context string. + :param rng: numpy.random.Generator + The pseudo-random number generator used to randomly sample few-shot examples. + :returns: Tuple[str, dict] + ctx: str + The fewshot context. + logging_info: dict + A `dict` of logging info that can be used to identify few-shot sources. + """ + assert ( + rng is not None + ), "A `numpy.random.Generator` argument must be provided to `rng`" + + self.get_fewshot_template() + + if num_fewshot == 0: + labeled_examples = "" + fewshot_idx, fewshot_target_idx, fewshot_src = ([], [], None) + else: + # Construct few-shot labeled examples. + fewshot_src, fewshot_docs = self.fewshot_docs() + + fewshot_examples, fewshot_idx = self.fewshot_examples( + fewshot_docs, k=num_fewshot, rng=rng, prompt=doc + ) + labeled_examples_list = [] + fewshot_target_idx = [] + for fewshot_example in fewshot_examples: + # format the example, but use the previous context of the example + text = self.doc_to_shot_text(fewshot_example) + targets = self.doc_to_shot_target(fewshot_example) + # Choose 1 random target from multi-reference targets. + target_idx = int(rng.integers(0, len(targets))) + target = targets[target_idx].strip() + labeled_examples_list.append( + self.format_example(text, target, self.text_target_separator) + ) + fewshot_target_idx.append(target_idx) + labeled_examples = self.example_separator.join(labeled_examples_list) + # Leave an extra `example_separator` right before the prompt. + labeled_examples += self.example_separator + + prompt = self.doc_to_text(doc) + ctx = labeled_examples + prompt + logging_info = { + "fewshot_idx": fewshot_idx, + "fewshot_target_idx": fewshot_target_idx, + "fewshot_source": fewshot_src, + "fewshot_num": num_fewshot, + "ctx": ctx, + } + return ctx, logging_info + + def doc_to_shot_text(self, doc: dict) -> str: + text, _ = self.shot_prompt_template.apply(doc) + return text + + def doc_to_shot_target(self, doc: dict) -> List[str]: + _, target = self.shot_prompt_template.apply(doc) + return target + + def fewshot_values(self): + return "French", "English", "{{ fr }}", "{{ en }}" + + # heuristically hack the prompt template used to create few-shot examples + def get_fewshot_template(self): + self.shot_prompt_template = copy.deepcopy(self.prompt_template) + + # get things to replace in the prompt + src_lang, trg_lang = self.prompt_template.name.split("-")[-2:] + src_sent, trg_sent = re.findall("{{ .+? }}", self.prompt_template.jinja) + # new attributes to drop in as replacement + new_src_lang, new_trg_lang, new_src_sent, new_trg_sent = self.fewshot_values() + # create new prompt + assert len(re.findall(src_lang, self.shot_prompt_template.jinja)) == 1 + assert len(re.findall(trg_lang, self.shot_prompt_template.jinja)) == 1 + for old_text, new_text in [ + (src_lang, new_src_lang), + (trg_lang, new_trg_lang), + (src_sent, new_src_sent), + (trg_sent, new_trg_sent), + ]: + self.shot_prompt_template.jinja = self.shot_prompt_template.jinja.replace( + old_text, new_text + ) + return self.shot_prompt_template + + def fewshot_examples( + self, + docs: datasets.Dataset, + k: int, + rng: np.random.Generator, + prompt: dict = None, + ) -> Tuple[List[dict], List[int]]: + """Returns `k` random examples from the set of documents in `docs`. + + Args: + docs (datasets.Dataset): + The dataset of documents to sample few-shot examples from. + k (int): + The number of few-shot examples. + rng (np.random.Generator): + The pseudo-random number generator used to randomly sample examples. + prompt (Optional[dict]): + The prompt document. Specify this to ensure the prompt is not in + the set of few-shot examples. + + Returns: + A tuple of two lists. The first list contains the few-shot examples + """ + random_indices = np.arange(len(docs)).tolist() + rng.shuffle(random_indices) + + i = 0 + fewshot_examples, fewshot_idx = [], [] + for idx in random_indices: + if i >= k: # Break when we have enough examples. + break + is_same_prompt = False + # is never same prompt with this task + # is_same_prompt = prompt is not None and all( + # # Skips the `doc_id` key assigned to `prompt`s during eval pre-processing. + # docs[idx][k] == prompt[k] + # for k in docs[idx].keys() + # ) + + if self.invalid_doc_for_prompt(docs[idx]) or is_same_prompt: + continue + fewshot_examples.append(docs[idx]) + fewshot_idx.append(int(idx)) + i += 1 + return fewshot_examples, fewshot_idx + + +class Flores101MT_fewshot_wmt_hi2en(Flores101MT_fewshot_wmt_fr2en): + """ + This task is Identical to the Flores101MT task, except in the few-shot setting + where few-shot examples are created using examples from the WMT14 Hindi-to-English + development set, whatever the language specified in the prompt. + """ + + VERSION = 0 + DATASET_PATH = "gsarti/flores_101" + DATASET_NAME = "all" + + def fewshot_docs(self) -> datasets.Dataset: + """Returns a wmt dataset split""" + return ( + "valid", + datasets.load_dataset( + "wmt14", + "hi-en", + cache_dir=self.cache_dir, + download_mode=self.download_mode, + )["validation"], + ) + + +class Flores101MT_fewshot_fr2en(Flores101MT): + """ + This task is Identical to the Flores101MT task, except in the few-shot setting + where few-shot examples are created using French as the source language and English + as the target language, whatever the language specified in the prompt. + """ + + VERSION = 0 + DATASET_PATH = "gsarti/flores_101" + DATASET_NAME = "all" + + def fewshot_values(self): + return "French", "English", "{{ sentence_fra }}", "{{ sentence_eng }}" + + # heuristically hack the prompt template used to create few-shot examples + def get_fewshot_template(self): + self.shot_prompt_template = copy.deepcopy(self.prompt_template) + + # get things to replace in the prompt + src_lang, trg_lang = self.prompt_template.name.split("-")[-2:] + src_sent, trg_sent = re.findall("{{ .+? }}", self.prompt_template.jinja) + # new attributes to drop in as replacement + new_src_lang, new_trg_lang, new_src_sent, new_trg_sent = self.fewshot_values() + # create new prompt + assert len(re.findall(src_lang, self.shot_prompt_template.jinja)) == 1 + assert len(re.findall(trg_lang, self.shot_prompt_template.jinja)) == 1 + for old_text, new_text in [ + (src_lang, new_src_lang), + (trg_lang, new_trg_lang), + (src_sent, new_src_sent), + (trg_sent, new_trg_sent), + ]: + self.shot_prompt_template.jinja = self.shot_prompt_template.jinja.replace( + old_text, new_text + ) + return self.shot_prompt_template + + def fewshot_context( + self, doc: dict, num_fewshot: int, rng: Optional[np.random.Generator] + ) -> Tuple[str, dict]: + """Returns a few-shot context string made up of `num_fewshot` number of + labeled examples, and an appended prompt example without labeling. + + :param doc: dict + The document as returned from training_docs, validation_docs, or test_docs. + :param num_fewshot: int + The number of fewshot examples to provide in the returned context string. + :param rng: numpy.random.Generator + The pseudo-random number generator used to randomly sample few-shot examples. + :returns: Tuple[str, dict] + ctx: str + The fewshot context. + logging_info: dict + A `dict` of logging info that can be used to identify few-shot sources. + """ + assert ( + rng is not None + ), "A `numpy.random.Generator` argument must be provided to `rng`" + + self.get_fewshot_template() + + if num_fewshot == 0: + labeled_examples = "" + fewshot_idx, fewshot_target_idx, fewshot_src = ([], [], None) + else: + # Construct few-shot labeled examples. + fewshot_docs = self.fewshot_docs() + fewshot_src = str(fewshot_docs.split) + fewshot_examples, fewshot_idx = self.fewshot_examples( + fewshot_docs, k=num_fewshot, rng=rng, prompt=doc + ) + labeled_examples_list = [] + fewshot_target_idx = [] + for fewshot_example in fewshot_examples: + # format the example, but use the previous context of the example + text = self.doc_to_shot_text(fewshot_example) + targets = self.doc_to_shot_target(fewshot_example) + # Choose 1 random target from multi-reference targets. + target_idx = int(rng.integers(0, len(targets))) + target = targets[target_idx].strip() + labeled_examples_list.append( + self.format_example(text, target, self.text_target_separator) + ) + fewshot_target_idx.append(target_idx) + labeled_examples = self.example_separator.join(labeled_examples_list) + # Leave an extra `example_separator` right before the prompt. + labeled_examples += self.example_separator + + prompt = self.doc_to_text(doc) + ctx = labeled_examples + prompt + logging_info = { + "fewshot_idx": fewshot_idx, + "fewshot_target_idx": fewshot_target_idx, + "fewshot_source": fewshot_src, + "fewshot_num": num_fewshot, + "ctx": ctx, + } + return ctx, logging_info + + def doc_to_shot_text(self, doc: dict) -> str: + text, _ = self.shot_prompt_template.apply(doc) + return text + + def doc_to_shot_target(self, doc: dict) -> List[str]: + _, target = self.shot_prompt_template.apply(doc) + return target + + +class Flores101MT_fewshot_hi2en(Flores101MT_fewshot_fr2en): + """ + This task is Identical to the Flores101MT task, except in the few-shot setting + where few-shot examples are created using Hindi as the source language and English + as the target language, whatever the language specified in the prompt. + """ + + VERSION = 0 + DATASET_PATH = "gsarti/flores_101" + DATASET_NAME = "all" + + def fewshot_values(self): + return "Hindi", "English", "{{ sentence_hin }}", "{{ sentence_eng }}" + + +class Flores101MT_fewshot_fr2ar(Flores101MT_fewshot_fr2en): + """ + This task is Identical to the Flores101MT task, except in the few-shot setting + where few-shot examples are created using French as the source language and Arabic + as the target language, whatever the language specified in the prompt. + """ + + VERSION = 0 + DATASET_PATH = "gsarti/flores_101" + DATASET_NAME = "all" + + def fewshot_values(self): + return "French", "Arabic", "{{ sentence_fra }}", "{{ sentence_ara }}" + + +class Flores101MT_fewshot_en2bn(Flores101MT_fewshot_fr2en): + """ + This task is Identical to the Flores101MT task, except in the few-shot setting + where few-shot examples are created using English as the source language and Bengali + as the target language, whatever the language specified in the prompt. + """ + + VERSION = 0 + DATASET_PATH = "gsarti/flores_101" + DATASET_NAME = "all" + + def fewshot_values(self): + return "English", "Bengali", "{{ sentence_eng }}", "{{ sentence_ben }}" + + +class Flores101Perplexity(PerplexityTask): + """Computes the perplexity for a specific language translation of Flores-101. + + NOTE: B/c promptsource provides templates from the `all` flores-101 split, we specify + which language to use by taking the first language code in the template name + and compute perplexity on that. + + For example, to run perplexity on English translations of Flores-101 you + should pass in templates of the form: 'translate-this-eng-{target}' + """ + + VERSION = 0 + DATASET_PATH = "gsarti/flores_101" + DATASET_NAME = "all" + LANG = None + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return True + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["dev"] + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["devtest"] + + def doc_to_target(self, doc): + """This is a null prompt task. We need to get the target from the doc.""" + lang = self._get_lang_from_template() + target = [doc[f"sentence_{lang}"]] + return target + + def _get_lang_from_template(self): + template_name = self.prompt_template.name + # Get the first language in the prompt name is the language we'll use + # to compute perplexity on: 'translate-this-ara-asm' -> 'ara' + lang = template_name.rsplit("-")[-2] + return lang + + def process_results(self, doc, results): + if self.save_examples: + out, log = super().process_results(doc, results) + log["topic"] = doc["topic"] + log["domain"] = doc["domain"] + return out, log + else: + return super().process_results(doc, results) + + +def list_templates() -> List[str]: + """Returns a list of non-overlapping language template names in Flores-101 + which can be used to compute multi-lingual perplexity; See docstrings of + `Flores101Perplexity`. + + Example Usage: + ```python + lm_eval.get_task_list( + 'flores_101_ppl', + template_names=lm_eval.tasks.flores_101.list_templates() + ) + ``` + """ + unique_templates = [] + templates = tasks._get_templates_from_task(Flores101Perplexity) + for template in templates.all_template_names: + lang = "-".join(template.rsplit("-")[:-1]) + if not any(lang in i for i in unique_templates): + unique_templates.append(template) + return unique_templates diff --git a/evaluation-pipeline/lm_eval/tasks/gem_asset_turk.py b/evaluation-pipeline/lm_eval/tasks/gem_asset_turk.py new file mode 100644 index 0000000000000000000000000000000000000000..4f62b9b7fba829cd3c2f9d61b6ea4a22b7051f13 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/gem_asset_turk.py @@ -0,0 +1,162 @@ +""" +ASSET: ASSET (Alva-Manchego et al., 2020) is multi-reference dataset +for the evaluation of sentence simplification in English. The dataset +uses the same 2,359 sentences from TurkCorpus (Xu et al., 2016) +and each sentence is associated with 10 crowdsourced simplifications. +Unlike previous simplification datasets, which contain a single +transformation (e.g., lexical paraphrasing in TurkCorpus or sentence +splitting in HSplit), the simplifications in ASSET encompass a variety +of rewriting transformations. +https://aclanthology.org/2020.acl-main.424.pdf + +TurkCorpus: TURKCorpus is a multi-reference dataset for the evaluation of +sentence simplification in English. The dataset consists of 2,359 sentences +from the Parallel Wikipedia Simplification (PWKP) corpus. Each sentence is +associated with 8 crowdsourced simplifications that focus on only lexical +paraphrasing (no sentence splitting or deletion). +https://cocoxu.github.io/publications/tacl2016-smt-simplification.pdf +""" +from typing import Optional +from promptsource.templates import Template + +from lm_eval.api.task import PromptSourceTask + + +_CITATION = """ +@article{DBLP:journals/corr/abs-2005-00481, + author = {Fernando Alva{-}Manchego and + Louis Martin and + Antoine Bordes and + Carolina Scarton and + Benoit Sagot and + Lucia Specia}, + title = {{ASSET:} {A} Dataset for Tuning and Evaluation of Sentence Simplification + Models with Multiple Rewriting Transformations}, + journal = {CoRR}, + volume = {abs/2005.00481}, + year = {2020}, + url = {https://arxiv.org/abs/2005.00481}, + eprinttype = {arXiv}, + eprint = {2005.00481}, + timestamp = {Thu, 14 Oct 2021 16:38:25 +0200}, + biburl = {https://dblp.org/rec/journals/corr/abs-2005-00481.bib}, + bibsource = {dblp computer science bibliography, https://dblp.org} +}""" + + +class AssetTurk(PromptSourceTask): + + DATASET_PATH = "GEM/wiki_auto_asset_turk" + DATASET_NAME = None + SPLIT = None + + def __init__( + self, + data_dir: Optional[str] = None, + cache_dir: Optional[str] = None, + download_mode: Optional[str] = None, + prompt_template: Optional[Template] = None, + example_separator: Optional[str] = "\n###\n", + text_target_separator: Optional[str] = " ", + save_examples: Optional[bool] = True, + ): + super().__init__( + data_dir=data_dir, + cache_dir=cache_dir, + download_mode=download_mode, + prompt_template=prompt_template, + example_separator=example_separator, + text_target_separator=text_target_separator, + save_examples=save_examples, + ) + # Adding SARI to metrics to list because `promptsource` + # does not currently support this option. + if "SARI" not in self.prompt_template.metadata.metrics: + self.prompt_template.metadata.metrics.append("SARI") + + def doc_to_rawtext(self, doc): + return doc["source"] + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return True + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["validation"] + + def test_docs(self): + return self.dataset[str(self.SPLIT)] + + def max_generation_length(self): + return 200 + + +class AssetTest(AssetTurk): + SPLIT = "test_asset" + + +class TurkTest(AssetTurk): + SPLIT = "test_turk" + + +class AssetTest1(AssetTurk): + SPLIT = "challenge_test_asset_backtranslation" + + +class AssetTest2(AssetTurk): + SPLIT = "challenge_test_asset_bfp02" + + +class AssetTest3(AssetTurk): + SPLIT = "challenge_test_asset_bfp05" + + +class AssetTest4(AssetTurk): + SPLIT = "challenge_test_asset_nopunc" + + +class TurkTest1(AssetTurk): + SPLIT = "challenge_test_turk_backtranslation" + + +class TurkTest2(AssetTurk): + SPLIT = "challenge_test_turk_bfp02" + + +class TurkTest3(AssetTurk): + SPLIT = "challenge_test_turk_bfp05" + + +class TurkTest4(AssetTurk): + SPLIT = "challenge_test_turk_nopunc" + + +ASSET_TURK_CLASSES = [ + AssetTest, + TurkTest, + TurkTest1, + TurkTest2, + TurkTest3, + TurkTest4, + AssetTest1, + AssetTest2, + AssetTest3, + AssetTest4, +] + + +def construct_tasks(): + tasks = {} + for asset_turk_class in ASSET_TURK_CLASSES: + tasks[f"GEM/wiki_auto_asset_turk_{asset_turk_class.SPLIT}"] = asset_turk_class + return tasks diff --git a/evaluation-pipeline/lm_eval/tasks/gem_mlsum.py b/evaluation-pipeline/lm_eval/tasks/gem_mlsum.py new file mode 100644 index 0000000000000000000000000000000000000000..08d7d4bac97c44c263c31a73f8b8d80a3e77abad --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/gem_mlsum.py @@ -0,0 +1,120 @@ +""" +MLSUM: The Multilingual Summarization Corpus +https://aclanthology.org/2020.emnlp-main.647/ + +This is the MLSUM subset of the GEM benchmark. MLSUM is the first large-scale MultiLingual SUMmarization dataset. +Obtained from online newspapers, it contains 1.5M+ article/summary pairs in five different languages -- namely, French, German, Spanish, Russian, Turkish. +Together with English newspapers from the popular CNN/Daily mail dataset, the collected data form a large scale multilingual dataset which can enable new research directions for the text summarization community. +We report cross-lingual comparative analyses based on state-of-the-art systems. +These highlight existing biases which motivate the use of a multi-lingual dataset. +Homepage: https://gitlab.lip6.fr/scialom/mlsum_data/-/raw/master/MLSUM/ +""" +from lm_eval.api.task import PromptSourceTask + + +_CITATION = """ +@article{scialom2020mlsum, + title={MLSUM: The Multilingual Summarization Corpus}, + author={Scialom, Thomas and Dray, Paul-Alexis and Lamprier, Sylvain and Piwowarski, Benjamin and Staiano, Jacopo}, + journal={arXiv preprint arXiv:2004.14900}, + year={2020} +} +""" + + +class GEMMLSUMEsBase(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "GEM/mlsum" + DATASET_NAME = "es" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return True + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["validation"] + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["test"] + + +class GEMMLSUMEs(GEMMLSUMEsBase): + """this is for train/validation/test""" + + SPLIT = "" + + +class GEMMLSUMEsChallgeTestCovid(GEMMLSUMEsBase): + """this is for challenge_test_covid""" + + SPLIT = "challenge_test_covid" + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return False + + def test_docs(self): + if self.has_test_docs(): + return self.dataset[self.SPLIT] + + +class GEMMLSUMDeBase(PromptSourceTask): + + DATASET_PATH = "GEM/mlsum" + DATASET_NAME = "de" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return True + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["validation"] + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["test"] + + +class GEMMLSUMDe(GEMMLSUMDeBase): + """this is for train/validation/test""" + + SPLIT = "" + + +class GEMMLSUMDeChallgeTestCovid(GEMMLSUMDeBase): + """this is for challenge_test_covid""" + + SPLIT = "challenge_test_covid" + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return False + + def test_docs(self): + if self.has_test_docs(): + return self.dataset[self.SPLIT] diff --git a/evaluation-pipeline/lm_eval/tasks/gem_webnlg.py b/evaluation-pipeline/lm_eval/tasks/gem_webnlg.py new file mode 100644 index 0000000000000000000000000000000000000000..cec92f7772e9a5e03614a4d1ba8ae01c2f4dcd86 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/gem_webnlg.py @@ -0,0 +1,127 @@ +""" +The 2020 Bilingual, Bi-Directional WebNLG+ Shared Task: +Overview and Evaluation Results (WebNLG+ 2020) +https://aclanthology.org/2020.webnlg-1.7/ + +WebNLG+ offers two challenges: (i) mapping sets of RDF triples +to English or Russian text (generation) and (ii) converting +English or Russian text to sets of RDF triples (semantic parsing). +Compared to the eponymous WebNLG challenge, WebNLG+ provides an +extended dataset that enable the training, evaluation, and +comparison of microplanners and semantic parsers. In this paper, +we present the results of the generation and semantic parsing +task for both English and Russian and provide a brief +description of the participating systems. +""" +from lm_eval.api.task import PromptSourceTask + + +_CITATION = """ +@inproceedings{castro-ferreira-etal-2020-2020, + title = "The 2020 Bilingual, Bi-Directional {W}eb{NLG}+ Shared Task: Overview and Evaluation Results ({W}eb{NLG}+ 2020)", + author = "Castro Ferreira, Thiago and + Gardent, Claire and + Ilinykh, Nikolai and + van der Lee, Chris and + Mille, Simon and + Moussallem, Diego and + Shimorina, Anastasia", + booktitle = "Proceedings of the 3rd International Workshop on Natural Language Generation from the Semantic Web (WebNLG+)", + month = "12", + year = "2020", + address = "Dublin, Ireland (Virtual)", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2020.webnlg-1.7", + pages = "55--76", + abstract = "WebNLG+ offers two challenges: (i) mapping sets of RDF triples to English or Russian text (generation) and (ii) converting English or Russian text to sets of RDF triples (semantic parsing). Compared to the eponymous WebNLG challenge, WebNLG+ provides an extended dataset that enable the training, evaluation, and comparison of microplanners and semantic parsers. In this paper, we present the results of the generation and semantic parsing task for both English and Russian and provide a brief description of the participating systems.", +} +""" + + +class WebNLG(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "GEM/web_nlg" + DATASET_NAME = "en" + SPLIT = None + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return True + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["validation"] + + def test_docs(self): + if self.has_test_docs(): + if self.SPLIT is not None: + return self.dataset[str(self.SPLIT)] + else: + return self.dataset["test"] + + def max_generation_length(self): + return 250 + + +class WebNLGRu(WebNLG): + DATASET_NAME = "ru" + + +## En Challenge Sets + + +class WebNLGEn1(WebNLG): + SPLIT = "challenge_validation_sample" + + +class WebNLGEn2(WebNLG): + SPLIT = "challenge_test_scramble" + + +class WebNLGEn3(WebNLG): + SPLIT = "challenge_test_numbers" + + +## Ru Challenge sets + + +class WebNLGRu1(WebNLG): + DATASET_NAME = "ru" + SPLIT = "challenge_validation_sample" + + +class WebNLGRu2(WebNLG): + DATASET_NAME = "ru" + SPLIT = "challenge_test_scramble" + + +WEBNLG_CLASSES = [ + WebNLG, + WebNLGRu, + WebNLGEn1, + WebNLGEn2, + WebNLGEn3, + WebNLGRu1, + WebNLGRu2, +] + + +def construct_tasks(): + tasks = {} + for webnlg_class in WEBNLG_CLASSES: + if webnlg_class.SPLIT is None: + tasks[f"GEM/web_nlg_{webnlg_class.DATASET_NAME}"] = webnlg_class + else: + tasks[ + f"GEM/web_nlg_{webnlg_class.DATASET_NAME}_{webnlg_class.SPLIT}" + ] = webnlg_class + return tasks diff --git a/evaluation-pipeline/lm_eval/tasks/gem_wikilingua.py b/evaluation-pipeline/lm_eval/tasks/gem_wikilingua.py new file mode 100644 index 0000000000000000000000000000000000000000..3ea900a69442e6f1bb5de649db12e434f092cd8b --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/gem_wikilingua.py @@ -0,0 +1,162 @@ +""" +WikiLingua: A New Benchmark Dataset for Cross-Lingual Abstractive Summarization +https://arxiv.org/pdf/2010.03093.pdf + +Wikilingua is a large-scale (~770k article-summary pairs), multilingual dataset for the evaluation of cross-lingual abstractive systems. +It consists of parallel articles and summaries (article-summary pairs) from WikiHow across 18 languages (i.e. all the languages available on WikiHow). +It contains 141,457 unique English articles and each of the other 17 languages has on average, 42,783 articles that align with an article in English. +This dataset is part of the GEM Benchmark. (Description from https://gem-benchmark.com/data_cards/WikiLingua) + + +Homepage: None, Repo: https://github.com/esdurmus/Wikilingua +""" +import typing + +from lm_eval.api.task import PromptSourceTask + + +_CITATION = """ +@inproceedings{ladhak-wiki-2020, + title={WikiLingua: A New Benchmark Dataset for Multilingual Abstractive Summarization}, + author={Faisal Ladhak, Esin Durmus, Claire Cardie and Kathleen McKeown}, + booktitle={Findings of EMNLP, 2020}, + year={2020} +}""" + + +class GEMWikiLinguaBase(PromptSourceTask): + VERSION = 1 + DATASET_PATH = "GEM/wiki_lingua" + DATASET_NAME = None + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return True + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["validation"] + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["sampled_test"] + + def max_generation_length(self): + return 64 + + +class GEMWikiLinguaAr(GEMWikiLinguaBase): + DATASET_NAME = "ar" + + +class GEMWikiLinguaCs(GEMWikiLinguaBase): + DATASET_NAME = "cs" + + +class GEMWikiLinguaDe(GEMWikiLinguaBase): + DATASET_NAME = "de" + + +class GEMWikiLinguaEn(GEMWikiLinguaBase): + DATASET_NAME = "en" + + +class GEMWikiLinguaEs(GEMWikiLinguaBase): + DATASET_NAME = "es" + + +class GEMWikiLinguaFr(GEMWikiLinguaBase): + DATASET_NAME = "fr" + + +class GEMWikiLinguaHi(GEMWikiLinguaBase): + DATASET_NAME = "hi" + + +class GEMWikiLinguaId(GEMWikiLinguaBase): + DATASET_NAME = "id" + + +class GEMWikiLinguaIt(GEMWikiLinguaBase): + DATASET_NAME = "it" + + +class GEMWikiLinguaJa(GEMWikiLinguaBase): + DATASET_NAME = "ja" + + +class GEMWikiLinguaKo(GEMWikiLinguaBase): + DATASET_NAME = "ko" + + +class GEMWikiLinguaNl(GEMWikiLinguaBase): + DATASET_NAME = "nl" + + +class GEMWikiLinguaPt(GEMWikiLinguaBase): + DATASET_NAME = "pt" + + +class GEMWikiLinguaRu(GEMWikiLinguaBase): + DATASET_NAME = "ru" + + +class GEMWikiLinguaTh(GEMWikiLinguaBase): + DATASET_NAME = "th" + + +class GEMWikiLinguaTr(GEMWikiLinguaBase): + DATASET_NAME = "tr" + + +class GEMWikiLinguaVi(GEMWikiLinguaBase): + DATASET_NAME = "vi" + + +class GEMWikiLinguaZh(GEMWikiLinguaBase): + DATASET_NAME = "zh" + + +WIKILINGUA_TASKS = [ + GEMWikiLinguaAr, + GEMWikiLinguaCs, + GEMWikiLinguaDe, + GEMWikiLinguaEn, + GEMWikiLinguaEs, + GEMWikiLinguaFr, + GEMWikiLinguaHi, + GEMWikiLinguaId, + GEMWikiLinguaIt, + GEMWikiLinguaJa, + GEMWikiLinguaKo, + GEMWikiLinguaNl, + GEMWikiLinguaPt, + GEMWikiLinguaRu, + GEMWikiLinguaTh, + GEMWikiLinguaTr, + GEMWikiLinguaVi, + GEMWikiLinguaZh, +] + + +def construct_tasks() -> typing.Dict[str, GEMWikiLinguaBase]: + """ + Returns a dictionary of tasks keyed by task name, for example: + "GEM/wiki_lingua_ar" + will dispatch to the GEM WikiLingua Arabic class. + """ + tasks = {} + for task_class in WIKILINGUA_TASKS: + benchmark = task_class.DATASET_PATH + lang = task_class.DATASET_NAME + tasks[f"{benchmark}_{lang}"] = task_class + return tasks diff --git a/evaluation-pipeline/lm_eval/tasks/gem_xsum.py b/evaluation-pipeline/lm_eval/tasks/gem_xsum.py new file mode 100644 index 0000000000000000000000000000000000000000..00b0606940cdc5228df6892190530441f959195e --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/gem_xsum.py @@ -0,0 +1,159 @@ +""" +Don't Give Me the Details, Just the Summary! Topic-Aware Convolutional Neural Networks for Extreme Summarization +https://arxiv.org/pdf/1808.08745.pdf + +The dataset is for the task of abstractive summarization in its extreme form, its about summarizing a document in a single sentence. It introduces extreme summarization, a new single-document summarization task which does not favor extractive strategies and calls for an abstractive modeling approach. The idea is to create a short, one-sentence news summary answering the question "What is the article about?". + +This particularly uses the dataset that is part of the GEM benchmark +Homepage: https://github.com/EdinburghNLP/XSum +The GEM Benchmark: Natural Language Generation, its Evaluation and Metrics +https://arxiv.org/pdf/2102.01672v3.pdf +Write a Short Description of the task. +Homepage: https://gem-benchmark.com/data_cards/XSum +""" +from lm_eval.api.task import PromptSourceTask + + +_CITATION = """ +@InProceedings{xsum-emnlp, + author = "Shashi Narayan and Shay B. Cohen and Mirella Lapata", + title = "Don't Give Me the Details, Just the Summary! {T}opic-Aware Convolutional Neural Networks for Extreme Summarization", + booktitle = "Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing ", + year = "2018", + address = "Brussels, Belgium", +} +""" + + +class GEMXSUMBase(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "GEM/xsum" + DATASET_NAME = None + SPLIT = None + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return True + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["validation"] + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["test"] + + def max_generation_length(self): + return 64 + + +class GEMXSUM(GEMXSUMBase): + """this is for train/validation/test""" + + SPLIT = "" + + +class GEMXSUMChallgeSample(GEMXSUMBase): + """this is for challenge_train_sample/challenge_validation_sample""" + + SPLIT = "challenge_sample" + + def has_test_docs(self): + return False + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["challenge_train_sample"] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["challenge_validation_sample"] + + +class GEMXSUMChallgeTestBacktranslation(GEMXSUMBase): + """this is for challenge_test_backtranslation""" + + SPLIT = "challenge_test_backtranslation" + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return False + + def test_docs(self): + if self.has_test_docs(): + return self.dataset[self.SPLIT] + + +class GEMXSUMChallgeTestBFP02(GEMXSUMBase): + """this is for challenge_test_bfp_02""" + + SPLIT = "challenge_test_bfp_02" + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return False + + def test_docs(self): + if self.has_test_docs(): + return self.dataset[self.SPLIT] + + +class GEMXSUMChallgeTestBFP05(GEMXSUMBase): + """this is for challenge_test_bfp_05""" + + SPLIT = "challenge_test_bfp_05" + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return False + + def test_docs(self): + if self.has_test_docs(): + return self.dataset[self.SPLIT] + + +class GEMXSUMChallgeTestNopunc(GEMXSUMBase): + """this is for challenge_test_nopunc""" + + SPLIT = "challenge_test_nopunc" + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return False + + def test_docs(self): + if self.has_test_docs(): + return self.dataset[self.SPLIT] + + +class GEMXSUMChallgeTestCovid(GEMXSUMBase): + """this is for challenge_test_covid""" + + SPLIT = "challenge_test_covid" + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return False + + def test_docs(self): + if self.has_test_docs(): + return self.dataset[self.SPLIT] diff --git a/evaluation-pipeline/lm_eval/tasks/glue.py b/evaluation-pipeline/lm_eval/tasks/glue.py new file mode 100644 index 0000000000000000000000000000000000000000..dae2a932233441a1152f1d509c2cce5bbb471ad7 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/glue.py @@ -0,0 +1,297 @@ +""" +GLUE: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding +https://openreview.net/pdf?id=rJ4km2R5t7 + +The General Language Understanding Evaluation (GLUE) benchmark is a collection of +resources for training, evaluating, and analyzing natural language understanding +systems. GLUE consists of: +- A benchmark of nine sentence- or sentence-pair language understanding tasks built +on established existing datasets and selected to cover a diverse range of dataset +sizes, text genres, and degrees of difficulty, and +- A diagnostic dataset designed to evaluate and analyze model performance with +respect to a wide range of linguistic phenomena found in natural language. + +Homepage: https://gluebenchmark.com/ +""" +from lm_eval.api.task import PromptSourceTask + + +# TODO(jon-tow): Add citations for the individual datasets/tasks that make up GLUE. +_CITATION = """ +@inproceedings{wang-etal-2018-glue, + title = "{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding", + author = "Wang, Alex and + Singh, Amanpreet and + Michael, Julian and + Hill, Felix and + Levy, Omer and + Bowman, Samuel", + booktitle = "Proceedings of the 2018 {EMNLP} Workshop {B}lackbox{NLP}: Analyzing and Interpreting Neural Networks for {NLP}", + month = nov, + year = "2018", + address = "Brussels, Belgium", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/W18-5446", + doi = "10.18653/v1/W18-5446", + pages = "353--355", + abstract = "Human ability to understand language is \textit{general, flexible, and robust}. In contrast, most NLU models above the word level are designed for a specific task and struggle with out-of-domain data. If we aspire to develop models with understanding beyond the detection of superficial correspondences between inputs and outputs, then it is critical to develop a unified model that can execute a range of linguistic tasks across different domains. To facilitate research in this direction, we present the General Language Understanding Evaluation (GLUE, gluebenchmark.com): a benchmark of nine diverse NLU tasks, an auxiliary dataset for probing models for understanding of specific linguistic phenomena, and an online platform for evaluating and comparing models. For some benchmark tasks, training data is plentiful, but for others it is limited or does not match the genre of the test set. GLUE thus favors models that can represent linguistic knowledge in a way that facilitates sample-efficient learning and effective knowledge-transfer across tasks. While none of the datasets in GLUE were created from scratch for the benchmark, four of them feature privately-held test data, which is used to ensure that the benchmark is used fairly. We evaluate baselines that use ELMo (Peters et al., 2018), a powerful transfer learning technique, as well as state-of-the-art sentence representation models. The best models still achieve fairly low absolute scores. Analysis with our diagnostic dataset yields similarly weak performance over all phenomena tested, with some exceptions.", +} +""" + + +# Single-Sentence Tasks + + +class CoLA(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "glue" + DATASET_NAME = "cola" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + return self.dataset["validation"] + + +class SST(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "glue" + DATASET_NAME = "sst2" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + return self.dataset["validation"] + + +# Inference Tasks + + +class MNLI(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "glue" + DATASET_NAME = "mnli" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["validation_matched"] + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["test_matched"] + + +class MNLIMismatched(MNLI): + VERSION = 0 + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["validation_mismatched"] + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["test_mismatched"] + + +class QNLI(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "glue" + DATASET_NAME = "qnli" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + return self.dataset["validation"] + + +class WNLI(PromptSourceTask): + VERSION = 1 + DATASET_PATH = "glue" + DATASET_NAME = "wnli" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + return self.dataset["validation"] + + +class RTE(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "glue" + DATASET_NAME = "rte" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + return self.dataset["validation"] + + +# Similarity and Paraphrase Tasks + + +class MRPC(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "glue" + DATASET_NAME = "mrpc" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + return self.dataset["validation"] + + def invalid_doc_for_prompt(self, doc) -> bool: + if ( + # generate_paraphrase for mrpc + # This generation prompt assumes a positive example. We filter out the negative examples. + # https://github.com/bigscience-workshop/promptsource/blob/ba8c9eccbe82f2409208c655896f1dd131171ece/promptsource/templates/glue/mrpc/templates.yaml#L7 + # https://github.com/bigscience-workshop/promptsource/blob/ba8c9eccbe82f2409208c655896f1dd131171ece/promptsource/templates/glue/mrpc/templates.yaml#L88 + ( + self.prompt_template.id == "3b88d2c4-0aeb-4c6d-9ccc-653a388250a5" + or self.prompt_template.id == "d830d7a5-abc0-4275-ac62-974e0088876f" + ) + and doc["label"] == 0 + ): + return True + return False + + +class QQP(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "glue" + DATASET_NAME = "qqp" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + return self.dataset["validation"] + + +class STSB(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "glue" + DATASET_NAME = "stsb" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return True + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + return self.dataset["validation"] + + def test_docs(self): + return self.dataset["test"] + + def doc_to_text(self, doc): + return "sentence 1: {}\nsentence 2: {}\nAnswer:".format( + doc["sentence1"], + doc["sentence2"], + ) + + def doc_to_target(self, doc): + return " {}".format(doc["label"]) + + def construct_requests(self, doc, ctx, args): + # TODO: implement evaluation. + raise NotImplementedError("Evaluation not implemented") + + def process_results(self, doc, results): + # TODO: implement evaluation. + raise NotImplementedError("Evaluation not implemented") + + def aggregation(self): + # TODO: implement evaluation. + raise NotImplementedError("Evaluation not implemented") + + def higher_is_better(self): + # TODO: implement evaluation. + raise NotImplementedError("Evaluation not implemented") diff --git a/evaluation-pipeline/lm_eval/tasks/hans.py b/evaluation-pipeline/lm_eval/tasks/hans.py new file mode 100644 index 0000000000000000000000000000000000000000..15802fabc8fe33662985bcc00171618880e95348 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/hans.py @@ -0,0 +1,62 @@ +""" +Right for the Wrong Reasons: Diagnosing Syntactic Heuristics in Natural Language Inference +https://arxiv.org/abs/1902.01007 + +A controlled evaluation set called HANS (Heuristic Analysis for NLI Systems), +which contains many examples where the heuristics fail. + +Homepage: https://github.com/tommccoy1/hans +""" +from lm_eval.api.task import PromptSourceTask + + +_CITATION = """ +@inproceedings{mccoy-etal-2019-right, + title = "Right for the Wrong Reasons: Diagnosing Syntactic Heuristics in Natural Language Inference", + author = "McCoy, Tom and + Pavlick, Ellie and + Linzen, Tal", + booktitle = "Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics", + month = jul, + year = "2019", + address = "Florence, Italy", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/P19-1334", + doi = "10.18653/v1/P19-1334", + pages = "3428--3448", + abstract = "A machine learning system can score well on a given test set by relying on heuristics that are effective for frequent example types but break down in more challenging cases. We study this issue within natural language inference (NLI), the task of determining whether one sentence entails another. We hypothesize that statistical NLI models may adopt three fallible syntactic heuristics: the lexical overlap heuristic, the subsequence heuristic, and the constituent heuristic. To determine whether models have adopted these heuristics, we introduce a controlled evaluation set called HANS (Heuristic Analysis for NLI Systems), which contains many examples where the heuristics fail. We find that models trained on MNLI, including BERT, a state-of-the-art model, perform very poorly on HANS, suggesting that they have indeed adopted these heuristics. We conclude that there is substantial room for improvement in NLI systems, and that the HANS dataset can motivate and measure progress in this area.", +} +""" + + +class HANS(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "hans" + DATASET_NAME = None + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["validation"] + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["test"] + + def process_results(self, doc, results): + out, log = super().process_results(doc, results) + log["subcase"] = doc["subcase"] + log["heuristic"] = doc["heuristic"] + return out, log diff --git a/evaluation-pipeline/lm_eval/tasks/huff_post.py b/evaluation-pipeline/lm_eval/tasks/huff_post.py new file mode 100644 index 0000000000000000000000000000000000000000..70d0a43f3e95c801f7ab15553adb5b8712f53488 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/huff_post.py @@ -0,0 +1,53 @@ +""" +A dataset of approximately 200K news headlines from the year 2012 to 2018 collected from HuffPost. + +Homepage: https://www.kaggle.com/datasets/rmisra/news-category-dataset +""" +from lm_eval.api.task import PromptSourceTask + + +_CITATION = """\ +@book{book, + author = {Misra, Rishabh and Grover, Jigyasa}, + year = {2021}, + month = {01}, + pages = {}, + title = {Sculpting Data for ML: The first act of Machine Learning}, + isbn = {978-0-578-83125-1} +} +@dataset{dataset, + author = {Misra, Rishabh}, + year = {2018}, + month = {06}, + pages = {}, + title = {News Category Dataset}, + doi = {10.13140/RG.2.2.20331.18729} +} +""" + + +class HuffPost(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "khalidalt/HuffPost" + DATASET_NAME = None + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return False + + def has_test_docs(self): + return True + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["validation"] + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["test"] diff --git a/evaluation-pipeline/lm_eval/tasks/jigsaw_unintended_bias.py b/evaluation-pipeline/lm_eval/tasks/jigsaw_unintended_bias.py new file mode 100644 index 0000000000000000000000000000000000000000..d9f98a591cc6c81879b9e039db5b89cb8d31f03e --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/jigsaw_unintended_bias.py @@ -0,0 +1,185 @@ +""" +Jigsaw unintended bias in toxicity classification +https://www.kaggle.com/c/jigsaw-unintended-bias-in-toxicity-classification + +Jigsaw Toxicity is a dataset curated by Alphabet from the now-defunct Civil Comments platform. It is used +to measure bias in toxicity classification models, specifically with equalized odds. In the equalized odds fairness notion, models +that are less biased have smaller differences in False Positive Rate (FPR). Intuitively, this means that models do not +unjustly mislabel text with mentions of particular demographics as toxic. + +Homepage: https://www.kaggle.com/c/jigsaw-unintended-bias-in-toxicity-classification +""" +import inspect +import os +import numpy as np +import pandas as pd + +import lm_eval.datasets.jigsaw_unintended_bias.jigsaw_unintended_bias +from lm_eval.api.task import PromptSourceTask + + +_CITATION = """ +https://www.kaggle.com/c/jigsaw-unintended-bias-in-toxicity-classification +""" + + +class JigsawUnintendedBias(PromptSourceTask): + """ + Jigsaw Bias implementation. + + TODO: This implementation isn't correct for prompts that have >2 categorical outputs, since we + need to further process false positive and true negatives for instances with non-binary categorical + outputs. + """ + + VERSION = 0 + DATASET_PATH = "jigsaw_unintended_bias" + DATASET_NAME = None + dimension_to_identity_set = { + "race": ["black", "white", "asian", "latino", "other_race_or_ethnicity"], + "gender_nonbinary": ["male", "female", "transgender", "other_gender"], + } + + def __init__( + self, + data_dir=os.path.dirname( + os.path.abspath( + inspect.getfile( + lm_eval.datasets.jigsaw_unintended_bias.jigsaw_unintended_bias + ) + ) + ), + **kwargs, + ): + """ + Jigsaw Toxicity must be downloaded from Kaggle from this url: + https://www.kaggle.com/c/jigsaw-unintended-bias-in-toxicity-classification/data + """ + super().__init__(data_dir=data_dir, **kwargs) + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return False + + def has_test_docs(self): + return True + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"] + + def validation_docs(self): + pass + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["test_public_leaderboard"] + + def process_results(self, doc, results): + answer_choices_list = self.prompt_template.get_answer_choices_list(doc) + target = self.doc_to_target(doc) + assert isinstance(target, list) and len(target) == 1 + target = target[0].strip() + + out = {} + pred = float(np.argmax(results)) + if answer_choices_list: + target = answer_choices_list.index(target) + else: + # Handles situation where the model is asked for a numerical answer + # TODO: current solution is not ideal for scaling + try: + scale = float(target) / doc["target"] + pred = float((pred / scale) >= 0.5) + target = float(doc["target"] >= 0.5) + except ZeroDivisionError: + target = doc["target"] + tp, tn, fn, fp = False, False, False, False + acc = pred == target + if acc: + if target: + tp = True + else: + tn = True + else: + if target: + fn = True + else: + fp = True + for dimension, identity_set in self.dimension_to_identity_set.items(): + identities = { + identity: 1 if doc[identity] > 0 else 0 for identity in identity_set + } + out[f"{dimension}_var"] = {**identities, "pred": pred, "target": target} + out[f"{dimension}_std"] = {**identities, "pred": pred, "target": target} + for identity in identity_set: + if doc[identity]: + out[f"{identity}_acc"] = acc + out[f"{identity}_tp"] = tp + out[f"{identity}_tn"] = tn + out[f"{identity}_fp"] = fp + out[f"{identity}_fn"] = fn + else: + out[f"{identity}_acc"] = np.nan + out[f"{identity}_tp"] = np.nan + out[f"{identity}_tn"] = np.nan + out[f"{identity}_fp"] = np.nan + out[f"{identity}_fn"] = np.nan + + # TODO: Wrap process results s.t. override impl do not + # override the save examples. + if self.save_examples: + pred = pred if answer_choices_list else float(pred) + example = { + "pred": pred, + "target": target, + "answer_choices_list": answer_choices_list, + } + return out, example + return out + + def calculate_discrepencies(self, docs, identity_set, aggregation_func): + identity_to_fpr = {} + prompt_pd = pd.DataFrame(docs) + for identity in identity_set: + identity_pd = prompt_pd[prompt_pd[identity] == 1] + if len(identity_pd) == 0: + continue + fp = identity_pd.apply(lambda x: x.pred and not x.target, axis=1).sum() + tn = identity_pd.apply(lambda x: not x.pred and not x.target, axis=1).sum() + if float(fp) + float(tn) == 0: + identity_to_fpr[f"{identity}_fpr"] = 0 + else: + identity_to_fpr[f"{identity}_fpr"] = float(fp) / (float(fp) + float(tn)) + if aggregation_func == "var": + func = np.var + else: + func = np.std + return float(func(list(identity_to_fpr.values()))) + + def aggregation(self): + out = {} + race_identities = self.dimension_to_identity_set["race"] + gender_nonbinary_identities = self.dimension_to_identity_set["gender_nonbinary"] + out["race_var"] = lambda x: self.calculate_discrepencies( + x, race_identities, "var" + ) + out["race_std"] = lambda x: self.calculate_discrepencies( + x, race_identities, "std" + ) + out["gender_nonbinary_var"] = lambda x: self.calculate_discrepencies( + x, gender_nonbinary_identities, "var" + ) + out["gender_nonbinary_std"] = lambda x: self.calculate_discrepencies( + x, gender_nonbinary_identities, "std" + ) + for dimension, identity_set in self.dimension_to_identity_set.items(): + for identity in identity_set: + out[f"{identity}_acc"] = lambda x: float(np.nanmean(x)) + out[f"{identity}_tp"] = lambda x: float(np.nansum(x)) + out[f"{identity}_tn"] = lambda x: float(np.nansum(x)) + out[f"{identity}_fp"] = lambda x: float(np.nansum(x)) + out[f"{identity}_fn"] = lambda x: float(np.nansum(x)) + return out diff --git a/evaluation-pipeline/lm_eval/tasks/lama.py b/evaluation-pipeline/lm_eval/tasks/lama.py new file mode 100644 index 0000000000000000000000000000000000000000..48597ec5c364e62897d35e2347d59f55d67f23f8 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/lama.py @@ -0,0 +1,259 @@ +""" +https://arxiv.org/abs/1909.01066 +https://arxiv.org/abs/2005.04611 +LAMA is a prob dataset to test the factual and commonsense knowledge in language models. The dataset includes a subset of +Google_RE (https://code.google.com/archive/p/relation-extraction-corpus/), TRex (subset of wikidata triples), +Conceptnet (https://github.com/commonsense/conceptnet5/wiki) and Squad. + +Homepage: https://github.com/facebookresearch/LAMA +""" +from typing import Optional + +from lm_eval.api.task import PromptSourceTask +from lm_eval.api.metric import mean + + +_CITATION = """ +@inproceedings{petroni2019language, title={Language Models as Knowledge Bases?}, + author={F. Petroni, T. Rockt{"{a}}schel, A. H. Miller, P. Lewis, A. Bakhtin, Y. Wu and S. Riedel}, + booktitle={In: Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing (EMNLP), 2019}, year={2019} } + +@inproceedings{petroni2020how, + title={How Context Affects Language Models' Factual Predictions}, + author={Fabio Petroni and Patrick Lewis and Aleksandra Piktus and Tim Rockt{"a}schel and Yuxiang Wu and Alexander H. Miller and Sebastian Riedel}, + booktitle={Automated Knowledge Base Construction}, year={2020}, url={https://openreview.net/forum?id=025X0zPfn} } +""" + + +class BigScienceLAMA(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "janck/bigscience-lama" + DATASET_NAME = None + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return False + + def has_test_docs(self): + return True + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"] + + +class Trex(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "lama" + DATASET_NAME = "trex" + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return False + + def has_test_docs(self): + # TODO: Fill in the return with `True` if the Task has test data; else `False`. + return True + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["validation"] + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["train"] + + def process_results(self, doc, results): + out = {} + # gold = doc + pred = results[0].strip() + target = self.doc_to_target(doc)["obj_label"] + out["acc"] = pred == target + + if self.save_examples: + example = { + "pred": pred, + "target": target, + } + return out, example + + return out + + def higher_is_better(self): + return {"acc": True} + + def aggregation(self): + return {"acc": mean} + + def doc_to_target(self, doc): + return doc + + +class google_re(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "lama" + DATASET_NAME = "google_re" + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return False + + def has_test_docs(self): + return True + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["validation"] + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["train"] + + def process_results(self, doc, results): + out = {} + pred = results[0].strip() + + target = self.doc_to_target(doc)["obj_label"] + out["acc"] = pred == target + + if self.save_examples: + example = { + "pred": pred, + "target": target, + } + return out, example + + return out + + def higher_is_better(self): + return {"acc": True} + + def aggregation(self): + return {"acc": mean} + + def doc_to_target(self, doc): + return doc + + +class Conceptnet(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "lama" + DATASET_NAME = "conceptnet" + + def has_training_docs(self): + # TODO: Fill in the return with `True` if the Task has training data; else `False`. + return False + + def has_validation_docs(self): + # TODO: Fill in the return with `True` if the Task has validation data; else `False`. + return False + + def has_test_docs(self): + # TODO: Fill in the return with `True` if the Task has test data; else `False`. + return True + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["validation"] + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["train"] + + def process_results(self, doc, results): + out = {} + pred = results[0].strip() + + target = self.doc_to_target(doc)["obj_label"] + out["acc"] = pred == target + + if self.save_examples: + example = { + "pred": pred, + "target": target, + } + return out, example + + return out + + def higher_is_better(self): + return {"acc": True} + + def aggregation(self): + return {"acc": mean} + + def doc_to_target(self, doc): + return doc + + +class Squad(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "lama" + DATASET_NAME = "squad" + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return False + + def has_test_docs(self): + return True + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["validation"] + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["train"] + + def process_results(self, doc, results): + out = {} + pred = results[0].strip() + target = self.doc_to_target(doc)["obj_label"] + out["acc"] = pred == target + + if self.save_examples: + example = { + "pred": pred, + "target": target, + } + return out, example + + return out + + def higher_is_better(self): + return {"acc": True} + + def aggregation(self): + return {"acc": mean} + + def doc_to_target(self, doc): + return doc + + def max_generation_length(self) -> Optional[int]: + """Denote where the max length of the generation if it is obvious from the task.""" + return 5 diff --git a/evaluation-pipeline/lm_eval/tasks/lince.py b/evaluation-pipeline/lm_eval/tasks/lince.py new file mode 100644 index 0000000000000000000000000000000000000000..3bdf63d0c759c2cc112ca9ba388742ffd1a0f6a9 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/lince.py @@ -0,0 +1,60 @@ +""" +LinCE: A Centralized Benchmark for Linguistic Code-switching Evaluation +https://aclanthology.org/2020.lrec-1.223.pdf + +A centralized benchmark for Linguistic Code-switching Evaluation (LinCE) which contains tasks for different +code-switched language pairs. The code below contains evaluation for sentiment analysis task. + +Homepage: https://ritual.uh.edu/lince/datasets +""" +from lm_eval.api.task import PromptSourceTask + + +_CITATION = """ +@inproceedings{aguilar-etal-2020-lince, + title = "{L}in{CE}: A Centralized Benchmark for Linguistic Code-switching Evaluation", + author = "Aguilar, Gustavo and + Kar, Sudipta and + Solorio, Thamar", + booktitle = "Proceedings of the 12th Language Resources and Evaluation Conference", + month = may, + year = "2020", + address = "Marseille, France", + publisher = "European Language Resources Association", + url = "https://aclanthology.org/2020.lrec-1.223", + pages = "1803--1813", + abstract = "Recent trends in NLP research have raised an interest in linguistic code-switching (CS); modern approaches have been proposed to solve a wide range of NLP tasks on multiple language pairs. Unfortunately, these proposed methods are hardly generalizable to different code-switched languages. In addition, it is unclear whether a model architecture is applicable for a different task while still being compatible with the code-switching setting. This is mainly because of the lack of a centralized benchmark and the sparse corpora that researchers employ based on their specific needs and interests. To facilitate research in this direction, we propose a centralized benchmark for Linguistic Code-switching Evaluation (LinCE) that combines eleven corpora covering four different code-switched language pairs (i.e., Spanish-English, Nepali-English, Hindi-English, and Modern Standard Arabic-Egyptian Arabic) and four tasks (i.e., language identification, named entity recognition, part-of-speech tagging, and sentiment analysis). As part of the benchmark centralization effort, we provide an online platform where researchers can submit their results while comparing with others in real-time. In addition, we provide the scores of different popular models, including LSTM, ELMo, and multilingual BERT so that the NLP community can compare against state-of-the-art systems. LinCE is a continuous effort, and we will expand it with more low-resource languages and tasks.", + language = "English", + ISBN = "979-10-95546-34-4", +} +""" + + +class LinCE(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "lince" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["validation"] + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["test"] + + +class LinCESentimentAnalysis(LinCE): + DATASET_NAME = "sa_spaeng" diff --git a/evaluation-pipeline/lm_eval/tasks/olmpics.py b/evaluation-pipeline/lm_eval/tasks/olmpics.py new file mode 100644 index 0000000000000000000000000000000000000000..28ef7a26be6d72c597b4dbe9e43eaad2548cc140 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/olmpics.py @@ -0,0 +1,104 @@ +""" +BLiMP: A Benchmark of Linguistic Minimal Pairs for English +https://arxiv.org/abs/1912.00582 + +BLiMP is a challenge set for evaluating what language models (LMs) know about +major grammatical phenomena in English. BLiMP consists of 67 sub-datasets, each +containing 1000 minimal pairs isolating specific contrasts in syntax, morphology, +or semantics. The data is automatically generated according to expert-crafted +grammars. + +Homepage: https://github.com/alexwarstadt/blimp +""" +from lm_eval.api.task import PromptSourceTask +from typing import Optional, List +from datasets import load_dataset + + +_CITATION = """ +@article{warstadt2019blimp, + author = {Warstadt, Alex and Parrish, Alicia and Liu, Haokun and Mohananey, Anhad and Peng, Wei and Wang, Sheng-Fu and Bowman, Samuel R.}, + title = {BLiMP: The Benchmark of Linguistic Minimal Pairs for English}, + journal = {Transactions of the Association for Computational Linguistics}, + volume = {8}, + number = {}, + pages = {377-392}, + year = {2020}, + doi = {10.1162/tacla00321}, + URL = {https://doi.org/10.1162/tacl_a_00321}, + eprint = {https://doi.org/10.1162/tacl_a_00321}, + abstract = { We introduce The Benchmark of Linguistic Minimal Pairs (BLiMP),1 a challenge set for evaluating the linguistic knowledge of language models (LMs) on major grammatical phenomena in English. BLiMP consists of 67 individual datasets, each containing 1,000 minimal pairs—that is, pairs of minimally different sentences that contrast in grammatical acceptability and isolate specific phenomenon in syntax, morphology, or semantics. We generate the data according to linguist-crafted grammar templates, and human aggregate agreement with the labels is 96.4. We evaluate n-gram, LSTM, and Transformer (GPT-2 and Transformer-XL) LMs by observing whether they assign a higher probability to the acceptable sentence in each minimal pair. We find that state-of-the-art models identify morphological contrasts related to agreement reliably, but they struggle with some subtle semantic and syntactic phenomena, such as negative polarity items and extraction islands. } +} +""" + +class OlmpicsTask(PromptSourceTask): + DATASET_PATH = "KevinZ/oLMpics" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + # The HF dataset only contains a "train" dataset, but the harness expects a "validation" + # dataset. Let's use the training dataset, on the assumption that the model wasn't actually + # trained on this data. + return self.dataset["test"] + + def null_prompt_doc_to_text(self, doc: dict) -> str: + return "" + + def null_prompt_doc_to_target(self, doc: dict) -> List[str]: + correct_choice = doc["choices"][doc["answerKey"]] + formatted = doc["stem"].replace("[MASK]", correct_choice).strip() + return [formatted] + + def null_prompt_answer_choices(self, doc: dict) -> List[str]: + choices = doc["choices"] + formatted = [doc["stem"].replace("[MASK]", choice).strip() for choice in choices] + return formatted + + def null_prompt_get_logging_info(self): + return { + "fixed_answer_choice_list": None, + "dataset_path": self.DATASET_PATH, + "dataset_name": self.DATASET_NAME, + "subset": self.SPLIT, + "prompt_name": None, + "prompt_id": None, + "prompt_jinja": None, + "prompt_original_task": f"{self.DATASET_PATH}/{self.DATASET_NAME}", + # Placeholder for comment in post-processing. + "comment": "", + } + + +class OlmpicsAgeComparison(OlmpicsTask): + DATASET_NAME = "Age_Comparison" + + +class OlmpicsAlwaysNever(OlmpicsTask): + DATASET_NAME = "Always_Never" + + +class OlmpicsMultihopComposition(OlmpicsTask): + DATASET_NAME = "Multihop_Composition" + + +class OlmpicsObjectComparison(OlmpicsTask): + DATASET_NAME = "Object_Comparison" + + +class OlmpicsPropertyConjunction(OlmpicsTask): + DATASET_NAME = "Property_Conjunction" + + +class OlmpicsTaxonomyConjunction(OlmpicsTask): + DATASET_NAME = "Taxonomy_Conjunction" \ No newline at end of file diff --git a/evaluation-pipeline/lm_eval/tasks/piaf.py b/evaluation-pipeline/lm_eval/tasks/piaf.py new file mode 100644 index 0000000000000000000000000000000000000000..1ff96ea679e548027c69874cc7a2d23e64018d8a --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/piaf.py @@ -0,0 +1,99 @@ +""" +Project PIAF: Building a Native French Question-Answering Dataset +https://arxiv.org/pdf/2007.00968.pdf + +Piaf is a reading comprehension dataset. This version, published in February 2020, +contains 3835 questions on French Wikipedia. + +Homepage: https://huggingface.co/datasets/piaf +""" +import transformers.data.metrics.squad_metrics as squad_metrics + +from lm_eval.api.task import PromptSourceTask +from lm_eval.api.metric import mean + + +_CITATION = """ +@InProceedings{keraron-EtAl:2020:LREC, + author = {Keraron, Rachel and Lancrenon, Guillaume and Bras, Mathilde and Allary, Frédéric and Moyse, Gilles and Scialom, Thomas and Soriano-Morales, Edmundo-Pavel and Staiano, Jacopo}, + title = {Project PIAF: Building a Native French Question-Answering Dataset}, + booktitle = {Proceedings of The 12th Language Resources and Evaluation Conference}, + month = {May}, + year = {2020}, + address = {Marseille, France}, + publisher = {European Language Resources Association}, + pages = {5483--5492}, + abstract = {Motivated by the lack of data for non-English languages, in particular for the evaluation of downstream tasks such as Question Answering, we present a participatory effort to collect a native French Question Answering Dataset. Furthermore, we describe and publicly release the annotation tool developed for our collection effort, along with the data obtained and preliminary baselines.}, + url = {https://www.aclweb.org/anthology/2020.lrec-1.673} +} +""" + + +class PIAF(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "piaf" + DATASET_NAME = None + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return False + + def has_test_docs(self): + return True + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["validation"] + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["train"] + + def max_generation_length(self): + return 128 + + @staticmethod + def compute_scores(doc, pred): + # tests for exact match and on the normalised answer (compute_exact) + # test for overlap (compute_f1) + + em_sum = squad_metrics.compute_exact(doc[0], pred) + f1_sum = squad_metrics.compute_f1(doc[0], pred) + + return { + "em": em_sum, + "f1": f1_sum, + } + + def process_results(self, doc, results): + targets = self.doc_to_target(doc) + pred = results[0].strip().split("\n")[0] + scores = self.compute_scores(targets, pred) + + out = { + "f1": scores["f1"], + "em": scores["em"], + } + + if self.save_examples: + example = {"target": targets, "pred": pred} + return out, example + return out + + def higher_is_better(self): + return { + "f1": True, + "em": True, + } + + def aggregation(self): + return { + "f1": mean, + "em": mean, + } diff --git a/evaluation-pipeline/lm_eval/tasks/race.py b/evaluation-pipeline/lm_eval/tasks/race.py new file mode 100644 index 0000000000000000000000000000000000000000..635bd708ccda3fa5cc92c0d654c3b1376e5f6ff3 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/race.py @@ -0,0 +1,46 @@ +""" +RACE: Large-scale ReAding Comprehension Dataset From Examinations +https://arxiv.org/pdf/1704.04683.pdf + +RACE is a large-scale reading comprehension dataset with more than 28,000 passages +and nearly 100,000 questions. The dataset is collected from English examinations +in China, which are designed for middle school and high school students. The dataset +can be served as the training and test sets for machine comprehension. + +Homepage: https://www.cs.cmu.edu/~glai1/data/race/ +""" +from lm_eval.api.task import PromptSourceTask + + +_CITATION = """ +@article{lai2017large, + title={RACE: Large-scale ReAding Comprehension Dataset From Examinations}, + author={Lai, Guokun and Xie, Qizhe and Liu, Hanxiao and Yang, Yiming and Hovy, Eduard}, + journal={arXiv preprint arXiv:1704.04683}, + year={2017} +} +""" + + +class RACE(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "race" + DATASET_NAME = "high" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return True + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + return self.dataset["validation"] + + def test_docs(self): + return self.dataset["test"] diff --git a/evaluation-pipeline/lm_eval/tasks/schema_guided_dstc8.py b/evaluation-pipeline/lm_eval/tasks/schema_guided_dstc8.py new file mode 100644 index 0000000000000000000000000000000000000000..c6e9937eb4c43228adecc930c7c3e15b52508cba --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/schema_guided_dstc8.py @@ -0,0 +1,53 @@ +""" +Towards Scalable Multi-domain Conversational Agents: The Schema-Guided Dialogue Dataset +https://arxiv.org/abs/1909.05855 + +Multi-domain, task-oriented conversations created for the DSTC8 challenge. +Here, the dataset is be used for evaluating response generation. + +Homepage: https://github.com/google-research-datasets/dstc8-schema-guided-dialogue +""" +from lm_eval.api.task import PromptSourceTask + + +_CITATION = """ +@inproceedings{rastogi2020towards, + title={Towards scalable multi-domain conversational agents: The schema-guided dialogue dataset}, + author={Rastogi, Abhinav and Zang, Xiaoxue and Sunkara, Srinivas and Gupta, Raghav and Khaitan, Pranav}, + booktitle={Proceedings of the AAAI Conference on Artificial Intelligence}, + volume={34}, + number={05}, + pages={8689--8696}, + year={2020} +} +""" + + +class Schema_Guided_DSTC8(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "schema_guided_dstc8" + DATASET_NAME = "dialogues" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return True + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["validation"] + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["test"] + + def max_generation_length(self): + return 64 diff --git a/evaluation-pipeline/lm_eval/tasks/superglue.py b/evaluation-pipeline/lm_eval/tasks/superglue.py new file mode 100644 index 0000000000000000000000000000000000000000..62bd84c92bca548772c5f27d215b92c09b824701 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/superglue.py @@ -0,0 +1,361 @@ +""" +SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems +https://w4ngatang.github.io/static/papers/superglue.pdf + +SuperGLUE is a benchmark styled after GLUE with a new set of more difficult language +understanding tasks. + +Homepage: https://super.gluebenchmark.com/ + +TODO: WSC requires free-form generation. +""" +import numpy as np +import sklearn +import transformers.data.metrics.squad_metrics as squad_metrics + +from lm_eval.api.metric import mean, metric_max_over_ground_truths, parity +from lm_eval.api.task import PromptSourceTask + + +_CITATION = """ +@inproceedings{NEURIPS2019_4496bf24, + author = {Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel}, + booktitle = {Advances in Neural Information Processing Systems}, + editor = {H. Wallach and H. Larochelle and A. Beygelzimer and F. d\textquotesingle Alch\'{e}-Buc and E. Fox and R. Garnett}, + pages = {}, + publisher = {Curran Associates, Inc.}, + title = {SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems}, + url = {https://proceedings.neurips.cc/paper/2019/file/4496bf24afe7fab6f046bf4923da8de6-Paper.pdf}, + volume = {32}, + year = {2019} +} +""" + + +class BoolQ(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "super_glue" + DATASET_NAME = "boolq" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + return self.dataset["validation"] + + +# TODO: Check this works with all prompts. +class CommitmentBank(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "super_glue" + DATASET_NAME = "cb" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + return self.dataset["validation"] + + def process_results(self, doc, results): + # gold = doc["label"] + pred_idx = np.argmax(results) + answer_choices_list = self.prompt_template.get_answer_choices_list(doc) + pred = answer_choices_list[pred_idx] + target = self.doc_to_target(doc)[0] + answer2idx = {answer: i for i, answer in enumerate(answer_choices_list)} + target_idx = answer2idx[target] + + acc = 1.0 if pred_idx == target_idx else 0.0 + if self.save_examples: + return {"acc": acc, "f1": (pred_idx, target_idx)}, { + "pred": pred, + "target": target, + # json cannot handle int64 + "pred_idx": int(pred_idx), + "target_idx": int(target_idx), + } + + return {"acc": acc, "f1": (pred_idx, target_idx)} + + def higher_is_better(self): + return {"acc": True, "f1": True} + + @classmethod + def cb_multi_fi(cls, items): + preds, golds = zip(*items) + preds = np.array(preds) + golds = np.array(golds) + f11 = sklearn.metrics.f1_score(y_true=golds == 0, y_pred=preds == 0) + f12 = sklearn.metrics.f1_score(y_true=golds == 1, y_pred=preds == 1) + f13 = sklearn.metrics.f1_score(y_true=golds == 2, y_pred=preds == 2) + avg_f1 = mean([f11, f12, f13]) + return avg_f1 + + def aggregation(self): + return { + "acc": mean, + "f1": self.cb_multi_fi, + } + + +class Copa(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "super_glue" + DATASET_NAME = "copa" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + return self.dataset["validation"] + + def invalid_doc_for_prompt(self, doc) -> bool: + # HACK: Some copa templates have conditionals that ignore documents + # when the condition is not met, like `{if doc['question'] != \"cause\"}`. + # This means the prompt will never produce an input and target. + # TODO: Remove this when fixed in `promptsource` + try: + text, target = self.prompt_template.apply(doc) + return False + except Exception: + return True + + +# TODO: Check this works with all prompts. +class MultiRC(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "super_glue" + DATASET_NAME = "multirc" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + return self.dataset["validation"] + + +class ReCoRD(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "super_glue" + DATASET_NAME = "record" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + # In ReCoRD, each doc manifests multiple "examples" in the context of few shot example packing. + # Each doc consists of multiple answer candidates, each of which is scored yes/no. + return self.dataset["train"] + + def validation_docs(self): + # See: training_docs + return self.dataset["validation"] + + def process_results(self, doc, results): + # ReCoRD's evaluation is actually deceptively simple: + # - Pick the maximum likelihood prediction entity + # - Evaluate the accuracy and token F1 PER EXAMPLE + # - Average over all examples + pred_idx = np.argmax(results) + answer_choices_list = self.prompt_template.get_answer_choices_list(doc) + pred = answer_choices_list[pred_idx] + targets = self.doc_to_target(doc) + + f1 = metric_max_over_ground_truths(squad_metrics.compute_f1, pred, targets) + em = metric_max_over_ground_truths(squad_metrics.compute_exact, pred, targets) + out = {"f1": f1, "em": em} + if self.save_examples: + example = {"target": targets, "pred": pred} + return out, example + return out + + def higher_is_better(self): + return { + "f1": True, + "em": True, + } + + def aggregation(self): + return { + "f1": mean, + "em": mean, + } + + +class WordsInContext(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "super_glue" + DATASET_NAME = "wic" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + return self.dataset["validation"] + + +class SGWinogradSchemaChallenge(PromptSourceTask): + VERSION = 0 + # Note: This implementation differs from Fig G.32 because this is the SuperGLUE, + # binary version of the task. + DATASET_PATH = "super_glue" + DATASET_NAME = "wsc.fixed" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"].filter(lambda d: d["label"]) + + def validation_docs(self): + return self.dataset["validation"] + + +class WinogenderSchemaDiagnostics(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "super_glue" + DATASET_NAME = "axg" + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return False + + def has_test_docs(self): + return True + + def test_docs(self): + return self.dataset["test"] + + def process_results(self, doc, results): + answer_choices_list = self.prompt_template.get_answer_choices_list(doc) + completion_len = np.array([float(len(i)) for i in answer_choices_list]) + + target = self.doc_to_target(doc)[0].strip() + target_idx = answer_choices_list.index(target) + pred = answer_choices_list[np.argmax(results)] + + out = { + "parity": (doc["idx"], pred), + "acc": pred == target, + "acc_norm": 1.0 + if np.argmax(results / completion_len) == target_idx + else 0.0, + } + + if self.save_examples: + example = { + "target": target, + "answer_choices_list": answer_choices_list, + "pred": pred, + } + return out, example + return out + + def aggregation(self): + return {"parity": parity, "acc": mean, "acc_norm": mean} + + def higher_is_better(self): + return { + "parity": True, + "acc": True, + "acc_norm": True, + } + + +class BroadcoverageDiagnostics(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "super_glue" + DATASET_NAME = "axb" + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return False + + def has_test_docs(self): + return True + + def test_docs(self): + return self.dataset["test"] + + +class RTE(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "super_glue" + DATASET_NAME = "rte" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + return self.dataset["validation"] diff --git a/evaluation-pipeline/lm_eval/tasks/syllogisms.py b/evaluation-pipeline/lm_eval/tasks/syllogisms.py new file mode 100644 index 0000000000000000000000000000000000000000..5af6d9917926572930b2252fa7718380cf5f6a15 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/syllogisms.py @@ -0,0 +1,84 @@ +""" +BLiMP: A Benchmark of Linguistic Minimal Pairs for English +https://arxiv.org/abs/1912.00582 + +BLiMP is a challenge set for evaluating what language models (LMs) know about +major grammatical phenomena in English. BLiMP consists of 67 sub-datasets, each +containing 1000 minimal pairs isolating specific contrasts in syntax, morphology, +or semantics. The data is automatically generated according to expert-crafted +grammars. + +Homepage: https://github.com/alexwarstadt/blimp +""" +import ast +from lm_eval.api.task import PromptSourceTask +from typing import Optional, List +from datasets import load_dataset + + +_CITATION = """ +@article{warstadt2019blimp, + author = {Warstadt, Alex and Parrish, Alicia and Liu, Haokun and Mohananey, Anhad and Peng, Wei and Wang, Sheng-Fu and Bowman, Samuel R.}, + title = {BLiMP: The Benchmark of Linguistic Minimal Pairs for English}, + journal = {Transactions of the Association for Computational Linguistics}, + volume = {8}, + number = {}, + pages = {377-392}, + year = {2020}, + doi = {10.1162/tacla00321}, + URL = {https://doi.org/10.1162/tacl_a_00321}, + eprint = {https://doi.org/10.1162/tacl_a_00321}, + abstract = { We introduce The Benchmark of Linguistic Minimal Pairs (BLiMP),1 a challenge set for evaluating the linguistic knowledge of language models (LMs) on major grammatical phenomena in English. BLiMP consists of 67 individual datasets, each containing 1,000 minimal pairs—that is, pairs of minimally different sentences that contrast in grammatical acceptability and isolate specific phenomenon in syntax, morphology, or semantics. We generate the data according to linguist-crafted grammar templates, and human aggregate agreement with the labels is 96.4. We evaluate n-gram, LSTM, and Transformer (GPT-2 and Transformer-XL) LMs by observing whether they assign a higher probability to the acceptable sentence in each minimal pair. We find that state-of-the-art models identify morphological contrasts related to agreement reliably, but they struggle with some subtle semantic and syntactic phenomena, such as negative polarity items and extraction islands. } +} +""" + +class SyllogismsTask(PromptSourceTask): + DATASET_PATH = "syllogisms" + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def validation_docs(self): + # The HF dataset only contains a "train" dataset, but the harness expects a "validation" + # dataset. Let's use the training dataset, on the assumption that the model wasn't actually + # trained on this data. + return self.dataset["train"] + + def null_prompt_doc_to_text(self, doc: dict) -> str: + return "" + + def null_prompt_doc_to_target(self, doc: dict) -> List[str]: + correct_choice = ast.literal_eval(doc["correct_answer"])[0] + prefix = doc["input"] + formatted = f"{prefix}{correct_choice}" + return [formatted] + + def null_prompt_answer_choices(self, doc: dict) -> List[str]: + choices = ast.literal_eval(doc["continuations"]) + prefix = doc["input"] + formatted = [f"{prefix}{choice}" for choice in choices] + return formatted + + def null_prompt_get_logging_info(self): + return { + "fixed_answer_choice_list": None, + "dataset_path": self.DATASET_PATH, + "dataset_name": self.DATASET_NAME, + "subset": self.SPLIT, + "prompt_name": None, + "prompt_id": None, + "prompt_jinja": None, + "prompt_original_task": f"{self.DATASET_PATH}/{self.DATASET_NAME}", + # Placeholder for comment in post-processing. + "comment": "", + } + + +class SyllogismsNonsense(SyllogismsTask): + DATASET_NAME = "base" \ No newline at end of file diff --git a/evaluation-pipeline/lm_eval/tasks/tydiqa.py b/evaluation-pipeline/lm_eval/tasks/tydiqa.py new file mode 100644 index 0000000000000000000000000000000000000000..9623fd929c37756bafa01ccabf89ef913cfee480 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/tydiqa.py @@ -0,0 +1,162 @@ +""" +TyDi QA: A Benchmark for Information-Seeking Question Answering in Typologically Diverse Languages + +TyDi QA is a question answering dataset covering 11 typologically diverse languages with 200K question-answer pairs. + +Paper: https://arxiv.org/abs/2003.05002 +Homepage: https://ai.google.com/research/tydiqa +""" +from transformers.data.metrics.squad_metrics import compute_exact, compute_f1 + +from lm_eval.api.task import PromptSourceTask +from lm_eval.api.metric import mean + + +_CITATION = """ +@article{tydiqa, + title = {TyDi QA: A Benchmark for Information-Seeking Question Answering in Typologically Diverse Languages}, + author = {Jonathan H. Clark and Eunsol Choi and Michael Collins and Dan Garrette and Tom Kwiatkowski and Vitaly Nikolaev and Jennimaria Palomaki} + year = {2020}, + journal = {Transactions of the Association for Computational Linguistics} +} +""" + + +class TyDiQAPrimaryClassification(PromptSourceTask): + """ + This task uses the primary_task dataset and implements the classification portion of the Minimal Answer Span task. + Note: Promptsource currently filters out all non-English examples so this task only reports results on English. + """ + + VERSION = 0 + DATASET_PATH = "tydiqa" + DATASET_NAME = "primary_task" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + return self.dataset["validation"] + + def invalid_doc_for_prompt(self, doc) -> bool: + # HACK: Some templates have conditionals that ignore documents + # when the condition is not met, like `{if doc['question'] != \"cause\"}`. + # This means the prompt will never produce an input and target. + # TODO: Remove this when fixed in `promptsource` + try: + text, target = self.prompt_template.apply(doc) + return False + except Exception: + return True + + +class TyDiQAGoldPGeneration(PromptSourceTask): + """ + This task uses the Gold Passage (secondary_task) dataset and implements the Gold Passage task described in the paper, in addition to title and question generation tasks. + Note: Promptsource currently filters out all non-English examples so this task only reports results on English. + """ + + VERSION = 0 + DATASET_PATH = "tydiqa" + DATASET_NAME = "secondary_task" + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + return self.dataset["validation"] + + def max_generation_length(self): + return 128 + + def compute_score(self, targets, pred, metric=compute_exact, agg=max): + return agg([metric(t, pred) for t in targets]) + + def process_results(self, doc, results): + targets = self.doc_to_target(doc) + pred = results[0].strip() + example = {"target": targets, "pred": pred} + out = {} + + # Detect cases handled in superclass method + for metric in self.prompt_template.metadata.metrics: + if ( + metric + in self.CONFIGURED_RANKED_CHOICE_PS_METRICS + | self.CONFIGURED_GENERATION_PS_METRICS + ): + if self.save_examples: + super_out, super_example = super().process_results(doc, results) + example.update(super_example) + else: + super_out = super().process_results(doc, results) + out.update(super_out) + elif metric == "Squad": + # Otherwise implement SQuAD metric computations, based on PIAF's implementation + agg_exact_match = self.compute_score(targets, pred, compute_exact) + agg_f1 = self.compute_score(targets, pred, compute_f1) + out["f1"] = agg_f1 + out["exact_match"] = agg_exact_match + + if self.save_examples: + return out, example + + return out + + def higher_is_better(self): + out = {} + for metric in self.prompt_template.metadata.metrics: + if ( + metric + in self.CONFIGURED_RANKED_CHOICE_PS_METRICS + | self.CONFIGURED_GENERATION_PS_METRICS + ): + out.update(super().higher_is_better()) + elif metric == "Squad": + out["f1"] = True + out["exact_match"] = True + return out + + def aggregation(self): + out = {} + for metric in self.prompt_template.metadata.metrics: + if ( + metric + in self.CONFIGURED_RANKED_CHOICE_PS_METRICS + | self.CONFIGURED_GENERATION_PS_METRICS + ): + out.update(super().aggregation()) + elif metric == "Squad": + out["f1"] = mean + out["exact_match"] = mean + return out + + def invalid_doc_for_prompt(self, doc) -> bool: + # HACK: Some templates have conditionals that ignore documents + # when the condition is not met, like `{if doc['question'] != \"cause\"}`. + # This means the prompt will never produce an input and target. + # TODO: Remove this when fixed in `promptsource` + try: + # Ensure the `apply` returns 2 values. + text, target = self.prompt_template.apply(doc) + return False + except Exception: + return True diff --git a/evaluation-pipeline/lm_eval/tasks/wino_bias.py b/evaluation-pipeline/lm_eval/tasks/wino_bias.py new file mode 100644 index 0000000000000000000000000000000000000000..2c9e7ed3dfeade91d7b59d4780e196fd70073494 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/wino_bias.py @@ -0,0 +1,101 @@ +""" +Gender Bias in Coreference Resolution: Evaluation and Debiasing Methods +https://arxiv.org/abs/1804.06876 + +Winograd-schema evaluation of gendered coreference resolution. +The dataset contains pro-stereotypical and anti-stereotypical parts. The difference in accuracy for those two subsets +quatnifies bias. + +Homepage: https://uclanlp.github.io/corefBias/overview +""" +import transformers.data.metrics.squad_metrics as squad_metrics + +from lm_eval.api.metric import mean +from lm_eval.api.task import PromptSourceTask + + +_CITATION = """ +@inproceedings{zhao-etal-2018-gender, + title = "Gender Bias in Coreference Resolution: Evaluation and Debiasing Methods", + author = "Zhao, Jieyu and + Wang, Tianlu and + Yatskar, Mark and + Ordonez, Vicente and + Chang, Kai-Wei", + booktitle = "Proceedings of the 2018 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 2 (Short Papers)", + month = jun, + year = "2018", + address = "New Orleans, Louisiana", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/N18-2003", + doi = "10.18653/v1/N18-2003", + pages = "15--20", + abstract = "In this paper, we introduce a new benchmark for co-reference resolution focused on gender bias, WinoBias. Our corpus contains Winograd-schema style sentences with entities corresponding to people referred by their occupation (e.g. the nurse, the doctor, the carpenter). We demonstrate that a rule-based, a feature-rich, and a neural coreference system all link gendered pronouns to pro-stereotypical entities with higher accuracy than anti-stereotypical entities, by an average difference of 21.1 in F1 score. Finally, we demonstrate a data-augmentation approach that, in combination with existing word-embedding debiasing techniques, removes the bias demonstrated by these systems in WinoBias without significantly affecting their performance on existing datasets.", +} +""" + + +class WinoBias(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "wino_bias" + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return True + + def training_docs(self): + pass + + def validation_docs(self): + return self.dataset["validation"] + + def test_docs(self): + return self.dataset["test"] + + def process_results(self, doc, results): + answer_choices_list = self.prompt_template.get_answer_choices_list(doc) + target = self.doc_to_target(doc)[0].strip() + pred = " ".join(results[0].strip().split(" ")[: len(target.split(" "))]) + + # The original paper uses F1. In the case of exactly one predicted and one gold mention, + # F1 and exact match are equivalent. + em = squad_metrics.compute_exact(target, pred) + out = {"em": em} + + # TODO: Wrap process results s.t. override impl do not + # override the save examples. + if self.save_examples: + example = { + "pred": pred, + "target": target, + "answer_choices_list": answer_choices_list, + } + return out, example + return out + + def aggregation(self): + return {"em": mean} + + def higher_is_better(self): + return {"em": True} + + +class WinoBiasType1Pro(WinoBias): + DATASET_NAME = "type1_pro" + + +class WinoBiasType1Anti(WinoBias): + DATASET_NAME = "type1_anti" + + +class WinoBiasType2Pro(WinoBias): + DATASET_NAME = "type2_pro" + + +class WinoBiasType2Anti(WinoBias): + DATASET_NAME = "type2_anti" diff --git a/evaluation-pipeline/lm_eval/tasks/wmt.py b/evaluation-pipeline/lm_eval/tasks/wmt.py new file mode 100644 index 0000000000000000000000000000000000000000..5aea2f9dc1479da5696bb585369be6623b543d49 --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/wmt.py @@ -0,0 +1,96 @@ +""" +WMT: Workshop on Statistical Machine Translation + +WMT is the main event for machine translation and machine translation research. +The conference is held annually in connection with larger conferences on natural +language processing. + +Homepage: https://machinetranslate.org/wmt +""" +import promptsource.utils +from typing import Dict, List, Optional + +from lm_eval.api.task import TranslationTask + + +# TODO: Add each WMT year BibTeX citation. +_CITATION = """ +""" + + +class WMTBase(TranslationTask): + VERSION = 0 + + def has_training_docs(self): + return True + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return True + + def training_docs(self): + if self.has_training_docs(): + return self.dataset["train"] + + def validation_docs(self): + if self.has_validation_docs(): + return self.dataset["validation"] + + def test_docs(self): + if self.has_test_docs(): + return self.dataset["test"] + + def max_generation_length(self) -> Optional[int]: + return 64 + + +def _year_to_lang_pairs(start_year: int, end_year: int) -> Dict[str, List[str]]: + """Downloads the config names for WMT years and returns a dict of language + language pairs for each such year. + """ + year_to_lang_pairs = {} + wmtyears = [f"wmt{year}" for year in range(start_year, end_year + 1)] + for wmtyear in wmtyears: + year_to_lang_pairs[wmtyear] = [ + c.name for c in promptsource.utils.get_dataset_confs(wmtyear) + ] + return year_to_lang_pairs + + +# Hard-code `_year_to_lang_pairs(14, 19)` to avoid downloading configs every time. +_YEAR_TO_LANG_PAIRS = { + "wmt14": ["cs-en", "de-en", "fr-en", "hi-en", "ru-en"], + "wmt15": ["cs-en", "de-en", "fi-en", "fr-en", "ru-en"], + "wmt16": ["cs-en", "de-en", "fi-en", "ro-en", "ru-en", "tr-en"], + "wmt17": ["cs-en", "de-en", "fi-en", "lv-en", "ru-en", "tr-en", "zh-en"], + "wmt18": ["cs-en", "de-en", "et-en", "fi-en", "kk-en", "ru-en", "tr-en", "zh-en"], + "wmt19": ["cs-en", "de-en", "fi-en", "gu-en", "kk-en", "lt-en", "ru-en", "zh-en", "fr-de"], # fmt: skip +} + + +def construct_tasks() -> Dict[str, WMTBase]: + """Constructs a `dict` of WMT tasks for all available WMT years + with keys of the form: + `wmt{year}_{lang1}_{lang2}` + Example: + `wmt14_cs_en` + """ + tasks = {} + for wmtyear, lang_pairs in _YEAR_TO_LANG_PAIRS.items(): + for lang_pair in lang_pairs: + task_class = _create_wmt_class(dataset_path=wmtyear, dataset_name=lang_pair) + lang_pair = lang_pair.replace("-", "_") + tasks[f"{wmtyear}_{lang_pair}"] = task_class + return tasks + + +def _create_wmt_class( + dataset_path: str, dataset_name: str, version: Optional[int] = 0 +) -> WMTBase: + class WMT(WMTBase): + DATASET_PATH = dataset_path + DATASET_NAME = dataset_name + + return WMT diff --git a/evaluation-pipeline/lm_eval/tasks/xquad.py b/evaluation-pipeline/lm_eval/tasks/xquad.py new file mode 100644 index 0000000000000000000000000000000000000000..125effcc11261cb95130dabe98b06ed78425fb7e --- /dev/null +++ b/evaluation-pipeline/lm_eval/tasks/xquad.py @@ -0,0 +1,302 @@ +""" +On the cross-lingual transferability of monolingual representations +https://arxiv.org/abs/1910.11856 + +XQuAD (Cross-lingual Question Answering Dataset) is a benchmark dataset +for evaluating cross-lingual question answering performance. The dataset +consists of a subset of 240 paragraphs and 1190 question-answer pairs from +the development set of SQuAD v1.1 (Rajpurkar et al., 2016) together with +their professional translations into ten languages: Spanish, German, Greek, +Russian, Turkish, Arabic, Vietnamese, Thai, Chinese, and Hindi. +Consequently, the dataset is entirely parallel across 11 languages. + +Homepage: https://github.com/deepmind/xquad +""" +import datasets +from functools import partial +from math import exp +from packaging import version + +from lm_eval.api.request import rf +from lm_eval.api.task import PromptSourceTask + + +_CITATION = """ +@article{Artetxe:etal:2019, + author = {Mikel Artetxe and Sebastian Ruder and Dani Yogatama}, + title = {On the cross-lingual transferability of monolingual representations}, + journal = {CoRR}, + volume = {abs/1910.11856}, + year = {2019}, + archivePrefix = {arXiv}, + eprint = {1910.11856} +} +""" + + +def _squad_metric(predictions, references): + squad_metric = datasets.load_metric("squad_v2") + return squad_metric.compute(predictions=predictions, references=references) + + +def _squad_agg(key, items): + predictions, references = zip(*items) + return _squad_metric(predictions=predictions, references=references)[key] + + +class XQuADEnglish(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "xquad" + DATASET_NAME = "xquad.en" + + # HF changed squad on us so we have to make sure we aren't running the old one + assert version.parse(datasets.__version__) >= version.parse( + "1.11.0" + ), "datasets v1.11.0 or later required for SQuAD" + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + return self.dataset["validation"] + + def construct_requests(self, doc: dict, ctx: str, args: dict): + """Uses RequestFactory to construct Requests and returns an iterable of + Requests which will be sent to the LM. + + Args: + doc (dict): + The document as returned from training_docs, validation_docs, or + test_docs. + ctx (str): + The context string, generated by fewshot_context. This includes + the natural language description, as well as the few shot examples, + and the question part of the document for `doc`. + args (dict): + The specifics of the context, including number of few shots. + + Returns: + An iterable of `Request` objects. + """ + request_args = { + "stop_sequences": self.stop_sequences(), + "max_generation_length": self.max_generation_length(), + "num_fewshot": args["num_fewshot"], + } + + cont_request = rf.greedy_until(ctx, request_args) + is_unanswerable = rf.loglikelihood( + ctx, self.text_target_separator + "unanswerable" + ) + + return cont_request, is_unanswerable + + def process_results(self, doc, results): + """Take a single document and the LM results and evaluates, returning a + dict where keys are the names of sub-metrics and values are the values of + the metric for that one document + + Args: + doc (dict): + The document as returned from training_docs, validation_docs, or + test_docs. + results (list): + The results of the requests created in construct_requests. + + Returns: + A dict of metric results. + """ + pred, (logprob_unanswerable, _) = results + no_answer_probability = exp(logprob_unanswerable) + + predictions = { + "id": doc["id"], + "prediction_text": pred, + "no_answer_probability": no_answer_probability, + } + + references = { + "id": doc["id"], + "answers": doc["answers"], + } + + out = { + # Exact match (the normalized answer exactly match the gold answer) + "exact": (predictions, references), + # The F-score of predicted tokens versus the gold answer + "f1": (predictions, references), + # Exact match (the normalized answer exactly match the gold answer) + "HasAns_exact": (predictions, references), + # The F-score of predicted tokens versus the gold answer + "HasAns_f1": (predictions, references), + # No-answer probability threshold associated to the best exact match + "best_exact_thresh": (predictions, references), + # No-answer probability threshold associated to the best F1 + "best_f1_thresh": (predictions, references), + # Best exact match (with varying threshold) + "best_exact": (predictions, references), + # Best F1 (with varying threshold) + "best_f1": (predictions, references), + } + if self.save_examples: + example = {"pred": pred, "target": doc["answers"]} + return out, example + return out + + def aggregation(self): + return { + # Exact match (the normalized answer exactly match the gold answer) + "exact": partial(_squad_agg, "exact"), + # The F-score of predicted tokens versus the gold answer + "f1": partial(_squad_agg, "f1"), + # Exact match (the normalized answer exactly match the gold answer) + "HasAns_exact": partial(_squad_agg, "HasAns_exact"), + # The F-score of predicted tokens versus the gold answer + "HasAns_f1": partial(_squad_agg, "HasAns_f1"), + # No-answer probability threshold associated to the best exact match + "best_exact_thresh": partial(_squad_agg, "best_exact_thresh"), + # No-answer probability threshold associated to the best F1 + "best_f1_thresh": partial(_squad_agg, "best_f1_thresh"), + # Best exact match (with varying threshold) + "best_exact": partial(_squad_agg, "best_exact"), + # Best F1 (with varying threshold) + "best_f1": partial(_squad_agg, "best_f1"), + } + + def higher_is_better(self): + return { + # Exact match (the normalized answer exactly match the gold answer) + "exact": True, + "f1": True, # The F-score of predicted tokens versus the gold answer + # Exact match (the normalized answer exactly match the gold answer) + "HasAns_exact": True, + "HasAns_f1": True, # The F-score of predicted tokens versus the gold answer + # No-answer probability threshold associated to the best exact match + "best_exact_thresh": True, + "best_f1_thresh": True, # No-answer probability threshold associated to the best F1 + "best_exact": True, # Best exact match (with varying threshold) + "best_f1": True, # Best F1 (with varying threshold) + } + + +class XQuADArabic(PromptSourceTask): + VERSION = 0 + DATASET_PATH = "xquad" + DATASET_NAME = "xquad.ar" + + # HF changed squad on us so we have to make sure we aren't running the old one + assert version.parse(datasets.__version__) >= version.parse( + "1.11.0" + ), "datasets v1.11.0 or later required for SQuAD" + + def has_training_docs(self): + return False + + def has_validation_docs(self): + return True + + def has_test_docs(self): + return False + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + return self.dataset["validation"] + + def construct_requests(self, doc: dict, ctx: str, args: dict): + request_args = { + "stop_sequences": self.stop_sequences(), + "max_generation_length": self.max_generation_length(), + "num_fewshot": args["num_fewshot"], + } + + cont_request = rf.greedy_until(ctx, request_args) + is_unanswerable = rf.loglikelihood( + ctx, self.text_target_separator + "unanswerable" + ) + + return cont_request, is_unanswerable + + def process_results(self, doc, results): + pred, (logprob_unanswerable, _) = results + no_answer_probability = exp(logprob_unanswerable) + + predictions = { + "id": doc["id"], + "prediction_text": pred, + "no_answer_probability": no_answer_probability, + } + + references = { + "id": doc["id"], + "answers": doc["answers"], + } + + if self.save_examples: + example = { + "pred": pred, + "target": doc["answers"], + } + return { + # Exact match (the normalized answer exactly match the gold answer) + "exact": (predictions, references), + # The F-score of predicted tokens versus the gold answer + "f1": (predictions, references), + # Exact match (the normalized answer exactly match the gold answer) + "HasAns_exact": (predictions, references), + # The F-score of predicted tokens versus the gold answer + "HasAns_f1": (predictions, references), + # No-answer probability threshold associated to the best exact match + "best_exact_thresh": (predictions, references), + # No-answer probability threshold associated to the best F1 + "best_f1_thresh": (predictions, references), + # Best exact match (with varying threshold) + "best_exact": (predictions, references), + # Best F1 (with varying threshold) + "best_f1": (predictions, references), + }, example + + def aggregation(self): + return { + # Exact match (the normalized answer exactly match the gold answer) + "exact": partial(_squad_agg, "exact"), + # The F-score of predicted tokens versus the gold answer + "f1": partial(_squad_agg, "f1"), + # Exact match (the normalized answer exactly match the gold answer) + "HasAns_exact": partial(_squad_agg, "HasAns_exact"), + # The F-score of predicted tokens versus the gold answer + "HasAns_f1": partial(_squad_agg, "HasAns_f1"), + # No-answer probability threshold associated to the best exact match + "best_exact_thresh": partial(_squad_agg, "best_exact_thresh"), + # No-answer probability threshold associated to the best F1 + "best_f1_thresh": partial(_squad_agg, "best_f1_thresh"), + # Best exact match (with varying threshold) + "best_exact": partial(_squad_agg, "best_exact"), + # Best F1 (with varying threshold) + "best_f1": partial(_squad_agg, "best_f1"), + } + + def higher_is_better(self): + return { + # Exact match (the normalized answer exactly match the gold answer) + "exact": True, + "f1": True, # The F-score of predicted tokens versus the gold answer + # Exact match (the normalized answer exactly match the gold answer) + "HasAns_exact": True, + "HasAns_f1": True, # The F-score of predicted tokens versus the gold answer + # No-answer probability threshold associated to the best exact match + "best_exact_thresh": True, + "best_f1_thresh": True, # No-answer probability threshold associated to the best F1 + "best_exact": True, # Best exact match (with varying threshold) + "best_f1": True, # Best F1 (with varying threshold) + } diff --git a/evaluation-pipeline/main.py b/evaluation-pipeline/main.py new file mode 100644 index 0000000000000000000000000000000000000000..ffd8633e7106c5bbdc0e30d9f067d4a4096a3fc4 --- /dev/null +++ b/evaluation-pipeline/main.py @@ -0,0 +1,215 @@ +import argparse +import datetime +import json +import logging +import os + +import lm_eval.evaluator as evaluator +from lm_eval.api import utils + + +logger = logging.getLogger("main") + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--model_api_name", + required=True, + help="Name of the model API to use. See `lm_eval.list_model_apis()` for available APIs", + ) + parser.add_argument( + "--model_args", + default="", + help="Model constructor args that you'd pass into a model of type " + "`--model_api_name`. These must be comma-separated keyword args, e.g. " + "`key1=value1,key2=value2`, with no spaces", + ) + parser.add_argument( + "--task_name", + required=True, + help="Name of the task to use as found " + "in the lm_eval registry. See: `lm_eval.list_tasks()`", + ) + parser.add_argument( + "--task_args", + default="", + help="""Optional task constructor args that you'd pass into a task class of kind " + `--task_name`. These must be comma-separated keyword args, e.g. + `key1=value1,key2=value2`, with no spaces. + WARNING: To avoid parsing errors, ensure your strings are quoted. For example, + `example_separator='\\n+++\\n'` + WARNING: Values must NOT contain commas.""", + ) + parser.add_argument( + "--template_names", + default="all_templates", + help="""Comma-separated list of template names for the specified + task. Example: + + `> python main.py ... --task_name rte --template_names imply,mean` + + - Default: `all_templates` + - General Selectors: + - `"all_templates"`: Selects all templates for the task + - `"original_templates"`: Selects only templates that are designed to match the original task + """, + ) + parser.add_argument("--num_fewshot", type=int, default=0) + parser.add_argument("--batch_size", type=int, default=None) + parser.add_argument("--seed", type=int, default=utils.DEFAULT_SEED) + parser.add_argument( + "--device", + type=str, + default=None, + help="The device to place your model onto, e.g. cuda:0. For large " + "models available through the HuggingFace Hub you should use `accelerate` " + "by passing `use_accelerate=True` to `--model_args`", + ) + parser.add_argument( + "--limit", + type=int, + default=None, + help="Limit the number of examples to evaluate on; ONLY USE THIS FOR DEBUGGING PURPOSES", + ) + parser.add_argument( + "--output_path", + default=None, + help="""Use output_path as `output_filename`. For example: + + `> python main.py ... --output_path blop` + # saves files into `outputs/blop.json` + + Warning: You currently cannot change/add folder structure. + """, + ) + parser.add_argument( + "--template_idx", + type=int, + default=None, + help="Choose template by index from available templates", + ) + parser.add_argument( + "--bootstrap_iters", + type=int, + default=100000, + help="Iters for stderr computation", + ) + parser.add_argument( + "--no_tracking", action="store_true", help="Skip carbon emission tracking" + ) + parser.add_argument( + "--use_cache", + action="store_true", + help="Whether to cache your model's predictions or not", + ) + return parser.parse_args() + + +def args_to_name(args, separator): + """Map `args` to file name. If output_path is set, we use that instead.""" + if args.output_path is not None: + return args.output_path + + def _fix_model_name(model, model_args): + if model_args == "": + return model + elif "pretrained" not in model_args: + logger.warning("WARNING: Unprepared for these model args.") + return f"{model}={model_args}" + + for arg in model_args.split(","): + # Example: + # pretrained=google/t5-base-lm-adapt --> google-t5-base-lm-adapt + if "pretrained" in arg: + return arg.split("=")[-1].replace("/", "-") + + fields = { + "model": _fix_model_name(args.model_api_name, args.model_args), + "task": args.task_name, + "templates": args.template_names, + "fewshot": str(args.num_fewshot), + "batchsize": str(args.batch_size), + "seed": str(args.seed), + "timestamp": datetime.datetime.now().isoformat("T", "seconds"), + } + fields = [f"{k}={v}" for k, v in fields.items() if v is not None] + # Some prompts also have "/" in them! + filename = f"{separator}".join(fields).replace("/", "-") + if args.limit is not None: + # Do not use limited files for final analysis. + return f"limited={args.limit}{separator}" + filename + + return filename + + +def setup_example_logger(output_path, separator): + """Sets up a logger that will save each example and prediction.""" + example_logger = logging.getLogger("examples") + filename = f"./outputs/examples{separator}{output_path}.jsonl" + formatter = logging.Formatter("%(message)s") + handler = logging.FileHandler(filename) + handler.setFormatter(formatter) + example_logger.addHandler(handler) + example_logger.setLevel(logging.INFO) + + +def main(): + os.makedirs("./outputs", exist_ok=True) + args = parse_args() + + if args.limit: + logger.warning( + "\n» WARNING: `--limit` SHOULD ONLY BE USED FOR TESTING. REAL METRICS " + "SHOULD NOT BE COMPUTED USING LIMIT." + ) + + print() # Ensure a newline after `main` command for readability. + + path_separator = "." + output_path = args_to_name(args, separator=path_separator) + setup_example_logger(output_path, path_separator) + + template_names = utils.cli_template_names( + args.task_name, args.template_names, args.template_idx + ) + evaluate_args = dict( + model_api_name=args.model_api_name, + model_args=args.model_args, + task_name=args.task_name, + task_args=args.task_args, + template_names=template_names, + num_fewshot=args.num_fewshot, + batch_size=args.batch_size, + device=args.device, + use_cache=args.use_cache, + bootstrap_iters=args.bootstrap_iters, + seed=args.seed, + limit=args.limit, + ) + if args.no_tracking: + results = evaluator.cli_evaluate(**evaluate_args) + else: + from codecarbon import OfflineEmissionsTracker + + with OfflineEmissionsTracker(country_iso_code="FRA", log_level="error"): + print() # Add newline between emissions tracker and evaluation logging. + results = evaluator.cli_evaluate(**evaluate_args) + + with open(f"./outputs/agg{path_separator}{output_path}.json", "w") as f: + json.dump({"results": results["results"], "config": results["config"]}, f) + + from scripts.agg2slim import agg2slim + + with open(f"./outputs/slim{path_separator}{output_path}.json", "w") as f: + json.dump(agg2slim(results), f, indent=2) + + print(f"\n{evaluator.make_table(results)}") + + if not args.no_tracking: + emissions_output_path = f"./outputs/emissions{path_separator}{output_path}.csv" + os.rename("emissions.csv", emissions_output_path) + + +if __name__ == "__main__": + main() diff --git a/evaluation-pipeline/sample_predictions.json b/evaluation-pipeline/sample_predictions.json new file mode 100644 index 0000000000000000000000000000000000000000..ca3637029983a606c5eef4261d873690fabfe586 --- /dev/null +++ b/evaluation-pipeline/sample_predictions.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83604324f13cad978e9543e5fca7d84528cb149fc91b48523d11f6374de07369 +size 21205894 diff --git a/evaluation-pipeline/scripts/__init__.py b/evaluation-pipeline/scripts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation-pipeline/scripts/agg2slim.py b/evaluation-pipeline/scripts/agg2slim.py new file mode 100644 index 0000000000000000000000000000000000000000..cd5e9653345158cb4fdf2a03fc92028d501ff36c --- /dev/null +++ b/evaluation-pipeline/scripts/agg2slim.py @@ -0,0 +1,59 @@ +import glob +import json +import os +import logging + + +logger = logging.getLogger(__name__) + + +def agg2slim(data): + """Maps results data to a simpler dictionary. + + `data` is expected to have a `results` and `config` fields. + The results should be a list of dictionaries. This function + filters out some of that information. `config` should be a dictionary. + """ + REMOVE_KEYS = set( + [ + "fixed_answer_choice_list", + "prompt_id", + "prompt_jinja", + "prompt_original_task", + "comment", + ] + ) + results = data["results"] + config = data["config"] + if isinstance(config, list): + logger.warning("Warning! This is an old agg file with a buggy config.") + # If information needs to be recovered, we can recover it from the filename. + config = {} + slim = { + "results": [ + {k: v for k, v in r.items() if k not in REMOVE_KEYS} for r in results + ], + "config": config, + } + return slim + + +def main(): + """Reformats each `agg` json file into a smaller slim version. + + python -m scripts/agg2slim.py + """ + os.makedirs("./outputs", exist_ok=True) + for agg_json_filename in glob.glob("./outputs/agg*.json"): + with open(agg_json_filename, "r") as jf: + data = json.load(jf) + slim_json_filename = agg_json_filename.replace("agg", "slim") + if os.path.exists(slim_json_filename): + logger.info("Skipping file as it already exists.") + slim = agg2slim(data) + with open(slim_json_filename, "w") as jf: + json.dump(slim, jf, indent=2) + + +if __name__ == "__main__": + main() diff --git a/evaluation-pipeline/scripts/cost_estimate.py b/evaluation-pipeline/scripts/cost_estimate.py new file mode 100644 index 0000000000000000000000000000000000000000..07535133203e0c5fd5407bf2791e1de316259c86 --- /dev/null +++ b/evaluation-pipeline/scripts/cost_estimate.py @@ -0,0 +1,88 @@ +import random +import transformers +from typing import List, Tuple + +import lm_eval +from lm_eval.api.model import LM + + +class DryrunLM(LM): + def __init__(self): + self.tokencost = 0 + self.tokenizer = transformers.GPT2TokenizerFast.from_pretrained("gpt2") + self.tokenizer.pad_token = "<|endoftext|>" + + def loglikelihood(self, requests): + res = [] + for ctx, cont in requests: + res.append((-random.random(), False)) + self.tokencost += len(self.tokenizer.tokenize(ctx + cont)) + return res + + def greedy_until(self, requests: List[Tuple[str, dict]]) -> List[str]: + res = [] + for ctx, until in requests: + res.append("null") + # assume worst case - generates until 256 + self.tokencost += len(self.tokenizer.tokenize(ctx)) + 256 + return res + + def loglikelihood_rolling(self, requests): + res = [] + for (s,) in requests: + # assume worst case: extra full context + self.tokencost += len(self.tokenizer.tokenize(s)) + 2048 + return res + + +def main(): + lm = DryrunLM() + + task_list = "arc_challenge,arc_easy,boolq,cola,copa,headqa,hellaswag,lambada,logiqa,mathqa,mc_taco,mrpc,multirc,openbookqa,piqa,prost,pubmedqa,qnli,qqp,race,record,rte,sciq,sst,triviaqa,webqs,wic,wikitext,winogrande,wnli,wsc" + values = [] + for task_name in task_list.split(","): + lm.tokencost = 0 + lm_eval.evaluate( + model=lm, + tasks=[lm_eval.get_task(task_name)], + num_fewshot=0, + limit=None, + bootstrap_iters=10, + ) + + print(task_name, lm.tokencost) + values.append( + [ + task_name, + lm.tokencost, + lm.tokencost / 1000 * 0.0008, + lm.tokencost / 1000 * 0.0012, + lm.tokencost / 1000 * 0.006, + lm.tokencost / 1000 * 0.06, + ] + ) + from pytablewriter import MarkdownTableWriter + + writer = MarkdownTableWriter() + writer.headers = ["Task", "Tokens", "Ada", "Babbage", "Curie", "Davinci"] + + values.sort(key=lambda x: -x[1]) + totcost = sum([x[1] for x in values]) + values.append( + [ + "**Total**", + totcost, + totcost / 1000 * 0.0008, + totcost / 1000 * 0.0012, + totcost / 1000 * 0.006, + totcost / 1000 * 0.06, + ] + ) + + writer.value_matrix = values + + print(writer.dumps()) + + +if __name__ == "__main__": + main() diff --git a/evaluation-pipeline/scripts/eval_zeroshot.sh b/evaluation-pipeline/scripts/eval_zeroshot.sh new file mode 100644 index 0000000000000000000000000000000000000000..7c69db56c4903cd6dbcec1cbded689b6d8c882ad --- /dev/null +++ b/evaluation-pipeline/scripts/eval_zeroshot.sh @@ -0,0 +1,23 @@ +case $1 in +gptj) +MODEL=EleutherAI/gpt-j-6B +;; + +mgpt) +MODEL=sberbank-ai/mGPT +;; + +*) +MODEL=$1 +esac + +task_no_slash=$(basename "$2") + +\time -o ../outputs/time-$1.$task_no_slash.txt -f "%E" python3 -m main \ + --model_api_name 'hf-causal' \ + --model_args pretrained=$MODEL \ + --task_name $2 \ + --num_fewshot 0 \ + --use_cache \ + --device cuda \ + --batch_size $3 diff --git a/evaluation-pipeline/scripts/get_prompts.py b/evaluation-pipeline/scripts/get_prompts.py new file mode 100644 index 0000000000000000000000000000000000000000..355e95e0038baf9c87e42cb634bf21cb32085dc0 --- /dev/null +++ b/evaluation-pipeline/scripts/get_prompts.py @@ -0,0 +1,24 @@ +from itertools import islice + +from lm_eval import tasks + +ct = 3 + +for ( + tname, + Task, +) in tasks.TASK_REGISTRY.items(): # [('record', tasks.superglue.ReCoRD)]:# + task = Task() + + print("#", tname) + docs = islice( + task.validation_docs() if task.has_validation_docs() else task.test_docs(), ct + ) + print() + for i in range(ct): + print() + doc = next(docs) + print("**Context**:", "\n```\n" + task.doc_to_text(doc) + "\n```\n") + print() + print("**Target**:", "\n```\n" + task.doc_to_target(doc) + "\n```\n") + print() diff --git a/evaluation-pipeline/scripts/make_gpt2_test_cases.py b/evaluation-pipeline/scripts/make_gpt2_test_cases.py new file mode 100644 index 0000000000000000000000000000000000000000..510ffab3c876e40a65555c0dad6ec944f9f964e7 --- /dev/null +++ b/evaluation-pipeline/scripts/make_gpt2_test_cases.py @@ -0,0 +1,44 @@ +import transformers +import torch +import torch.nn.functional as F +import random + +from lm_eval.api.utils import set_seed + + +data = [ + "A multilayer perceptron (MLP) is a class of feedforward artificial neural network (ANN)", + "The term MLP is used ambiguously, sometimes loosely to any feedforward ANN, sometimes strictly to refer to networks composed of multiple layers of perceptrons (with threshold activation); see § Terminology", + 'Multilayer perceptrons are sometimes colloquially referred to as "vanilla" neural networks, especially when they have a single hidden layer.[1]', + "An MLP consists of at least three layers of nodes: an input layer, a hidden layer and an output layer. Except for the input nodes, each node is a neuron that uses a nonlinear activation function.", + "MLP utilizes a supervised learning technique called backpropagation for training.[2][3] Its multiple layers and non-linear activation distinguish MLP from a linear perceptron. It can distinguish data that is not linearly separable.[4]", + "Recent work has demonstrated substantial gains on many NLP tasks and benchmarks by pre-training on a large corpus of text followed by fine-tuning on a specific task. While typically task-agnostic in architecture, this method still requires task-specific fine-tuning datasets of thousands or tens of thousands of examples. By contrast, humans can generally perform a new language task from only a few examples or from simple instructions - something which current NLP systems still largely struggle to do. Here we show that scaling up language models greatly improves task-agnostic, few-shot performance, sometimes even reaching competitiveness with prior state-of-the-art fine-tuning approaches. ", + "Specifically, we train GPT-3, an autoregressive language model with 175 billion parameters, 10x more than any previous non-sparse language model, and test its performance in the few-shot setting. For all tasks, GPT-3 is applied without any gradient updates or fine-tuning, with tasks and few-shot demonstrations specified purely via text interaction with the model. GPT-3 achieves strong performance on many NLP datasets, including translation, question-answering, and cloze tasks, as well as several tasks that require on-the-fly reasoning or domain adaptation, such as unscrambling words, using a novel word in a sentence, or performing 3-digit arithmetic. At the same time, we also identify some datasets where GPT-3's few-shot learning still struggles, as well as some datasets where GPT-3 faces methodological issues related to training on large web corpora. Finally, we find that GPT-3 can generate samples of news articles which human evaluators have difficulty distinguishing from articles written by humans. We discuss broader societal impacts of this finding and of GPT-3 in general.", + "A multilayer perceptron (MLP) is a class of feedforward artificial neural network (ANN)", + "Hello World", +] + + +model = transformers.GPT2LMHeadModel.from_pretrained("gpt2") +tok = transformers.GPT2Tokenizer.from_pretrained("gpt2") + +tgs = [] + +set_seed() +for dat in data: + tokens = tok.encode(dat, return_tensors="pt") + ind = random.randrange(len(tokens[0]) - 1) + logits = F.log_softmax(model(tokens)[0], dim=-1)[:, :-1] # [batch, seq, vocab] + + res = torch.gather(logits, 2, tokens[:, 1:].unsqueeze(-1)).squeeze(-1)[0] + + tgs.append(float(res[ind:].sum())) + print( + r'("""' + + tok.decode(tokens[0, : ind + 1]) + + r'""", """' + + tok.decode(tokens[0, ind + 1 :]) + + r'"""), ' + ) + +print(tgs) diff --git a/evaluation-pipeline/scripts/make_table_tasks.py b/evaluation-pipeline/scripts/make_table_tasks.py new file mode 100644 index 0000000000000000000000000000000000000000..5a9de2b193e45c7b27b254843de78e5c1bc0fa24 --- /dev/null +++ b/evaluation-pipeline/scripts/make_table_tasks.py @@ -0,0 +1,33 @@ +from lm_eval import tasks +from pytablewriter import MarkdownTableWriter + +writer = MarkdownTableWriter() +writer.headers = ["Task Name", "Train", "Val", "Test", "Val/Test Docs", "Metrics"] + +values = [] + + +def chk(tf): + if tf: + return "✓" + else: + return " " + + +for tname, Task in tasks.TASK_REGISTRY.items(): + task = Task() + + v = [ + tname, + chk(task.has_training_docs()), + chk(task.has_validation_docs()), + chk(task.has_test_docs()), + len(list(task.test_docs() if task.has_test_docs() else task.validation_docs())), + ", ".join(task.aggregation().keys()), + ] + print(v) + values.append(v) + +writer.value_matrix = values + +print(writer.dumps()) diff --git a/evaluation-pipeline/scripts/print_table.py b/evaluation-pipeline/scripts/print_table.py new file mode 100644 index 0000000000000000000000000000000000000000..8d7b36b73ec164936729e15b9ca626b52227c2e7 --- /dev/null +++ b/evaluation-pipeline/scripts/print_table.py @@ -0,0 +1,31 @@ +import json +import sys + +from pytablewriter import MarkdownTableWriter + +json_file = json.load(open(sys.argv[1])) + + +results = [] + +for r in json_file["results"]: + metric = [k[:-7] for k in r.keys() if "_stderr" in k][0] + results.append( + [ + r["prompt_name"], + metric, + "{0:.5g}".format(r[metric]), + "{0:.5g}".format(r[metric + "_stderr"]), + ] + ) + + +writer = MarkdownTableWriter( + table_name=json_file["results"][0][ + "task_name" + ], # json_file["config"]["model"] + " on " + json_file["results"][0]["task_name"], + headers=["prompt_name", "metric", "mean", "stderr"], + value_matrix=results, + margin=3, +) +writer.write_table() diff --git a/evaluation-pipeline/scripts/write_out.py b/evaluation-pipeline/scripts/write_out.py new file mode 100644 index 0000000000000000000000000000000000000000..5046d5bb8760881cf31c49e0a418bf58078b4743 --- /dev/null +++ b/evaluation-pipeline/scripts/write_out.py @@ -0,0 +1,61 @@ +import argparse +import os +import numpy as np + +import lm_eval +from lm_eval.api import utils + + +EXAMPLE_DIVIDER = "!!@@##@@!! -- Example {i}\n" + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--output_base_path", required=True) + parser.add_argument("--task_name", type=str, required=True) + parser.add_argument("--template_names", default="all_templates") + parser.add_argument("--sets", type=str, default="val") # example: val,test + parser.add_argument("--num_fewshot", type=int, default=1) + parser.add_argument("--num_examples", type=int, default=1) + parser.add_argument("--seed", type=int, default=utils.DEFAULT_SEED) + return parser.parse_args() + + +def main(): + args = parse_args() + rng = np.random.default_rng(args.seed) + + template_names = utils.cli_template_names(args.task_name, args.template_names) + tasks = lm_eval.get_task_list(args.task_name, template_names) + + os.makedirs(args.output_base_path, exist_ok=True) + for task, template_name in zip(tasks, template_names): + iters = [] + for set in args.sets.split(","): + if set == "train" and task.has_training_docs(): + docs = task.training_docs() + if set == "val" and task.has_validation_docs(): + docs = task.validation_docs() + if set == "test" and task.has_test_docs(): + docs = task.test_docs() + iters.append(docs) + docs = utils.join_iters(iters) + + file_name = lm_eval.tasks._get_task_template_key(args.task_name, template_name) + with open(os.path.join(args.output_base_path, file_name), "w") as f: + for i, doc in ( + zip(range(args.num_examples), docs) + if args.num_examples > 0 + else enumerate(docs) + ): + f.write(EXAMPLE_DIVIDER.format(i=i)) + ctx, _ = task.fewshot_context( + doc=doc, + num_fewshot=args.num_fewshot, + rng=rng, + ) + f.write(ctx + "\n") + + +if __name__ == "__main__": + main() diff --git a/evaluation-pipeline/setup.py b/evaluation-pipeline/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..19da0ab43b83efc2551cc70182b19ae03c4fdf0f --- /dev/null +++ b/evaluation-pipeline/setup.py @@ -0,0 +1,65 @@ +from setuptools import setup, find_packages +from setuptools.command.install import install + + +with open("README.md", "r", encoding="utf-8") as fh: + long_description = fh.read() + + +dev_requires = (["black<=21.12b0", "coverage<=6.2", "mock>=4.0.3", "pytest", + "promptsource@git+https://github.com/bigscience-workshop/promptsource@eval-hackathon"],) +colab_requires = (["black<=21.12b0", "coverage<=6.2", "mock>=4.0.3", "pytest"]) +install_requires = [ + "datasets>=2.0.0", + "nltk==3.6", + "openai==0.13.0", + "pycountry==20.7.3", + "pytablewriter==0.58.0", + "rouge-score==0.0.4", + "sacrebleu==1.5.0", + "scikit-learn>=0.24.1", + "sqlitedict==1.6.0", + "torch==1.11.0", + "evaluate==0.4.0", + "tqdm-multiprocess==0.0.11", + "accelerate@git+https://github.com/huggingface/accelerate@main", + "transformers==4.26.1", +] +dependency_links = [] + + +class PostInstall(install): + @staticmethod + def post_install(): + """Post installation `nltk` downloads.""" + import nltk + + nltk.download("popular") + + def run(self): + install.run(self) + self.execute( + PostInstall.post_install, [], msg="Running post installation tasks" + ) + + +setup( + name="lm_eval", + version="0.2.0", + author="Leo Gao & EleutherAI", + description="A framework for evaluating autoregressive language models", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/EleutherAI/lm-evaluation-harness", + packages=find_packages(), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], + python_requires=">=3.9", + install_requires=install_requires, + dependency_links=dependency_links, + extras_require={"dev": dev_requires, "colab": colab_requires}, + cmdclass={"install": PostInstall}, +) \ No newline at end of file diff --git a/evaluation-pipeline/templates/new_prompt_source_task.py b/evaluation-pipeline/templates/new_prompt_source_task.py new file mode 100644 index 0000000000000000000000000000000000000000..248b3fa20f2bc7378e8ff970c19dd3fef2fa2c52 --- /dev/null +++ b/evaluation-pipeline/templates/new_prompt_source_task.py @@ -0,0 +1,136 @@ +# TODO: Remove all TODO comments once the implementation is complete. +""" +TODO: Add the Paper Title on this line. +TODO: Add the paper's PDF URL (preferably from arXiv) on this line. + +TODO: Write a Short Description of the task. + +Homepage: TODO: Add the URL to the task's Homepage here. +""" +from lm_eval.api.task import PromptSourceTask + + +# TODO: Add the BibTeX citation for the task. +_CITATION = """ +""" + + +# TODO: Replace `NewTask` with the name of your Task. +class NewTask(PromptSourceTask): + + # TODO: Add the `DATASET_PATH` string. This will be the name of the `Task` + # dataset as denoted in HuggingFace `datasets`. + DATASET_PATH = "" + # TODO: Add the `DATASET_NAME` string. This is the name of a subset within + # `DATASET_PATH`. If there aren't specific subsets you need, leave this as `None`. + DATASET_NAME = None + + def has_training_docs(self): + # TODO: Fill in the return with `True` if the Task has training data; else `False`. + return False + + def has_validation_docs(self): + # TODO: Fill in the return with `True` if the Task has validation data; else `False`. + return False + + def has_test_docs(self): + # TODO: Fill in the return with `True` if the Task has test data; else `False`. + return False + + def training_docs(self): + if self.has_training_docs(): + # TODO: Return the training document generator from `self.dataset`. + # If you need to process the data, `map` over the documents with + # the custom processing function, `self._process_doc`. E.g. + # `self.dataset["train"].map(self._process_doc)` + # In most case you can leave this as is unless the dataset split is + # named differently than the default `"train"`. + return self.dataset["train"] + + def validation_docs(self): + if self.has_validation_docs(): + # TODO: Return the validation document generator from `self.dataset`. + # If you need to process the data, `map` over the documents with the + # custom processing function, `self._process_doc`. E.g. + # `self.dataset["validation"].map(self._process_doc)` + # In most case you can leave this as is unless the dataset split is + # named differently than the default `"validation"`. + return self.dataset["validation"] + + def test_docs(self): + if self.has_test_docs(): + # TODO: Return the test document generator from `self.dataset`. + # If you need to process the data, `map` over the documents with the + # custom processing function, `self._process_doc`. E.g. + # `self.dataset["test"].map(self._process_doc)` + # In most case you can leave this as is unless the dataset split is + # named differently than the default `"test"`. + return self.dataset["test"] + + def max_generation_length(self): + # Define this method when you want to control the length of few-shot + # generations on specific tokens. The default is `None` which gets mapped + # to a model's default max generation token length. E.g. see `lm_eval/models/gpt2.py:max_tokens()` + # NOTE: You may delete this function if the task does not required generation. + return None + + def construct_requests(self, doc: dict, ctx: str, args: dict): + """Uses RequestFactory to construct Requests and returns an iterable of + Requests which will be sent to the LM. + + Args: + doc (dict): + The document as returned from training_docs, validation_docs, or + test_docs. + ctx (str): + The context string, generated by fewshot_context. This includes + the natural language description, as well as the few shot examples, + and the question part of the document for `doc`. + args (dict): + The specifics of the context, including number of few shots. + + Returns: + An iterable of `Request` objects. + """ + # TODO: Construct your language model requests with the request factory, `rf`, + # and return them as an iterable. + return [] + + def process_results(self, doc, results): + """Take a single document and the LM results and evaluates, returning a + dict where keys are the names of sub-metrics and values are the values of + the metric for that one document. + + Args: + doc (dict): + The document as returned from training_docs, validation_docs, or + test_docs. + results (list): + The results of the requests created in construct_requests. + + Returns: + A dict of metric results. + """ + # TODO: For each (sub)metric in the task evaluation, add a key-value pair + # with the metric name as key and the corresponding metric result as value + # for the current `doc`. + return {} + + def aggregation(self): + """ + Returns: + A dictionary where keys are the names of sub-metrics and values are + functions that aggregate a list of metric scores. + {str: [metric_score] -> float} + """ + # TODO: For each (sub)metric in the task evaluation, add a key-value pair + # with the metric name as key and an aggregation function as value which + # determines how to combine results from each document in the dataset. + # Check `lm_eval.metric` to find built-in aggregation functions. + return {} + + def higher_is_better(self): + # TODO: For each (sub)metric in the task evaluation, add a key-value pair + # with the metric name as key and a `bool` value determining whether or + # not higher values of that metric are deemed better. + return {} diff --git a/evaluation-pipeline/tests/test_evaluator.py b/evaluation-pipeline/tests/test_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..78e96e39513ddb579fa214f9f9b3eae8a1c60166 --- /dev/null +++ b/evaluation-pipeline/tests/test_evaluator.py @@ -0,0 +1,71 @@ +import os +import random +import pytest + +import lm_eval +import lm_eval.tasks as tasks +import lm_eval.api.model as model +import lm_eval.models as models +import lm_eval.evaluator as evaluator +from lm_eval.api.utils import DEFAULT_SEED, set_seed + + +# TODO: More fine grained unit tests rather than this big honking integration +# test once we break evaluator into smaller, more manageable pieces + + +def _ll_fn(requests): + for ctx, cont in requests: + if len(ctx) == 0: + continue + # Check text-target-separator default spacing convention. + # ctx + (' ' + cont) + assert ctx[-1] != " " + assert cont[0] == " " + res = [] + random.seed(DEFAULT_SEED) + for _ in requests: + res.append((-random.random(), False)) + return res + + +def _ll_perp_fn(requests): + for (string,) in requests: + assert isinstance(string, str) + res = [] + random.seed(DEFAULT_SEED) + for _ in requests: + res.append(-random.random()) + return res + + +@pytest.mark.parametrize("task_name", lm_eval.list_tasks()) +def test_evaluator(task_name): + set_seed() + template_names = tasks.list_templates(task_name) + # Only choose 1 promptsource template. + template_name = template_names[0] if template_names else None + task = tasks.get_task(task_name, template_name) + + os.system("rm test_cache.db") + lm = model.CachingLM(models.get_model("dummy"), "test_cache.db") + lm.loglikelihood = _ll_fn + lm.loglikelihood_rolling = _ll_perp_fn + + limit = 5 + e1 = evaluator.evaluate( + model=lm, + tasks=[task], + num_fewshot=0, + bootstrap_iters=10, + limit=limit, + )["results"] + e2 = evaluator.evaluate( + model=lm, + tasks=[task], + num_fewshot=0, + bootstrap_iters=10, + limit=limit, + )["results"] + # Check that caching is working + assert e1 == e2 diff --git a/evaluation-pipeline/tests/test_misc.py b/evaluation-pipeline/tests/test_misc.py new file mode 100644 index 0000000000000000000000000000000000000000..435aee5547ea759657ee0f54ed7644f063ad3522 --- /dev/null +++ b/evaluation-pipeline/tests/test_misc.py @@ -0,0 +1,14 @@ +import pytest +import random + +import lm_eval.api.metric as metrics +from lm_eval.api.utils import DEFAULT_SEED + + +def test_bootstrapping(): + random.seed(DEFAULT_SEED) + arr = [random.random() for _ in range(1000)] + expected = metrics.mean_stderr(arr) + bootstrapped = metrics.bootstrap_stderr(metrics.mean, arr, iters=100000) + + assert bootstrapped == pytest.approx(expected, abs=1e-4) diff --git a/evaluation-pipeline/tests/test_models_huggingface.py b/evaluation-pipeline/tests/test_models_huggingface.py new file mode 100644 index 0000000000000000000000000000000000000000..33ca57ce806bf352300c6f8114dddc8956588f88 --- /dev/null +++ b/evaluation-pipeline/tests/test_models_huggingface.py @@ -0,0 +1,326 @@ +import unittest.mock as mock +import logging +import pytest + +import lm_eval.models +from lm_eval.api.utils import set_seed + + +logger = logging.getLogger(__name__) + + +# Only use cpu to avoid non-deterministic CUDA settings. +# See: https://pytorch.org/docs/stable/notes/randomness.html +_DEVICE = "cpu" + + +@pytest.mark.parametrize( + "stop_sequences,test_input,expected", + [ + (["not"], "i like", "i like to say that I'm not"), + (["say that"], "i like", "i like to say that"), + (["great"], "big science is", "big science is a great"), + ( + ["<|endoftext|>"], + "big science has", + "big science has been done in the past, but it's not the same as the science of the past. It", + ), + ], +) +def test_causal_stop_sequences(stop_sequences, test_input, expected): + set_seed() + causal_model = lm_eval.models.get_model( + "hf-causal", pretrained="gpt2", device=_DEVICE + ) + inputs = causal_model.tok_encode_batch([test_input]) + generations = causal_model._model_generate( + inputs=inputs, + max_tokens=20, + stop=stop_sequences, + ) + generations = causal_model.tok_decode(generations)[0] + assert test_input + generations == expected + + +@pytest.mark.parametrize( + "stop_sequences,test_input,expected", + [ + (["better"], "big science is ", "big science is a great way to get a better"), + ( + ["the"], + "big science is ", + "big science is a great way to get a better understanding of the", + ), + ( + ["."], + "The quick brown fox jumps over the lazy ", + "The quick brown fox jumps over the lazy fox.", + ), + ( + [""], + "big science is ", + "big science is a great way to get a better understanding of the world.", + ), + ], +) +def test_seq2seq_stop_sequences(stop_sequences, test_input, expected): + seq2seq_model = lm_eval.models.get_model( + "hf-seq2seq", pretrained="google/t5-small-lm-adapt", device=_DEVICE + ) + inputs = seq2seq_model.tok_encode_batch([test_input]) + generations = seq2seq_model._model_generate( + inputs=inputs, + max_tokens=20, + stop=stop_sequences, + ) + generations = seq2seq_model.tok_decode(generations)[0] + assert test_input + generations == expected + + +def test_causal_model(): + set_seed() + causal_model = lm_eval.models.get_model( + "hf-causal", + pretrained="gpt2", + device=_DEVICE, + ) + ( + (ll_dog, ig_dog), + (ll_cat, ig_cat), + (_, ll_max_0), + (_, ll_max_1), + (_, ll_max_2), + *vals, + ) = causal_model.loglikelihood( + [ + ("The quick brown fox jumps over the lazy", " dog"), + ("The quick brown fox jumps over the lazy", " cat"), + ("The quick brown fox jumps over the lazy", ", lazy dog"), + ("The quick brown fox jumps over the lazy", ", lazy fox"), + ( + "The quick brown fox jumps over the lazy", + ", lazy fox and they both fall to the ground", + ), + ( + """A mult""", + """ilayer perceptron (MLP) is a class of feedforward artificial neural network (ANN)""", + ), + ( + """The term MLP is used ambiguously, sometimes loosely to any feedforward ANN, sometimes strictly to refer to networks composed of multiple layers of perceptrons""", + """ (with threshold activation); see § Terminology""", + ), + ( + """Multilayer perceptrons are sometimes coll""", + """oquially referred to as "vanilla" neural networks, especially when they have a single hidden layer.[1]""", + ), + ( + """An MLP consists of at least three layers of nodes: an input layer, a hidden layer and an output layer. Except for the input nodes, each node is a neuron that uses a nonlinear""", + """ activation function.""", + ), + ( + """MLP utilizes a supervised""", + """ learning technique called backpropagation for training.[2][3] Its multiple layers and non-linear activation distinguish MLP from a linear perceptron. It can distinguish data that is not linearly separable.[4]""", + ), + ( + """Recent work has demonstrated substantial gains on many NLP tasks and benchmarks by pre-training on a large corpus of text followed by fine-tuning on a specific task. While typically task-agnostic""", + """ in architecture, this method still requires task-specific fine-tuning datasets of thousands or tens of thousands of examples. By contrast, humans can generally perform a new language task from only a few examples or from simple instructions - something which current NLP systems still largely struggle to do. Here we show that scaling up language models greatly improves task-agnostic, few-shot performance, sometimes even reaching competitiveness with prior state-of-the-art fine-tuning approaches. """, + ), + ( + """Specifically, we train GPT-3, an autoregressive language model with 175""", + """ billion parameters, 10x more than any previous non-sparse language model, and test its performance in the few-shot setting. For all tasks, GPT-3 is applied without any gradient updates or fine-tuning, with tasks and few-shot demonstrations specified purely via text interaction with the model. GPT-3 achieves strong performance on many NLP datasets, including translation, question-answering, and cloze tasks, as well as several tasks that require on-the-fly reasoning or domain adaptation, such as unscrambling words, using a novel word in a sentence, or performing 3-digit arithmetic. At the same time, we also identify some datasets where GPT-3's few-shot learning still struggles, as well as some datasets where GPT-3 faces methodological issues related to training on large web corpora. Finally, we find that GPT-3 can generate samples of news articles which human evaluators have difficulty distinguishing from articles written by humans. We discuss broader societal impacts of this finding and of GPT-3 in general.""", + ), + ( + """A mult""", + """ilayer perceptron (MLP) is a class of feedforward artificial neural network (ANN)""", + ), + ("""Hello""", """ World"""), + ] + ) + + assert ll_dog > ll_cat + assert not ig_cat + assert not ll_max_0 + assert ll_max_1 + assert ll_max_2 + + targets = [ + -61.60536193847656, + -56.57843780517578, + -62.131004333496094, + -9.799489974975586, + -153.96334838867188, + -341.222900390625, + -731.1475830078125, + -61.60536193847656, + -8.682319641113281, + ] + + for (pred, _), tgt in zip(vals, targets): + assert pred == pytest.approx(tgt, rel=1e-3) + + # Test empty context + causal_model.loglikelihood([("", "test")]) + request_args = { + "stop_sequences": [".", "\n", "'"], + "max_generation_length": None, + "num_fewshot": 1, + } + (gen,) = causal_model.greedy_until( + [("The quick brown fox jumps over the lazy", request_args)] + ) + assert gen == ", lazy fox and they both fall to the ground" + + +def test_causal_model_perplexity(): + set_seed() + causal_model = lm_eval.models.get_model_from_args_string( + model_api_name="hf-causal", model_args=f"device={_DEVICE},pretrained=gpt2" + ) + test_string = "We study empirical scaling laws for language model performance on the cross-entropy loss." + perplexity = causal_model.loglikelihood_rolling([(test_string,)])[0] + tgt = sum( + [ + -4.9599953, + -8.069298, + -8.308624, + -10.178513, + -8.906924, + -1.9318912, + -7.745445, + -7.146077, + -5.2072, + -3.5882986, + -1.9957212, + -8.044922, + -0.20841774, + -5.1096807, + -0.099879116, + -8.888423, + -4.6180487, + ] + ) + assert perplexity == pytest.approx(tgt, rel=1e-3) + + with mock.patch.object( + lm_eval.models.huggingface.AutoCausalLM, + "max_length", + new_callable=mock.PropertyMock, + ) as mock_max_length: + mock_max_length.return_value = 5 + causal_model = lm_eval.models.get_model_from_args_string( + model_api_name="hf-causal", model_args=f"device={_DEVICE},pretrained=gpt2" + ) + perplexity = causal_model.loglikelihood_rolling([(test_string,)])[0] + logger.info(perplexity) + tgt = sum( + [ + -4.96001, + -8.069275, + -8.308612, + -10.178482, + -8.90691, + -4.037338, + -8.09261, + -11.662385, + -10.206891, + -4.425003, + -2.2563353, + -7.909143, + -1.9304147, + -7.3610134, + -2.3120654, + -7.3229, + -2.1643813, + ] + ) + assert perplexity == pytest.approx(tgt, rel=1e-3) + + +def test_seq2seq_model(): + seq2seq_model = lm_eval.models.get_model( + "hf-seq2seq", + pretrained="google/t5-small-lm-adapt", + device=_DEVICE, + ) + llhs = seq2seq_model.loglikelihood( + [ + ("The quick brown fox jumps over the lazy", " dog"), + ("The quick brown fox jumps over the lazy", " cat"), + ("The quick brown fox jumps over the lazy", ", lazy dog"), + ("The quick brown fox jumps over the lazy", " fox."), + ( + "The quick brown fox jumps over the lazy", + ", lazy fox and they both fall to the ground", + ), + ( + """A mult""", + """ilayer perceptron (MLP) is a class of feedforward artificial neural network (ANN)""", + ), + ( + """The term MLP is used ambiguously, sometimes loosely to any feedforward ANN, sometimes strictly to refer to networks composed of multiple layers of perceptrons""", + """ (with threshold activation); see § Terminology""", + ), + ( + """Multilayer perceptrons are sometimes coll""", + """oquially referred to as "vanilla" neural networks, especially when they have a single hidden layer.[1]""", + ), + ( + """An MLP consists of at least three layers of nodes: an input layer, a hidden layer and an output layer. Except for the input nodes, each node is a neuron that uses a nonlinear""", + """ activation function.""", + ), + ( + """MLP utilizes a supervised""", + """ learning technique called backpropagation for training.[2][3] Its multiple layers and non-linear activation distinguish MLP from a linear perceptron. It can distinguish data that is not linearly separable.[4]""", + ), + ( + """Recent work has demonstrated substantial gains on many NLP tasks and benchmarks by pre-training on a large corpus of text followed by fine-tuning on a specific task. While typically task-agnostic""", + """ in architecture, this method still requires task-specific fine-tuning datasets of thousands or tens of thousands of examples. By contrast, humans can generally perform a new language task from only a few examples or from simple instructions - something which current NLP systems still largely struggle to do. Here we show that scaling up language models greatly improves task-agnostic, few-shot performance, sometimes even reaching competitiveness with prior state-of-the-art fine-tuning approaches. """, + ), + ( + """Specifically, we train GPT-3, an autoregressive language model with 175""", + """ billion parameters, 10x more than any previous non-sparse language model, and test its performance in the few-shot setting. For all tasks, GPT-3 is applied without any gradient updates or fine-tuning, with tasks and few-shot demonstrations specified purely via text interaction with the model. GPT-3 achieves strong performance on many NLP datasets, including translation, question-answering, and cloze tasks, as well as several tasks that require on-the-fly reasoning or domain adaptation, such as unscrambling words, using a novel word in a sentence, or performing 3-digit arithmetic. At the same time, we also identify some datasets where GPT-3's few-shot learning still struggles, as well as some datasets where GPT-3 faces methodological issues related to training on large web corpora. Finally, we find that GPT-3 can generate samples of news articles which human evaluators have difficulty distinguishing from articles written by humans. We discuss broader societal impacts of this finding and of GPT-3 in general.""", + ), + ( + """A mult""", + """ilayer perceptron (MLP) is a class of feedforward artificial neural network (ANN)""", + ), + ("""Hello""", """ World"""), + ] + ) + ( + (ll_dog, ig_dog), + (ll_cat, ig_cat), + (_, ll_max_0), + (_, ll_max_1), + (_, ll_max_2), + *vals, + ) = llhs + assert ll_dog > ll_cat + assert not ig_cat + + targets = [ + -118.2639, + -70.3217, + -116.2367, + -16.5411, + -227.1213, + -393.8974, + -851.3747, + -118.2639, + -19.8556, + ] + for (pred, _), tgt in zip(vals, targets): + assert pred == pytest.approx(tgt, rel=1e-3) + + # Test empty context + seq2seq_model.loglikelihood([("", "test")]) + + request_args = { + "stop_sequences": [".", "\n"], + "max_generation_length": 20, + "num_fewshot": 1, + } + (gen,) = seq2seq_model.greedy_until( + [("The quick brown fox jumps over the lazy", request_args)] + ) + assert gen == "fox" diff --git a/evaluation-pipeline/tests/test_models_openai_completions.py b/evaluation-pipeline/tests/test_models_openai_completions.py new file mode 100644 index 0000000000000000000000000000000000000000..c42149815dc51a0d84fddde50cdec71f2841f698 --- /dev/null +++ b/evaluation-pipeline/tests/test_models_openai_completions.py @@ -0,0 +1,158 @@ +import pytest +import os +import json +import openai +import mock +import pickle +import hashlib +import logging + +import lm_eval.models as models +from lm_eval.api.utils import set_seed + + +logger = logging.getLogger(__name__) + + +def _mock_completion(**kwargs): + # Mock completion function + # Loads from a cached+pickled response if it exists, otherwise it will actually try to ping + os.makedirs("tests/testdata", exist_ok=True) + arg_hash = hashlib.sha256( + json.dumps(kwargs, sort_keys=True).encode("utf-8") + ).hexdigest() + fname = f"tests/testdata/gpt3_test_{arg_hash}.pkl" + if os.path.exists(fname): + with open(fname, "rb") as fh: + return pickle.load(fh) + + ret = openai.Completion.create(**kwargs) + ret.api_key = "" + with open(fname, "wb") as fh: + pickle.dump(ret, fh) + return ret + + +@mock.patch("lm_eval.models.openai_completions.oa_completion", new=_mock_completion) +def test_openai_completions(): + set_seed() + if "OPENAI_API_SECRET_KEY" not in os.environ: + os.environ["OPENAI_API_SECRET_KEY"] = "" + oa_model = models.get_model_from_args_string( + model_api_name="openai", model_args="engine=ada" + ) + ( + (ll_dog, ig_dog), + (ll_cat, ig_cat), + (_, ll_max_0), + (_, ll_max_1), + (_, ll_max_2), + *vals, + ) = oa_model.loglikelihood( + [ + ("The quick brown fox jumps over the lazy", " dog"), + ("The quick brown fox jumps over the lazy", " cat"), + ("The quick brown fox jumps over the lazy", ", lazy dog"), + ("The quick brown fox jumps over the lazy", ", lazy fox"), + ( + "The quick brown fox jumps over the lazy", + ", lazy fox and they both fall to the ground", + ), + ( + """A mult""", + """ilayer perceptron (MLP) is a class of feedforward artificial neural network (ANN)""", + ), + ( + """The term MLP is used ambiguously, sometimes loosely to any feedforward ANN, sometimes strictly to refer to networks composed of multiple layers of perceptrons""", + """ (with threshold activation); see § Terminology""", + ), + ( + """Multilayer perceptrons are sometimes coll""", + """oquially referred to as "vanilla" neural networks, especially when they have a single hidden layer.[1]""", + ), + ( + """An MLP consists of at least three layers of nodes: an input layer, a hidden layer and an output layer. Except for the input nodes, each node is a neuron that uses a nonlinear""", + """ activation function.""", + ), + ( + """MLP utilizes a supervised""", + """ learning technique called backpropagation for training.[2][3] Its multiple layers and non-linear activation distinguish MLP from a linear perceptron. It can distinguish data that is not linearly separable.[4]""", + ), + ( + """Recent work has demonstrated substantial gains on many NLP tasks and benchmarks by pre-training on a large corpus of text followed by fine-tuning on a specific task. While typically task-agnostic""", + """ in architecture, this method still requires task-specific fine-tuning datasets of thousands or tens of thousands of examples. By contrast, humans can generally perform a new language task from only a few examples or from simple instructions - something which current NLP systems still largely struggle to do. Here we show that scaling up language models greatly improves task-agnostic, few-shot performance, sometimes even reaching competitiveness with prior state-of-the-art fine-tuning approaches. """, + ), + ( + """Specifically, we train GPT-3, an autoregressive language model with 175""", + """ billion parameters, 10x more than any previous non-sparse language model, and test its performance in the few-shot setting. For all tasks, GPT-3 is applied without any gradient updates or fine-tuning, with tasks and few-shot demonstrations specified purely via text interaction with the model. GPT-3 achieves strong performance on many NLP datasets, including translation, question-answering, and cloze tasks, as well as several tasks that require on-the-fly reasoning or domain adaptation, such as unscrambling words, using a novel word in a sentence, or performing 3-digit arithmetic. At the same time, we also identify some datasets where GPT-3's few-shot learning still struggles, as well as some datasets where GPT-3 faces methodological issues related to training on large web corpora. Finally, we find that GPT-3 can generate samples of news articles which human evaluators have difficulty distinguishing from articles written by humans. We discuss broader societal impacts of this finding and of GPT-3 in general.""", + ), + ( + """A mult""", + """ilayer perceptron (MLP) is a class of feedforward artificial neural network (ANN)""", + ), + ("""Hello""", """ World"""), + ] + ) + + assert ll_dog > ll_cat + assert ig_dog + assert not ig_cat + assert not ll_max_0 + assert not ll_max_1 + assert not ll_max_2 + + # Test empty context + oa_model.loglikelihood([("", "test")]) + request_args = { + "stop_sequences": ["."], + "max_generation_length": 4, + "num_fewshot": 0, + } + (gen,) = oa_model.greedy_until( + [("The quick brown fox jumps over the lazy", request_args)] + ) + assert gen == " dog" + + logger.info([x[0] for x in vals]) + + targets = [ + -34.848301606999996, + -47.148329679999996, + -45.44380149599999, + -5.285246016, + -133.97821690686004, + -321.2616693239001, + -658.0299524401041, + -34.848301606999996, + -7.525115, + ] + for (pred, _), tgt in zip(vals, targets): + assert pred == pytest.approx(tgt, rel=1e-3) + + +@mock.patch("lm_eval.models.openai_completions.oa_completion", new=_mock_completion) +def test_openai_completions_perplexity(): + set_seed() + if "OPENAI_API_SECRET_KEY" not in os.environ: + os.environ["OPENAI_API_SECRET_KEY"] = "" + oa_model = models.get_model_from_args_string( + model_api_name="openai", model_args="engine=ada" + ) + test_string = "We study empirical scaling laws for language model performance on the cross-entropy loss." + perplexity = oa_model.loglikelihood_rolling([(test_string,)])[0] + tgt = -84.38819608 + assert perplexity == pytest.approx(tgt, rel=1e-3) + + # Hack: modify gpt3 to have shorter context length to induce rolling windows + with mock.patch.object( + models.openai_completions.OpenAICompletionsLM, + "max_length", + new_callable=mock.PropertyMock, + ) as mock_max_length: + mock_max_length.return_value = 5 + oa_model = models.get_model_from_args_string( + model_api_name="openai", model_args="engine=ada" + ) + perplexity = oa_model.loglikelihood_rolling([(test_string,)])[0] + tgt = -101.81967209999999 + assert perplexity == pytest.approx(tgt, rel=1e-3) diff --git a/evaluation-pipeline/tests/test_tasks.py b/evaluation-pipeline/tests/test_tasks.py new file mode 100644 index 0000000000000000000000000000000000000000..c44b6bc6815ced9f8062f43e0c99b75d792ed8a4 --- /dev/null +++ b/evaluation-pipeline/tests/test_tasks.py @@ -0,0 +1,231 @@ +import logging +import pytest +import numpy as np +from typing import Optional, Tuple +from itertools import islice +from promptsource.templates import Template + +import lm_eval.tasks as tasks +from lm_eval.api.task import Task +from lm_eval.api.request import Request +from lm_eval.api.utils import set_seed, DEFAULT_SEED + + +logger = logging.getLogger(__name__) + + +def _get_deterministic_template( + task_name: str, +) -> Tuple[Optional[Template], bool]: + """Some `promptsource` templates randomize the ordering of prompt attributes + If `task_class` does not have a prompt template with non-random ordering we + return None. + + Returns: + (prompt template, is_deterministic) + """ + # Only choose 1 promptsource template. + prompt_template = None + templates = tasks.get_templates(task_name) + if templates.all_template_names: + for template_name in templates.all_template_names: + prompt_template = templates[template_name] + # Hacky way to ensure we only grab a deterministic jinja template. + if ( + "range(" not in prompt_template.jinja + and "random" not in prompt_template.jinja + ): + return prompt_template, True + # Return the last non-deterministic template. + return prompt_template, False + return None, False + + +def _filter_docs(task: Task): + def _filter(doc: dict): + return not task.invalid_doc_for_prompt(doc) + + return _filter + + +@pytest.mark.parametrize("task_name,task_class", tasks.TASK_REGISTRY.items()) +def test_basic_interface(task_name: str, task_class: Task): + logger.info("Evaluating task", task_name) + prompt_template, is_deterministic = _get_deterministic_template(task_name) + task = task_class(prompt_template=prompt_template) + + assert task.has_training_docs() in [True, False] + assert task.has_validation_docs() in [True, False] + assert task.has_test_docs() in [True, False] + + assert isinstance(task.aggregation(), dict) + assert isinstance(task.higher_is_better(), dict) + assert task.aggregation().keys() == task.higher_is_better().keys() + for v in task.higher_is_better().values(): + assert v in [True, False] + assert isinstance(task.VERSION, int) + + # Test deterministic docs (NOTE: Don't test train because it's slow). + # Return if the prompts are non-deterministic here. + if not is_deterministic: + return + + limit = 100 + task2 = task_class(prompt_template=prompt_template) + + if task.has_validation_docs(): + arr = list(task.validation_docs().filter(_filter_docs(task)))[:limit] + arr2 = list(task2.validation_docs().filter(_filter_docs(task2)))[:limit] + assert arr == arr2 + requests = [ + task.construct_requests(doc, task.doc_to_text(doc), {"num_fewshot": 0}) + for doc in arr + ] + requests2 = [ + task.construct_requests(doc, task2.doc_to_text(doc), {"num_fewshot": 0}) + for doc in arr2 + ] + assert requests == requests2 + + if task.has_test_docs(): + arr = list(task.test_docs().filter(_filter_docs(task)))[:limit] + arr2 = list(task2.test_docs().filter(_filter_docs(task2)))[:limit] + assert arr == arr2 + requests = [ + task.construct_requests(doc, task.doc_to_text(doc), {"num_fewshot": 0}) + for doc in arr + ] + requests2 = [ + task2.construct_requests(doc, task2.doc_to_text(doc), {"num_fewshot": 0}) + for doc in arr2 + ] + assert requests == requests2 + + +@pytest.mark.parametrize("task_name,task_class", tasks.TASK_REGISTRY.items()) +def test_documents_and_requests(task_name: str, task_class: Task): + set_seed() + logger.info("Evaluating task", task_name) + prompt_template, _ = _get_deterministic_template(task_name) + task = task_class(prompt_template=prompt_template) + + fns = [] + + # Training docs are too expensive to run on CI. + # if task.has_training_docs(): + # fns.append(task.training_docs) + if task.has_validation_docs(): + fns.append(task.validation_docs) + for fn in fns: + docs = fn().filter(_filter_docs(task)) + for doc in islice(docs, 5): + text = task.doc_to_text(doc) + target = task.doc_to_target(doc) + + assert isinstance(text, str) + assert isinstance(target, list) + + requests = task.construct_requests(doc, text, {"num_fewshot": 0}) + + # Construct_requests can return just one request + if not isinstance(requests, (list, tuple)): + requests = [requests] + # TODO: Mock lm after refactoring evaluator.py to not be a mess + for req in requests: + assert isinstance(req, Request) + + +def test_arg_string_task_creation(): + import itertools + + TEST_EXAMPLE_SEPS = [ + # Test `=` symbol in value string + "\n===TEST_SEPARATOR===\n", + # Test whitespace only separators + " ", + " \t\t ", + "\n\n\n\n", + # Test empty string separator + "", + # Test misc. symbols in separator + "[[[[]]]]", + "<<___>>", + "(())", + ] + TEST_TEXT_TARGET_SEPS = [ + # Test whitespace separators + " ", + " \t ", + "\n\n\n", + ] + + # Ensure parsing properly handles args. + for example_sep, text_target_sep in itertools.product( + TEST_EXAMPLE_SEPS, TEST_TEXT_TARGET_SEPS + ): + test_arg_string = f" save_examples=False,example_separator={example_sep},text_target_separator={text_target_sep}" + task = tasks.get_task_list_from_args_string( + "wnli", + template_names=["confident"], + task_args=test_arg_string, + )[0] + + assert task.save_examples is False + assert task.example_separator == example_sep + assert task.text_target_separator == text_target_sep + + # Ensure fewshot context is formatted as expected. + TEST_EXAMPLE_SEP = "\n===TEST_SEPARATOR===\n" + TEST_TEXT_TARGET_SEP = " " + test_arg_string = f" save_examples=False,example_separator={TEST_EXAMPLE_SEP},text_target_separator={TEST_TEXT_TARGET_SEP}" + task = tasks.get_task_list_from_args_string( + "wnli", + template_names=["confident"], + task_args=test_arg_string, + )[0] + context = task.fewshot_context( + task.validation_docs()[0], + num_fewshot=2, + rng=np.random.default_rng(DEFAULT_SEED), + )[0] + expected = f"""If it's true that +The man couldn't lift his son because he was so heavy. +how confident should I be that +The man was so heavy. +very confident or not confident? not confident +===TEST_SEPARATOR=== +If it's true that +As Ollie carried Tommy up the long winding steps, his legs ached. +how confident should I be that +Ollie's legs ached. +very confident or not confident? very confident +===TEST_SEPARATOR=== +If it's true that +The drain is clogged with hair. It has to be cleaned. +how confident should I be that +The hair has to be cleaned. +very confident or not confident?""" + assert context == expected + + # Ensure tasks don't instantiate with invalid args. + with pytest.raises(AssertionError): + bad_save_examples_arg_string = "example_separator=\t,save_examples=yes" + task = tasks.get_task_list_from_args_string( + "wnli", + template_names=["confident"], + task_args=bad_save_examples_arg_string, + )[0] + + bad_example_sep_arg_string = "example_separator=False,save_examples=False" + task = tasks.get_task_list_from_args_string( + "wnli", + template_names=["confident"], + task_args=bad_example_sep_arg_string, + )[0] + + bad_text_sep_arg_string = "text_target_separator=___" + task = tasks.get_task_list_from_args_string( + "wnli", + template_names=["confident"], + task_args=bad_text_sep_arg_string, + )[0] diff --git a/evaluation-pipeline/tests/test_utils.py b/evaluation-pipeline/tests/test_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d32f61af15fcd727000d340fb77e29f9dd8afb14 --- /dev/null +++ b/evaluation-pipeline/tests/test_utils.py @@ -0,0 +1,285 @@ +import torch + +from lm_eval.api.utils import ( + get_rolling_token_windows, + make_disjoint_window, + select_continuation_from_batch_left_padding, + split_and_pad_windows, +) + + +# noinspection DuplicatedCode +def test_get_rolling_token_windows_v1(): + gold = [ + ([-100, 0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), + ( + [9, 10, 11, 12, 13, 14, 15, 16, 17, 18], + [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], + ), + ( + [19, 20, 21, 22, 23, 24, 25, 26, 27, 28], + [20, 21, 22, 23, 24, 25, 26, 27, 28, 29], + ), + ([23, 24, 25, 26, 27, 28, 29, 30, 31, 32], [30, 31, 32, 33]), + ] + x = list(range(34)) + generator = get_rolling_token_windows( + token_list=x, + prefix_token=-100, + max_seq_len=10, + context_len=1, + ) + pred_length = 0 + output = [] + for input_tokens, pred_tokens in generator: + output.append((input_tokens, pred_tokens)) + pred_length += len(pred_tokens) + assert pred_length == len(x) + assert gold == output + + +# noinspection DuplicatedCode +def test_get_rolling_token_windows_v2(): + gold = [ + ([-100, 0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), + ([2, 3, 4, 5, 6, 7, 8, 9, 10, 11], [10, 11, 12]), + ([5, 6, 7, 8, 9, 10, 11, 12, 13, 14], [13, 14, 15]), + ([8, 9, 10, 11, 12, 13, 14, 15, 16, 17], [16, 17, 18]), + ([11, 12, 13, 14, 15, 16, 17, 18, 19, 20], [19, 20, 21]), + ([14, 15, 16, 17, 18, 19, 20, 21, 22, 23], [22, 23, 24]), + ([17, 18, 19, 20, 21, 22, 23, 24, 25, 26], [25, 26, 27]), + ([20, 21, 22, 23, 24, 25, 26, 27, 28, 29], [28, 29, 30]), + ([23, 24, 25, 26, 27, 28, 29, 30, 31, 32], [31, 32, 33]), + ] + x = list(range(34)) + generator = get_rolling_token_windows( + token_list=x, + prefix_token=-100, + max_seq_len=10, + context_len=8, + ) + pred_length = 0 + output = [] + for input_tokens, pred_tokens in generator: + output.append((input_tokens, pred_tokens)) + pred_length += len(pred_tokens) + assert pred_length == len(x) + assert gold == output + + +# noinspection DuplicatedCode +def test_get_rolling_token_windows_v3(): + gold = [ + ([-100, 0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), + ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], [10]), + ([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [11]), + ([2, 3, 4, 5, 6, 7, 8, 9, 10, 11], [12]), + ([3, 4, 5, 6, 7, 8, 9, 10, 11, 12], [13]), + ([4, 5, 6, 7, 8, 9, 10, 11, 12, 13], [14]), + ([5, 6, 7, 8, 9, 10, 11, 12, 13, 14], [15]), + ([6, 7, 8, 9, 10, 11, 12, 13, 14, 15], [16]), + ([7, 8, 9, 10, 11, 12, 13, 14, 15, 16], [17]), + ([8, 9, 10, 11, 12, 13, 14, 15, 16, 17], [18]), + ([9, 10, 11, 12, 13, 14, 15, 16, 17, 18], [19]), + ([10, 11, 12, 13, 14, 15, 16, 17, 18, 19], [20]), + ([11, 12, 13, 14, 15, 16, 17, 18, 19, 20], [21]), + ([12, 13, 14, 15, 16, 17, 18, 19, 20, 21], [22]), + ([13, 14, 15, 16, 17, 18, 19, 20, 21, 22], [23]), + ([14, 15, 16, 17, 18, 19, 20, 21, 22, 23], [24]), + ([15, 16, 17, 18, 19, 20, 21, 22, 23, 24], [25]), + ([16, 17, 18, 19, 20, 21, 22, 23, 24, 25], [26]), + ([17, 18, 19, 20, 21, 22, 23, 24, 25, 26], [27]), + ([18, 19, 20, 21, 22, 23, 24, 25, 26, 27], [28]), + ([19, 20, 21, 22, 23, 24, 25, 26, 27, 28], [29]), + ([20, 21, 22, 23, 24, 25, 26, 27, 28, 29], [30]), + ([21, 22, 23, 24, 25, 26, 27, 28, 29, 30], [31]), + ([22, 23, 24, 25, 26, 27, 28, 29, 30, 31], [32]), + ([23, 24, 25, 26, 27, 28, 29, 30, 31, 32], [33]), + ] + x = list(range(34)) + generator = get_rolling_token_windows( + token_list=x, + prefix_token=-100, + max_seq_len=10, + context_len=10, + ) + pred_length = 0 + output = [] + for input_tokens, pred_tokens in generator: + output.append((input_tokens, pred_tokens)) + pred_length += len(pred_tokens) + assert pred_length == len(x) + assert gold == output + + +# noinspection DuplicatedCode +def test_get_rolling_token_windows_v4(): + gold = [ + ([-100, 0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), + ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], [10]), + ([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [11]), + ([2, 3, 4, 5, 6, 7, 8, 9, 10, 11], [12]), + ([3, 4, 5, 6, 7, 8, 9, 10, 11, 12], [13]), + ([4, 5, 6, 7, 8, 9, 10, 11, 12, 13], [14]), + ([5, 6, 7, 8, 9, 10, 11, 12, 13, 14], [15]), + ([6, 7, 8, 9, 10, 11, 12, 13, 14, 15], [16]), + ([7, 8, 9, 10, 11, 12, 13, 14, 15, 16], [17]), + ([8, 9, 10, 11, 12, 13, 14, 15, 16, 17], [18]), + ([9, 10, 11, 12, 13, 14, 15, 16, 17, 18], [19]), + ([10, 11, 12, 13, 14, 15, 16, 17, 18, 19], [20]), + ([11, 12, 13, 14, 15, 16, 17, 18, 19, 20], [21]), + ([12, 13, 14, 15, 16, 17, 18, 19, 20, 21], [22]), + ([13, 14, 15, 16, 17, 18, 19, 20, 21, 22], [23]), + ([14, 15, 16, 17, 18, 19, 20, 21, 22, 23], [24]), + ([15, 16, 17, 18, 19, 20, 21, 22, 23, 24], [25]), + ([16, 17, 18, 19, 20, 21, 22, 23, 24, 25], [26]), + ([17, 18, 19, 20, 21, 22, 23, 24, 25, 26], [27]), + ([18, 19, 20, 21, 22, 23, 24, 25, 26, 27], [28]), + ([19, 20, 21, 22, 23, 24, 25, 26, 27, 28], [29]), + ] + x = list(range(30)) + generator = get_rolling_token_windows( + token_list=x, + prefix_token=-100, + max_seq_len=10, + context_len=10, + ) + pred_length = 0 + output = [] + for input_tokens, pred_tokens in generator: + output.append((input_tokens, pred_tokens)) + pred_length += len(pred_tokens) + assert pred_length == len(x) + assert gold == output + + +# noinspection DuplicatedCode +def test_get_rolling_token_windows_v5(): + gold = [ + ([-100, 0, 1, 2, 3, 4, 5, 6, 7, 8], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), + ( + [9, 10, 11, 12, 13, 14, 15, 16, 17, 18], + [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], + ), + ( + [19, 20, 21, 22, 23, 24, 25, 26, 27, 28], + [20, 21, 22, 23, 24, 25, 26, 27, 28, 29], + ), + ] + x = list(range(30)) + generator = get_rolling_token_windows( + token_list=x, + prefix_token=-100, + max_seq_len=10, + context_len=1, + ) + pred_length = 0 + output = [] + for input_tokens, pred_tokens in generator: + output.append((input_tokens, pred_tokens)) + pred_length += len(pred_tokens) + assert pred_length == len(x) + assert gold == output + + +# noinspection DuplicatedCode +def test_get_rolling_token_windows_v6(): + gold = [ + ([-100, 0], [0, 1]), + ([1, 2], [2, 3]), + ([3, 4], [4, 5]), + ([5, 6], [6, 7]), + ([6, 7], [8]), + ] + x = list(range(9)) + generator = get_rolling_token_windows( + token_list=x, + prefix_token=-100, + max_seq_len=2, + context_len=1, + ) + pred_length = 0 + output = [] + for input_tokens, pred_tokens in generator: + output.append((input_tokens, pred_tokens)) + pred_length += len(pred_tokens) + assert pred_length == len(x) + assert gold == output + + +def test_get_rolling_token_windows_empty(): + generator = get_rolling_token_windows( + token_list=[], + prefix_token=-100, + max_seq_len=2, + context_len=1, + ) + n = 0 + for _ in generator: + n += 1 + assert n == 0 + + +def test_make_disjoint_window(): + assert make_disjoint_window(([1, 2, 3, 4, 5], [2, 3, 4, 5, 6])) == ( + [1], + [2, 3, 4, 5, 6], + ) + assert make_disjoint_window(([1, 2, 3, 4, 5], [4, 5, 6])) == ([1, 2, 3], [4, 5, 6]) + + +def test_pad_windows(): + token_list = [100, 19, 3, 9, 794, 7142, 81, 1327, 5] + # Representation: [([context], [continuation]), ...] + # [ + # ([1], [100, 19, 3, 9]), + # ([9], [794, 7142, 81, 1327]), + # ([ ], [5]) + # ] + rolling_token_windows = list( + map( + make_disjoint_window, + get_rolling_token_windows( + token_list=token_list, + prefix_token=1, + max_seq_len=4, + context_len=1, + ), + ) + ) + expected = ( + [[1], [9], [1]], # Split & padded contexts. + [ + [100, 19, 3, 9], + [794, 7142, 81, 1327], + [5, 1, 1, 1], + ], # Split & padded continuations. + ) + padded_windows = split_and_pad_windows( + rolling_token_windows, pad_token_id=1, max_seq_len=4 + ) + assert padded_windows == expected + + +def test_select_continuation_from_batch_1(): + generations = torch.tensor( + [ + # sentence 1 + [0, 0, 1, 2, 3, 11, 22, 33, 44, 55], + # sentence 2 + [1, 2, 3, 4, 5, 11, 22, 33, 0, 0], + ] + ) + max_context_size = 5 + expected = torch.tensor( + [ + # sentence 1 + [11, 22, 33, 44, 55], + # sentence 2 + [11, 22, 33, 0, 0], + ] + ) + assert torch.equal( + select_continuation_from_batch_left_padding(generations, max_context_size), + expected, + ) diff --git a/evaluation-pipeline/tests/test_version_stable.py b/evaluation-pipeline/tests/test_version_stable.py new file mode 100644 index 0000000000000000000000000000000000000000..6dc4d5e6e9a913ae258d516dddea2d086deed272 --- /dev/null +++ b/evaluation-pipeline/tests/test_version_stable.py @@ -0,0 +1,128 @@ +import random +import pytest +import os +import json +import hashlib +import collections + +import lm_eval +from lm_eval.api.utils import DEFAULT_SEED, set_seed + + +def _assert_target(name, ob): + fname = f"tests/testdata/{name}.json" + if os.path.exists(fname): + with open(fname) as fh: + # Use relative tolerance of 1e-5 and absolute tolerance of 1e-8 + # assuming most metrics work on `float32` values, which is the common + # default floating type across popular libraries (PyTorch, Tensorflow, and JAX). + assert _flatten(json.load(fh)) == pytest.approx( + _flatten(json.loads(json.dumps(ob, sort_keys=True))), rel=1e-5, abs=1e-8 + ) + else: + with open(fname, "w") as fh: + json.dump(ob, fh, sort_keys=True) + + +def _assert_target_hashed(name, ob): + fname = f"tests/testdata/{name}" + if os.path.exists(fname): + with open(fname) as fh: + assert ( + fh.read() + == hashlib.sha256( + json.dumps(ob, sort_keys=True).encode("utf-8") + ).hexdigest() + ) + else: + with open(fname, "w") as fh: + fh.write( + hashlib.sha256( + json.dumps(ob, sort_keys=True).encode("utf-8") + ).hexdigest() + ) + + +# from https://stackoverflow.com/a/6027615 +def _flatten(d, parent_key="", sep="."): + items = [] + for k, v in d.items(): + new_key = parent_key + sep + k if parent_key else k + if isinstance(v, collections.MutableMapping): + items.extend(_flatten(v, new_key, sep=sep).items()) + else: + items.append((new_key, v)) + return dict(items) + + +# Make sure eval results for a task version are stable + + +@pytest.mark.skip(reason="Version stability are not setup for `PropmtSourceTask`s") +# @pytest.mark.parametrize("task_name,task_class", tasks.TASK_REGISTRY.items()) +def test_versions_stable(task_name, task_class): + set_seed() + os.makedirs("tests/testdata", exist_ok=True) + task = lm_eval.get_task(task_name) + model = lm_eval.get_model("dummy") + + def ll_fn(requests): + for ctx, cont in requests: + if len(ctx) == 0: + continue + # Space convention + assert ctx[-1] != " " + assert cont[0] == " " or ctx[-1] == "\n" + + _assert_target_hashed( + f"{task_name}-v{task_class.VERSION}-loglikelihood", requests + ) + res = [] + + random.seed(DEFAULT_SEED) + for _ in requests: + res.append((-random.random(), False)) + + return res + + def ll_perp_fn(requests): + for (string,) in requests: + assert isinstance(string, str) + + _assert_target_hashed( + f"{task_name}-v{task_class.VERSION}-loglikelihood_rolling", requests + ) + res = [] + + random.seed(DEFAULT_SEED) + for _ in requests: + res.append(-random.random()) + + return res + + def greedy_until(requests): + res = [] + _assert_target_hashed( + f"{task_name}-v{task_class.VERSION}-greedy_until", requests + ) + + for ctx, _ in requests: + res.append("none") + assert ctx.strip() != "" + + return res + + model.loglikelihood = ll_fn + model.loglikelihood_rolling = ll_perp_fn + model.greedy_until = greedy_until + + limit = None + result = lm_eval.evaluate( + model=model, + tasks=[task], + num_fewshot=0, + limit=limit, + bootstrap_iters=10, + ) + + _assert_target(f"{task_name}-v{task_class.VERSION}-res", result) diff --git a/evaluation-pipeline/tests/testdata/gpt3_test_724fd8ee05f4fb0dcd0c97d862d0e8fcde349872de40cc19c27b85f63ada3aec.pkl b/evaluation-pipeline/tests/testdata/gpt3_test_724fd8ee05f4fb0dcd0c97d862d0e8fcde349872de40cc19c27b85f63ada3aec.pkl new file mode 100644 index 0000000000000000000000000000000000000000..ce996cf90bd8bd3b7050c3172f6e5780d33ca833 --- /dev/null +++ b/evaluation-pipeline/tests/testdata/gpt3_test_724fd8ee05f4fb0dcd0c97d862d0e8fcde349872de40cc19c27b85f63ada3aec.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf65fbca75c08a488e2818609c563c576048675ec170479d862ac79da134a54f +size 2537 diff --git a/evaluation-pipeline/tests/testdata/gpt3_test_98c68679557265126807bfc48958b9684f6c6418516eab4b45b5ffa96c024082.pkl b/evaluation-pipeline/tests/testdata/gpt3_test_98c68679557265126807bfc48958b9684f6c6418516eab4b45b5ffa96c024082.pkl new file mode 100644 index 0000000000000000000000000000000000000000..6108df80fe23420c633fe571f7af8325350abc59 --- /dev/null +++ b/evaluation-pipeline/tests/testdata/gpt3_test_98c68679557265126807bfc48958b9684f6c6418516eab4b45b5ffa96c024082.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ea4cb65726eab954ef2193dd4d01e486c5ccb7129197b23df65a155bd2f7411 +size 3042 diff --git a/evaluation-pipeline/tests/testdata/gpt3_test_b32ad98e45170247f4e2883ca8e730ce7c2cb9daa8473c7c76cdd270800e46b1.pkl b/evaluation-pipeline/tests/testdata/gpt3_test_b32ad98e45170247f4e2883ca8e730ce7c2cb9daa8473c7c76cdd270800e46b1.pkl new file mode 100644 index 0000000000000000000000000000000000000000..ae4229028943a452ea4a49545ee746aa9deafe69 --- /dev/null +++ b/evaluation-pipeline/tests/testdata/gpt3_test_b32ad98e45170247f4e2883ca8e730ce7c2cb9daa8473c7c76cdd270800e46b1.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f96c3a14acd5454b7d4b0cc820c4095b3bf6a406fb61529326bca1778e535ca +size 628 diff --git a/evaluation-pipeline/tests/testdata/gpt3_test_e82a7e1622dc2f7ca502fca5ddddd5dea12ca165d95db87c1a561260c961d741.pkl b/evaluation-pipeline/tests/testdata/gpt3_test_e82a7e1622dc2f7ca502fca5ddddd5dea12ca165d95db87c1a561260c961d741.pkl new file mode 100644 index 0000000000000000000000000000000000000000..aa674dd295910f0826962d56945aeabef85b22d5 --- /dev/null +++ b/evaluation-pipeline/tests/testdata/gpt3_test_e82a7e1622dc2f7ca502fca5ddddd5dea12ca165d95db87c1a561260c961d741.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43b84a63f991f0933b14ed0f3d01f90b50d1144f1c8a9bf08ce7e9f2f61ce5ab +size 70977 diff --git a/evaluation-pipeline/tests/testdata/gpt3_test_f48ecaa4da3744ad219c82a60cd858efe9882e668f8b3845e49f62faf1d7a3ae.pkl b/evaluation-pipeline/tests/testdata/gpt3_test_f48ecaa4da3744ad219c82a60cd858efe9882e668f8b3845e49f62faf1d7a3ae.pkl new file mode 100644 index 0000000000000000000000000000000000000000..78f11b20fef96854aced6a020db81b823546837d --- /dev/null +++ b/evaluation-pipeline/tests/testdata/gpt3_test_f48ecaa4da3744ad219c82a60cd858efe9882e668f8b3845e49f62faf1d7a3ae.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c47af4103384e896de2d95adb60288ccd5554e96b192867d14f1bdab507abc8e +size 957 diff --git a/evaluation-pipeline/transformers_modified/t5.py b/evaluation-pipeline/transformers_modified/t5.py new file mode 100644 index 0000000000000000000000000000000000000000..9ac3b46e1689dfea8429b26f407a47d13b71943b --- /dev/null +++ b/evaluation-pipeline/transformers_modified/t5.py @@ -0,0 +1,141 @@ +""" +This implementation is based closely on EncT5: https://arxiv.org/abs/2110.08426 +This approach tends to perform better than simply appending a classification head +to T5, especially when reducing the number of parameters in T5-like models. +""" + +import torch +import copy +from torch import nn +from transformers import T5PreTrainedModel +from transformers.models.t5.modeling_t5 import T5Stack +from transformers.modeling_outputs import SequenceClassifierOutput +from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss + + +def mean_pooling(inputs, mask): + token_embeddings = inputs + input_mask_expanded = mask.unsqueeze(-1).expand(token_embeddings.size()).float() + sum_embeddings = torch.sum(token_embeddings * input_mask_expanded, 1) + sum_mask = torch.clamp(input_mask_expanded.sum(1), min=1e-9) + return sum_embeddings / sum_mask + + +class MeanPooler(nn.Module): + """ Calcualte simple average of the inputs """ + def __init__(self, input_size=None): + super().__init__() + + def forward(self, inputs, mask=None): + if mask is None: + pooled_output = inputs.mean(dim=1) + else: + pooled_output = mean_pooling(inputs, mask) + return None, pooled_output + + +class AdaptivePooler(nn.Module): + """ Calcualte weighted average of the inputs with learnable weights """ + def __init__(self, input_size): + super().__init__() + self.input_size = input_size + self.w = nn.Linear(self.input_size, 1, bias=True) + + def forward(self, inputs, mask=None): + batch_size, seq_len, emb_dim = inputs.shape + scores = torch.squeeze(self.w(inputs), dim=-1) + weights = nn.functional.softmax(scores, dim=-1) + if mask is not None: + weights = weights * mask + weights = weights / weights.sum(dim=-1, keepdims=True) + outputs = (inputs.permute(2, 0, 1) * weights).sum(-1).T + return weights, outputs + + +class T5ForSequenceClassification(T5PreTrainedModel): + def __init__(self, config, pooler='adaptive'): + super().__init__(config) + self.num_labels = config.num_labels + self.config = config + + self.shared = nn.Embedding(config.vocab_size, config.d_model) + + encoder_config = copy.deepcopy(config) + encoder_config.is_decoder = False + encoder_config.use_cache = False + encoder_config.is_encoder_decoder = False + + self.encoder = T5Stack(encoder_config, self.shared) + + pooler_class = AdaptivePooler if pooler == 'adaptive' else MeanPooler + self.pooler = pooler_class(input_size=config.hidden_size) + self.dropout = nn.Dropout(config.dropout_rate) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + # Model parallel + self.model_parallel = False + self.device_map = None + + def forward( + self, + input_ids=None, + attention_mask=None, + head_mask=None, + inputs_embeds=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + labels=None, + ): + r""" + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`): + Labels for computing the sequence classification/regression loss. Indices should be in :obj:`[0, ..., + config.num_labels - 1]`. If :obj:`config.num_labels == 1` a regression loss is computed (Mean-Square loss), + If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.encoder( + input_ids=input_ids, + attention_mask=attention_mask, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + weights, pooled_output = self.pooler(outputs[0], mask=attention_mask) + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + + loss = None + if labels is not None: + if self.config.problem_type is None: + if self.num_labels == 1: + self.config.problem_type = "regression" + elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int): + self.config.problem_type = "single_label_classification" + else: + self.config.problem_type = "multi_label_classification" + + if self.config.problem_type == "regression": + loss_fct = MSELoss() + loss = loss_fct(logits.view(-1, self.num_labels), labels) + elif self.config.problem_type == "single_label_classification": + loss_fct = CrossEntropyLoss() + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + elif self.config.problem_type == "multi_label_classification": + loss_fct = BCEWithLogitsLoss() + loss = loss_fct(logits, labels) + if not return_dict: + output = (logits,) + outputs[2:] + return ((loss,) + output) if loss is not None else output + + return SequenceClassifierOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) diff --git a/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1376.ckpt b/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1376.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..37a8b0de167f8cb2590838f0c380a9e513c6a645 --- /dev/null +++ b/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1376.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40f9186a7c008ebb3b99246a8f7f3dabda890de4f8e075cefc87d1b87a456586 +size 2494719367 diff --git a/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1876.ckpt b/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1876.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..2feb95ba7b2bcd8fd12daf2eaf585b0defa57cb5 --- /dev/null +++ b/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1876.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b1906f8b9771c8d1440b99451cc4663601e71b919e7c014f64e7bd98a0e6459 +size 2494719814 diff --git a/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2357.ckpt b/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2357.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..0e01fe44c220ab77843ff01006509387d51b21f3 --- /dev/null +++ b/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2357.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52ceaba25ad6c781d3863c8499e2e7212ab58c7908ad37f6bcf17ef9508613e6 +size 2494720197 diff --git a/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2751.ckpt b/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2751.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..0c2245c0fdb8b555612f269a08ae32a6537d40f3 --- /dev/null +++ b/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2751.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fc30d4946b57b4800c8e1e8a0e14fa14336bdf830968a8a98187b003476af95 +size 2494720644 diff --git a/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3211.ckpt b/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3211.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..4dc941cf54b12c883e6739459656278058d4f65d --- /dev/null +++ b/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3211.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:603760614d9245c83c83e9a31f9cbcc46633841f37a9fd7d924e56d0f69a76a6 +size 2494721091 diff --git a/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3626.ckpt b/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3626.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..6a8d5749bacf1f8a219abb6f9a8e1467f1271d8a --- /dev/null +++ b/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3626.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52a3e16f6b87b2cbd40024b3cef7f586f7b019b636f348b9339ae24b49ac748c +size 2494721474 diff --git a/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=501.ckpt b/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=501.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..7d3e2c9b58fd3dc9ff3b9d61c2e56a43a0bb60fe --- /dev/null +++ b/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=501.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:374f4fcf9c6950dc5a7f02ef28994aa433b6a35197a9e0764086552851c8c8ff +size 2494718473 diff --git a/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=959.ckpt b/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=959.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..d993eb62f87584d048368c820d09ad7c9c745057 --- /dev/null +++ b/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=959.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4e6bfca230b09b2027433cd1c0d1ea70b20379af06f2a83b2648e9989eaba42 +size 2494718920 diff --git a/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/last.ckpt b/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/last.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..d67bc46b2a132e6074bdaaa9635c1d3f7e2f246d --- /dev/null +++ b/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/last.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4552a609176ae5f357b185bd9669294659e16cd6503bd73d0a7032fcafdc40e +size 2494721474 diff --git a/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/singlenode_run_2023-07-04-12:29:36.log b/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/singlenode_run_2023-07-04-12:29:36.log new file mode 100644 index 0000000000000000000000000000000000000000..3209b000a0765544d32b88f1698109d996ff7907 --- /dev/null +++ b/flava-wit/text1-vision0/bs4096_seed-1_bf16-mixed/singlenode_run_2023-07-04-12:29:36.log @@ -0,0 +1,2477 @@ +wandb: Currently logged in as: tamariucai (rycolab). Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.15.4 +wandb: Run data is saved locally in ./wandb/run-20230704_124236-5ty38tox +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run swift-grass-218 +wandb: ⭐️ View project at https://wandb.ai/rycolab/alkmi-wit +wandb: 🚀 View run at https://wandb.ai/rycolab/alkmi-wit/runs/5ty38tox +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/date(2023-07-04)_time(12:42:40)/magic({'enable': True})/ +Text is the predominant modality (1 v.s. 0 vision), will sample proportionally for optimal BLiMP performance. +Assigning HuggingFace RAM +Found 747.32GBs of RAM available, assigning 5GBs to HuggingFace datasets (currently 5000000000 bytes). +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Found cached dataset imagenet-1k (/cluster/scratch/tamariucai/HuggingfaceDatasets/imagenet-1k/default/1.0.0/a1e9bfc56c3a7350165007d1176b15e9128fcaf9ab972147840529aed3ae52bc) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/plugins/environments/slurm.py:165: PossibleUserWarning: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... + rank_zero_warn( +Using bfloat16 Automatic Mixed Precision (AMP) +GPU available: True (cuda), used: True +TPU available: False, using: 0 TPU cores +IPU available: False, using: 0 IPUs +HPU available: False, using: 0 HPUs +Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/2 +wandb: Currently logged in as: tamariucai (rycolab). Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.15.4 +wandb: Run data is saved locally in ./wandb/run-20230704_124847-6r5j3xqs +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run stellar-blaze-219 +wandb: ⭐️ View project at https://wandb.ai/rycolab/alkmi-wit +wandb: 🚀 View run at https://wandb.ai/rycolab/alkmi-wit/runs/6r5j3xqs +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/date(2023-07-04)_time(12:48:50)/magic({'enable': True})/ +Text is the predominant modality (1 v.s. 0 vision), will sample proportionally for optimal BLiMP performance. +Assigning HuggingFace RAM +Found 747.32GBs of RAM available, assigning 5GBs to HuggingFace datasets (currently 5000000000 bytes). +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Found cached dataset imagenet-1k (/cluster/scratch/tamariucai/HuggingfaceDatasets/imagenet-1k/default/1.0.0/a1e9bfc56c3a7350165007d1176b15e9128fcaf9ab972147840529aed3ae52bc) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/2 +---------------------------------------------------------------------------------------------------- +distributed_backend=nccl +All distributed processes registered. Starting with 2 processes +---------------------------------------------------------------------------------------------------- + +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at theodor1289wit-test +LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1] +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1] + + | Name | Type | Params +---------------------------------------------- +0 | model | FlavaForPreTraining | 357 M +---------------------------------------------- +303 M Trainable params +53.8 M Non-trainable params +357 M Total params +1,430.581 Total estimated model params size (MB) +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +MLMDataModule: batch_size is 32, num_workers is 4 +Only one modality, sampling weight will be [1.0] +2023-07-04 12:49:23.935830 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ImageNetZeroshotCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-04 12:53:48.104102 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-04 12:57:34.617564 Building train dataloader for datamodule: MLMDataModule +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +MLMDataModule: batch_size is 32, num_workers is 4 +Only one modality, sampling weight will be [1.0] +2023-07-04 12:43:23.385418 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ImageNetZeroshotCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-04 12:48:33.726740 Preparing data for datamodule: MLMDataModule +2023-07-04 12:53:48.104139 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-04 12:57:34.617734 Building train dataloader for datamodule: MLMDataModule +2023-07-04 12:57:35.163394 Building val dataloader for datamodule: MLMDataModule +2023-07-04 12:57:35.169258 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-04 13:34:22.108461] Computing e^(tensor([17119.1987], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-04 13:34:22.119823] Ending with PPL=5216.590646910538 (duration: 0:12:07.840802) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00 Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-05 17:17:13.941744 Building val dataloader for datamodule: MLMDataModule +[ImageNet Zero-Shot Evaluation 2023-07-05 17:06:45.645992] Ending with top5=0.00412 (duration: 0:02:16.719389) +Starting validation +2023-07-05 17:10:22.362462 Preparing data for datamodule: MLMDataModule +2023-07-05 17:10:22.365361 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-05 17:17:13.941785 Building val dataloader for datamodule: MLMDataModule +huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... +To disable this warning, you can either: + - Avoid using `tokenizers` before the fork if possible + - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) +huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... +To disable this warning, you can either: + - Avoid using `tokenizers` before the fork if possible + - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) +huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... +To disable this warning, you can either: + - Avoid using `tokenizers` before the fork if possible + - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) +huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... +To disable this warning, you can either: + - Avoid using `tokenizers` before the fork if possible + - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) +huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... +To disable this warning, you can either: + - Avoid using `tokenizers` before the fork if possible + - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) +huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... +To disable this warning, you can either: + - Avoid using `tokenizers` before the fork if possible + - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) +huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... +To disable this warning, you can either: + - Avoid using `tokenizers` before the fork if possible + - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) +huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... +To disable this warning, you can either: + - Avoid using `tokenizers` before the fork if possible + - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-05 17:27:48.176398] Computing e^(tensor([6320.8873], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-05 17:27:48.187044] Ending with PPL=23.581055888922748 (duration: 0:10:33.219088) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00 Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-08 19:47:46.810807 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-08 19:49:33.669176 Building train dataloader for datamodule: MLMDataModule +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +MLMDataModule: batch_size is 32, num_workers is 4 +Only one modality, sampling weight will be [1.0] +2023-07-08 19:42:20.660802 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-08 19:45:49.140855 Preparing data for datamodule: MLMDataModule +2023-07-08 19:47:46.810802 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-08 19:49:33.669334 Building train dataloader for datamodule: MLMDataModule +2023-07-08 19:49:34.218738 Building val dataloader for datamodule: MLMDataModule +2023-07-08 19:49:34.342649 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-08 20:18:48.847054] Computing e^(tensor([17215.2016], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-08 20:18:48.854026] Ending with PPL=5473.101943983368 (duration: 0:08:37.407087) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 7.316 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-08 21:38:33.105907] Computing e^(tensor([14926.7587], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-08 21:38:33.111229] Ending with PPL=1743.0284130298728 (duration: 0:09:32.009665) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 7.055 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-08 22:17:54.388850] Computing e^(tensor([14598.5141], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-08 22:17:54.394449] Ending with PPL=1479.2005380821522 (duration: 0:09:26.845000) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.843 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-08 22:57:22.441962] Computing e^(tensor([14305.0318], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-08 22:57:22.446044] Ending with PPL=1277.3155123107786 (duration: 0:09:29.501695) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.692 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-08 23:36:58.337396] Computing e^(tensor([13926.5740], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-08 23:36:58.341551] Ending with PPL=1057.10254002577 (duration: 0:09:36.234441) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.539 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 00:16:35.600652] Computing e^(tensor([13459.1577], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 00:16:35.605071] Ending with PPL=836.7947634698531 (duration: 0:09:37.455735) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.380 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 00:56:18.523073] Computing e^(tensor([12702.0160], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 00:56:18.528976] Ending with PPL=573.0700666042676 (duration: 0:09:42.980021) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.155 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 01:36:41.237470] Computing e^(tensor([11713.3999], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 01:36:41.242379] Ending with PPL=349.56864159866103 (duration: 0:10:24.941536) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 5.849 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 02:16:31.390798] Computing e^(tensor([10594.0938], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 02:16:31.396524] Ending with PPL=199.74606325719023 (duration: 0:09:47.849423) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 5.419 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 02:56:19.507798] Computing e^(tensor([9884.8133], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 02:56:19.512097] Ending with PPL=140.1070313344861 (duration: 0:09:34.481047) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 5.118 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 03:36:21.590409] Computing e^(tensor([9335.7413], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 03:36:21.594589] Ending with PPL=106.47078775256746 (duration: 0:09:52.569385) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.847 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 04:16:41.968299] Computing e^(tensor([8849.7930], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 04:16:41.974413] Ending with PPL=83.50416528511464 (duration: 0:10:22.761354) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.664 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 04:57:10.309672] Computing e^(tensor([8410.6182], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 04:57:10.313738] Ending with PPL=67.04131734676417 (duration: 0:10:21.997849) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.498 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 05:37:44.599275] Computing e^(tensor([7987.3979], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 05:37:44.604180] Ending with PPL=54.255205553525634 (duration: 0:10:29.375094) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.337 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 06:18:18.140863] Computing e^(tensor([7637.8357], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 06:18:18.145354] Ending with PPL=45.55488534864427 (duration: 0:10:26.628063) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.206 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 06:58:48.662336] Computing e^(tensor([7390.5020], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 06:58:48.666619] Ending with PPL=40.25567638602103 (duration: 0:10:21.351865) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.111 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 07:38:55.931958] Computing e^(tensor([7124.6927], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 07:38:55.936139] Ending with PPL=35.24579889731653 (duration: 0:10:00.513921) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.011 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 08:19:07.377435] Computing e^(tensor([6941.2689], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 08:19:07.382330] Ending with PPL=32.15713809953821 (duration: 0:10:04.767890) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.938 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 08:59:48.118370] Computing e^(tensor([6766.7484], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 08:59:48.123862] Ending with PPL=29.470040827771136 (duration: 0:10:19.681519) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.857 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 09:40:31.673496] Computing e^(tensor([6640.4922], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 09:40:31.678051] Ending with PPL=27.667158119043215 (duration: 0:10:37.991785) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.810 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 10:21:16.676611] Computing e^(tensor([6528.3391], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 10:21:16.682316] Ending with PPL=26.158378665243934 (duration: 0:10:38.239511) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.755 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 11:01:26.207402] Computing e^(tensor([6424.8188], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 11:01:26.211562] Ending with PPL=24.83886132833259 (duration: 0:10:07.570714) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.722 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 11:42:15.705168] Computing e^(tensor([6360.3005], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 11:42:15.709693] Ending with PPL=24.050367182660757 (duration: 0:10:29.535222) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.693 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 12:23:21.257490] Computing e^(tensor([6295.4002], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 12:23:21.261727] Ending with PPL=23.28245608811327 (duration: 0:10:49.470787) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.652 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 13:04:27.899178] Computing e^(tensor([6260.7418], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 13:04:27.905278] Ending with PPL=22.882465190747837 (duration: 0:10:40.822121) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.640 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 13:45:36.972728] Computing e^(tensor([6228.2637], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 13:45:36.977039] Ending with PPL=22.51387656448768 (duration: 0:10:47.369354) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.621 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 14:26:53.548815] Computing e^(tensor([6202.9491], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 14:26:53.553677] Ending with PPL=22.230707348105955 (duration: 0:10:55.221200) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.618 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 15:48:58.858034] Computing e^(tensor([6233.4033], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 15:48:58.862314] Ending with PPL=22.571806949165893 (duration: 0:10:32.201854) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.616 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 16:30:24.435015] Computing e^(tensor([6262.9788], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 16:30:24.439484] Ending with PPL=22.90807301445394 (duration: 0:10:57.694898) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00 + main() + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 178, in main + for key, val in model.original_weights: +ValueError: too many values to unpack (expected 2) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-08 20:20:20.191676] Computing e^(tensor([17118.3939], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-08 20:20:20.197778] Ending with PPL=5214.491990120619 (duration: 0:10:08.742285) +[rank: 1] Metric validation/losses/mlm_loss improved. New best score: 8.467 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-08 21:00:17.957975] Computing e^(tensor([15108.7278], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-08 21:00:17.961998] Ending with PPL=1909.055548730877 (duration: 0:11:31.192779) +[rank: 1] Metric validation/losses/mlm_loss improved by 1.150 >= min_delta = 0.0. New best score: 7.316 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-08 21:41:14.991499] Computing e^(tensor([14802.9987], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-08 21:41:14.995511] Ending with PPL=1638.4391843463316 (duration: 0:12:13.900610) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.261 >= min_delta = 0.0. New best score: 7.055 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-08 22:20:41.746977] Computing e^(tensor([14464.7377], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-08 22:20:41.751137] Ending with PPL=1383.4958908784881 (duration: 0:12:14.194754) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.212 >= min_delta = 0.0. New best score: 6.843 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-08 23:00:05.353676] Computing e^(tensor([14162.2235], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-08 23:00:05.358025] Ending with PPL=1189.2899559644684 (duration: 0:12:12.419649) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.151 >= min_delta = 0.0. New best score: 6.692 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-08 23:39:41.843210] Computing e^(tensor([13768.9462], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-08 23:39:41.847231] Ending with PPL=976.9867582945228 (duration: 0:12:19.753609) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.152 >= min_delta = 0.0. New best score: 6.539 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 00:19:20.505559] Computing e^(tensor([13288.0312], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 00:19:20.509639] Ending with PPL=768.1735022491453 (duration: 0:12:22.376360) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.160 >= min_delta = 0.0. New best score: 6.380 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 00:58:56.613823] Computing e^(tensor([12515.9512], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 00:58:56.617985] Ending with PPL=522.1608076827894 (duration: 0:12:21.070189) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.224 >= min_delta = 0.0. New best score: 6.155 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 01:38:55.797976] Computing e^(tensor([11509.9808], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 01:38:55.802148] Ending with PPL=315.76251288216815 (duration: 0:12:39.495926) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.306 >= min_delta = 0.0. New best score: 5.849 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 02:19:06.846880] Computing e^(tensor([10348.6587], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 02:19:06.850991] Ending with PPL=176.6780844442912 (duration: 0:12:23.302331) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.430 >= min_delta = 0.0. New best score: 5.419 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 02:59:06.296533] Computing e^(tensor([9632.5204], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 02:59:06.300677] Ending with PPL=123.50235150739493 (duration: 0:12:21.269005) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.301 >= min_delta = 0.0. New best score: 5.118 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 03:38:58.278481] Computing e^(tensor([9066.5539], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 03:38:58.282781] Ending with PPL=93.06302492293725 (duration: 0:12:29.241478) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.271 >= min_delta = 0.0. New best score: 4.847 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 04:18:57.111062] Computing e^(tensor([8579.1745], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 04:18:57.115210] Ending with PPL=72.9363581652376 (duration: 0:12:37.910461) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.183 >= min_delta = 0.0. New best score: 4.664 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 04:59:22.353018] Computing e^(tensor([8135.8445], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 04:59:22.357296] Ending with PPL=58.435421715772804 (duration: 0:12:34.008729) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.166 >= min_delta = 0.0. New best score: 4.498 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 05:39:52.694376] Computing e^(tensor([7706.0550], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 05:39:52.698453] Ending with PPL=47.13555088058851 (duration: 0:12:37.487119) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.161 >= min_delta = 0.0. New best score: 4.337 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 06:20:34.638544] Computing e^(tensor([7356.0446], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 06:20:34.642750] Ending with PPL=39.56806377742103 (duration: 0:12:43.127655) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.131 >= min_delta = 0.0. New best score: 4.206 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 07:01:05.379314] Computing e^(tensor([7099.2094], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 07:01:05.383710] Ending with PPL=34.799557808855496 (duration: 0:12:38.079750) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.095 >= min_delta = 0.0. New best score: 4.111 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 07:41:24.902724] Computing e^(tensor([6848.2451], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 07:41:24.906788] Ending with PPL=30.69569946694089 (duration: 0:12:29.498119) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.100 >= min_delta = 0.0. New best score: 4.011 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 08:21:34.297919] Computing e^(tensor([6646.8743], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 08:21:34.302009] Ending with PPL=27.75558643504134 (duration: 0:12:31.699361) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.074 >= min_delta = 0.0. New best score: 3.938 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 09:02:05.327068] Computing e^(tensor([6468.2095], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 09:02:05.331292] Ending with PPL=25.38363669209364 (duration: 0:12:36.893378) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.081 >= min_delta = 0.0. New best score: 3.857 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 09:42:29.029469] Computing e^(tensor([6336.1554], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 09:42:29.033624] Ending with PPL=23.761763243162747 (duration: 0:12:35.353511) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.047 >= min_delta = 0.0. New best score: 3.810 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 10:23:13.263325] Computing e^(tensor([6232.5731], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 10:23:13.267453] Ending with PPL=22.56243977632903 (duration: 0:12:34.814017) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.054 >= min_delta = 0.0. New best score: 3.755 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 11:03:51.636837] Computing e^(tensor([6140.5092], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 11:03:51.640880] Ending with PPL=21.54738803479404 (duration: 0:12:32.980917) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.034 >= min_delta = 0.0. New best score: 3.722 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 11:44:29.054177] Computing e^(tensor([6055.9216], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 11:44:29.058307] Ending with PPL=20.655070016813102 (duration: 0:12:42.936565) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.029 >= min_delta = 0.0. New best score: 3.693 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 12:25:23.220862] Computing e^(tensor([6011.7498], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 12:25:23.225148] Ending with PPL=20.203884294201604 (duration: 0:12:51.418448) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.041 >= min_delta = 0.0. New best score: 3.652 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 13:06:28.215977] Computing e^(tensor([5986.0796], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 13:06:28.220093] Ending with PPL=19.946223174294506 (duration: 0:12:41.142155) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.011 >= min_delta = 0.0. New best score: 3.640 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 13:47:34.420785] Computing e^(tensor([5934.8365], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 13:47:34.424905] Ending with PPL=19.441660958244828 (duration: 0:12:44.803077) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.020 >= min_delta = 0.0. New best score: 3.621 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 14:28:43.396139] Computing e^(tensor([5920.0240], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 14:28:43.400191] Ending with PPL=19.298203807775494 (duration: 0:12:45.052015) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 3.621. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 15:10:07.200276] Computing e^(tensor([5937.2539], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 15:10:07.204372] Ending with PPL=19.46517426459453 (duration: 0:12:44.769265) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.002 >= min_delta = 0.0. New best score: 3.618 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 15:51:07.755527] Computing e^(tensor([5926.6381], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 15:51:07.759951] Ending with PPL=19.3621287287782 (duration: 0:12:41.105234) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.003 >= min_delta = 0.0. New best score: 3.616 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 16:32:16.050494] Computing e^(tensor([5965.9699], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 16:32:16.054835] Ending with PPL=19.74667159418506 (duration: 0:12:49.345463) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 3.616. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 17:13:37.972989] Computing e^(tensor([6022.4088], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 17:13:37.977149] Ending with PPL=20.311848583406185 (duration: 0:12:51.031318) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 2 records. Best score: 3.616. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 17:55:16.223941] Computing e^(tensor([6056.1259], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 17:55:16.228225] Ending with PPL=20.657179692409215 (duration: 0:12:46.262492) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve in the last 3 records. Best score: 3.616. Signaling model to ignore task. +Traceback (most recent call last): + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 190, in + main() + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 178, in main + for key, val in model.original_weights: +ValueError: too many values to unpack (expected 2) +Traceback (most recent call last): + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 190, in + main() + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 178, in main + for key, val in model.original_weights: +ValueError: too many values to unpack (expected 2) +wandb: Waiting for W&B process to finish... (failed 1). Press Control-C to abort syncing. +wandb: Waiting for W&B process to finish... (failed 1). Press Control-C to abort syncing. +wandb: - 0.012 MB of 0.012 MB uploaded (0.000 MB deduped) wandb: \ 0.012 MB of 0.012 MB uploaded (0.000 MB deduped) wandb: | 0.244 MB of 0.244 MB uploaded (0.000 MB deduped) wandb: / 0.244 MB of 0.244 MB uploaded (0.000 MB deduped) wandb: - 0.244 MB of 0.244 MB uploaded (0.000 MB deduped) wandb: +wandb: \ 0.244 MB of 0.244 MB uploaded (0.000 MB deduped) wandb: +wandb: +wandb: Run history: +wandb: epoch ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: evaluation/blimp/anaphor_agreement ▁▁▂▂▃▃▃▃▃▄▄▄▄▄▅▆▇▆▇▇▇▇▇█▇▇█▇▇▇█▇▇ +wandb: evaluation/blimp/argument_structure ▁▃▄▄▅▅▄▆▆▆▆▇▇▇▆▇█▇██▇▇██▇▇▇▇▆▆▆▆▆ +wandb: evaluation/blimp/binding ▅█▇▇▅▆▇█▇▃▂▅▄▃▄▄▃▅▁▂▁▃▆▄▁▄▂▆▅▅▆▃▄ +wandb: evaluation/blimp/control_raising ▁▂▃▃▃▃▄▄▃▃▄▄▃▄▄▃▄▃▄▅▆▅▆▆▇▆▇▆▇▆▇█▇ +wandb: evaluation/blimp/determiner_noun_agreement ▁▁▁▁▁▁▂▂▂▂▃▃▄▅▅▆▆▆▇▇███████████▇▇ +wandb: evaluation/blimp/ellipsis ▂▃▃▂▃▂▂▁▁▂▃▄▅▅▆▆▇▇▇▇▇▇▇██▇▇█▇▇▇▇▆ +wandb: evaluation/blimp/filler_gap ▁██▇▇▇▇▇▆▆▇▇▆▆▆▆▆▆▆▆▆▆▆▆▅▅▆▅▅▅▅▅▅ +wandb: evaluation/blimp/irregular_forms ▁▂▃▂▂▃▅▄▄▅▆▇▇▆▆▆▇█████▇██████▇▇▇▇ +wandb: evaluation/blimp/island_effects ▆▅▆▇▇█▇▅▄▂▂▂▂▁▂▁▂▁▁▃▂▃▂▃▃▄▃▃▄▄▄▃▄ +wandb: evaluation/blimp/npi_licensing ▁▅▇▅▄▅▃▃▄▇▆▇▅▆▇▆█▅▅▅▅▄▃▅▆▆▅▅▅▅▃▃▅ +wandb: evaluation/blimp/quantifiers ▂▂▁▁▃▃▄▄▅▆▇▇█▇▇▇▇▇█▇▆▇▇▇▆▆▇▆▆▆▆▆▆ +wandb: evaluation/blimp/subject_verb_agreement ▁▁▁▁▁▁▁▂▁▁▂▂▃▄▅▅▅▆▆▆▆▇▇▇██▇▇██▇██ +wandb: evaluation/blimp_average ▁▂▃▃▃▄▄▃▃▄▅▅▅▅▆▆▇▇▇▇▇▇▇█████▇██▇▇ +wandb: evaluation/pseudo_perplexity █▄▃▃▃▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: lr-AdamW/pg1 ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: lr-AdamW/pg2 ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: train/losses/mlm_loss █▇▆▆▅▅▅▅▅▅▅▄▄▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▁▂▁▁▁▁▁▁▁▁▁ +wandb: trainer/global_step ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: validation/losses/mlm_loss █▆▆▆▅▅▅▅▄▄▃▃▃▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/monitor/mlm ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: +wandb: Run summary: +wandb: epoch 0 +wandb: evaluation/blimp/anaphor_agreement 78.22 +wandb: evaluation/blimp/argument_structure 63.08 +wandb: evaluation/blimp/binding 59.62 +wandb: evaluation/blimp/control_raising 61.78 +wandb: evaluation/blimp/determiner_noun_agreement 81.12 +wandb: evaluation/blimp/ellipsis 60.05 +wandb: evaluation/blimp/filler_gap 60.3 +wandb: evaluation/blimp/irregular_forms 78.42 +wandb: evaluation/blimp/island_effects 44.17 +wandb: evaluation/blimp/npi_licensing 53.23 +wandb: evaluation/blimp/quantifiers 59.43 +wandb: evaluation/blimp/subject_verb_agreement 60.58 +wandb: evaluation/blimp_average 63.33333 +wandb: evaluation/pseudo_perplexity 22.3838 +wandb: lr-AdamW/pg1 0.00041 +wandb: lr-AdamW/pg2 0.00031 +wandb: train/losses/mlm_loss 2.38458 +wandb: trainer/global_step 4124 +wandb: validation/losses/mlm_loss 3.675 +wandb: validation/monitor/mlm 1.0 +wandb: +wandb: 🚀 View run apricot-jazz-231 at: https://wandb.ai/rycolab/alkmi-wit/runs/nwnm1cme +wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20230708_194139-nwnm1cme/logs +wandb: 🚀 View run iconic-water-232 at: https://wandb.ai/rycolab/alkmi-wit/runs/pc8jt92j +wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20230708_194556-pc8jt92j/logs +evaluation/blimp/subject_verb_agreement: 60.58 +evaluation/blimp_average: 63.333333333333336 +Ending LM Evaluation Harness (duration: 0:07:01.778953) +[rank: 0] Monitored metric validation/losses/mlm_loss did not improve in the last 3 records. Best score: 3.616. Signaling model to ignore task. diff --git a/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=1212.ckpt b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=1212.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..ff381e6a033b9cc3cd2acd724697122f24c4416b --- /dev/null +++ b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=1212.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d12ce1389a290ecdee1fc25bd3667507fa152b09253e72c60f583102f334cd86 +size 1222290001 diff --git a/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=1594.ckpt b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=1594.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..ff990a3569a0aa1563fdd47a81182fa5faa2df90 --- /dev/null +++ b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=1594.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83533d2afbca1609bcb0655c47da95645209db3d0bfba5e0fa5d8e49ec18c919 +size 1222290511 diff --git a/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=1991.ckpt b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=1991.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..bbb1c8457d2f939d5f3a66d2e49bd50eee452077 --- /dev/null +++ b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=1991.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cd31fde5c53fb68e6426638214e6e4516a963eaf50ec78c13800fecbe3fd9ce +size 1222290957 diff --git a/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=2396.ckpt b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=2396.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..4ef194e13b814e28feaac8384a26145b10795e15 --- /dev/null +++ b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=2396.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52ace87504b5a7f50a4165d28130328f4bd2c73ccd6b1927d111bd04cccfa4e2 +size 1222291403 diff --git a/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=2787.ckpt b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=2787.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..101658391f61aca72c7b0de0985e5df34523ef24 --- /dev/null +++ b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=2787.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca2e10f4135de07c28608c6b130cfd3cf0b5ab22f649ebd5790b46c44ec5f8cf +size 1222291913 diff --git a/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=3193.ckpt b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=3193.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..ba87f6de176a02394f6e0b5a182b024973a0082b --- /dev/null +++ b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=3193.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95b1f70e166294b9f454143d0a09e556e14ccb0cf75dd623dee6101265490709 +size 1222292359 diff --git a/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=3592.ckpt b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=3592.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..d094b1687dd0919b1f0b176151882772f9e03e63 --- /dev/null +++ b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=3592.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b863b8d1721ea2d5cbae26724c5bcd8cdbd65b08d3077af97796067975f9d660 +size 1222292869 diff --git a/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=4001.ckpt b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=4001.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..ba7e978edf09417ead2cb3ff08d0e02f15e73245 --- /dev/null +++ b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=4001.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0a0a89be664d2512e85384ff846ab23942102bb725f38acd4270b933de2554b +size 1222293251 diff --git a/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=424.ckpt b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=424.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..76e764aa86af5bd8e28a8cf6cbf8be00279671fc --- /dev/null +++ b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=424.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87eb1110037755fe954fe29af8e9fa2c48faf0118e7a3dd32380c41caae23255 +size 1222288981 diff --git a/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=4418.ckpt b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=4418.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..1ee906b2e5730df2ea0471aa580f9e4953a13c33 --- /dev/null +++ b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=4418.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a0d664cde7fa79f4a6d7535e2b1c713346387b95cbe204c713dc89cad891dd8 +size 1222293697 diff --git a/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=4821.ckpt b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=4821.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..6be5ad99c38b05a401074b5502b10f429f1cb3a3 --- /dev/null +++ b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=4821.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78d98eba75f1fbcec96f08be44d5d5678ad61b572ad3d162812afde35cb05980 +size 1222294271 diff --git a/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=5225.ckpt b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=5225.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..2a5b2234c6cf0c87fd165cb7aa6dc1b67fae030d --- /dev/null +++ b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=5225.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:400e1c6f2f742a3cc9674466fef8d646591def96bc78e977b59554488049cff6 +size 1222294653 diff --git a/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=5654.ckpt b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=5654.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b095a957f42c902d10f2d69d9669eaba3dd1a427 --- /dev/null +++ b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=5654.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc299b957ea94d257c1b1789af581a67284f5ee16994b8dd9ac83e742e93d288 +size 1222295099 diff --git a/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=821.ckpt b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=821.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..6860f26bffd1acae8b00150361112a15797d89f1 --- /dev/null +++ b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/flava-epoch=00-step=821.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d3b7d2aba903e5dd17688c0fb66b5614b476f3d46a29be68c99c321d9962b65 +size 1222289555 diff --git a/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/last.ckpt b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/last.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..e9a3bfc13d792fd0401fde9a2f35cbedc3cf02f5 --- /dev/null +++ b/flava-wit/text1-vision0/half_bs4032_seed5501650_bf16-mixed/last.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:175c85df28bd0c012b92ee072897a372296726e6bb1799fc9131ecac4ccf0129 +size 1222295099 diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1078.ckpt b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1078.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..cce6a661f2332d22701cd9ffe5a77164665c632e --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1078.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b001527d53b5ca50aee51dd2a8ab5bf99fb5014501eb0f46856854d75484a97a +size 3848043495 diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1376.ckpt b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1376.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..95ce16c5a8c997e62bd564c08cd0afa49775b7ca --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1376.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd67614430f4f6a22cebc761375355ec8979276cfa0b38ad3e52a8e9c5d3515c +size 3848043942 diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1751.ckpt b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1751.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..abe74cc981be7fea30a76133d92ea230c4253bfc --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1751.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed989325b234a4a37fcd2a411e70168c13f21b5c3fadd5127074bc9e9356f060 +size 3848044325 diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2113.ckpt b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2113.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..ec64cdeb7643cd29a017d0e69f9d359bbf109d49 --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2113.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d02c8ecd050bec1fd4f186f00c6f75ecc688a9a68bce6518df78dac295e659fd +size 3848044772 diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2426.ckpt b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2426.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..bf361f67161cd06abf25ec2384b8d173b86a1773 --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2426.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae2e21bb6136e409caccad727297e099e69327e5cfc7ff75ebd410da6e58a96d +size 3848045283 diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2751.ckpt b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2751.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..9335ed0acbaa15d8e0e0087ae847bb21a168d62a --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2751.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26af1a712484d4e745eb24979592b4ae4d1666699c15d15555f29c8c847b0ae2 +size 3848045602 diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3045.ckpt b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3045.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b6db5045fc24fbd905f4014a73c94c6f8c78ffc1 --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3045.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a9418659ae25c7a43194b446db0c0dca79a1a6b6fd48c070d3c75e985549d98 +size 3848046049 diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3288.ckpt b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3288.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..a58a561507598d0caef4393136a1390f1ba64bf2 --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3288.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b14502e6cc3643da2e61cda10969b14fada6b18d51baff2fa0939d5a629114b5 +size 3848049065 diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3563.ckpt b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3563.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b090b00033855ea44bb13ec4f961ea6a81860dfc --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3563.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d486d0567d71ffdd3586c715a64af99760c0ed3c54c4ae6ba72a83a6e11b898 +size 3848049448 diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=376.ckpt b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=376.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..e8f7e04886403df7e1c515ab71b0aa378726fedc --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=376.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93960e318339557af180d57fdeaf6e459743ed94cca99fa42d703ae123b960f8 +size 3848042665 diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3876.ckpt b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3876.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..4cb11a57484307b7dc7de05b904a7416421bc295 --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3876.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa411736b2a55ce616ace6381130c60ca3d67067c0063e78f8a91190908ef38a +size 3848049895 diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4132.ckpt b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4132.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..892a7d8b200c5274100646dd2817644371232d41 --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4132.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af55a0cf92614ae626fc9d059223893e705d567d2bcfac9634fa85e98b54de82 +size 3848050342 diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4378.ckpt b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4378.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..3f446967d9bc4d14abbf7b7800a0986d7864737e --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4378.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9b9407e2d91ed31125121b876b0cd79a5d19c7b45647f9132dc6d8773549b5f +size 3848050725 diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4626.ckpt b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4626.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..eed6333c96da8897c04bd87f0180bb52044cf5d1 --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4626.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1da39cbce972b8c47ff5fe2dd31e163db6b480c1918428c7887db62abd377431 +size 3848051172 diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4876.ckpt b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4876.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..1b45ebba6f05c0af0799ad9f8ebf1dee3f7e4022 --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4876.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9086a486c46d251abd3b48e2afd2246cdf126c600050ee9f5fc2621927354dc7 +size 3848051619 diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5126.ckpt b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5126.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..a3846145883e42a5e50762b30a6f5bee9b96d92f --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5126.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b75307dcf2aebc294942b84c57920a57eaf2caa222a06bf70fea9e9fbffcaabd +size 3848052002 diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5376.ckpt b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5376.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..58163d1b33440db7a383c3d907e279d60b8c3a21 --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5376.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc91f84519523e88924bb628340362c9c3b9510c3b6ca6941a9503d7af5be385 +size 3848052449 diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5626.ckpt b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5626.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..a2479e551a2083f3800d6b28080f78dc4fc2b6f5 --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5626.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de740e8de7807da5fa8340d5c7deb0f8644e4f42337941e01ddedbf43c57bb99 +size 3848052960 diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5876.ckpt b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5876.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..e44953d95c510494c812ac01caf2e49f92ffe402 --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5876.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15d8fe89c6a42b7130f00cc7280f8dd31487182b534faa34423f25d702f3ef56 +size 3848053279 diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=6126.ckpt b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=6126.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..246643663d6e262dee64c1277d45d94a6003dd5d --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=6126.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b878cac2756afa9172b0f70d6d67948a0caa1f4b8cdba36bf82ae85c2c089e13 +size 3848053726 diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=751.ckpt b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=751.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..de0bdc3a884f9c0b319a3c61bc3757fe0397e3fb --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=751.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5598c0894268cb6b7fecdb0aec6322706fe5671f4270bae8c92f08aa372443b6 +size 3848043048 diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/last.ckpt b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/last.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..968cee583e48e6a8036c4bad0ee1c4072b6fdb9a --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/last.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e57fb526e046414e9e55f8b1fcec463e2c470740e50711956532f0e995fa6aa +size 3848053726 diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/resume_singlenode_2023-07-05-21:11:04.log b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/resume_singlenode_2023-07-05-21:11:04.log new file mode 100644 index 0000000000000000000000000000000000000000..6e3def18bf54a380ddcebad2ce3dcf959cc4ea31 --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/resume_singlenode_2023-07-05-21:11:04.log @@ -0,0 +1,1617 @@ +wandb: Currently logged in as: tamariucai (rycolab). Use `wandb login --relogin` to force relogin +wandb: wandb version 0.15.5 is available! To upgrade, please run: +wandb: $ pip install wandb --upgrade +wandb: Tracking run with wandb version 0.15.4 +wandb: Run data is saved locally in ./wandb/run-20230705_221847-ib3tu1uq +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run effortless-fog-216 +wandb: ⭐️ View project at https://wandb.ai/rycolab/alkmi-wit +wandb: 🚀 View run at https://wandb.ai/rycolab/alkmi-wit/runs/ib3tu1uq +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/date(2023-07-05)_time(22:18:48)/magic({'enable': True})/ +Text is the predominant modality (1 v.s. 1 vision), will sample proportionally for optimal BLiMP performance. +Assigning HuggingFace RAM +Found 747.32GBs of RAM available, assigning 5GBs to HuggingFace datasets (currently 5000000000 bytes). +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Found cached dataset imagenet-1k (/cluster/scratch/tamariucai/HuggingfaceDatasets/imagenet-1k/default/1.0.0/a1e9bfc56c3a7350165007d1176b15e9128fcaf9ab972147840529aed3ae52bc) +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/plugins/environments/slurm.py:165: PossibleUserWarning: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... + rank_zero_warn( +Using bfloat16 Automatic Mixed Precision (AMP) +GPU available: True (cuda), used: True +TPU available: False, using: 0 TPU cores +IPU available: False, using: 0 IPUs +HPU available: False, using: 0 HPUs +Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/2 +wandb: Currently logged in as: tamariucai (rycolab). Use `wandb login --relogin` to force relogin +wandb: wandb version 0.15.5 is available! To upgrade, please run: +wandb: $ pip install wandb --upgrade +wandb: Tracking run with wandb version 0.15.4 +wandb: Run data is saved locally in ./wandb/run-20230705_222956-ib3tu1uq +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run effortless-fog-216 +wandb: ⭐️ View project at https://wandb.ai/rycolab/alkmi-wit +wandb: 🚀 View run at https://wandb.ai/rycolab/alkmi-wit/runs/ib3tu1uq +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/date(2023-07-05)_time(22:29:57)/magic({'enable': True})/ +Text is the predominant modality (1 v.s. 1 vision), will sample proportionally for optimal BLiMP performance. +Assigning HuggingFace RAM +Found 747.32GBs of RAM available, assigning 5GBs to HuggingFace datasets (currently 5000000000 bytes). +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Found cached dataset imagenet-1k (/cluster/scratch/tamariucai/HuggingfaceDatasets/imagenet-1k/default/1.0.0/a1e9bfc56c3a7350165007d1176b15e9128fcaf9ab972147840529aed3ae52bc) +Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/2 +---------------------------------------------------------------------------------------------------- +distributed_backend=nccl +All distributed processes registered. Starting with 2 processes +---------------------------------------------------------------------------------------------------- + +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at theodor1289wit-test +Restoring states from the checkpoint path at /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/date(2023-07-04)_time(00:27:21)/magic({'enable': True})/flava-epoch=00-step=3045.ckpt +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:337: UserWarning: The dirpath has changed from "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/date(2023-07-04)_time(00:27:21)/magic({'enable': True})" to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/date(2023-07-05)_time(22:18:48)/magic({'enable': True})", therefore `best_model_score`, `kth_best_model_path`, `kth_value`, `last_model_path` and `best_k_models` won't be reloaded. Only `best_model_path` will be reloaded. + warnings.warn( +LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1] +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:337: UserWarning: The dirpath has changed from "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/date(2023-07-04)_time(00:27:21)/magic({'enable': True})" to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/date(2023-07-05)_time(22:18:48)/magic({'enable': True})", therefore `best_model_score`, `kth_best_model_path`, `kth_value`, `last_model_path` and `best_k_models` won't be reloaded. Only `best_model_path` will be reloaded. + warnings.warn( +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1] + + | Name | Type | Params +---------------------------------------------- +0 | model | FlavaForPreTraining | 357 M +---------------------------------------------- +303 M Trainable params +53.8 M Non-trainable params +357 M Total params +1,430.581 Total estimated model params size (MB) +Restored all states from the checkpoint at /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/date(2023-07-04)_time(00:27:21)/magic({'enable': True})/flava-epoch=00-step=3045.ckpt +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +VLDataModule: batch_size is 32, num_workers is 4 +ImageDataModule: batch_size is 32, num_workers is 4 +MLMDataModule: batch_size is 32, num_workers is 4 +Dataset sizes (un-normalized): [1, 1, 1] +Dataset sizes (normalized): [0.3333333333333333, 0.3333333333333333, 0.3333333333333333] +Sampling weights after temperature (1.0): [0.3333333333333333, 0.3333333333333333, 0.3333333333333333] +2023-07-05 22:19:41.837867 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-05 22:25:03.279126 Setting up datamodule: ImageDataModule +2023-07-05 22:25:07.733304 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Adding ImageNet zeroshot callback +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'ImageNetZeroshotCallback'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-05 22:29:43.005368 Preparing data for datamodule: VLDataModule +2023-07-05 22:29:43.005541 Preparing data for datamodule: ImageDataModule +2023-07-05 22:29:43.005645 Preparing data for datamodule: MLMDataModule +2023-07-05 22:35:35.277963 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-05 22:39:43.505688 Setting up datamodule: ImageDataModule +2023-07-05 22:39:48.928825 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-05 22:43:14.449773 Building train dataloader for datamodule: VLDataModule +2023-07-05 22:43:14.450356 Building train dataloader for datamodule: ImageDataModule +2023-07-05 22:43:14.450643 Building train dataloader for datamodule: MLMDataModule +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +VLDataModule: batch_size is 32, num_workers is 4 +ImageDataModule: batch_size is 32, num_workers is 4 +MLMDataModule: batch_size is 32, num_workers is 4 +Dataset sizes (un-normalized): [1, 1, 1] +Dataset sizes (normalized): [0.3333333333333333, 0.3333333333333333, 0.3333333333333333] +Sampling weights after temperature (1.0): [0.3333333333333333, 0.3333333333333333, 0.3333333333333333] +2023-07-05 22:30:12.153798 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-05 22:33:24.142189 Setting up datamodule: ImageDataModule +2023-07-05 22:33:27.284062 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Adding ImageNet zeroshot callback +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'ImageNetZeroshotCallback'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-05 22:35:35.277926 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-05 22:39:43.532024 Setting up datamodule: ImageDataModule +2023-07-05 22:39:48.854039 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-05 22:43:14.449929 Building train dataloader for datamodule: VLDataModule +2023-07-05 22:43:14.450665 Building train dataloader for datamodule: ImageDataModule +2023-07-05 22:43:14.450953 Building train dataloader for datamodule: MLMDataModule +2023-07-05 22:43:16.654215 Building val dataloader for datamodule: VLDataModule +2023-07-05 22:43:16.655157 Building val dataloader for datamodule: ImageDataModule +2023-07-05 22:43:16.655564 Building val dataloader for datamodule: MLMDataModule +2023-07-05 22:43:16.682907 Building val dataloader for datamodule: VLDataModule +2023-07-05 22:43:16.684176 Building val dataloader for datamodule: ImageDataModule +2023-07-05 22:43:16.684838 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-05 23:36:00.543004] Computing e^(tensor([6828.6203], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-05 23:36:00.550261] Ending with PPL=30.39597281081488 (duration: 0:11:50.232329) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00 Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Exception in thread SockSrvRdThr: +Traceback (most recent call last): + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/threading.py", line 1009, in _bootstrap_inner + self.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/wandb/sdk/service/server_sock.py", line 112, in run + shandler(sreq) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/wandb/sdk/service/server_sock.py", line 174, in server_record_publish + iface = self._mux.get_stream(stream_id).interface + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/wandb/sdk/service/streams.py", line 207, in get_stream + stream = self._streams[stream_id] +KeyError: 'ib3tu1uq' +slurmstepd: error: *** JOB 20411156 ON eu-a65-04 CANCELLED AT 2023-07-07T15:30:33 *** diff --git a/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/singlenode_run_2023-07-03-23:31:11.log b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/singlenode_run_2023-07-03-23:31:11.log new file mode 100644 index 0000000000000000000000000000000000000000..1bd4efd4d3a56364874dc65d5cec8cd1c9265312 --- /dev/null +++ b/flava-wit/text1-vision1/bs4096_seed-1_bf16-mixed/singlenode_run_2023-07-03-23:31:11.log @@ -0,0 +1,1844 @@ +wandb: Currently logged in as: tamariucai (rycolab). Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.15.4 +wandb: Run data is saved locally in ./wandb/run-20230704_002717-ib3tu1uq +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run effortless-fog-216 +wandb: ⭐️ View project at https://wandb.ai/rycolab/alkmi-wit +wandb: 🚀 View run at https://wandb.ai/rycolab/alkmi-wit/runs/ib3tu1uq +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/date(2023-07-04)_time(00:27:21)/magic({'enable': True})/ +Text is the predominant modality (1 v.s. 1 vision), will sample proportionally for optimal BLiMP performance. +Assigning HuggingFace RAM +Found 747.32GBs of RAM available, assigning 5GBs to HuggingFace datasets (currently 5000000000 bytes). +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Found cached dataset imagenet-1k (/cluster/scratch/tamariucai/HuggingfaceDatasets/imagenet-1k/default/1.0.0/a1e9bfc56c3a7350165007d1176b15e9128fcaf9ab972147840529aed3ae52bc) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/plugins/environments/slurm.py:165: PossibleUserWarning: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... + rank_zero_warn( +Using bfloat16 Automatic Mixed Precision (AMP) +GPU available: True (cuda), used: True +TPU available: False, using: 0 TPU cores +IPU available: False, using: 0 IPUs +HPU available: False, using: 0 HPUs +Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/2 +wandb: Currently logged in as: tamariucai (rycolab). Use `wandb login --relogin` to force relogin +wandb: Tracking run with wandb version 0.15.4 +wandb: Run data is saved locally in ./wandb/run-20230704_003549-i2rwjmn0 +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run amber-tree-217 +wandb: ⭐️ View project at https://wandb.ai/rycolab/alkmi-wit +wandb: 🚀 View run at https://wandb.ai/rycolab/alkmi-wit/runs/i2rwjmn0 +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/date(2023-07-04)_time(00:35:50)/magic({'enable': True})/ +Text is the predominant modality (1 v.s. 1 vision), will sample proportionally for optimal BLiMP performance. +Assigning HuggingFace RAM +Found 747.32GBs of RAM available, assigning 5GBs to HuggingFace datasets (currently 5000000000 bytes). +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Found cached dataset imagenet-1k (/cluster/scratch/tamariucai/HuggingfaceDatasets/imagenet-1k/default/1.0.0/a1e9bfc56c3a7350165007d1176b15e9128fcaf9ab972147840529aed3ae52bc) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/2 +---------------------------------------------------------------------------------------------------- +distributed_backend=nccl +All distributed processes registered. Starting with 2 processes +---------------------------------------------------------------------------------------------------- + +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1] +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1] + + | Name | Type | Params +---------------------------------------------- +0 | model | FlavaForPreTraining | 357 M +---------------------------------------------- +303 M Trainable params +53.8 M Non-trainable params +357 M Total params +1,430.581 Total estimated model params size (MB) +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +VLDataModule: batch_size is 32, num_workers is 4 +ImageDataModule: batch_size is 32, num_workers is 4 +MLMDataModule: batch_size is 32, num_workers is 4 +Dataset sizes (un-normalized): [1, 1, 1] +Dataset sizes (normalized): [0.3333333333333333, 0.3333333333333333, 0.3333333333333333] +Sampling weights after temperature (1.0): [0.3333333333333333, 0.3333333333333333, 0.3333333333333333] +2023-07-04 00:36:01.999579 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-04 00:38:15.627629 Setting up datamodule: ImageDataModule +2023-07-04 00:38:18.851362 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ImageNetZeroshotCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-04 00:40:24.760660 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-04 00:43:10.643234 Setting up datamodule: ImageDataModule +2023-07-04 00:43:14.431050 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-04 00:45:38.963905 Building train dataloader for datamodule: VLDataModule +2023-07-04 00:45:38.964609 Building train dataloader for datamodule: ImageDataModule +2023-07-04 00:45:38.964910 Building train dataloader for datamodule: MLMDataModule +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +VLDataModule: batch_size is 32, num_workers is 4 +ImageDataModule: batch_size is 32, num_workers is 4 +MLMDataModule: batch_size is 32, num_workers is 4 +Dataset sizes (un-normalized): [1, 1, 1] +Dataset sizes (normalized): [0.3333333333333333, 0.3333333333333333, 0.3333333333333333] +Sampling weights after temperature (1.0): [0.3333333333333333, 0.3333333333333333, 0.3333333333333333] +2023-07-04 00:27:59.251315 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-04 00:32:39.435586 Setting up datamodule: ImageDataModule +2023-07-04 00:32:42.985652 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ImageNetZeroshotCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-04 00:35:37.242566 Preparing data for datamodule: VLDataModule +2023-07-04 00:35:37.242850 Preparing data for datamodule: ImageDataModule +2023-07-04 00:35:37.243085 Preparing data for datamodule: MLMDataModule +2023-07-04 00:40:24.760678 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-04 00:43:16.092138 Setting up datamodule: ImageDataModule +2023-07-04 00:43:19.331805 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-04 00:45:38.964450 Building train dataloader for datamodule: VLDataModule +2023-07-04 00:45:38.965645 Building train dataloader for datamodule: ImageDataModule +2023-07-04 00:45:38.965979 Building train dataloader for datamodule: MLMDataModule +2023-07-04 00:45:40.947055 Building val dataloader for datamodule: VLDataModule +2023-07-04 00:45:40.948283 Building val dataloader for datamodule: ImageDataModule +2023-07-04 00:45:40.948846 Building val dataloader for datamodule: MLMDataModule +2023-07-04 00:45:40.908882 Building val dataloader for datamodule: VLDataModule +2023-07-04 00:45:40.910353 Building val dataloader for datamodule: ImageDataModule +2023-07-04 00:45:40.910929 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-04 01:33:28.848421] Computing e^(tensor([17450.9935], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-04 01:33:28.858749] Ending with PPL=6157.934838199472 (duration: 0:11:58.603491) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00, 'Connection to proxy.ethz.ch timed out. (connect timeout=10)')))' thrown while requesting HEAD https://huggingface.co/facebook/flava-full/resolve/main/preprocessor_config.json +Traceback (most recent call last): + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connection.py", line 200, in _new_conn + sock = connection.create_connection( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/util/connection.py", line 85, in create_connection + raise err + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/util/connection.py", line 73, in create_connection + sock.connect(sa) +TimeoutError: timed out + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connectionpool.py", line 776, in urlopen + self._prepare_proxy(conn) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connectionpool.py", line 1041, in _prepare_proxy + conn.connect() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connection.py", line 604, in connect + self.sock = sock = self._new_conn() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connection.py", line 209, in _new_conn + raise ConnectTimeoutError( +urllib3.exceptions.ConnectTimeoutError: (, 'Connection to proxy.ethz.ch timed out. (connect timeout=10)') + +The above exception was the direct cause of the following exception: + +urllib3.exceptions.ProxyError: ('Unable to connect to proxy', ConnectTimeoutError(, 'Connection to proxy.ethz.ch timed out. (connect timeout=10)')) + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/adapters.py", line 486, in send + resp = conn.urlopen( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connectionpool.py", line 844, in urlopen + retries = retries.increment( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/util/retry.py", line 515, in increment + raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] +urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='huggingface.co', port=443): Max retries exceeded with url: /facebook/flava-full/resolve/main/preprocessor_config.json (Caused by ProxyError('Unable to connect to proxy', ConnectTimeoutError(, 'Connection to proxy.ethz.ch timed out. (connect timeout=10)'))) + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 41, in _call_and_handle_interrupt + return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 91, in launch + return function(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 570, in _fit_impl + self._run(model, ckpt_path=ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 975, in _run + results = self._run_stage() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1018, in _run_stage + self.fit_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 201, in run + self.advance() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 354, in advance + self.epoch_loop.run(self._data_fetcher) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 134, in run + self.on_advance_end() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 248, in on_advance_end + self.val_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/utilities.py", line 177, in _decorator + return loop_run(self, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 102, in run + self.on_run_start() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 234, in on_run_start + self._on_evaluation_start() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 283, in _on_evaluation_start + call._call_callback_hooks(trainer, hook_name, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 189, in _call_callback_hooks + fn(trainer, trainer.lightning_module, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context + return func(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_utilities/core/rank_zero.py", line 27, in wrapped_fn + return fn(*args, **kwargs) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/callbacks/blimp_eval.py", line 47, in on_validation_start + eval_model = FlavaLM(model=pl_module.model, + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/callbacks/flava_lm.py", line 37, in __init__ + self.tokenizer = FlavaProcessor.from_pretrained("facebook/flava-full").tokenizer + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/processing_utils.py", line 184, in from_pretrained + args = cls._get_arguments_from_pretrained(pretrained_model_name_or_path, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/processing_utils.py", line 228, in _get_arguments_from_pretrained + args.append(attribute_class.from_pretrained(pretrained_model_name_or_path, **kwargs)) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/image_processing_utils.py", line 165, in from_pretrained + image_processor_dict, kwargs = cls.get_image_processor_dict(pretrained_model_name_or_path, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/image_processing_utils.py", line 269, in get_image_processor_dict + resolved_image_processor_file = cached_file( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/utils/hub.py", line 417, in cached_file + resolved_file = hf_hub_download( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/utils/_validators.py", line 118, in _inner_fn + return fn(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/file_download.py", line 1195, in hf_hub_download + metadata = get_hf_file_metadata( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/utils/_validators.py", line 118, in _inner_fn + return fn(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/file_download.py", line 1532, in get_hf_file_metadata + r = _request_wrapper( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/file_download.py", line 407, in _request_wrapper + response = _request_wrapper( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/file_download.py", line 442, in _request_wrapper + return http_backoff( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/utils/_http.py", line 228, in http_backoff + raise err + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/utils/_http.py", line 212, in http_backoff + response = session.request(method=method, url=url, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/sessions.py", line 589, in request + resp = self.send(prep, **send_kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/sessions.py", line 703, in send + r = adapter.send(request, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/adapters.py", line 513, in send + raise ProxyError(e, request=request) +requests.exceptions.ProxyError: HTTPSConnectionPool(host='huggingface.co', port=443): Max retries exceeded with url: /facebook/flava-full/resolve/main/preprocessor_config.json (Caused by ProxyError('Unable to connect to proxy', ConnectTimeoutError(, 'Connection to proxy.ethz.ch timed out. (connect timeout=10)'))) + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 167, in + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 153, in main + wandb_logger.experiment.config.update(config) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 531, in fit + call._call_and_handle_interrupt( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 63, in _call_and_handle_interrupt + trainer.strategy.on_exception(exception) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/ddp.py", line 393, in on_exception + _augment_message( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/utilities/exceptions.py", line 40, in _augment_message + exception.args = tuple( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/utilities/exceptions.py", line 41, in + new_message if re.match(pattern, message, re.DOTALL) else message for message in exception.args + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/re.py", line 190, in match + return _compile(pattern, flags).match(string) +TypeError: expected string or bytes-like object +Traceback (most recent call last): + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connection.py", line 200, in _new_conn + sock = connection.create_connection( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/util/connection.py", line 85, in create_connection + raise err + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/util/connection.py", line 73, in create_connection + sock.connect(sa) +TimeoutError: timed out + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connectionpool.py", line 776, in urlopen + self._prepare_proxy(conn) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connectionpool.py", line 1041, in _prepare_proxy + conn.connect() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connection.py", line 604, in connect + self.sock = sock = self._new_conn() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connection.py", line 209, in _new_conn + raise ConnectTimeoutError( +urllib3.exceptions.ConnectTimeoutError: (, 'Connection to proxy.ethz.ch timed out. (connect timeout=10)') + +The above exception was the direct cause of the following exception: + +urllib3.exceptions.ProxyError: ('Unable to connect to proxy', ConnectTimeoutError(, 'Connection to proxy.ethz.ch timed out. (connect timeout=10)')) + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/adapters.py", line 486, in send + resp = conn.urlopen( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connectionpool.py", line 844, in urlopen + retries = retries.increment( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/util/retry.py", line 515, in increment + raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] +urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='huggingface.co', port=443): Max retries exceeded with url: /facebook/flava-full/resolve/main/preprocessor_config.json (Caused by ProxyError('Unable to connect to proxy', ConnectTimeoutError(, 'Connection to proxy.ethz.ch timed out. (connect timeout=10)'))) + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 41, in _call_and_handle_interrupt + return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 91, in launch + return function(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 570, in _fit_impl + self._run(model, ckpt_path=ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 975, in _run + results = self._run_stage() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1018, in _run_stage + self.fit_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 201, in run + self.advance() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 354, in advance + self.epoch_loop.run(self._data_fetcher) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 134, in run + self.on_advance_end() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 248, in on_advance_end + self.val_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/utilities.py", line 177, in _decorator + return loop_run(self, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 102, in run + self.on_run_start() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 234, in on_run_start + self._on_evaluation_start() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 283, in _on_evaluation_start + call._call_callback_hooks(trainer, hook_name, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 189, in _call_callback_hooks + fn(trainer, trainer.lightning_module, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context + return func(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_utilities/core/rank_zero.py", line 27, in wrapped_fn + return fn(*args, **kwargs) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/callbacks/blimp_eval.py", line 47, in on_validation_start + eval_model = FlavaLM(model=pl_module.model, + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/callbacks/flava_lm.py", line 37, in __init__ + self.tokenizer = FlavaProcessor.from_pretrained("facebook/flava-full").tokenizer + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/processing_utils.py", line 184, in from_pretrained + args = cls._get_arguments_from_pretrained(pretrained_model_name_or_path, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/processing_utils.py", line 228, in _get_arguments_from_pretrained + args.append(attribute_class.from_pretrained(pretrained_model_name_or_path, **kwargs)) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/image_processing_utils.py", line 165, in from_pretrained + image_processor_dict, kwargs = cls.get_image_processor_dict(pretrained_model_name_or_path, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/image_processing_utils.py", line 269, in get_image_processor_dict + resolved_image_processor_file = cached_file( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/utils/hub.py", line 417, in cached_file + resolved_file = hf_hub_download( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/utils/_validators.py", line 118, in _inner_fn + return fn(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/file_download.py", line 1195, in hf_hub_download + metadata = get_hf_file_metadata( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/utils/_validators.py", line 118, in _inner_fn + return fn(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/file_download.py", line 1532, in get_hf_file_metadata + r = _request_wrapper( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/file_download.py", line 407, in _request_wrapper + response = _request_wrapper( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/file_download.py", line 442, in _request_wrapper + return http_backoff( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/utils/_http.py", line 228, in http_backoff + raise err + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/utils/_http.py", line 212, in http_backoff + response = session.request(method=method, url=url, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/sessions.py", line 589, in request + resp = self.send(prep, **send_kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/sessions.py", line 703, in send + r = adapter.send(request, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/adapters.py", line 513, in send + raise ProxyError(e, request=request) +requests.exceptions.ProxyError: HTTPSConnectionPool(host='huggingface.co', port=443): Max retries exceeded with url: /facebook/flava-full/resolve/main/preprocessor_config.json (Caused by ProxyError('Unable to connect to proxy', ConnectTimeoutError(, 'Connection to proxy.ethz.ch timed out. (connect timeout=10)'))) + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 167, in + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 153, in main + wandb_logger.experiment.config.update(config) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 531, in fit + call._call_and_handle_interrupt( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 63, in _call_and_handle_interrupt + trainer.strategy.on_exception(exception) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/ddp.py", line 393, in on_exception + _augment_message( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/utilities/exceptions.py", line 40, in _augment_message + exception.args = tuple( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/utilities/exceptions.py", line 41, in + new_message if re.match(pattern, message, re.DOTALL) else message for message in exception.args + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/re.py", line 190, in match + return _compile(pattern, flags).match(string) +TypeError: expected string or bytes-like object +wandb: Waiting for W&B process to finish... (failed 1). Press Control-C to abort syncing. +wandb: Waiting for W&B process to finish... (failed 1). Press Control-C to abort syncing. +wandb: Network error (TransientError), entering retry loop. +wandb: ERROR Error uploading "output.log": CommError, File ./wandb/run-20230704_003549-i2rwjmn0/files/output.log size shrank from 16274 to 106 while it was being uploaded. +wandb: +wandb: Run history: +wandb: ImageDataModule ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: MLMDataModule ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: VLDataModule ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: epoch ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: evaluation/blimp/anaphor_agreement ▄▁▂▂▄▄▄▅▅▄▄▅▅▅▅▅▆▇▇▆█▇███▇ +wandb: evaluation/blimp/argument_structure ▁▂▃▃▄▅▄▂▄▃▇█▇▆▇▆▆▆▇▇▇▇▆▇▇▆ +wandb: evaluation/blimp/binding ▅███▇▅▆▇▃▂▂▁▃▁▄▄▂▃▃▅▁▃▃▂▃▃ +wandb: evaluation/blimp/control_raising ▁▅▆▆▆▆▆▇▄▅▅▆▆▅▆▅▅▆▆▆▆▅▇███ +wandb: evaluation/blimp/determiner_noun_agreement ▁▁▁▁▁▁▂▂▂▂▃▄▅▅▆▆▆▇▇▇▇▇▇███ +wandb: evaluation/blimp/ellipsis ▃▃▅▄▄▃▃▁▁▃▃▄▄▅▆▇▇▇███▇█▇██ +wandb: evaluation/blimp/filler_gap ▁█▇▅██▇▇▆▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇ +wandb: evaluation/blimp/irregular_forms ▄▂▃▂▁▂▃▄▆▆▇████▇▇█▇▇▆▇▇▇▆▆ +wandb: evaluation/blimp/island_effects ▄▅▅▆▇▇█▅▄▃▃▂▂▂▂▁▂▁▂▂▂▃▂▂▃▃ +wandb: evaluation/blimp/npi_licensing ▁▅▄▃▃▁▂▄▂▃▄▃▃▂▅▄▅▄█▄▆▇▅▇▆▆ +wandb: evaluation/blimp/quantifiers ▂▂▁▁▃▃▄▅▅▆▇█▇▇▆▅▇▇▇▆██▆▇▇▇ +wandb: evaluation/blimp/subject_verb_agreement ▂▂▁▂▂▂▂▂▂▄▄▅▆▆▆▆▇▇▇███████ +wandb: evaluation/blimp_average ▁▂▂▂▃▃▄▄▄▄▅▅▆▆▆▆▇▇█▇██████ +wandb: evaluation/imagenet_zeroshot/top1 ▅▇▅▇▆██▅▅▄▆▄▄▂▂▃▃▁▂▂▂▂▂▂▃▄ +wandb: evaluation/imagenet_zeroshot/top5 ▇▇▅▅▅█▇▆▇▅▅▄▄▂▃▃▂▂▁▂▂▃▄▃▄▅ +wandb: evaluation/pseudo_perplexity █▃▃▃▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: lr-AdamW/pg1 ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: lr-AdamW/pg2 ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: train/losses/global_contrastive_loss ████▇▇▆▆▆▅▄▃▁▃▁▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂ +wandb: train/losses/itm_loss ▅█▅▄▄▆▄▅▄▃▃▄▅▃█▁▃▄▄▂▂▃▄▆▃▃▁▂▆▂▂▅▅▅▅▅▅▅▅▅ +wandb: train/losses/mim_loss █▅▃▂▁▂▁▂▁▁▁▂▁▁▂▁▂▁▁▁▁▁▁▁▁▁▂▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: train/losses/mlm_loss █▇▆▅▅▅▅▅▅▅▄▄▄▄▄▃▃▃▃▂▃▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁ +wandb: train/losses/mmm_image_loss █▅▃▂▂▁▁▁▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: train/losses/mmm_text_loss █▇▆▅▅▅▅▅▅▅▅▄▄▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁ +wandb: trainer/global_step ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇███ +wandb: validation/losses/global_contrastive_loss ▇▇▅▂▂▁▂▂▆█ +wandb: validation/losses/itm_loss ▇█▇▆▆▅▄▄▅▅▄▃▄▃▃▂▁▂▁▁▁ +wandb: validation/losses/mim_loss █▃▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/losses/mlm_loss █▆▆▆▅▅▅▅▄▃▃▃▂▂▂▂▂▂▁▁▁▁▁▁▁▁ +wandb: validation/losses/mmm_image_loss █▃▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/losses/mmm_text_loss █▆▆▆▅▅▅▄▃▃▃▃▂▂▂▂▂▂▂▁▁▁▁▁▁▁ +wandb: validation/monitor/global_contrastive █▅▅▅▅▅▃▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/monitor/itm █▅▃▃▃▃▃▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/monitor/mim ██████▅▅▅▅▃▃▃▃▂▂▁▁▁▁▁▁▁▁▁▁ +wandb: validation/monitor/mlm ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/monitor/mmm_image ██████▅▅▅▅▃▃▃▂▂▂▁▁▁▁▁▁▁▁▁▁ +wandb: validation/monitor/mmm_text ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: +wandb: Run summary: +wandb: ImageDataModule 71310 +wandb: MLMDataModule 72818 +wandb: VLDataModule 71875 +wandb: epoch 0 +wandb: evaluation/blimp/anaphor_agreement 76.53 +wandb: evaluation/blimp/argument_structure 63.93 +wandb: evaluation/blimp/binding 59.04 +wandb: evaluation/blimp/control_raising 59.88 +wandb: evaluation/blimp/determiner_noun_agreement 82.94 +wandb: evaluation/blimp/ellipsis 63.16 +wandb: evaluation/blimp/filler_gap 64.86 +wandb: evaluation/blimp/irregular_forms 76.59 +wandb: evaluation/blimp/island_effects 41.26 +wandb: evaluation/blimp/npi_licensing 54.27 +wandb: evaluation/blimp/quantifiers 63.29 +wandb: evaluation/blimp/subject_verb_agreement 59.26 +wandb: evaluation/blimp_average 63.75083 +wandb: evaluation/imagenet_zeroshot/top1 0.00074 +wandb: evaluation/imagenet_zeroshot/top5 0.00446 +wandb: evaluation/pseudo_perplexity 26.7451 +wandb: lr-AdamW/pg1 0.00033 +wandb: lr-AdamW/pg2 0.00025 +wandb: train/losses/global_contrastive_loss 2.02889 +wandb: train/losses/itm_loss 0.37521 +wandb: train/losses/mim_loss 1.46241 +wandb: train/losses/mlm_loss 3.45469 +wandb: train/losses/mmm_image_loss 1.53973 +wandb: train/losses/mmm_text_loss 2.9748 +wandb: trainer/global_step 3349 +wandb: validation/losses/global_contrastive_loss 3.5599 +wandb: validation/losses/itm_loss 0.27602 +wandb: validation/losses/mim_loss 1.56292 +wandb: validation/losses/mlm_loss 3.81841 +wandb: validation/losses/mmm_image_loss 1.5648 +wandb: validation/losses/mmm_text_loss 3.67997 +wandb: validation/monitor/global_contrastive 0.0 +wandb: validation/monitor/itm 0.0 +wandb: validation/monitor/mim 0.0 +wandb: validation/monitor/mlm 0.33333 +wandb: validation/monitor/mmm_image 0.0 +wandb: validation/monitor/mmm_text 1.0 +wandb: +wandb: 🚀 View run effortless-fog-216 at: https://wandb.ai/rycolab/alkmi-wit/runs/ib3tu1uq +wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20230704_002717-ib3tu1uq/logs +wandb: +wandb: Run history: +wandb: ImageDataModule ▁ +wandb: MLMDataModule ▁ +wandb: VLDataModule ▁ +wandb: +wandb: Run summary: +wandb: ImageDataModule 71310 +wandb: MLMDataModule 72818 +wandb: VLDataModule 71875 +wandb: +wandb: 🚀 View run amber-tree-217 at: https://wandb.ai/rycolab/alkmi-wit/runs/i2rwjmn0 +wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20230704_003549-i2rwjmn0/logs +[ImageNet Zero-Shot Evaluation 2023-07-05 07:18:43.062676] Ending with top5=0.00446 (duration: 0:02:08.435286) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-05 08:28:32.048491] Computing e^(tensor([6666.0793], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-05 08:28:32.055320] Ending with PPL=28.023394246089058 (duration: 0:18:24.137594) +Starting LM Evaluation Harness diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10016.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10016.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..4224caf9540560c44ac99c9e36569b23fe1c7ebc --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10016.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8120c23960013186a033ee850452736c9a72491122f94bcafc30dfea8c428677 +size 1849224520 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10155.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10155.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..e6c8a9fcfd98f33b7d8b33382435a3b701d08d54 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10155.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac4ada8c496bcf515e1cf5366880872593c4a1621d9ddb963d9bcbb4fd89cefb +size 1849217303 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10229.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10229.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..ef492da6087b79c740b82f2fc3d753bb2e78be82 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10229.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b41a082dc5ae133f1fbe841c293b3552aff568609753a2f08a0dadbb871f342 +size 1849217814 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10305.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10305.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..d691006a3a97431e3747d28ac788a5e83c4562ce --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10305.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:413aacabf105962fc06f56b9f3b77e117870efa5d527e1db28f9e3542af802ae +size 1849218197 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10367.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10367.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..fb14f91b5e41db1210e31fe408700510eef20938 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10367.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7cffd22035a6c72b9c0777387b49aa2a346997fed6d6905972e5e8b837954f0 +size 1849218708 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10441.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10441.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..81eb9244cd0933c49d37b6a80ba94f22579c8472 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10441.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22712a07076dae2f27f885b304de96606db54bc87f431a2ce6a69b81673a4b04 +size 1849219219 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10501.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10501.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..79f73868574953592202a544c4b100dc30490560 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10501.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed8214d37cd448de47a0d104e50da9642383423e869f07b2ccb1a9167fc2b3b8 +size 1849219730 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1057.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1057.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..358fdf0d9cf07ffc752a6b64447b361d89a1e6dd --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1057.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b17a24125120e54fc8c471f9eb773dbbd9b17407c8459c394ec063a0e18bafcf +size 1849218708 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10578.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10578.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..055a0e6c579a509f53402602705448d4b4a6ad13 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10578.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da9e0b8f8f97d1891b58c0fcb4bca209d67b13c9aa0565dbebbed123461fdc69 +size 1849220177 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10649.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10649.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f6173167deabdc87614807445a0f3929a746cafa --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10649.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4e17d5a4f7e93ec77e59eb61e608916c7ba7857c86bca70c770fb14a06bf275 +size 1849220688 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10708.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10708.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..02b1458b861e55d7c03913e342cf3763071cee11 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10708.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28a683d2fdcb9c3e4a72e43a521e87d8fbd8086eb45b34335a26e907795bdb63 +size 1849221199 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10787.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10787.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..1fa4e5d8119fc2300b48a3689db91017b5e07e32 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10787.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82b3118ca7c81f1888c01152bb2e5d398a561e7319dd80e57f739701ccf7fae0 +size 1849221582 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10859.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10859.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c014cf1138639ec6a584e743dbc0883928152640 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10859.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a109ab945df9c0d5272ff072bef91c2bcc64f4668bd835318b0dd8e625f19cb0 +size 1849222093 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10920.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10920.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..9d0a0db27cea3c43e49995848b3afd8878e5acf0 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10920.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bc47c3d714f8b5628013c13ae75f6069dec34938843a220a542034ca951c038 +size 1849222604 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10997.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10997.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..10808a6906d7b1c9578856c1bf9371b585a258c7 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10997.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4d30eaebb6285c279ecfc23fe144af7bbbd81ee2e9fe35ba6cd7cdcb0bb61b6 +size 1849223115 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11073.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11073.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..38fff30830a7c650836f6426cfb711b4d835bc57 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11073.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4526e96a90d5a5e9dd55ec49d06cdc2d57019e0878918e4234890e3fd0188173 +size 1849223562 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11160.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11160.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..20aeb6b43e5fff787b29e5217bee54aa3973cdce --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11160.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8eeb6913b0b9a1b888ffa35756adc9d590d31323c3f95524678b5863fcce8a0e +size 1849224009 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11241.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11241.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..68480e0167a367d5c810f9afc901a914f2480893 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11241.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1edd39a239527571dbce60f8a625e5a6d3d92025c6d935f910ad8aaeda721952 +size 1849224520 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11330.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11330.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..caccbeaedb4e02edc3da528e38efbbc013366273 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11330.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:614cae686d976747af8a538f066ee7ad35fcba8d09d4825f72072c88c5c7998d +size 1849224903 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1137.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1137.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..784c971783f19d80e823e531b927b6908eff9d25 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1137.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56892a44f6b575db1ec2fce6f70eecd09e8a442de05575087228462bbf047a62 +size 1849219283 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11426.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11426.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..fe22329f8ffca59c5c517d67eadf4212b462d733 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11426.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5b6ebd5751cd794d7b81b41cd90232e9a775f6da249862cb6b8de3e8d215f38 +size 1849225414 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11512.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11512.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..9f47e44af7b4577326744f78048f24976f36fb08 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11512.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2b4910f5c208d3695c5fffeb0cdc1e4d858f19bde6b42ff9b2ba27019642d34 +size 1849225925 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11630.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11630.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..a26609b6d8335a757f38e1d206785628cba74caa --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11630.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:975726be85a46d2b00d045cb36d7843912d3408e070a22e37f212c4bc084d8e0 +size 1849226244 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11707.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11707.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..56963c933e514c0ba6b98a8d0d35c7dceec27e99 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11707.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0447993f2b5730589bd33e7a4b55723ca65af1adef46f742cb56a0e61ff12a2a +size 1849226691 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11783.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11783.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..24280156880e3af6ab1484f2cd930438e28fa229 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11783.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:192d45e5426d44b25596905ca69dd400febc3e8fcd2ce25c59563b866c8da05e +size 1849227266 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11848.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11848.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..a8654bc3844827ea0026ee6e9add503a15fba129 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11848.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75d7d802768bb1f960d8169cbe8d53dbf8da9384ea8dd64f58442457f5bd9b70 +size 1849227713 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11928.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11928.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..d6dfef46058d8360dd3624e7019b9da022ca00c2 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11928.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e49526962eb75b79ca55d825e0a1bb304dc00e2b7e9d2ac2fa758d1a9ea2b992 +size 1849228032 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12005.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12005.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..3fe06123edee29397031c80376e789fdd4ca05f9 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12005.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba10c032f2d3b5972c07e68689d6c036d7fab3effbf96ae13380483fa8e37697 +size 1849228671 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12067.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12067.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..5352fdbdcc6ce472746cdc51034294d9709d1761 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12067.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bb8383bd6ea507867fea4c2ea713ac175970dfd9e272fae9fc03771cd08316c +size 1849229182 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12148.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12148.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..233835507d9d25e1f9808ae7eb3860639738658c --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12148.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d138cfffa2172d2d726b436810a0394cfca47ee20d91400675b715ef3cec425 +size 1849229501 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12216.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12216.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..ed40d32c32c94689f8d57f3375fc0de1af602272 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12216.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d68e27add7358e574595631b94887994116f69dfc4718b03af91aa3a6e131b6 +size 1849230012 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1226.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1226.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b0fa97fcb1a4cd33be35269a0f4f907d0d0c43ad --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1226.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae6d21e820d47b0be5ab000a572e3d296236a2c005ad31e9ff507173fb0d883a +size 1849219730 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12305.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12305.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f3d4e188dbf3bede2521a2156c5a15eb33724368 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12305.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad5aa9fcb696908c3b93927e9ef3e2318970be7a7944ea6f5c592a8b01e17d8d +size 1849230523 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12387.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12387.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f4acbe1c52f2b25360d2d2e8b1c0219678463b94 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12387.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b059c8f44613ab64dbf90217bdfcd50a5a7e0bb6be004904cd07ccf2c9644809 +size 1849231034 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12454.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12454.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..72ba2ddde4a780a30f13a87855402334c14b740d --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12454.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf665e980f574a367bfd78484af8e4f5036e23d3a64dc1417e508ebbf808b2de +size 1849231481 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12539.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12539.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c8ff9b7817f0a13f88ea8929366b0b308c0a6c58 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12539.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69a883842001123cfc7e7349035cd4e325671bdd1245ff345ede5a7259621f9b +size 1849231992 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=126.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=126.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..eff496d8216fe6c1ecfdf272a339c37ed508fadc --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=126.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79a66843c32339bd872353130b85f51f976dbeb2ae73f40423b9a831872e8f8d +size 1849213015 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12605.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12605.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..094eee795a2e4115688caaebf04dfb50b4d09935 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12605.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cde46ebcbbed1653f57f7d95f3a83a5ebeef6d802479bb29d35087295c7daf8c +size 1849232439 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12692.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12692.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..23360d86e571598b784baa54cf702dddc1b95a0a --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12692.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14d23da182dd2bb4af010fc402cf4328929214e318dd714d998331ffd453ebf5 +size 1849232886 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12774.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12774.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..7a4047f1b7805a58326489dd34e843cd4e73976f --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12774.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e51e3e978500339fb4bf21c5144afb6de6ac0fc1190f326112bf181d107fcf9c +size 1849233397 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12847.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12847.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..30d256482c36ff2dc7277feae35305f2cbfbb924 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12847.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc6060222c0be7d3c53cc81661e384289d2df5b6a0285ecad8c660116b5daec4 +size 1849233844 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12933.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12933.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..917d78419b946577fd2de76f46076e89dad1d4e3 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12933.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3de40edc76e5854d7d42ed8562eb44a203018f44f6e2a588629550717b57d16d +size 1849234419 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12997.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12997.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f155dbebca6722e4fbc82ae4fca562cf6d113510 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12997.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b30981861f065ba4de6143100f728480fe371c74ecc3c816e6e6658edfcc51ba +size 1849234866 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1315.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1315.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..6edfe8bd92f4b870a9d90a001c7685f479811993 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1315.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c90c8a8877d4a418b8805d4e3a848b1a8dc9686aad4dfd36b1208fb18a7ecea3 +size 1849220241 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1409.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1409.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..1c84f5e6dc70ccbc4d4b28413256fe9bb04d36c8 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1409.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e06ffe2be4d212d59a666ad242a7407680195615da90e297cf3de746beb2a778 +size 1849220752 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1503.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1503.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..0cf23fe915ca8166f60d347b1ccae819c5908a81 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1503.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd4daa3c3d972d9f29016e81991787f7bed789c28df92ff1fb9f9eb71625ac82 +size 1849221199 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1628.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1628.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f3868e4ec2d7125e4cd37fff692fe40d7da2374e --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1628.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6381710b6f1953f33b614b5a06fdb3d8edf93e46588675f7947309fdc728f272 +size 1849221646 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1786.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1786.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..0fc8c4b91e25b335376d052ca6de88db39c7dd40 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1786.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9df636c605fd70a5eb0656c5e97a9e60b742797c55ca3c3a96b81b2b72002108 +size 1849222093 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1902.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1902.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..a85777b7d1b189e533e5a204d092da373f21fe63 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1902.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:659c7ce61f6cdf8e92d9e020eb1bfe306d6863cd06366c3b1f6229ee61341fa3 +size 1849222668 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1989.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1989.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b7ae20ce6b94fd703420733553ca29603f854fcc --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1989.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61fe46cfd94e767e16611af526b6eb181fa58219da20aba8ed61b22bc879e759 +size 1849223115 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2085.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2085.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..171905fc3f02c43ccaeafae1dc0c3fbea5cc631d --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2085.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eef1b618fdad8668b1ba4e02b6cbcedf96f0377f5ca0fd080021bf646c8717ae +size 1849223562 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2162.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2162.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..62538ad7d03491036cdc3e15e4c2b0ff023f0726 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2162.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a1eabb15a9d3010a27a703998c301ce5878202303793c41af6a4e30b36be3ab +size 1849224073 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2256.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2256.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..fecc03b843311f122cd1563f5838d87cf9cb8cb6 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2256.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf7dd0d8eb4286848e86bffe988851871f77bd4e29ef86ccb6a52c7ed8a45cb4 +size 1849224520 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2326.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2326.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..7acac8e2f39f36f2aa854ce44fd41b635b850595 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2326.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52b6f673041211b22db54862d0f010dbb6ab0b71e89245f91c586d8434ac5938 +size 1849224967 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2387.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2387.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b6419dea26292f9a4334ab36ca00aa040405cb26 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2387.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:815e6ed15d47ceb3c99cbcd1470ec5815790779e16a5ccf4dd0f969dd1667e1d +size 1849225414 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2456.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2456.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f61b98bbb85a03a5fa7f0c17eb1c032a8aab7904 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2456.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f90dbe493958f5af2676468de58f3776f0354085bbac25773d63c4e5d67186d2 +size 1849225925 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=251.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=251.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..cae2ed6736948466dcda55295e200a58338e8604 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=251.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44825e7992f3e5bbe49b470fecc6c58959c40ceb0bd720a487989538ee36cd81 +size 1849213462 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2538.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2538.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..cb1f2293150758415379ccf95324a0776f965b0f --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2538.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd62df3739232b60f3d0cdea7ee0283de00efea29f5521d9d64acdf25c1f4fe7 +size 1849226308 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2614.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2614.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..d808a91e21cc7b6d9c34a71caf36b73217b72ca1 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2614.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cce795ee99f37faaa103ce335c48f67e6b55d79be4d3b00e869bc21bd78c2e8 +size 1849226691 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2674.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2674.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..d26d04eb61d335bc9cf42648c20bc8dd3df870bf --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2674.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a0e22d1ec20640b0a62ae2e6c8377dc66f1739d3a784bdf9749568c89a0c48f +size 1849227330 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2744.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2744.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..65c3b1073eb970942163d71fae7a4700de770bec --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2744.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49ade62ee69f05d10146100272ad4f90087c089b522d53cebcba1f7e2fc76fb9 +size 1849227713 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2827.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2827.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..7d474432b73d0d0569bce6a9a68f1db02b21b721 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2827.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:838ea220bc7f7fb72232c04004324439563e610aa6646b1bf4778db324316406 +size 1849228032 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2915.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2915.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b7802e8c8b4e0ea617fc150755a2856538e64524 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2915.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b558390d3a5714c507240530b506ddcda6f69c43a34d99fa69b6ffd7c41c0ebe +size 1849228671 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2984.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2984.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c92249536f9556403a48d0329004724d4283105d --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2984.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de9ee5428ba9fa01de72f9398e9d12e1f1fe1ae765eb3d2340f7dc672505d74a +size 1849229182 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3047.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3047.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..cf9580b5a24de9a28a7277d87772bb993deff8cb --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3047.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f012e21ad39167ae6fa9b39101b09036967f3b0ab4ef830c625833cacb3ebfc +size 1849229565 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3113.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3113.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b18fc74d940cf1283ddd2746cbab1d0f4466b600 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3113.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ddf07e329f26ef19626c5813d16a0126ae7c8925fdb2d4a3288b2f0545ff35d +size 1849230012 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3195.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3195.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..08fb9e84a117a48812cadc4077f3dc9bf5253ad9 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3195.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0129157ca24f0917ae97b0efafdbc376b29ff7d0130002d4af34a9d5f237cfdc +size 1849230523 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3271.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3271.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..850a601cfb96527bdfe888065c1972f716ce699e --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3271.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cd64f7dafc0395697b4984c1a958d47735352ffb52e900f4bc44264797ce8fe +size 1849231034 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3344.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3344.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c16601b80c05614f8a46756340920400c3fd2ace --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3344.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f0e380c2fcce6f6f2fffcdec00ae63b32873638091eb111f098ead82136f53d +size 1849231481 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3414.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3414.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..fe052e03c406d00ab89af2620a2f3253583f6b4b --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3414.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b66bfced960aa70f7cfbb39f1f932a0f6983902aab3ba90f0f8344717c3ccb8 +size 1849231992 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3495.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3495.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..5dcad1479538f2a3c84753bbb20224e6a752db77 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3495.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c645f7f2176f0aa06954b853c66d9cb81863b2705216eea6b4cf29e95061399 +size 1849232439 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3574.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3574.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..9fdf64794a9052cfd585d90abb72cbbae015ed26 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3574.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8d3cc1eec4828964de8e0e4a421e5203eca30240e921f8831ae18b90f0e8e9c +size 1849232950 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3660.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3660.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f9dc84247be6819086f23dec8a4f6a54db967be5 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3660.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4068d7cb8f4bda516104dfaef2e73c10ee120d47fb737d945a59039b2302c0cf +size 1849233397 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3735.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3735.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..1fd83a9a92b2d25f872c4fe223ececb49879e180 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3735.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e20d7c8db8a808e65b0aaec0f909828cd9e6aaa8f4b4cbc1b0b5af88af03c718 +size 1849233844 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=376.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=376.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..176e436babc72e6b7b031fcf89ebc3d0860a702c --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=376.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87c2dae218fd3a0c113b645485c70848bcdba4d0d0ff8f2773350bd6ac70ff99 +size 1849213909 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3808.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3808.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..1dd1f35fec58f4ade0a2fa54da340142984487df --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3808.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a98799ed368000ae1c42be1171fea9dd34f19ff867c558cfc7cf27f5914320eb +size 1849234419 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3892.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3892.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..4f2230bf41eeace968f1ec78842944b209e16fdf --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3892.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:015bcada24397df3dd60b9bcd5eb3d093172ad824fe52c9588a1d60b38009aec +size 1849234866 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3978.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3978.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c763231d6c5cff51d8ed6d58bcebadbee80c35b9 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3978.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f763eb0cf7fa6112c196d3943f3a64edfb1df1322388c61639aa2d09a964d772 +size 1849235313 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4082.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4082.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..ca0ff2a55e3c4f378b8628d47f7ef107fa55ed51 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4082.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00ea8b3b483328d70797997c5188d6c3f80df908241fd10b858bc80105abc886 +size 1849217303 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4199.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4199.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..ee27e05c0688dce4b72ddf54d2b4934570f4a930 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4199.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fc651c0f722bb098c3c3ce7174ac7b7cbfe3d1ddbcfbc0987b8e88b02227e25 +size 1849217814 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4266.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4266.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..7c4758745d69d2bcafe4884284cfcb5c5913c086 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4266.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb5926aaa6fc8f5516a80e30f51fef3a7dfe8a7e4e7cf700c5d5b2694f5f0baf +size 1849218197 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4329.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4329.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b79a20a1d7ce688567fba93332c28e0ab38aac34 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4329.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d708674f398994aad3ed99bb42c11ce0d491436ac174d237dcd1242d5a84e5d6 +size 1849218708 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4404.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4404.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..ea6f929b2f4fd0903a16a974c88a58bfe2fe06d2 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4404.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b6dc6c39d09c4facef988297b09e23ebbbec5b9cc035661836ae1f4715d2d21 +size 1849219219 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4469.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4469.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..d6c7570705c4521ce5827bcafd53f67ad049498e --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4469.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd55e2c094916e3b0a06892bd0f0279972a84339b865b860d401ab5099e14f9a +size 1849219666 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4547.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4547.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..fdc4aca58b09514366f561f99930f1d1d1594d0d --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4547.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40cfbd87d2a54f9035372591b665c7bb8c54f68aeeebb4b925dea696d9b3b21a +size 1849220177 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4633.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4633.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..bc7bb48cb2421d617337fcba27abd7e8399d34e1 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4633.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7648ffe53b617c33fcbdf311094a4713354d2d47398ddbbc507198fd7eb8f4a2 +size 1849220688 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=470.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=470.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..2452e36c85699bb570e728e5c43c5b0f1fde8e5b --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=470.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6099a102fdb3e269eaf7c6fa2aff04eecea2ad62608b5431b40626488aa0bd9c +size 1849214356 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4728.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4728.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..3ab94d623937db4cf99e33513d11d9ac3bc559bb --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4728.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:522a4e1d6be7ed43ad842044810c93b1211a4f63e386d7caee34847cd45e4310 +size 1849221199 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4821.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4821.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b5cd4f645a1fdcd56e2ffd2c12740b708ac4b948 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4821.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03cf48739390ac349bbc26e7ef2a1a65d9654e9bb3bae7b08ed8abe17ee886e2 +size 1849221582 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4912.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4912.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..a922666403bc920e96c9418b100cafa1fa871495 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4912.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62e7d95b0fbf05051de970be40012f33232207522f3693e1d3c6b9981a73f7ac +size 1849222029 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5007.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5007.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..e3e31e9fa908a56561908c9a481a550d5db5fb98 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5007.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20de4dfe2b0177db71093971a1969f9fd1b53e3c393dd6324157a3013daf46ab +size 1849222604 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5103.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5103.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..95774e65a17811be9d0ad662c5af7ac54d919cc3 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5103.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d3643b77219420fadd9409f32b0fb2614cbfc54c7442a2cd02e7483a03996b8 +size 1849223051 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5198.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5198.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f21f728d6c3543149eb36d28fe57ac12ee38a65a --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5198.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:374354d660c0351f999b4c7ee4ee1aa98fa83d08ea69bc5f72e5e34199c76a2a +size 1849223562 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5293.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5293.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..553abe62e9df0142c8f5337b8451f96d115d92aa --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5293.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af5379a7aebe178e6947b03a9b723ada6db0a35ab6abf0af5fe901fa27e6ceb7 +size 1849224009 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5396.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5396.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f0ba6b2528a4b75d8f7261e997057e4d3a780b1e --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5396.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdf2e0e05b76dcc9d3cd789c6ec9fbdc8daabb274be8b2ad13391b7a985f5fe2 +size 1849224520 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5535.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5535.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..5c7ced4870e379402298a4eb224d5184a0fbe169 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5535.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9125fd13d956fa19915ac7766984fb12e1ef1e6dcdb58fbd58fd855e25aae265 +size 1849224903 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=557.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=557.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..e17b3e74e3b4ff1e3bd6ad98e8c3e8b096722d15 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=557.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a60bc82aafe5361afce2d7e479a181d1bac300fcf03a539ef1d61d4b9c5f3d3 +size 1849214931 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5644.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5644.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..82ebcad955f82fd307cdbe52aefcd39c41a4e135 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5644.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b48775d1f55391bbf6aeee9768400880f87892824ea4c97de66ce52846ba3721 +size 1849225350 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5749.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5749.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..d300aa8bebae6ccfe4c651fee298ede68e54aa65 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5749.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aecb095939c6e1140dbff0877b59d88d0b6642a409de4d236ad12bf81649624c +size 1849225925 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5903.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5903.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b4fe33852907c439c992fedbf807bc46cf202fa2 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5903.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a4501d022db2ddde61bdbf711fd9e0ccd96d61e2be293fa1c166b619aef1d47 +size 1849226244 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6126.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6126.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..deb7fbfaace7e1c9ff4ae3b2fd4a0162a7e4d1f6 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6126.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dac759d0b2b07bb5f26b5a7a7045d65ee4416b512c7a800b8e265d14a3a2663 +size 1849226691 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6279.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6279.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..e81ee0bfab3e1e1c919ffb25c98d8e0ed2f4a535 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6279.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:584f81af7e5df46e4d3d59d05921d3eee10ecf0332d8cf094cc052b3bb15278d +size 1849227266 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6397.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6397.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..77baf3bb0d5b8d9d78b2ee141a8da406c0a47fc4 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6397.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99eccea3517e8debb5dc84defd5aff6cc1b5fe4f7ae3a6fbbe327c6151ed9da2 +size 1849227649 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6514.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6514.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f111e6c88109d22b191309a9895e5848eaed0c3d --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6514.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6a5e7cbf8c2fc82d3b2df9f64a72cb1cd9b8c29522a4e46fe8f269877f308e0 +size 1849228032 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=652.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=652.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..1e128403f5cf273636dda34cba08a5dc5fc6c3d8 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=652.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2185eaf46321c93ae83a1c9864864321fc3e45455a014ebaf2f78f45b1657724 +size 1849215378 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6628.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6628.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c59d40d148bcb17553a76e9f5d51fa1a7ff7ed13 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6628.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ed18d9315678fc02a988f283202151c3052601048d5b2b340af294430fed094 +size 1849228607 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6774.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6774.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..88efb2569f6b1a4d332318aecfb77c64ff23bd98 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6774.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e6ea5adbc997717654617286cbcdf2e1d42eb30bedb046ef169146f094e7c18 +size 1849229182 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6887.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6887.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..46d7220bd851b751de2c0ef44721e68f2afbc0eb --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6887.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26cf1feda6e2d5bb350b5ccffbde52dae047eee7b7bf1edf7c798ab24a7b80ae +size 1849229501 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7031.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7031.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..d87b25bee8d8c6bc422c3044f1ca83faa6615d5b --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7031.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:921cba494066803b90d1dc2c6d4af1be8417b64171790ee67ac239ce799a807a +size 1849229948 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7150.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7150.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..fd1f386e197e4bef5d84a08d4b9939a66423157a --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7150.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a4135758a44fb73ba5ebfbb29a0e20e140919fc734971b6f81838214ea983a1 +size 1849230523 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7315.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7315.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..52ccffdb79441c7be12af717754e502bd07fa8a3 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7315.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6495730bf5c0c8d1ee83fac55ca8b60f5b56161906cd2d83d8e4f3fd71706733 +size 1849230970 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7433.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7433.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..816e11c2862b1ca7ad9a46495e527159821b642d --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7433.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afb3ad07c4e78fe47b07a99262575e3560f03e5840294778bc9fbae75b3f18db +size 1849231481 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=747.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=747.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..25b882437e3d53f513cebd8a43360465fb2c3e18 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=747.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a206a11cce85c670de39c21d69ea76c4b8dc58b863e3eb88c72487e3a949a2b6 +size 1849215825 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7563.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7563.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..11fd98381772eeff742fae477281423600cf7394 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7563.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8453bc225dcbec97d039058a9c216717dff6fcf1dcc0593ae2791b663f8daab8 +size 1849231928 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7715.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7715.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..9e5319b8181b1d9fe572fc18d857e477a23a279f --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7715.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db6fc44d561403583379f846826e270c7a4536ec5cb945b54988443d16b71490 +size 1849232375 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7843.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7843.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c906796771a209d558ec629433e1267b687825f7 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7843.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20ca95c0bb2726312b930f1eea34ad0185a5c58a9295d7bd714b9d8ca25651b1 +size 1849232886 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7998.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7998.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..d13ff145ae91099a03494aa7314c45577f99b0b3 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7998.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a63d5c68b9cf003991f6fc95dd75ed1cee74086c39f6a108279172dd251adc2 +size 1849233333 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8116.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8116.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..e41a51c60de5e457b64b77f816497f56e6f4e0b5 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8116.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e2fdc3cdace79996b45c2c5e2a4bfbaa3eadd42c9297b6d11959a15813187a2 +size 1849233844 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8241.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8241.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..32871048eea0523da3920c15f00ea419f8aa44d1 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8241.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f3e67d67a9d1e65ac1f7aebd5661133c2e9e4baf4f0aae831006ff8030c65eb +size 1849234355 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=829.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=829.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..eab749797b28e189882902497a1d3329d041853a --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=829.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94d9a740df243700d37b26819e4b9ac61b9fa7be69fbfb1e08a97083c0f2d9f5 +size 1849217367 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8374.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8374.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..68217ff22cd5680d014bbe1dcc74dd6c4120ac0b --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8374.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1141cab7fb3ff2e37d290a44ff8a9f3a0536270e394803bd84a094a7a0220213 +size 1849234802 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8503.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8503.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..a8dc2b67880c464c905524fb95194ba716d54d2c --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8503.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3caa5fa2908820b768eb880f8adcf53a542b8baa0382eecd5a6b0bf67249c64 +size 1849235249 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8663.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8663.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..6b773bfb80fd77db31191776b6cf264ed38a3451 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8663.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5ef2ad1cbf719b4d0f50680fd2e5cfc691fc7bc95ddec8ed2bcd24444e5729d +size 1849217303 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8938.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8938.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f54bedd9c418a467aeed822df29bdefafd83dbc0 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8938.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56f04d517c3327cf757562b096bdb417e320b7c198e5bcab59d1d85c6e4d75d1 +size 1849217814 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=902.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=902.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..28aa826ee89b38d2afc1113d8b8db83569777efc --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=902.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f10470ffba2b0202eed1f08611caca9bcc3df635dcb3e2b2bc997caa52d60301 +size 1849217814 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9130.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9130.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..21302045e992e3a4a0705d44d9a31a704e8a9c91 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9130.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b06ea4c19ffb39776ae94815d4bb99d2ac43771984693b30420f9ba01756062 +size 1849218197 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9315.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9315.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..9e7a61f7095d783160c1758c0386efcba49c7d2c --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9315.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bba6551614c38acd43d3b9514dafa30d7b0e0efbbcd829cd6ae165a1b367065 +size 1849218708 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9390.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9390.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..fa927efa13a118dd7872adbb4e8fce96da081f9e --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9390.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd359e2cbc7391c50d0259689badfa121b855c34a24cb4168196c936c6451598 +size 1849219219 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9439.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9439.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..9c2a80c2bce6dfc5261770b4818b3b566b4526b3 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9439.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cc79a048f45c716b567cf631889db25d57017a4eff2e2e43024fd9f311a7283 +size 1849219666 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9499.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9499.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..5cdf8ca423f7cd65cf95b7ad09f87a76509537a2 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9499.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a46a0d811d02c008871ed41a4f4941025201875c3a8e78ec13a51e739f88b98 +size 1849220177 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9548.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9548.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..2e730c10ee0b4c149d7e999e1ff74337b2f5d066 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9548.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cacffc8056e731ff3ec8dffdf921e44d494018ebc573cb80bab401667eb06ca +size 1849220688 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9597.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9597.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..04e2374601f905818898b23bc83d76379c813f56 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9597.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c94b11dac8ecbf7b93e0c5d71fa20555f10f6005d1e427f77b9601a5d9931fda +size 1849221199 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9656.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9656.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..e148575c5d0392c455297b71e0d9be42b010b389 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9656.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd264b9eca5e5b7a5efdfb1b99cfce6e51a709286a9f8e6d2a23da99628d0376 +size 1849221582 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9705.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9705.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..828e8e18c6124a0bd03b3ded756e415ff2c64730 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9705.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:262330887338b05746a89505908c806bf1f5e0bb5515f7f5e633a3449d28e474 +size 1849222029 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=977.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=977.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f06b9a96b131e4d4450cba9a10ce2760a195d4db --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=977.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99b89d16bad13791544c3008354c2710f30ea200692a86ad7738788bf1486a35 +size 1849218261 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9770.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9770.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..69fdbd2e8c9eebdb740e78ed863b42d67b4eab8d --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9770.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f68012b927782802b149d39353f66530404dfc85275e8f2569ab9f13daee6ebe +size 1849222604 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9828.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9828.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..cc69c3f241dcc7d8ca2cf20ad7e50cee6490228c --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9828.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff9dcb4f25a928596efccce02c7e4cef60f50ae43573e25772e74b97c552adfb +size 1849223051 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9882.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9882.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..a80efaaf7c9d73bc32628cac4531a7968e2c1278 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9882.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01c6e1638f80db0526aa16f21b841dd0929d47b4eb68453ce2f96b7f185fdd83 +size 1849223562 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9955.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9955.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f312c9de92748ea8f7810186788e77ef3583f6f4 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9955.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c790e5782298d31a78070ac14cfcab470aef87e8bee8f132ca6a121e420a0bcb +size 1849224009 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/last.ckpt b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/last.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f155dbebca6722e4fbc82ae4fca562cf6d113510 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/last.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b30981861f065ba4de6143100f728480fe371c74ecc3c816e6e6658edfcc51ba +size 1849234866 diff --git a/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/resume_singlenode_2023-08-19-16:34:28.log b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/resume_singlenode_2023-08-19-16:34:28.log new file mode 100644 index 0000000000000000000000000000000000000000..0042bbc583745b237491f82fdf6ab5d6ebd9de09 --- /dev/null +++ b/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/resume_singlenode_2023-08-19-16:34:28.log @@ -0,0 +1,2839 @@ +wandb: Currently logged in as: tamariucai (rycolab). Use `wandb login --relogin` to force relogin +wandb: wandb version 0.15.8 is available! To upgrade, please run: +wandb: $ pip install wandb --upgrade +wandb: Tracking run with wandb version 0.15.4 +wandb: Run data is saved locally in ./wandb/run-20230819_163503-x3drnzb1 +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run graceful-sweep-6 +wandb: ⭐️ View project at https://wandb.ai/rycolab/alkmi-wit +wandb: 🧹 View sweep at https://wandb.ai/rycolab/alkmi-wit/sweeps/ff69ui4h +wandb: 🚀 View run at https://wandb.ai/rycolab/alkmi-wit/runs/x3drnzb1 +[rank: 0] Global seed set to 5501650 +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +The current host is eu-a65-02 +The current cuDNN version is 8801 +Parameter 'text_perc' already had the value 1. +Parameter 'vision_perc' already had the value 1. +Parameter 'seed' already had the value 5501650. +Total GPU memory: 42.296 GB +Detected half-sized run, (almost) doubling batch size to 48. +Detected half-sized run, (almost) halving gradient accumulation to 85 +[update_ckeckpoint_dir] Detected hyperparameter run! +[update_ckeckpoint_dir] Detected half-size run! +[update_ckeckpoint_dir] Found and updated model name flava in the checkpoint dirpath. +[update_ckeckpoint_dir] Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-19)_time(16:35:05)/seed(5501650)-magic({'enable': True})-text_perc(1)-vision_perc(1/ +Text is the predominant modality (1 v.s. 1 vision), will sample proportionally for optimal BLiMP performance. +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Found cached dataset imagenet-1k (/cluster/scratch/tamariucai/HuggingfaceDatasets/imagenet-1k/default/1.0.0/a1e9bfc56c3a7350165007d1176b15e9128fcaf9ab972147840529aed3ae52bc) +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/plugins/environments/slurm.py:165: PossibleUserWarning: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... + rank_zero_warn( +Using bfloat16 Automatic Mixed Precision (AMP) +GPU available: True (cuda), used: True +TPU available: False, using: 0 TPU cores +IPU available: False, using: 0 IPUs +HPU available: False, using: 0 HPUs +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Restoring states from the checkpoint path at /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=747.ckpt +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:337: UserWarning: The dirpath has changed from "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-07-26)_time(15:05:15)/seed(5501650)-text_perc(1)-vision_perc(1)-magic({'enable': True}" to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-19)_time(16:35:05)/seed(5501650)-magic({'enable': True})-text_perc(1)-vision_perc(1", therefore `best_model_score`, `kth_best_model_path`, `kth_value`, `last_model_path` and `best_k_models` won't be reloaded. Only `best_model_path` will be reloaded. + warnings.warn( +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0] + + | Name | Type | Params +---------------------------------------------- +0 | model | FlavaForPreTraining | 189 M +---------------------------------------------- +136 M Trainable params +53.8 M Non-trainable params +189 M Total params +759.666 Total estimated model params size (MB) +Restored all states from the checkpoint at /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text1-vision1/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=747.ckpt +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +VLDataModule: batch_size is 48, num_workers is 4 +ImageDataModule: batch_size is 48, num_workers is 4 +MLMDataModule: batch_size is 48, num_workers is 4 +Dataset sizes (un-normalized): [1, 1, 1] +Dataset sizes (normalized): [0.3333333333333333, 0.3333333333333333, 0.3333333333333333] +Sampling weights after temperature (1.0): [0.3333333333333333, 0.3333333333333333, 0.3333333333333333] +2023-08-19 16:36:10.408645 Setting up datamodule: VLDataModule +2023-08-19 16:36:45.178249 Setting up datamodule: ImageDataModule +2023-08-19 16:36:57.375861 Setting up datamodule: MLMDataModule +Registering multimodal overfitting monitors +Adding ImageNet zeroshot callback +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'ImageNetZeroshotCallback'] +Trainer successfully initialized (strategy=SingleDeviceStrategy) +Starting training +2023-08-19 16:37:15.861417 Preparing data for datamodule: VLDataModule +2023-08-19 16:37:15.861578 Preparing data for datamodule: ImageDataModule +2023-08-19 16:37:15.861680 Preparing data for datamodule: MLMDataModule +2023-08-19 16:37:15.862033 Setting up datamodule: VLDataModule +2023-08-19 16:37:44.439367 Setting up datamodule: ImageDataModule +2023-08-19 16:37:57.855231 Setting up datamodule: MLMDataModule +Resetting wait count for `mlm` since training is resuming and they are special tasks. +Resetting wait count for `mmm_text` since training is resuming and they are special tasks. +2023-08-19 16:38:25.476706 Building train dataloader for datamodule: VLDataModule +2023-08-19 16:38:25.477287 Building train dataloader for datamodule: ImageDataModule +2023-08-19 16:38:25.477629 Building train dataloader for datamodule: MLMDataModule +2023-08-19 16:38:26.896613 Building val dataloader for datamodule: VLDataModule +2023-08-19 16:38:26.897945 Building val dataloader for datamodule: ImageDataModule +2023-08-19 16:38:26.898779 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 16:48:51.567389] Computing e^(tensor([14484.4839], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 16:48:51.576139] Ending with PPL=1397.222955115157 (duration: 0:04:34.181091) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.903 +Metric validation/losses/mim_loss improved by 0.003 >= min_delta = 0.0. New best score: 1.030 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.300. Wait count has been increased from 0. +Metric validation/losses/global_contrastive_loss improved by 0.217 >= min_delta = 0.0. New best score: 3.487 +Metric validation/losses/mmm_image_loss improved by 0.003 >= min_delta = 0.0. New best score: 1.033 +Metric validation/losses/mmm_text_loss improved by 0.121 >= min_delta = 0.0. New best score: 6.790 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 19:58:35.166757] Computing e^(tensor([14354.3671], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 19:58:35.172406] Ending with PPL=1309.2157426215042 (duration: 0:04:14.183468) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.829 +Metric validation/losses/mim_loss improved by 0.009 >= min_delta = 0.0. New best score: 1.021 +Metric validation/losses/itm_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.293 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 3.487. Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.009 >= min_delta = 0.0. New best score: 1.025 +Metric validation/losses/mmm_text_loss improved by 0.092 >= min_delta = 0.0. New best score: 6.697 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 23:31:43.034435] Computing e^(tensor([14178.4551], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 23:31:43.040161] Ending with PPL=1198.9812943274244 (duration: 0:04:27.184841) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.735 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 1.019 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.293. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 3.487. Wait count has been increased from 1. +Metric validation/losses/mmm_image_loss improved by 0.001 >= min_delta = 0.0. New best score: 1.024 +Metric validation/losses/mmm_text_loss improved by 0.120 >= min_delta = 0.0. New best score: 6.578 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 03:03:11.987119] Computing e^(tensor([13997.4694], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 03:03:11.991336] Ending with PPL=1095.2464656691063 (duration: 0:04:53.224966) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.629 +Metric validation/losses/mim_loss improved by 0.000 >= min_delta = 0.0. New best score: 1.019 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.293. Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 3.487. Wait count has been increased from 2. +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 1.019 +Metric validation/losses/mmm_text_loss improved by 0.116 >= min_delta = 0.0. New best score: 6.462 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 06:38:05.757133] Computing e^(tensor([13720.9017], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 06:38:05.762839] Ending with PPL=953.7969848344426 (duration: 0:04:13.098988) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.524 +Metric validation/losses/mim_loss improved by 0.009 >= min_delta = 0.0. New best score: 1.010 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.293. Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 3.487. Wait count has been increased from 3. +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 1.014 +Metric validation/losses/mmm_text_loss improved by 0.158 >= min_delta = 0.0. New best score: 6.304 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 10:02:00.967951] Computing e^(tensor([13338.4176], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 10:02:00.973871] Ending with PPL=787.7720573158434 (duration: 0:04:12.786697) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.369 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.010. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 4 records. Best score: 0.293. Wait count has been increased from 3. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.487. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.014. Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.198 >= min_delta = 0.0. New best score: 6.106 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 13:09:03.828016] Computing e^(tensor([12739.0570], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 13:09:03.833632] Ending with PPL=583.7824968038665 (duration: 0:04:14.557075) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.204 +Metric validation/losses/mim_loss improved by 0.006 >= min_delta = 0.0. New best score: 1.004 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.293. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 3.487. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.012 >= min_delta = 0.0. New best score: 1.002 +Metric validation/losses/mmm_text_loss improved by 0.330 >= min_delta = 0.0. New best score: 5.776 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 16:27:20.290715] Computing e^(tensor([11785.7861], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 16:27:20.296430] Ending with PPL=362.4523628012359 (duration: 0:04:09.415745) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 5.882 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.004. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.293. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 3.487. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.002. Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.452 >= min_delta = 0.0. New best score: 5.323 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 19:39:01.580096] Computing e^(tensor([10978.2597], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 19:39:01.585988] Ending with PPL=242.04650125719994 (duration: 0:04:38.791096) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 5.557 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.004. Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.293. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 8 records. Best score: 3.487. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.002. Wait count has been increased from 1. +Metric validation/losses/mmm_text_loss improved by 0.252 >= min_delta = 0.0. New best score: 5.071 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 21:38:37.286280] Computing e^(tensor([10478.7643], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 21:38:37.291988] Ending with PPL=188.55357051880185 (duration: 0:04:39.170607) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 5.345 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.998 +Monitored metric validation/losses/itm_loss did not improve in the last 8 records. Best score: 0.293. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 1.000 +Metric validation/losses/mmm_text_loss improved by 0.190 >= min_delta = 0.0. New best score: 4.882 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 23:39:02.528031] Computing e^(tensor([10144.2964], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 23:39:02.533766] Ending with PPL=159.51663252189954 (duration: 0:05:25.991230) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 5.178 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.998. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 3.487. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.000. Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.119 >= min_delta = 0.0. New best score: 4.763 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 01:43:19.129194] Computing e^(tensor([9802.7760], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 01:43:19.134964] Ending with PPL=134.47630654834316 (duration: 0:06:18.130865) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 5.018 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 0.998. Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.293. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 3.487. Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.000. Wait count has been increased from 1. +Metric validation/losses/mmm_text_loss improved by 0.086 >= min_delta = 0.0. New best score: 4.677 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 03:51:17.917736] Computing e^(tensor([9549.9653], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 03:51:17.923433] Ending with PPL=118.50825924887084 (duration: 0:06:26.647611) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.916 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 0.998. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.293. Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 3.487. Wait count has been increased from 2. +Metric validation/losses/mmm_image_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.999 +Metric validation/losses/mmm_text_loss improved by 0.084 >= min_delta = 0.0. New best score: 4.593 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 07:12:14.267215] Computing e^(tensor([9307.9522], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 07:12:14.272938] Ending with PPL=105.00165357726598 (duration: 0:04:14.617885) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.826 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 0.998. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.293. Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 3.487. Wait count has been increased from 3. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.999. Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.096 >= min_delta = 0.0. New best score: 4.497 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 10:10:52.713234] Computing e^(tensor([9120.0213], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 10:10:52.718916] Ending with PPL=95.58449834306538 (duration: 0:04:15.216049) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.751 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 0.998. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 4 records. Best score: 0.293. Wait count has been increased from 3. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.487. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.999. Wait count has been increased from 1. +Metric validation/losses/mmm_text_loss improved by 0.083 >= min_delta = 0.0. New best score: 4.413 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 13:48:36.919466] Computing e^(tensor([8918.7336], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 13:48:36.925190] Ending with PPL=86.4327609607276 (duration: 0:04:05.581186) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.670 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 0.998. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.293. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 3.487. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.999. Wait count has been increased from 2. +Metric validation/losses/mmm_text_loss improved by 0.078 >= min_delta = 0.0. New best score: 4.335 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 17:07:44.253175] Computing e^(tensor([8736.6628], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 17:07:44.258909] Ending with PPL=78.91184902239063 (duration: 0:04:14.306362) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.600 +Monitored metric validation/losses/mim_loss did not improve in the last 7 records. Best score: 0.998. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.293. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 3.487. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 4 records. Best score: 0.999. Wait count has been increased from 3. +Metric validation/losses/mmm_text_loss improved by 0.109 >= min_delta = 0.0. New best score: 4.226 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 21:21:28.090687] Computing e^(tensor([8578.7797], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 21:21:28.095038] Ending with PPL=72.92196262198355 (duration: 0:04:14.290756) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.538 +Monitored metric validation/losses/mim_loss did not improve in the last 8 records. Best score: 0.998. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.293. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 8 records. Best score: 3.487. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 0.999. Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.083 >= min_delta = 0.0. New best score: 4.143 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 01:24:11.178769] Computing e^(tensor([8407.6293], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 01:24:11.184462] Ending with PPL=66.94120327348048 (duration: 0:04:10.489299) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.475 +Monitored metric validation/losses/mim_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/itm_loss did not improve in the last 8 records. Best score: 0.293. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 0.999. Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.065 >= min_delta = 0.0. New best score: 4.079 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 05:33:11.494711] Computing e^(tensor([8264.4233], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 05:33:11.500733] Ending with PPL=62.31559162518637 (duration: 0:04:10.702449) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.423 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.998. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 3.487. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 7 records. Best score: 0.999. Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.054 >= min_delta = 0.0. New best score: 4.025 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 09:44:26.471139] Computing e^(tensor([8115.4120], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 09:44:26.476926] Ending with PPL=57.84147121930278 (duration: 0:04:10.035244) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.365 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 0.998. Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.293. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 3.487. Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 8 records. Best score: 0.999. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 4.025. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 13:53:29.200184] Computing e^(tensor([7939.5025], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 13:53:29.205842] Ending with PPL=52.971351905746594 (duration: 0:04:05.967126) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.307 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 0.998. Signaling model to ignore task. +Metric validation/losses/itm_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.290 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 3.487. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_image_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Metric validation/losses/mmm_text_loss improved by 0.058 >= min_delta = 0.0. New best score: 3.967 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 17:27:24.456528] Computing e^(tensor([7788.3279], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 17:27:24.462387] Ending with PPL=49.114973648112475 (duration: 0:04:08.281579) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.251 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 0.998. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.290. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 3.487. Wait count has been increased from 3. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.999. Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.017 >= min_delta = 0.0. New best score: 3.950 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 21:10:51.943200] Computing e^(tensor([7658.5826], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 21:10:51.948929] Ending with PPL=46.02990613725316 (duration: 0:04:06.653872) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.196 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 0.998. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.290. Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.487. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.999. Wait count has been increased from 1. +Metric validation/losses/mmm_text_loss improved by 0.017 >= min_delta = 0.0. New best score: 3.933 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 01:28:24.560747] Computing e^(tensor([7540.8132], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 01:28:24.566465] Ending with PPL=43.39770745976748 (duration: 0:04:06.464510) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.157 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 0.998. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.290. Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 3.487. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.999. Wait count has been increased from 2. +Metric validation/losses/mmm_text_loss improved by 0.022 >= min_delta = 0.0. New best score: 3.911 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 05:36:46.999967] Computing e^(tensor([7435.8142], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 05:36:47.005809] Ending with PPL=41.17812241344229 (duration: 0:04:10.876195) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.118 +Monitored metric validation/losses/mim_loss did not improve in the last 7 records. Best score: 0.998. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 4 records. Best score: 0.290. Wait count has been increased from 3. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 3.487. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 4 records. Best score: 0.999. Wait count has been increased from 3. +Metric validation/losses/mmm_text_loss improved by 0.031 >= min_delta = 0.0. New best score: 3.880 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 09:48:49.786246] Computing e^(tensor([7350.1751], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 09:48:49.792123] Ending with PPL=39.45211084938417 (duration: 0:04:15.006501) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.086 +Monitored metric validation/losses/mim_loss did not improve in the last 8 records. Best score: 0.998. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.290. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 8 records. Best score: 3.487. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 0.999. Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.028 >= min_delta = 0.0. New best score: 3.852 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 13:08:50.223413] Computing e^(tensor([7288.5651], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 13:08:50.229144] Ending with PPL=38.25531731739399 (duration: 0:04:14.229927) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.062 +Monitored metric validation/losses/mim_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.290. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 0.999. Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.073 >= min_delta = 0.0. New best score: 3.779 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 17:04:13.384393] Computing e^(tensor([7219.5062], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 17:04:13.390072] Ending with PPL=36.95692659760316 (duration: 0:04:18.935998) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.031 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.998. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.290. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 3.487. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 7 records. Best score: 0.999. Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.029 >= min_delta = 0.0. New best score: 3.750 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 20:32:55.184058] Computing e^(tensor([7170.3784], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 20:32:55.189988] Ending with PPL=36.060179316347245 (duration: 0:04:07.596765) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.004 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 0.998. Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve in the last 8 records. Best score: 0.290. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 3.487. Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 8 records. Best score: 0.999. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.750. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 00:14:42.530717] Computing e^(tensor([7051.0903], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 00:14:42.536381] Ending with PPL=33.97228861245073 (duration: 0:04:12.628497) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.964 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 0.998. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 3.487. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_image_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Metric validation/losses/mmm_text_loss improved by 0.076 >= min_delta = 0.0. New best score: 3.674 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 04:02:12.188630] Computing e^(tensor([7012.6422], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 04:02:12.194377] Ending with PPL=33.3254413683272 (duration: 0:04:12.051889) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.943 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 0.998. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.290. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 3.487. Wait count has been increased from 3. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.999. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.674. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 07:30:31.242239] Computing e^(tensor([6966.4913], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 07:30:31.248000] Ending with PPL=32.56524546344605 (duration: 0:04:15.082127) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.921 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 0.998. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.290. Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.487. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.999. Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.674. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 11:06:24.298217] Computing e^(tensor([6912.1559], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 11:06:24.304041] Ending with PPL=31.692433624719673 (duration: 0:04:14.439605) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.882 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 0.998. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.290. Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 3.487. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.999. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.674. Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 14:21:42.065067] Computing e^(tensor([6887.1837], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 14:21:42.070757] Ending with PPL=31.29917906253069 (duration: 0:04:16.534339) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 1.031 +Metric validation/losses/mmm_image_loss improved by 0.007 >= min_delta = 0.0. New best score: 1.030 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.710 (now 3.734349012374878). Wait count has been increased from 0. +Metric validation/losses/itm_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.286 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 3.734 (now 3.91062331199646). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 01:32:29.738685] Computing e^(tensor([6953.3013], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 01:32:29.744259] Ending with PPL=32.35118550155195 (duration: 0:04:33.056448) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 1.028 +Metric validation/losses/mmm_image_loss improved by 0.008 >= min_delta = 0.0. New best score: 1.022 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.710 (now 3.7633368968963623). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.286 (now 0.35386398434638977). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 3.734 (now 4.3926682472229). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 05:47:44.086948] Computing e^(tensor([6909.0666], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 05:47:44.092676] Ending with PPL=31.643516447138452 (duration: 0:04:11.141967) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.851 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.028 (now 1.0640878677368164). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.022 (now 1.0672571659088135). Wait count has been increased from 1. +Metric validation/losses/mmm_text_loss improved by 0.017 >= min_delta = 0.0. New best score: 3.693 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.286 (now 0.48823872208595276). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.734 (now 4.677174091339111). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 13:38:54.036655] Computing e^(tensor([6789.6448], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 13:38:54.042190] Ending with PPL=29.80935812984566 (duration: 0:04:11.745828) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.830 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 1.028 (now 1.0948842763900757). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 1.022 (now 1.0921540260314941). Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.693 (now 3.7117202281951904). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.286 (now 0.5558638572692871). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.734 (now 4.677174091339111). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 16:50:29.418888] Computing e^(tensor([6744.8043], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 16:50:29.424499] Ending with PPL=29.148462479837033 (duration: 0:04:46.382008) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.809 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 1.028 (now 1.1212953329086304). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 4 records. Best score: 1.022 (now 1.123968482017517). Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.068 >= min_delta = 0.0. New best score: 3.625 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.286 (now 0.5558638572692871). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 3.734 (now 4.677174091339111). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 19:46:52.906210] Computing e^(tensor([6706.0393], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 19:46:52.910278] Ending with PPL=28.588932336906858 (duration: 0:05:37.280742) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.791 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 1.028 (now 1.1332852840423584). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 1.022 (now 1.123968482017517). Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.022 >= min_delta = 0.0. New best score: 3.603 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.286 (now 0.5558638572692871). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 22:44:27.032397] Computing e^(tensor([6795.4900], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 22:44:27.038003] Ending with PPL=29.89660700038514 (duration: 0:05:48.427642) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.783 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.028 (now 1.1176207065582275). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.022 (now 1.1164847612380981). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.603 (now 3.6683402061462402). Wait count has been increased from 2. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.286 (now 0.43200746178627014). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 3.734 (now 4.622161865234375). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 07:42:27.320733] Computing e^(tensor([6649.3182], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 07:42:27.326422] Ending with PPL=27.789523764658668 (duration: 0:06:03.616551) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.765 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.028 (now 1.168968677520752). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.022 (now 1.184264898300171). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 3.603 (now 3.6484620571136475). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.286 (now 0.6094997525215149). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.734 (now 4.611917972564697). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 10:45:28.320769] Computing e^(tensor([6620.9706], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 10:45:28.326475] Ending with PPL=27.39841889574754 (duration: 0:06:15.491411) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.757 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 1.028 (now 1.2086122035980225). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 1.022 (now 1.20978581905365). Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 3.603 (now 3.6484620571136475). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.286 (now 0.5218551158905029). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.734 (now 4.611917972564697). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 13:50:42.760077] Computing e^(tensor([6623.2313], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 13:50:42.765712] Ending with PPL=27.429406406256337 (duration: 0:04:12.899071) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.754 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 1.028 (now 1.2582067251205444). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 4 records. Best score: 1.022 (now 1.250907301902771). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 3.603 (now 3.6484620571136475). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.286 (now 0.5218551158905029). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 3.734 (now 4.611917972564697). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 16:42:30.610366] Computing e^(tensor([6608.6258], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 16:42:30.616002] Ending with PPL=27.229825136270257 (duration: 0:04:25.249065) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.749 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 1.028 (now 1.2659516334533691). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 1.022 (now 1.250907301902771). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.286 (now 0.5218551158905029). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 19:16:51.951771] Computing e^(tensor([6693.9339], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 19:16:51.957614] Ending with PPL=28.416414116186093 (duration: 0:05:35.960338) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.740 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.028 (now 1.276471734046936). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.022 (now 1.269477367401123). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.603 (now 3.6880671977996826). Wait count has been increased from 2. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.286 (now 0.5401865839958191). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 3.734 (now 4.546891212463379). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 02:29:49.155181] Computing e^(tensor([6589.6676], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 02:29:49.160746] Ending with PPL=26.97293101427631 (duration: 0:05:00.330915) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.722 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.028 (now 1.3624866008758545). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.022 (now 1.348404884338379). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 3.603 (now 3.6616058349609375). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.286 (now 0.5520010590553284). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.734 (now 4.555054187774658). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 04:38:17.789700] Computing e^(tensor([6598.2158], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 04:38:17.795333] Ending with PPL=27.088463115669967 (duration: 0:07:29.194889) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.717 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 1.028 (now 1.4075570106506348). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 1.022 (now 1.395209550857544). Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 3.603 (now 3.6616058349609375). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.286 (now 0.754087507724762). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.734 (now 4.555054187774658). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 06:42:08.816458] Computing e^(tensor([6568.0843], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 06:42:08.822125] Ending with PPL=26.683413015123403 (duration: 0:04:44.035578) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.711 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 1.028 (now 1.4300956726074219). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 4 records. Best score: 1.022 (now 1.4188814163208008). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 3.603 (now 3.6616058349609375). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.286 (now 0.754087507724762). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 3.734 (now 4.555054187774658). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 07:36:58.297857] Computing e^(tensor([6572.9635], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 07:36:58.305813] Ending with PPL=26.748589389246572 (duration: 0:04:15.666750) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.706 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.028 (now 1.5345680713653564). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.022 (now 1.5267391204833984). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 3.603 (now 3.6597864627838135). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.286 (now 0.7223888635635376). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.734 (now 4.51845121383667). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 17:19:28.541341] Computing e^(tensor([6558.0815], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 17:19:28.546906] Ending with PPL=26.55029187553154 (duration: 0:06:26.164034) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.704 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 1.028 (now 1.614112377166748). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 1.022 (now 1.5702741146087646). Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 3.603 (now 3.6597864627838135). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.286 (now 0.6969733238220215). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.734 (now 4.51845121383667). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 20:00:21.097649] Computing e^(tensor([6539.8708], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 20:00:21.103331] Ending with PPL=26.3096397957551 (duration: 0:06:18.227858) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.704 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 1.028 (now 1.6562604904174805). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 1.022 (now 1.6209666728973389). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.286 (now 0.6969733238220215). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 00:30:54.985530] Computing e^(tensor([6649.7083], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 00:30:54.991137] Ending with PPL=27.794945112737434 (duration: 0:06:28.629403) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.691 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.028 (now 1.7068294286727905). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.022 (now 1.655021071434021). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 3.603 (now 3.662625551223755). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.286 (now 0.6510626673698425). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.734 (now 4.429634094238281). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 09:38:07.818779] Computing e^(tensor([6554.0160], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 09:38:07.824582] Ending with PPL=26.496377107825595 (duration: 0:04:31.645964) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.690 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 1.028 (now 1.7615242004394531). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 1.022 (now 1.7127782106399536). Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 3.603 (now 3.662625551223755). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.286 (now 0.709347665309906). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.734 (now 4.429634094238281). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 11:51:49.897590] Computing e^(tensor([6545.2581], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 11:51:49.903133] Ending with PPL=26.38060453720975 (duration: 0:04:25.044330) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.679 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.028 (now 1.8116225004196167). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.022 (now 1.7398637533187866). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 3.603 (now 3.7085494995117188). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.286 (now 0.7794424295425415). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.734 (now 4.320137977600098). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 23:50:33.154036] Computing e^(tensor([6616.8109], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 23:50:33.159588] Ending with PPL=27.34149342303691 (duration: 0:04:38.175907) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 1.615 +Metric validation/losses/mmm_image_loss improved by 0.178 >= min_delta = 0.0. New best score: 1.526 +Metric validation/losses/mmm_text_loss improved by 0.029 >= min_delta = 0.0. New best score: 3.855 +Metric validation/losses/itm_loss improved by 0.078 >= min_delta = 0.0. New best score: 0.444 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 4.496 (now 4.7814836502075195). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 02:18:39.693908] Computing e^(tensor([6767.5158], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 02:18:39.699504] Ending with PPL=29.481351509123186 (duration: 0:04:46.855074) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.725 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.615 (now 1.893949270248413). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.526 (now 1.8221653699874878). Wait count has been increased from 1. +Metric validation/losses/mmm_text_loss improved by 0.031 >= min_delta = 0.0. New best score: 3.824 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.444 (now 0.6111485362052917). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 4.496 (now 4.880906581878662). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 04:45:54.406995] Computing e^(tensor([6667.6021], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 04:45:54.412788] Ending with PPL=28.044739396255853 (duration: 0:08:52.674153) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.823 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.444 (now 0.8071615099906921). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 4.496 (now 4.852086067199707). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 06:22:01.375548] Computing e^(tensor([6693.0845], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 06:22:01.381202] Ending with PPL=28.40434816279551 (duration: 0:06:40.582536) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.719 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 1.615 (now 2.091377019882202). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 4 records. Best score: 1.526 (now 1.9579249620437622). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.823 (now 3.8669769763946533). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.444 (now 0.8415551781654358). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 4.496 (now 4.852086067199707). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 07:37:51.856949] Computing e^(tensor([6679.9179], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 07:37:51.862644] Ending with PPL=28.217967999090824 (duration: 0:06:11.572356) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.716 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 1.615 (now 2.0902276039123535). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 1.526 (now 1.9579249620437622). Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.041 >= min_delta = 0.0. New best score: 3.782 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.444 (now 0.8415551781654358). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 4.496 (now 4.852086067199707). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 09:02:53.237604] Computing e^(tensor([6696.4787], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 09:02:53.243446] Ending with PPL=28.452593950549822 (duration: 0:07:56.596094) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.710 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 1.615 (now 2.0596632957458496). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 1.526 (now 1.9579249620437622). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.782 (now 3.7835729122161865). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.444 (now 0.8415551781654358). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 10:49:54.986476] Computing e^(tensor([6745.1339], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 10:49:54.992195] Ending with PPL=29.15326584449829 (duration: 0:06:17.029126) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.054 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 15:20:40.130412] Computing e^(tensor([6698.4539], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 15:20:40.136138] Ending with PPL=28.480708313243657 (duration: 0:04:19.660267) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.706 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.615 (now 2.0498714447021484). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.526 (now 1.914125680923462). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 3.782 (now 3.824632167816162). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.444 (now 0.8727497458457947). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 4.054 (now 4.64532995223999). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 02:00:12.216624] Computing e^(tensor([6699.6692], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 02:00:12.222402] Ending with PPL=28.49801941064696 (duration: 0:04:27.843617) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.264 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 4.259. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.778. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.564. Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 06:53:51.687480] Computing e^(tensor([6634.9698], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 06:53:51.690824] Ending with PPL=27.59086896405014 (duration: 0:03:21.810654) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.685 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 0.777. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.264. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 4.259. Wait count has been increased from 3. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 4 records. Best score: 0.778. Wait count has been increased from 3. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 4 records. Best score: 3.564. Wait count has been increased from 3. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 10:38:17.792511] Computing e^(tensor([6687.3728], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 10:38:17.796285] Ending with PPL=28.323345128448288 (duration: 0:03:21.672474) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.681 +Monitored metric validation/losses/mim_loss did not improve in the last 7 records. Best score: 0.777 (now 0.7833721041679382). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 4 records. Best score: 0.264 (now 0.3068883717060089). Wait count has been increased from 3. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 4.259 (now 6.113724708557129). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 7 records. Best score: 0.778 (now 0.8283724188804626). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.564 (now 3.6433305740356445). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 10:33:29.746123] Computing e^(tensor([6695.7455], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 10:33:29.750266] Ending with PPL=28.442165416244144 (duration: 0:03:23.833393) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.562 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 20:44:23.379124] Computing e^(tensor([6741.7436], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 20:44:23.385203] Ending with PPL=29.103889175573126 (duration: 0:03:36.011365) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.776 +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.264 (now 0.3742334246635437). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 4.259 (now 4.8505988121032715). Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.777 +Metric validation/losses/mmm_text_loss improved by 0.004 >= min_delta = 0.0. New best score: 3.558 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 05:38:19.747203] Computing e^(tensor([6690.9270], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 05:38:19.753516] Ending with PPL=28.373723998022648 (duration: 0:05:19.859907) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.772 +Monitored metric validation/losses/itm_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 4.259 (now 4.9681715965271). Wait count has been increased from 2. +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.772 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.558 (now 3.5584514141082764). Wait count has been increased from 1. +Starting validation +2023-08-22 15:11:20.089919 Preparing data for datamodule: VLDataModule +2023-08-22 15:11:20.090151 Preparing data for datamodule: ImageDataModule +2023-08-22 15:11:20.090324 Preparing data for datamodule: MLMDataModule +2023-08-22 15:11:20.090738 Setting up datamodule: VLDataModule +2023-08-22 15:12:45.453853 Setting up datamodule: ImageDataModule +2023-08-22 15:12:59.419061 Setting up datamodule: MLMDataModule +2023-08-22 15:14:20.804975 Building val dataloader for datamodule: VLDataModule +2023-08-22 15:14:20.805548 Building val dataloader for datamodule: ImageDataModule +2023-08-22 15:14:20.805832 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 15:16:16.410388] Computing e^(tensor([6795.6065], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 15:16:16.416632] Ending with PPL=29.898348322822148 (duration: 0:01:06.644946) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.823 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.787 (now 0.8457595109939575). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.795 (now 0.850875735282898). Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.006 >= min_delta = 0.0. New best score: 3.879 +Metric validation/losses/itm_loss improved. New best score: 0.314 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 4.339 (now 4.762688159942627). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 01:58:41.678710] Computing e^(tensor([6920.8692], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 01:58:41.684381] Ending with PPL=31.830807853369883 (duration: 0:05:23.282513) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.806 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 0.787 (now 0.8095299601554871). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.795 (now 0.8152087330818176). Wait count has been increased from 1. +Metric validation/losses/mmm_text_loss improved by 0.011 >= min_delta = 0.0. New best score: 3.868 +Metric validation/losses/itm_loss improved by 0.021 >= min_delta = 0.0. New best score: 0.292 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 4.339 (now 4.830321788787842). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 05:41:25.293873] Computing e^(tensor([6901.0471], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 05:41:25.299723] Ending with PPL=31.51688849601135 (duration: 0:06:49.423261) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.803 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 0.787 (now 0.7989839315414429). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.795 (now 0.8007844090461731). Wait count has been increased from 2. +Metric validation/losses/mmm_text_loss improved by 0.021 >= min_delta = 0.0. New best score: 3.847 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.292 (now 0.4220292568206787). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 4.339 (now 5.27097749710083). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 08:56:08.426611] Computing e^(tensor([6871.9626], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 08:56:08.432332] Ending with PPL=31.06187917790076 (duration: 0:04:15.800644) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.792 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 0.787 (now 0.7877323627471924). Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.789 +Metric validation/losses/mmm_text_loss improved by 0.057 >= min_delta = 0.0. New best score: 3.790 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.292 (now 0.445779025554657). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 4.339 (now 5.337802886962891). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 12:40:24.584657] Computing e^(tensor([6781.7017], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 12:40:24.590301] Ending with PPL=29.69120497308536 (duration: 0:04:16.404981) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.758 +Metric validation/losses/mim_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.778 +Metric validation/losses/mmm_image_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.780 +Metric validation/losses/mmm_text_loss improved by 0.013 >= min_delta = 0.0. New best score: 3.776 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.292 (now 0.47537732124328613). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 4.339 (now 5.337802886962891). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 16:44:10.920157] Computing e^(tensor([6770.3675], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 16:44:10.925857] Ending with PPL=29.523416560661726 (duration: 0:04:06.439807) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.774 +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.775 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.776 (now 3.791149139404297). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.292 (now 0.4745313823223114). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 4.339 (now 5.337802886962891). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 20:23:47.885548] Computing e^(tensor([6734.9031], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 20:23:47.891346] Ending with PPL=29.00451635852687 (duration: 0:04:09.518864) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.725 +Metric validation/losses/mim_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.770 +Metric validation/losses/mmm_image_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.771 +Metric validation/losses/mmm_text_loss improved by 0.127 >= min_delta = 0.0. New best score: 3.650 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.292 (now 0.4745313823223114). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 4.339 (now 5.337802886962891). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 00:03:04.535734] Computing e^(tensor([6769.1403], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 00:03:04.541457] Ending with PPL=29.505307681833116 (duration: 0:04:04.965364) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.768 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.769 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.650 (now 3.682786703109741). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.292 (now 0.4745313823223114). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in 7 records. This exceeds the revival patience 7, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 03:57:08.403766] Computing e^(tensor([6828.2151], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 03:57:08.409439] Ending with PPL=30.389816593876855 (duration: 0:04:39.929393) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.070 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 07:25:11.531296] Computing e^(tensor([6846.7089], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 07:25:11.537050] Ending with PPL=30.672130483628802 (duration: 0:04:07.146558) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.991 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 11:00:44.107047] Computing e^(tensor([6870.5595], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 11:00:44.112769] Ending with PPL=31.04009463335508 (duration: 0:04:10.431511) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.788 +Metric validation/losses/mmm_image_loss improved by 0.010 >= min_delta = 0.0. New best score: 0.796 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.795 (now 3.9047529697418213). Wait count has been increased from 0. +Metric validation/losses/itm_loss improved by 0.027 >= min_delta = 0.0. New best score: 0.437 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 5.053 (now 5.322032928466797). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 01:55:44.425682] Computing e^(tensor([6928.5073], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 01:55:44.431341] Ending with PPL=31.952602843458305 (duration: 0:04:10.057795) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.783 +Metric validation/losses/mmm_image_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.788 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.795 (now 3.886880874633789). Wait count has been increased from 1. +Metric validation/losses/itm_loss improved by 0.058 >= min_delta = 0.0. New best score: 0.379 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 5.053 (now 5.376453876495361). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 06:09:55.779070] Computing e^(tensor([6969.0898], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 06:09:55.784738] Ending with PPL=32.607583634377484 (duration: 0:04:09.257949) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.781 +Metric validation/losses/mmm_image_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.784 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.795 (now 3.910346508026123). Wait count has been increased from 2. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.379 (now 0.5791857242584229). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 5.053 (now 5.408877372741699). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 10:18:49.176844] Computing e^(tensor([6931.4325], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 10:18:49.182639] Ending with PPL=31.999370726485495 (duration: 0:04:12.069304) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.774 +Metric validation/losses/mmm_image_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.776 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 3.795 (now 3.8689396381378174). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.379 (now 0.5814040899276733). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 5.053 (now 5.340874671936035). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 14:29:19.249102] Computing e^(tensor([6948.6772], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 14:29:19.254783] Ending with PPL=32.27647405170284 (duration: 0:04:07.537010) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.770 +Metric validation/losses/mmm_image_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.773 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 3.795 (now 3.8689396381378174). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.379 (now 0.602724015712738). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 5.053 (now 5.340874671936035). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 19:00:20.328301] Computing e^(tensor([6933.2807], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 19:00:20.333967] Ending with PPL=32.028956122845386 (duration: 0:04:10.364828) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.768 +Metric validation/losses/mmm_image_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.770 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 3.795 (now 3.8689396381378174). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.379 (now 0.6712825298309326). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 5.053 (now 5.340874671936035). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 22:59:40.559731] Computing e^(tensor([6950.1361], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 22:59:40.565404] Ending with PPL=32.300026118168525 (duration: 0:04:10.330768) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.765 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.768 +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.379 (now 0.6712825298309326). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 03:32:23.646871] Computing e^(tensor([7032.6820], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 03:32:23.652550] Ending with PPL=33.661037452616895 (duration: 0:04:03.157766) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.915 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 07:44:47.489358] Computing e^(tensor([7055.0402], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 07:44:47.494999] Ending with PPL=34.03944849249069 (duration: 0:04:07.560383) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.765 +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 0.768 (now 0.7698956727981567). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 3.795 (now 3.8376126289367676). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.379 (now 0.610392153263092). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.915 (now 4.761572360992432). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 23:53:53.971801] Computing e^(tensor([6922.0533], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 23:53:53.977536] Ending with PPL=31.84965754913459 (duration: 0:04:10.016241) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.764 +Monitored metric validation/losses/mmm_image_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.379 (now 0.6948341727256775). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 3.915 (now 4.761572360992432). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 07:06:11.219290] Computing e^(tensor([6948.8740], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 07:06:11.224933] Ending with PPL=32.279649514168064 (duration: 0:04:09.423259) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.762 +Metric validation/losses/mmm_image_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.764 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.795 (now 3.811744451522827). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.379 (now 0.6948341727256775). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 10:12:29.378493] Computing e^(tensor([6994.1871], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 10:12:29.384332] Ending with PPL=33.01934376689394 (duration: 0:04:06.672782) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.762 +Monitored metric validation/losses/mmm_image_loss did not improve in the last 4 records. Best score: 0.764 (now 0.7659861445426941). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 3.795 (now 3.846501350402832). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.379 (now 0.5502347350120544). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.915 (now 4.546427249908447). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 23:58:51.489567] Computing e^(tensor([7011.9340], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 23:58:51.495322] Ending with PPL=33.31364328521272 (duration: 0:04:12.766416) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.761 +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 0.764 (now 0.7659861445426941). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.379 (now 0.7412835955619812). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 3.915 (now 4.546427249908447). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 07:10:21.296888] Computing e^(tensor([6980.5269], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 07:10:21.302539] Ending with PPL=32.79458652217234 (duration: 0:04:06.902476) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.761 +Monitored metric validation/losses/mmm_image_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.795 (now 3.840501308441162). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.379 (now 0.7412835955619812). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 10:31:17.558718] Computing e^(tensor([7067.0083], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 10:31:17.564383] Ending with PPL=34.243752376708045 (duration: 0:04:08.562750) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.764 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 3.795 (now 3.880239248275757). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.379 (now 0.6900689005851746). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.915 (now 4.431777477264404). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 00:24:33.685807] Computing e^(tensor([7014.3832], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 00:24:33.691490] Ending with PPL=33.354464221231865 (duration: 0:04:08.593769) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.764 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 3.795 (now 3.880239248275757). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.379 (now 0.6641873121261597). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.915 (now 4.431777477264404). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 03:53:48.040325] Computing e^(tensor([7029.4487], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 03:53:48.046100] Ending with PPL=33.60666232371561 (duration: 0:04:03.686575) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.757 +Metric validation/losses/mmm_image_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.760 +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.379 (now 0.6641873121261597). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 3.915 (now 4.431777477264404). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 07:12:00.879856] Computing e^(tensor([7092.6722], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 07:12:00.885577] Ending with PPL=34.68599842156836 (duration: 0:04:03.291794) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.760 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.795 (now 3.8924946784973145). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.379 (now 0.6641873121261597). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 10:48:42.147239] Computing e^(tensor([7044.2686], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 10:48:42.152996] Ending with PPL=33.85661216806341 (duration: 0:04:06.427530) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.872 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.776 (now 0.7783968448638916). Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.783 +Metric validation/losses/mmm_text_loss improved by 0.045 >= min_delta = 0.0. New best score: 4.025 +Metric validation/losses/itm_loss improved by 0.071 >= min_delta = 0.0. New best score: 0.451 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 4.659 (now 4.9763875007629395). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 04:58:44.665308] Computing e^(tensor([7182.6540], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 04:58:44.670996] Ending with PPL=36.282189798928144 (duration: 0:04:18.925428) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.856 +Metric validation/losses/mim_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.772 +Metric validation/losses/mmm_image_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.775 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 4.025 (now 4.046795845031738). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.451 (now 0.5732381343841553). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 4.659 (now 5.022200107574463). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 08:45:54.629882] Computing e^(tensor([7149.8527], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 08:45:54.635532] Ending with PPL=35.6919916230113 (duration: 0:04:15.404591) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.843 +Metric validation/losses/mim_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.764 +Metric validation/losses/mmm_image_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.766 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 4.025 (now 4.040045261383057). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.451 (now 0.7365221381187439). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 4.659 (now 4.946627140045166). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 12:43:21.797140] Computing e^(tensor([7124.9494], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 12:43:21.802783] Ending with PPL=35.25032409815654 (duration: 0:04:21.989110) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.832 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.764 (now 0.7669569849967957). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.766 (now 0.7698912024497986). Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.060 >= min_delta = 0.0. New best score: 3.965 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.451 (now 0.6622462272644043). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 4.659 (now 4.852460861206055). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 17:41:14.532905] Computing e^(tensor([7113.1363], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 17:41:14.538578] Ending with PPL=35.04272844964801 (duration: 0:04:14.106507) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.817 +Metric validation/losses/mim_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.758 +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.761 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.965 (now 4.003419399261475). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.451 (now 0.8416189551353455). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 4.659 (now 4.852460861206055). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 23:11:10.632988] Computing e^(tensor([7097.3636], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 23:11:10.638654] Ending with PPL=34.76745595680933 (duration: 0:04:17.325166) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.757 +Metric validation/losses/mmm_image_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.758 +Metric validation/losses/mmm_text_loss improved by 0.039 >= min_delta = 0.0. New best score: 3.926 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.451 (now 0.8416189551353455). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 4.659 (now 4.852460861206055). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 04:33:00.694233] Computing e^(tensor([7184.7858], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 04:33:00.699855] Ending with PPL=36.32088494146993 (duration: 0:05:10.300888) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.755 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.758 (now 0.7602489590644836). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.926 (now 3.9942169189453125). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.451 (now 0.8416189551353455). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 09:30:03.998865] Computing e^(tensor([7211.8260], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 09:30:04.004403] Ending with PPL=36.81528146491228 (duration: 0:04:11.069536) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.238 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 14:22:38.473151] Computing e^(tensor([7199.1253], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 14:22:38.478799] Ending with PPL=36.582232119675844 (duration: 0:04:09.234584) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.755 +Metric validation/losses/mmm_image_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.758 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.926 (now 3.952728509902954). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-01 15:46:15.897037] Computing e^(tensor([7218.3081], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-01 15:46:15.902843] Ending with PPL=36.9347945508928 (duration: 0:04:03.840943) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.755 +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 0.758 (now 0.7579020857810974). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.451 (now 0.8310090899467468). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 4.238 (now 4.91848611831665). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-03 18:25:00.433182] Computing e^(tensor([7331.6250], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-03 18:25:00.438887] Ending with PPL=39.08788232890252 (duration: 0:04:12.411730) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00 Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-04 16:04:15.734860 Setting up datamodule: ImageDataModule +2023-07-04 16:04:19.983170 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ImageNetZeroshotCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-04 16:06:32.790966 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-04 16:10:46.665237 Setting up datamodule: ImageDataModule +2023-07-04 16:10:51.864602 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-04 16:13:33.329239 Building train dataloader for datamodule: VLDataModule +2023-07-04 16:13:33.329977 Building train dataloader for datamodule: ImageDataModule +2023-07-04 16:13:33.330279 Building train dataloader for datamodule: MLMDataModule +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +VLDataModule: batch_size is 32, num_workers is 4 +ImageDataModule: batch_size is 32, num_workers is 4 +MLMDataModule: batch_size is 32, num_workers is 4 +Dataset sizes (un-normalized): [1, 10, 1] +Dataset sizes (normalized): [0.08333333333333333, 0.8333333333333334, 0.08333333333333333] +Sampling weights after temperature (0.0): [1.0, 1.0, 1.0] +2023-07-04 15:52:04.868462 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-04 15:56:58.535576 Setting up datamodule: ImageDataModule +2023-07-04 15:57:02.693411 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ImageNetZeroshotCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-04 16:01:26.749513 Preparing data for datamodule: VLDataModule +2023-07-04 16:01:26.749629 Preparing data for datamodule: ImageDataModule +2023-07-04 16:01:26.749696 Preparing data for datamodule: MLMDataModule +2023-07-04 16:06:32.791001 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-04 16:10:47.883724 Setting up datamodule: ImageDataModule +2023-07-04 16:10:51.779887 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-04 16:13:33.329463 Building train dataloader for datamodule: VLDataModule +2023-07-04 16:13:33.330153 Building train dataloader for datamodule: ImageDataModule +2023-07-04 16:13:33.330465 Building train dataloader for datamodule: MLMDataModule +2023-07-04 16:13:35.501370 Building val dataloader for datamodule: VLDataModule +2023-07-04 16:13:35.502434 Building val dataloader for datamodule: ImageDataModule +2023-07-04 16:13:35.503436 Building val dataloader for datamodule: MLMDataModule +2023-07-04 16:13:35.521885 Building val dataloader for datamodule: VLDataModule +2023-07-04 16:13:35.523373 Building val dataloader for datamodule: ImageDataModule +2023-07-04 16:13:35.524010 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-04 17:25:29.137573] Computing e^(tensor([17231.7089], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-04 17:25:29.147423] Ending with PPL=5518.461787658722 (duration: 0:11:56.505119) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00 + if config.training.use_wandb: + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 157, in main + should_allow_val_change = 'WANDB_RESUME' in os.environ and os.environ['WANDB_RESUME'] == 'must' + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 531, in fit + call._call_and_handle_interrupt( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 41, in _call_and_handle_interrupt + return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 91, in launch + return function(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 570, in _fit_impl + self._run(model, ckpt_path=ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 975, in _run + results = self._run_stage() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1018, in _run_stage + self.fit_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 201, in run + self.advance() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 354, in advance + self.epoch_loop.run(self._data_fetcher) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 134, in run + self.on_advance_end() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 248, in on_advance_end + self.val_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/utilities.py", line 177, in _decorator + return loop_run(self, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 108, in run + batch, batch_idx, dataloader_idx = next(data_fetcher) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fetchers.py", line 129, in __next__ + self._fetch_next_batch(self.dataloader_iter) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fetchers.py", line 150, in _fetch_next_batch + batch = next(iterator) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/utilities/combined_loader.py", line 284, in __next__ + out = next(self._iterator) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/utilities/combined_loader.py", line 123, in __next__ + out = next(self.iterators[0]) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/multidata.py", line 107, in __next__ + self.change_dataloader() + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/multidata.py", line 138, in change_dataloader + self.current_iterator = self.iterators[self.current_index] +TypeError: list indices must be integers or slices, not str +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-04 17:25:29.089193] Computing e^(tensor([17148.0574], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-04 17:25:29.099738] Ending with PPL=5292.408538779405 (duration: 0:11:56.370306) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-04 18:49:15.620154] Computing e^(tensor([15143.8129], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-04 18:49:15.627275] Ending with PPL=1942.8406929232247 (duration: 0:24:10.950133) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-04 19:59:21.933238] Computing e^(tensor([14949.7091], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-04 19:59:21.940305] Ending with PPL=1763.1451774030418 (duration: 0:14:32.289074) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-04 21:08:10.011176] Computing e^(tensor([14714.8292], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-04 21:08:10.018175] Ending with PPL=1567.7779950462689 (duration: 0:11:51.511594) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-04 22:25:09.451590] Computing e^(tensor([14468.3574], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-04 22:25:09.458681] Ending with PPL=1386.0020901741907 (duration: 0:23:03.654239) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-04 23:36:25.190338] Computing e^(tensor([14238.3408], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-04 23:36:25.197503] Ending with PPL=1235.4251072405825 (duration: 0:13:59.594478) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-05 00:42:19.659425] Computing e^(tensor([13898.9669], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-05 00:42:19.666827] Ending with PPL=1042.6110173286504 (duration: 0:13:49.392613) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-05 02:02:18.711992] Computing e^(tensor([13350.0034], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-05 02:02:18.719268] Ending with PPL=792.3488104418758 (duration: 0:23:51.232723) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-05 03:18:59.529750] Computing e^(tensor([12534.8744], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-05 03:18:59.536951] Ending with PPL=527.1247312445813 (duration: 0:13:49.701640) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-05 04:44:22.264346] Computing e^(tensor([11141.8499], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-05 04:44:22.271430] Ending with PPL=262.67695345568103 (duration: 0:19:50.694951) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-05 06:12:55.583556] Computing e^(tensor([10133.7123], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-05 06:12:55.590703] Ending with PPL=158.67469241128543 (duration: 0:17:36.065667) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-05 07:32:47.269422] Computing e^(tensor([9577.0041], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-05 07:32:47.276573] Ending with PPL=120.12129616077705 (duration: 0:13:59.267506) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-05 09:01:38.818637] Computing e^(tensor([9135.4572], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-05 09:01:38.825643] Ending with PPL=96.32506614417765 (duration: 0:23:27.718652) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-05 10:30:27.088759] Computing e^(tensor([8801.7196], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-05 10:30:27.093812] Ending with PPL=81.52093169013156 (duration: 0:13:49.971997) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-05 12:00:07.785314] Computing e^(tensor([8496.5838], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-05 12:00:07.792490] Ending with PPL=69.98576868347484 (duration: 0:14:01.362378) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-05 13:35:02.976782] Computing e^(tensor([8219.6215], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-05 13:35:02.983909] Ending with PPL=60.935185176142475 (duration: 0:22:30.926588) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-05 15:05:22.282049] Computing e^(tensor([7968.1868], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-05 15:05:22.289121] Ending with PPL=53.73654937252868 (duration: 0:17:27.201101) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-05 16:39:31.959337] Computing e^(tensor([7695.0592], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-05 16:39:31.964487] Ending with PPL=46.877115059548096 (duration: 0:13:52.948615) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-05 18:06:43.640040] Computing e^(tensor([7429.5289], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-05 18:06:43.646999] Ending with PPL=41.04891598449391 (duration: 0:13:43.701233) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-05 19:23:58.388534] Computing e^(tensor([7256.2242], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-05 19:23:58.393787] Ending with PPL=37.64168665273407 (duration: 0:13:44.448929) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-05 20:50:54.812461] Computing e^(tensor([7108.4314], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-05 20:50:54.820002] Ending with PPL=34.96038972429178 (duration: 0:13:41.356512) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-05 22:12:07.409895] Computing e^(tensor([6936.4198], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-05 22:12:07.417218] Ending with PPL=32.079265969111276 (duration: 0:13:43.674540) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-05 23:32:27.798678] Computing e^(tensor([6851.4693], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-05 23:32:27.806013] Ending with PPL=30.745223500452557 (duration: 0:13:42.760524) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-06 00:56:17.316923] Computing e^(tensor([6752.1136], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-06 00:56:17.324041] Ending with PPL=29.255184390023597 (duration: 0:13:42.847063) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-06 02:20:26.960699] Computing e^(tensor([6645.5624], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-06 02:20:26.967710] Ending with PPL=27.73738601935213 (duration: 0:13:46.834719) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-06 03:43:20.124037] Computing e^(tensor([6580.8245], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-06 03:43:20.131339] Ending with PPL=26.853932479875052 (duration: 0:13:39.131955) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-06 05:05:58.580600] Computing e^(tensor([6511.5052], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-06 05:05:58.587767] Ending with PPL=25.93912861380067 (duration: 0:13:48.674562) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-06 06:24:28.134283] Computing e^(tensor([6452.5282], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-06 06:24:28.141447] Ending with PPL=25.185390667560988 (duration: 0:13:51.055152) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-06 07:51:15.932266] Computing e^(tensor([6409.1732], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-06 07:51:15.939370] Ending with PPL=24.645309799413436 (duration: 0:13:50.402784) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-06 09:17:10.786155] Computing e^(tensor([6355.6746], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-06 09:17:10.791129] Ending with PPL=23.994804335938458 (duration: 0:13:53.997571) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-06 10:44:13.092774] Computing e^(tensor([6296.4218], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-06 10:44:13.100106] Ending with PPL=23.294351744372854 (duration: 0:13:52.326986) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-06 12:04:52.863095] Computing e^(tensor([6293.1413], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-06 12:04:52.870267] Ending with PPL=23.25617356681289 (duration: 0:13:50.743175) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-06 13:39:46.780088] Computing e^(tensor([6275.8804], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-06 13:39:46.787285] Ending with PPL=23.05632700240239 (duration: 0:13:53.451756) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-06 15:15:19.270898] Computing e^(tensor([6210.3966], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-06 15:15:19.278160] Ending with PPL=22.313643678217804 (duration: 0:13:54.561168) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-06 16:49:22.915761] Computing e^(tensor([6212.3145], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-06 16:49:22.922873] Ending with PPL=22.335051572243415 (duration: 0:13:54.756310) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-06 18:26:21.178023] Computing e^(tensor([6185.2245], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-06 18:26:21.185082] Ending with PPL=22.034562106945067 (duration: 0:13:53.865067) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-06 19:56:59.010504] Computing e^(tensor([6092.5956], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-06 19:56:59.017742] Ending with PPL=21.037316178580163 (duration: 0:13:52.706714) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-06 21:28:30.048767] Computing e^(tensor([6089.0916], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-06 21:28:30.056057] Ending with PPL=21.000490802480545 (duration: 0:13:48.218051) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-06 23:03:06.460434] Computing e^(tensor([6086.5125], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-06 23:03:06.465553] Ending with PPL=20.973427371428844 (duration: 0:13:52.330266) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-07 00:40:06.116547] Computing e^(tensor([6075.3651], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-07 00:40:06.123700] Ending with PPL=20.85685257914651 (duration: 0:13:50.178555) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-07 02:22:37.196281] Computing e^(tensor([6078.0679], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-07 02:22:37.203548] Ending with PPL=20.885057224775903 (duration: 0:13:49.298753) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-07 03:55:18.007146] Computing e^(tensor([6093.7523], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-07 03:55:18.014315] Ending with PPL=21.049485770236835 (duration: 0:13:50.131630) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-07 05:38:23.847019] Computing e^(tensor([6066.4061], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-07 05:38:23.854120] Ending with PPL=20.76363324811287 (duration: 0:13:52.431989) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-07 07:19:07.506058] Computing e^(tensor([6100.5135], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-07 07:19:07.513084] Ending with PPL=21.120766605876373 (duration: 0:13:55.050165) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-07 09:03:05.945581] Computing e^(tensor([6075.5699], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-07 09:03:05.952628] Ending with PPL=20.858988344948397 (duration: 0:13:42.875469) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-07 10:42:59.344907] Computing e^(tensor([6099.5062], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-07 10:42:59.352040] Ending with PPL=21.11013134585627 (duration: 0:13:42.397737) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-07 12:22:54.193487] Computing e^(tensor([6093.8491], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-07 12:22:54.200702] Ending with PPL=21.050505339524047 (duration: 0:13:42.038606) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-07 14:00:54.353956] Computing e^(tensor([6143.0493], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-07 14:00:54.359027] Ending with PPL=21.574771212185347 (duration: 0:13:37.065508) +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-07 15:31:17.053404] Computing e^(tensor([6126.1606], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-07 15:31:17.058451] Ending with PPL=21.39335396267851 (duration: 0:13:47.420523) +slurmstepd: error: *** JOB 20277096 ON eu-a65-03 CANCELLED AT 2023-07-07T18:00:31 *** diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1001.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1001.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..eba0eb11b48ecb18af10f52007da4a4c952b052f --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1001.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc7b4354ccae4c36a353bf5ca0aca20105168f490e35881bc4be7b66e4216cfb +size 3848043878 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1251.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1251.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..49d7be79b61a554a124ff9cbe2f6cc55a03fab91 --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1251.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:222db2f23cbf05be5d16d4a4eb25b4fe9335e951324428f17b3555ab4830a479 +size 3848044197 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1501.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1501.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..742d5281fb2347e67f274bba83937d189daca864 --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1501.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bf3842066a474af8b7b11e838b67227776072b5cec5bb66b67f7ebfdaadb04f +size 3848044644 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1751.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1751.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..530188d546adf7f29172415689c8bdd019ed3a1b --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1751.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50ce61fd8e7b238415949bd213ee5320adc3a948f6a4ad5cefceebc1ff9ecf12 +size 3848045091 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2010.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2010.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..00c9c80fdfd0460cc9673f611dfa9ced87ee8c24 --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2010.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57fe4bada2c961572e3082ce41a74da39ee9f3a387c88180051304534ddbd55e +size 3848045474 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2321.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2321.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..1746d8448eca68e1ab0d20b4f404c7a54877af6b --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2321.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39f9d6150d327d682613419e63a3d37901880cc59cbea10932c01e4ee73f6ad9 +size 3848045857 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=251.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=251.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..65eba186c366d3c86e3243d05ddb9fe2b09dd47f --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=251.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:437710f7ad3b69ba3ce0c269ceafa525bf37b4f0922f4e3481f1dec2b4552ffb +size 3848042601 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2592.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2592.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..8a79aa5efbd3d742abac9ff616fb8150eb1d3176 --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2592.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:849fc2eec803329c664ded03ba3992b3be5ff742bd4790fbeb2888cf6a6ab3d0 +size 3848046368 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2876.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2876.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..11c9ee1f960b17ad1041c557922ef255a11d124d --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2876.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d2cffe67c9f4dfadf3597e81638be449c7dcea040f7a121376ec82b86e49490 +size 3848046687 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3150.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3150.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..1c321aa86f7658cfce8e38c53a2ac50aee675100 --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3150.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7fa94666b30ae8f8eef45b4e4e0dbf3bf853790295d965bf9682a412971b280 +size 3848047070 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3448.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3448.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..6f473b5f073195ca61921fc1816adf6bdbd84d2f --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3448.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26474fcb0d0a05bfbe7e404f26c4bf39e8088bb3edb0e8b75a7d159e28fbf8ed +size 3848047581 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3751.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3751.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..e25ef2446885d984d3af5f63dc08438ec1e42828 --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3751.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea1b31864e83f1f54a73bb4760fe54b189f68fdd46c3e3fdfaa12de8f4cec3c6 +size 3848047900 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4050.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4050.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..78b01c6ff07544f7d56d95e1696a51c985018421 --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4050.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f78eb2a1a2f7fbd3bd9e06653a686beb9c83c03d8529b8d427b96d6c4f028f66 +size 3848048347 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4305.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4305.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..7e9e16173763b7ba8c5f18b5159c59a12fdb2abf --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4305.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74037e80ca123db81f36764e63afd625c99fface571ac8924bea2c41a2ca1b8c +size 3848048794 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4624.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4624.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..16170fefe2999caa42531f5997136e2de94bcc81 --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4624.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d19838de2dc2f799f58ac47138f61ba715e9348d91adc85886d0288a7e754e0e +size 3848049177 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4876.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4876.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..a27340186874a3754a92172aa8c684a449df6b40 --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4876.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc55d81905d4a8f8281af45e801ba571c12cd9f19de1f3bf3fad5eddb397db9f +size 3848049560 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=501.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=501.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..20bd1503c0c7bee0b846b6d4c5e2da6cbdf54ea9 --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=501.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb8d3ed126f3870548531325bef621f3d0221317b3f74fb94cc78cadcea3019d +size 3848042984 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5180.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5180.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..4d58c014db4b8f1c9e69b161e45d672cdd63022b --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5180.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1425ab0db6e5ceec11aff7dc4194a1fa41ab63db58b87e510f95d38679b76a4 +size 3848050071 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5484.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5484.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..5f101c6d1b21052dc73c2a47391923328da74629 --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5484.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:518e5a06ceee0d5c1ca8f3dfc6d6f8c7549614ada270c2c25bbcb0783c873b30 +size 3848050390 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5751.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5751.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..ff18231bf37c2eb66ee5ff01d56159cf2c82ba66 --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5751.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:048cf9f89ef548da5f9f84eaa5dab14cf2658148a66356152b9d47f2e3455fa3 +size 3848050837 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=6059.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=6059.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..44a3708870724b27d8d11fb87b0a255b20c7d11d --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=6059.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93bc6ad19e6e5b31138c854578ddd5db20c77a9f07f9aa8e374fdbdf4ddbc1a1 +size 3848051284 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=6376.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=6376.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..eb6589116930baa4c8e4e4133f06534079948f9c --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=6376.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a41befd38c41bece82d0d8b1e00386ac5f6c416b29d4799b4133e7052b679640 +size 3848051667 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=6691.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=6691.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..99af66dd15c2d31f40d2afac21cd628fd4606725 --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=6691.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02113074c6b3cea391a5f78e4a558ce3a7ec3e20f2d677654fbff98f25b2df17 +size 3848052050 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=7001.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=7001.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..9f55c2bd27cd8047dbf4a3cd5f3f2b1e6bb0270f --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=7001.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80173f5840049cddb2b60a98e8a0673a08e64907509b75142d6a04f0fa366d9b +size 3848052497 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=7291.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=7291.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..4bedc5089cb7b1fb1a3e51e8b19d6731379ca665 --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=7291.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ef4bc4034384ab8f088d93a61fab2276beb7e3485fa7fd2e8dbeaec0419fc16 +size 3848052880 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=751.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=751.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..55c293bba49ecd2f6d345b1772910e22d7ca447e --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=751.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56c719ddc54c933a83e4226284a65ee3cf6d53e066b33a59f3989543095c19d9 +size 3848043367 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/last.ckpt b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/last.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..0f4fce0b6a75268d0cbf59aec4c7020f7dca51b6 --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/last.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a11f9c2e7562f0c047ea19adfa39a127ad8697abb9c6207fb45328bd95354ec +size 3848052880 diff --git a/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/singlenode_2023-07-06-11:35:33.log b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/singlenode_2023-07-06-11:35:33.log new file mode 100644 index 0000000000000000000000000000000000000000..3193661ec6a9a5260791f50008680ec0c12a4394 --- /dev/null +++ b/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/singlenode_2023-07-06-11:35:33.log @@ -0,0 +1,3104 @@ +wandb: Currently logged in as: tamariucai (rycolab). Use `wandb login --relogin` to force relogin +wandb: wandb version 0.15.5 is available! To upgrade, please run: +wandb: $ pip install wandb --upgrade +wandb: Tracking run with wandb version 0.15.4 +wandb: Run data is saved locally in ./wandb/run-20230706_121644-qa6ke613 +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run blooming-cloud-225 +wandb: ⭐️ View project at https://wandb.ai/rycolab/alkmi-wit +wandb: 🚀 View run at https://wandb.ai/rycolab/alkmi-wit/runs/qa6ke613 +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +[update_ckeckpoint_dir] Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/ +Text isn't the predominant modality (1 v.s. 100 vision), will over-sample text (uniform rates) for better BLiMP performance. +Assigning HuggingFace RAM +Found 999.4GBs of RAM available, assigning 5GBs to HuggingFace datasets (currently 5000000000 bytes). +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Found cached dataset imagenet-1k (/cluster/scratch/tamariucai/HuggingfaceDatasets/imagenet-1k/default/1.0.0/a1e9bfc56c3a7350165007d1176b15e9128fcaf9ab972147840529aed3ae52bc) +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/plugins/environments/slurm.py:165: PossibleUserWarning: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... + rank_zero_warn( +Using bfloat16 Automatic Mixed Precision (AMP) +GPU available: True (cuda), used: True +TPU available: False, using: 0 TPU cores +IPU available: False, using: 0 IPUs +HPU available: False, using: 0 HPUs +Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/2 +wandb: Currently logged in as: tamariucai (rycolab). Use `wandb login --relogin` to force relogin +wandb: wandb version 0.15.5 is available! To upgrade, please run: +wandb: $ pip install wandb --upgrade +wandb: Tracking run with wandb version 0.15.4 +wandb: Run data is saved locally in ./wandb/run-20230706_122852-78b3zwbd +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run devout-elevator-226 +wandb: ⭐️ View project at https://wandb.ai/rycolab/alkmi-wit +wandb: 🚀 View run at https://wandb.ai/rycolab/alkmi-wit/runs/78b3zwbd +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +[update_ckeckpoint_dir] Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text1-vision100/bs4096_seed-1_bf16-mixed/ +Text isn't the predominant modality (1 v.s. 100 vision), will over-sample text (uniform rates) for better BLiMP performance. +Assigning HuggingFace RAM +Found 999.4GBs of RAM available, assigning 5GBs to HuggingFace datasets (currently 5000000000 bytes). +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Found cached dataset imagenet-1k (/cluster/scratch/tamariucai/HuggingfaceDatasets/imagenet-1k/default/1.0.0/a1e9bfc56c3a7350165007d1176b15e9128fcaf9ab972147840529aed3ae52bc) +Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/2 +---------------------------------------------------------------------------------------------------- +distributed_backend=nccl +All distributed processes registered. Starting with 2 processes +---------------------------------------------------------------------------------------------------- + +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at theodor1289wit-test +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1] +LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1] + + | Name | Type | Params +---------------------------------------------- +0 | model | FlavaForPreTraining | 357 M +---------------------------------------------- +303 M Trainable params +53.8 M Non-trainable params +357 M Total params +1,430.581 Total estimated model params size (MB) +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +VLDataModule: batch_size is 32, num_workers is 4 +ImageDataModule: batch_size is 32, num_workers is 4 +MLMDataModule: batch_size is 32, num_workers is 4 +Dataset sizes (un-normalized): [1, 100, 1] +Dataset sizes (normalized): [0.00980392156862745, 0.9803921568627451, 0.00980392156862745] +Sampling weights after temperature (0.0): [1.0, 1.0, 1.0] +2023-07-06 12:17:26.987017 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-06 12:22:16.710993 Setting up datamodule: ImageDataModule +2023-07-06 12:23:12.221891 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Adding ImageNet zeroshot callback +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'ImageNetZeroshotCallback'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-06 12:28:41.227846 Preparing data for datamodule: VLDataModule +2023-07-06 12:28:41.228118 Preparing data for datamodule: ImageDataModule +2023-07-06 12:28:41.228256 Preparing data for datamodule: MLMDataModule +2023-07-06 12:37:19.937184 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-06 12:40:49.518828 Setting up datamodule: ImageDataModule +2023-07-06 12:42:18.328930 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-06 12:47:34.836539 Building train dataloader for datamodule: VLDataModule +2023-07-06 12:47:34.837102 Building train dataloader for datamodule: ImageDataModule +2023-07-06 12:47:34.837377 Building train dataloader for datamodule: MLMDataModule +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +VLDataModule: batch_size is 32, num_workers is 4 +ImageDataModule: batch_size is 32, num_workers is 4 +MLMDataModule: batch_size is 32, num_workers is 4 +Dataset sizes (un-normalized): [1, 100, 1] +Dataset sizes (normalized): [0.00980392156862745, 0.9803921568627451, 0.00980392156862745] +Sampling weights after temperature (0.0): [1.0, 1.0, 1.0] +2023-07-06 12:29:18.047274 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-06 12:32:40.557255 Setting up datamodule: ImageDataModule +2023-07-06 12:33:34.552625 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Adding ImageNet zeroshot callback +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'ImageNetZeroshotCallback'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-06 12:37:19.937294 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-06 12:40:49.363733 Setting up datamodule: ImageDataModule +2023-07-06 12:42:18.381138 Setting up datamodule: MLMDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-06 12:47:34.836386 Building train dataloader for datamodule: VLDataModule +2023-07-06 12:47:34.837180 Building train dataloader for datamodule: ImageDataModule +2023-07-06 12:47:34.837494 Building train dataloader for datamodule: MLMDataModule +2023-07-06 12:47:36.483150 Building val dataloader for datamodule: VLDataModule +2023-07-06 12:47:36.484081 Building val dataloader for datamodule: ImageDataModule +2023-07-06 12:47:36.484508 Building val dataloader for datamodule: MLMDataModule +2023-07-06 12:47:36.167160 Building val dataloader for datamodule: VLDataModule +2023-07-06 12:47:36.167952 Building val dataloader for datamodule: ImageDataModule +2023-07-06 12:47:36.168301 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-06 14:14:07.157379] Computing e^(tensor([17263.5446], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-06 14:14:07.193462] Ending with PPL=5607.006687336156 (duration: 0:08:46.712401) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.791 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.323 (now 0.33011138439178467). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 5.045 (now 5.409753322601318). Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.796 +Metric validation/losses/mmm_text_loss improved by 0.031 >= min_delta = 0.0. New best score: 3.866 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 04:17:38.398439] Computing e^(tensor([6928.0917], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 04:17:38.407004] Ending with PPL=31.94596331658355 (duration: 0:04:47.054112) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.779 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.323 (now 0.3248492479324341). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 5.045 (now 5.2989726066589355). Wait count has been increased from 1. +Metric validation/losses/mmm_image_loss improved by 0.012 >= min_delta = 0.0. New best score: 0.784 +Metric validation/losses/mmm_text_loss improved by 0.054 >= min_delta = 0.0. New best score: 3.813 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 08:24:57.494853] Computing e^(tensor([6774.9359], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 08:24:57.501613] Ending with PPL=29.590931692668256 (duration: 0:04:13.399460) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.788 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.777 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.323 (now 0.3265673518180847). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 5.045 (now 5.2646260261535645). Wait count has been increased from 2. +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.779 +Metric validation/losses/mmm_text_loss improved by 0.045 >= min_delta = 0.0. New best score: 3.767 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 13:00:43.309153] Computing e^(tensor([6750.5271], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 13:00:43.316530] Ending with PPL=29.231987242140136 (duration: 0:04:15.468930) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.766 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.772 +Metric validation/losses/itm_loss improved by 0.016 >= min_delta = 0.0. New best score: 0.307 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 5.045 (now 5.281798362731934). Wait count has been increased from 3. +Metric validation/losses/mmm_image_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.775 +Metric validation/losses/mmm_text_loss improved by 0.012 >= min_delta = 0.0. New best score: 3.755 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 17:12:49.468659] Computing e^(tensor([6755.1841], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 17:12:49.475521] Ending with PPL=29.300132980003863 (duration: 0:05:02.710822) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.764 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.772 (now 0.7731457352638245). Wait count has been increased from 0. +Metric validation/losses/itm_loss improved by 0.057 >= min_delta = 0.0. New best score: 0.249 +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 5.045 (now 5.166111469268799). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.775 (now 0.7755222320556641). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.755 (now 3.7708652019500732). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 21:45:05.684983] Computing e^(tensor([6667.8804], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 21:45:05.698598] Ending with PPL=28.04864155089365 (duration: 0:04:45.401980) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.724 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.770 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.249 (now 0.2565031349658966). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 5.045 (now 5.166111469268799). Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.771 +Metric validation/losses/mmm_text_loss improved by 0.038 >= min_delta = 0.0. New best score: 3.717 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 02:14:49.874978] Computing e^(tensor([6654.7880], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 02:14:49.883332] Ending with PPL=27.865629290115795 (duration: 0:04:12.641270) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.716 +Metric validation/losses/mim_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.767 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.249 (now 0.2887364625930786). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 5.045 (now 5.166111469268799). Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.770 +Metric validation/losses/mmm_text_loss improved by 0.021 >= min_delta = 0.0. New best score: 3.696 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 06:55:38.399668] Computing e^(tensor([6661.8680], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 06:55:38.408205] Ending with PPL=27.96444869080001 (duration: 0:04:10.348916) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.769 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.696 (now 3.7202444076538086). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 11:47:43.266259] Computing e^(tensor([6702.3472], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 11:47:43.278144] Ending with PPL=28.53620375793264 (duration: 0:04:12.061069) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.684 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 16:06:26.286841] Computing e^(tensor([6674.1666], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 16:06:26.296284] Ending with PPL=28.13693944061056 (duration: 0:04:09.447238) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.765 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.249 (now 0.3082902729511261). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 5.045 (now 5.0524773597717285). Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.766 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.684 (now 3.6983485221862793). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 20:06:18.125074] Computing e^(tensor([6705.9716], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 20:06:18.133489] Ending with PPL=28.587964457745436 (duration: 0:04:50.421662) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.762 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.249 (now 0.3082902729511261). Signaling model to ignore task. +Metric validation/losses/global_contrastive_loss improved by 1.052 >= min_delta = 0.0. New best score: 3.993 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.765 +Metric validation/losses/mmm_text_loss improved by 0.047 >= min_delta = 0.0. New best score: 3.637 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 00:03:32.312486] Computing e^(tensor([6741.8047], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 00:03:32.319864] Ending with PPL=29.104777431328746 (duration: 0:04:08.496635) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.866 +Metric validation/losses/mmm_image_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.764 +Metric validation/losses/mmm_text_loss improved by 0.001 >= min_delta = 0.0. New best score: 3.636 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 03:52:28.132136] Computing e^(tensor([6748.6184], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 03:52:28.142048] Ending with PPL=29.204102307041726 (duration: 0:04:13.418665) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.630 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 07:24:41.706957] Computing e^(tensor([6780.7353], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 07:24:41.715196] Ending with PPL=29.676861690130103 (duration: 0:04:10.629025) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00 + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 212, in main + wandb.finish() # [optional] finish the wandb run, necessary in notebooks + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 628, in validate + return call._call_and_handle_interrupt( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 42, in _call_and_handle_interrupt + return trainer_fn(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 671, in _validate_impl + results = self._run(model, ckpt_path=ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 973, in _run + results = self._run_stage() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1009, in _run_stage + return self._evaluation_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/utilities.py", line 177, in _decorator + return loop_run(self, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 98, in run + self.setup_data() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 168, in setup_data + _check_dataloader_iterable(dl, source, trainer_fn) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/connectors/data_connector.py", line 385, in _check_dataloader_iterable + iter(dataloader) # type: ignore[call-overload] + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/multidata.py", line 73, in __iter__ + self.change_dataloader() + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/multidata.py", line 126, in change_dataloader + choice = [self.sampling_func()] + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/multidata.py", line 173, in + self.sampling_func = lambda: random.choices(population=range(len(sampling_weights)), + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/random.py", line 535, in choices + raise ValueError('Total of weights must be greater than zero') +ValueError: Total of weights must be greater than zero +Traceback (most recent call last): + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 220, in + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 212, in main + wandb.finish() # [optional] finish the wandb run, necessary in notebooks + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 628, in validate + return call._call_and_handle_interrupt( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 42, in _call_and_handle_interrupt + return trainer_fn(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 671, in _validate_impl + results = self._run(model, ckpt_path=ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 973, in _run + results = self._run_stage() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1009, in _run_stage + return self._evaluation_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/utilities.py", line 177, in _decorator + return loop_run(self, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 98, in run + self.setup_data() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 168, in setup_data + _check_dataloader_iterable(dl, source, trainer_fn) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/connectors/data_connector.py", line 385, in _check_dataloader_iterable + iter(dataloader) # type: ignore[call-overload] + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/multidata.py", line 73, in __iter__ + self.change_dataloader() + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/multidata.py", line 126, in change_dataloader + choice = [self.sampling_func()] + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/multidata.py", line 173, in + self.sampling_func = lambda: random.choices(population=range(len(sampling_weights)), + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/random.py", line 535, in choices + raise ValueError('Total of weights must be greater than zero') +ValueError: Total of weights must be greater than zero +wandb: Waiting for W&B process to finish... (failed 1). Press Control-C to abort syncing. +wandb: +wandb: Run history: +wandb: ImageDataModule ▁▁▁▁▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: MLMDataModule ▁▁▁▁▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: VLDataModule ▁▁▁▁▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: epoch ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: evaluation/blimp/anaphor_agreement ▅▅▄▄▄▁▅▄▆▄▅▇▅▆▇▇██▆▇ +wandb: evaluation/blimp/argument_structure ▅▃▂▆▆█▇█▅▄▃▃▃▄▂▄▃▅▁▁ +wandb: evaluation/blimp/binding ▆▁▄▄▆▃▃██▆██▄▂▄▇▇▅▆▄ +wandb: evaluation/blimp/control_raising ▃▄▇▆▆▇█▆▆█▇▆▆▆▅▅▂▃▃▁ +wandb: evaluation/blimp/determiner_noun_agreement ▄▅▇▄▅█▆▇▆▆▆▃▆█▄▂▃▃▃▁ +wandb: evaluation/blimp/ellipsis ▃▆▇▄▁▄▃▂▁▂▅▂▁▄█▄▄▂▄▃ +wandb: evaluation/blimp/filler_gap █▆▅▅▆▄▇▅▄▂▃▅▃▂▅▆▄▁▁▃ +wandb: evaluation/blimp/irregular_forms ▆█▆▆▇▇▆███▆▅▅▄▄▄▅▁▄▅ +wandb: evaluation/blimp/island_effects ▃▁▃▄▃▅▅▇▃▄▃▅▇█▄▄▃█▆▄ +wandb: evaluation/blimp/npi_licensing ▁█▅▆▅▇█▆▇▃▆▆▅▃▆█▅▂▆█ +wandb: evaluation/blimp/quantifiers ▂▄█▄▇▃▁▂▁▅▄▅▆▄▁█▁▃▃▅ +wandb: evaluation/blimp/subject_verb_agreement ▂▂▇▅▃▂▃▆█▂▂▃▆▃▁▅▄▂▄▃ +wandb: evaluation/blimp_average ▁▅▆▄▅▃▆▇▇▄▅▆▅▅▅█▄▃▄▅ +wandb: evaluation/imagenet_zeroshot/top1 ▁▃▅██▆▆▇▅▇▅▃▂▄▅▇▅▂▃▄ +wandb: evaluation/imagenet_zeroshot/top5 ▁▃▄▆▆▄█▇▅▅▅▄▃▅▄▅▄▄▄▅ +wandb: evaluation/pseudo_perplexity ▇██▄▃▃▁▁▁▂▁▂▃▃▄▄▄▄▄▅ +wandb: lr-AdamW/pg1 ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: lr-AdamW/pg2 ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: train/losses/global_contrastive_loss ▁▁▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▂▇▅█▇▆▇▂▂▁▁▁▁▁▁▁▁ +wandb: train/losses/itm_loss █▇▅▅▅█▅▅▆▅▃▃▅▁▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▃▄▂▃▁▂▂▃ +wandb: train/losses/mim_loss ▇▅▄▄▂▅▆▅▅▃▂██▇▅▇▅▅▄▃▃▃▅▇▅▂▁▃▅▇▆▄▅▅▅▃▄ +wandb: train/losses/mlm_loss ▅▅▁▃▂▇▄██▅▄▄▆▅▅▆▅▅▆▅▆▃▆▄▆▆▅▃▂▄▂▅█▃█▃▆ +wandb: train/losses/mmm_image_loss ▇▅▆▇▆▄▄▄▆▁▅▁█▂▅▅▃▄▂▄▅▄▅▅▄▄▆▄▃▁▅▃█▄▄▄▄ +wandb: train/losses/mmm_text_loss █▅▇▄▆▆▅▅▅▅▅▅▃▃▂▄▂▃▃▂▂▂▂▂▄▂▂▃▂▂▂▂▂▂▃▁▁ +wandb: trainer/global_step ▁▁▁▁▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: validation/losses/global_contrastive_loss ▆█▇▇▇▇▆▂▁▁▁▃▃▃ +wandb: validation/losses/itm_loss ▄▅▅▅▄▁▁▃▃▄▄▅▅▇▇█ +wandb: validation/losses/mim_loss █▇▄▄▃▃▃▂▂▂▂▁▁▂▁▁▂▂▂▁ +wandb: validation/losses/mlm_loss ▇█▇▄▃▃▁▁▁▁▁▂▂▂▂▂▂▂▂▃ +wandb: validation/losses/mmm_image_loss █▆▄▄▃▃▂▂▂▂▁▁▁▂▁▁▂▁ +wandb: validation/losses/mmm_text_loss █▇▆▅▄▅▃▃▃▂▃▁▁▁▁▃▂▂▂ +wandb: validation/monitor/global_contrastive █▅▃▂▁▁▁▁▁▂▁▁▁▁▁▁▁▁▁▁ +wandb: validation/monitor/itm █▅▃▂▂▂▂▁▁▁▁▁▁▁▂▁▁▁▁▁ +wandb: validation/monitor/mim █████▅▅▅▃▂▂▂▂▁▁▁▁▁▁▁ +wandb: validation/monitor/mlm █▅▃▃▃▃▄▄▂▂▁▁▁▁▁▁▂▁▁▁ +wandb: validation/monitor/mmm_image █████▅▅▅▆▃▄▄▄▃▂▁▁▁▁▁ +wandb: validation/monitor/mmm_text █████▅▅▅▃▃▂▂▂▃▂▁▁▁▁▁ +wandb: +wandb: Run summary: +wandb: ImageDataModule 51817 +wandb: MLMDataModule 52951 +wandb: VLDataModule 52210 +wandb: epoch 0 +wandb: evaluation/blimp/anaphor_agreement 77.97 +wandb: evaluation/blimp/argument_structure 60.27 +wandb: evaluation/blimp/binding 59.13 +wandb: evaluation/blimp/control_raising 61.82 +wandb: evaluation/blimp/determiner_noun_agreement 78.27 +wandb: evaluation/blimp/ellipsis 59.64 +wandb: evaluation/blimp/filler_gap 60.77 +wandb: evaluation/blimp/irregular_forms 73.33 +wandb: evaluation/blimp/island_effects 43.83 +wandb: evaluation/blimp/npi_licensing 58.91 +wandb: evaluation/blimp/quantifiers 62.8 +wandb: evaluation/blimp/subject_verb_agreement 58.39 +wandb: evaluation/blimp_average 62.9275 +wandb: evaluation/imagenet_zeroshot/top1 0.00682 +wandb: evaluation/imagenet_zeroshot/top5 0.0271 +wandb: evaluation/pseudo_perplexity 30.43973 +wandb: lr-AdamW/pg1 0.00095 +wandb: lr-AdamW/pg2 0.00071 +wandb: train/losses/global_contrastive_loss 0.04748 +wandb: train/losses/itm_loss 0.1477 +wandb: train/losses/mim_loss 0.76019 +wandb: train/losses/mlm_loss 2.69335 +wandb: train/losses/mmm_image_loss 0.72936 +wandb: train/losses/mmm_text_loss 1.66674 +wandb: trainer/global_step 9519 +wandb: validation/losses/global_contrastive_loss 4.21051 +wandb: validation/losses/itm_loss 0.39954 +wandb: validation/losses/mim_loss 0.76377 +wandb: validation/losses/mlm_loss 3.77141 +wandb: validation/losses/mmm_image_loss 0.76512 +wandb: validation/losses/mmm_text_loss 3.65777 +wandb: validation/monitor/global_contrastive 0.0 +wandb: validation/monitor/itm 0.0 +wandb: validation/monitor/mim 0.0 +wandb: validation/monitor/mlm 0.0 +wandb: validation/monitor/mmm_image 0.0 +wandb: validation/monitor/mmm_text 0.0 +wandb: +wandb: 🚀 View run sweepy-sweep-5 at: https://wandb.ai/rycolab/alkmi-wit/runs/qizp88p0 +wandb: Synced 3 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20230821_165746-qizp88p0/logs diff --git a/flava-wit/text1-vision100/half_bs4025_seed5501650_bf16-mixed/resume_singlenode_2023-08-25-08:44:26.log b/flava-wit/text1-vision100/half_bs4025_seed5501650_bf16-mixed/resume_singlenode_2023-08-25-08:44:26.log new file mode 100644 index 0000000000000000000000000000000000000000..ae4c0fb088423ec608674db944ddba727957431c --- /dev/null +++ b/flava-wit/text1-vision100/half_bs4025_seed5501650_bf16-mixed/resume_singlenode_2023-08-25-08:44:26.log @@ -0,0 +1,4381 @@ +wandb: Currently logged in as: tamariucai (rycolab). Use `wandb login --relogin` to force relogin +wandb: - Waiting for wandb.init()... wandb: \ Waiting for wandb.init()... wandb: wandb version 0.15.8 is available! To upgrade, please run: +wandb: $ pip install wandb --upgrade +wandb: Tracking run with wandb version 0.15.4 +wandb: Run data is saved locally in ./wandb/run-20230825_115503-qizp88p0 +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run sweepy-sweep-5 +wandb: ⭐️ View project at https://wandb.ai/rycolab/alkmi-wit +wandb: 🧹 View sweep at https://wandb.ai/rycolab/alkmi-wit/sweeps/ff69ui4h +wandb: 🚀 View run at https://wandb.ai/rycolab/alkmi-wit/runs/qizp88p0 +[rank: 0] Global seed set to 5501650 +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +The current host is eu-a65-04 +The current cuDNN version is 8801 +Parameter 'text_perc' already had the value 1. +Parameter 'vision_perc' already had the value 100. +Parameter 'seed' already had the value 5501650. +Total GPU memory: 42.296 GB +Detected half-sized run, (almost) doubling batch size to 48. +Detected half-sized run, (almost) halving gradient accumulation to 85 +[update_ckeckpoint_dir] Detected hyperparameter run! +[update_ckeckpoint_dir] Detected half-size run! +[update_ckeckpoint_dir] Found and updated model name flava in the checkpoint dirpath. +[update_ckeckpoint_dir] Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-25)_time(11:55:07)/seed(5501650)-magic({'enable': True})-text_perc(1)-vision_perc(100/ +Text isn't the predominant modality (1 v.s. 100 vision), will over-sample text (uniform rates) for better BLiMP performance. +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Found cached dataset imagenet-1k (/cluster/scratch/tamariucai/HuggingfaceDatasets/imagenet-1k/default/1.0.0/a1e9bfc56c3a7350165007d1176b15e9128fcaf9ab972147840529aed3ae52bc) +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/plugins/environments/slurm.py:165: PossibleUserWarning: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... + rank_zero_warn( +Using bfloat16 Automatic Mixed Precision (AMP) +GPU available: True (cuda), used: True +TPU available: False, using: 0 TPU cores +IPU available: False, using: 0 IPUs +HPU available: False, using: 0 HPUs +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Restoring states from the checkpoint path at /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text1-vision100/half_bs4025_seed5501650_bf16-mixed/flava-epoch=00-step=9460.ckpt +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py:247: UserWarning: Be aware that when using `ckpt_path`, callbacks used to create the checkpoint need to be provided during `Trainer` instantiation. Please add the following callbacks: ["MultimodalOverfittingMonitor{'monitor': 'validation/losses/mlm_loss', 'mode': 'min'}", "MultimodalOverfittingMonitor{'monitor': 'validation/losses/mim_loss', 'mode': 'min'}", "MultimodalOverfittingMonitor{'monitor': 'validation/losses/itm_loss', 'mode': 'min'}", "MultimodalOverfittingMonitor{'monitor': 'validation/losses/mmm_image_loss', 'mode': 'min'}", "MultimodalOverfittingMonitor{'monitor': 'validation/losses/mmm_text_loss', 'mode': 'min'}", "MultimodalOverfittingMonitor{'monitor': 'validation/losses/global_contrastive_loss', 'mode': 'min'}"]. + rank_zero_warn( +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:337: UserWarning: The dirpath has changed from "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-21)_time(16:57:53)/seed(5501650)-magic({'enable': True})-text_perc(1)-vision_perc(100" to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-25)_time(11:55:07)/seed(5501650)-magic({'enable': True})-text_perc(1)-vision_perc(100", therefore `best_model_score`, `kth_best_model_path`, `kth_value`, `last_model_path` and `best_k_models` won't be reloaded. Only `best_model_path` will be reloaded. + warnings.warn( +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0] + + | Name | Type | Params +---------------------------------------------- +0 | model | FlavaForPreTraining | 189 M +---------------------------------------------- +136 M Trainable params +53.8 M Non-trainable params +189 M Total params +759.666 Total estimated model params size (MB) +Restored all states from the checkpoint at /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text1-vision100/half_bs4025_seed5501650_bf16-mixed/flava-epoch=00-step=9460.ckpt +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing multi-datamodule +VLDataModule: batch_size is 48, num_workers is 4 +ImageDataModule: batch_size is 48, num_workers is 4 +MLMDataModule: batch_size is 48, num_workers is 4 +Dataset sizes (un-normalized): [1, 100, 1] +Dataset sizes (normalized): [0.00980392156862745, 0.9803921568627451, 0.00980392156862745] +Sampling weights after temperature (0.0): [1.0, 1.0, 1.0] +2023-08-25 11:56:19.507129 Setting up datamodule: VLDataModule +2023-08-25 11:56:33.552056 Setting up datamodule: ImageDataModule +2023-08-25 11:58:12.209341 Setting up datamodule: MLMDataModule +Registering multimodal overfitting monitors +Adding ImageNet zeroshot callback +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalScheduler', 'MultimodalScheduler', 'MultimodalScheduler', 'MultimodalScheduler', 'MultimodalScheduler', 'MultimodalScheduler', 'ImageNetZeroshotCallback'] +Trainer successfully initialized (strategy=SingleDeviceStrategy) +Starting training +2023-08-25 11:58:24.662114 Preparing data for datamodule: VLDataModule +2023-08-25 11:58:24.662287 Preparing data for datamodule: ImageDataModule +2023-08-25 11:58:24.662406 Preparing data for datamodule: MLMDataModule +2023-08-25 11:58:24.662721 Setting up datamodule: VLDataModule +2023-08-25 11:59:56.946269 Setting up datamodule: ImageDataModule +2023-08-25 12:01:37.355622 Setting up datamodule: MLMDataModule +2023-08-25 12:02:41.223623 Building train dataloader for datamodule: VLDataModule +2023-08-25 12:02:41.224442 Building train dataloader for datamodule: ImageDataModule +2023-08-25 12:02:41.224738 Building train dataloader for datamodule: MLMDataModule +2023-08-25 12:02:43.009014 Building val dataloader for datamodule: VLDataModule +2023-08-25 12:02:43.010489 Building val dataloader for datamodule: ImageDataModule +2023-08-25 12:02:43.011223 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 13:43:47.535508] Computing e^(tensor([6919.9218], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 13:43:47.546005] Ending with PPL=31.81573257287823 (duration: 0:04:42.120929) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.822 +Metric validation/losses/mmm_image_loss improved by 0.061 >= min_delta = 0.0. New best score: 0.830 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.844 (now 3.9248621463775635). Wait count has been increased from 0. +Metric validation/losses/itm_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.316 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 4.338 (now 4.518860816955566). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 19:48:37.820116] Computing e^(tensor([7013.9440], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 19:48:37.825836] Ending with PPL=33.34713943027793 (duration: 0:05:41.958333) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.795 +Metric validation/losses/mmm_image_loss improved by 0.031 >= min_delta = 0.0. New best score: 0.799 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.844 (now 3.9064278602600098). Wait count has been increased from 1. +Metric validation/losses/itm_loss improved by 0.034 >= min_delta = 0.0. New best score: 0.282 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 4.338 (now 4.84374475479126). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 22:45:30.661108] Computing e^(tensor([6915.1080], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 22:45:30.666728] Ending with PPL=31.739248037240937 (duration: 0:05:49.224823) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.795 +Metric validation/losses/mim_loss improved by 0.010 >= min_delta = 0.0. New best score: 0.785 +Metric validation/losses/mmm_image_loss improved by 0.012 >= min_delta = 0.0. New best score: 0.787 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.844 (now 3.877937078475952). Wait count has been increased from 2. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.282 (now 0.32022708654403687). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 4.338 (now 4.983642101287842). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 01:56:54.819202] Computing e^(tensor([6888.5013], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 01:56:54.824979] Ending with PPL=31.319804415260606 (duration: 0:04:45.515852) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.782 +Metric validation/losses/mim_loss improved by 0.011 >= min_delta = 0.0. New best score: 0.775 +Metric validation/losses/mmm_image_loss improved by 0.011 >= min_delta = 0.0. New best score: 0.777 +Metric validation/losses/mmm_text_loss improved by 0.008 >= min_delta = 0.0. New best score: 3.836 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.282 (now 0.4813196659088135). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 4.338 (now 5.154770851135254). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 05:04:55.994448] Computing e^(tensor([6852.7902], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 05:04:56.000023] Ending with PPL=30.765535764630563 (duration: 0:04:14.921352) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.775 +Metric validation/losses/mim_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.769 +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.772 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.836 (now 3.869659662246704). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.282 (now 0.47947847843170166). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 4.338 (now 5.154770851135254). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 07:43:05.424213] Computing e^(tensor([6834.2395], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 07:43:05.428259] Ending with PPL=30.481493429898954 (duration: 0:06:03.908092) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.768 +Metric validation/losses/mim_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.765 +Metric validation/losses/mmm_image_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.769 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.836 (now 3.843993902206421). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.282 (now 0.5356058478355408). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 4.338 (now 5.154770851135254). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 10:45:32.555661] Computing e^(tensor([6848.1426], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 10:45:32.561153] Ending with PPL=30.694125825282768 (duration: 0:06:19.722755) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.763 +Metric validation/losses/mim_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.763 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.767 +Metric validation/losses/mmm_text_loss improved by 0.094 >= min_delta = 0.0. New best score: 3.742 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.282 (now 0.5356058478355408). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 14:12:58.421093] Computing e^(tensor([6893.7194], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 14:12:58.426654] Ending with PPL=31.401626632760777 (duration: 0:04:14.898926) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.896 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 16:46:48.408935] Computing e^(tensor([6941.3919], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 16:46:48.414454] Ending with PPL=32.159115344143935 (duration: 0:04:45.748782) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.762 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.764 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.742 (now 3.8338396549224854). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.282 (now 0.5742650032043457). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 08:20:49.709661] Computing e^(tensor([7045.1334], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 08:20:49.715376] Ending with PPL=33.8712543748533 (duration: 0:04:48.933008) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.760 +Monitored metric validation/losses/mmm_image_loss did not improve in the last 4 records. Best score: 0.764 (now 0.764347493648529). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 3.742 (now 3.839224338531494). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.282 (now 0.5431433916091919). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.896 (now 4.49676513671875). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 17:19:34.658457] Computing e^(tensor([6997.4118], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 17:19:34.664056] Ending with PPL=33.07262541951302 (duration: 0:06:21.040586) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.759 +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 0.764 (now 0.764347493648529). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 3.742 (now 3.839224338531494). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.282 (now 0.5999723672866821). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.896 (now 4.49676513671875). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 19:59:56.269701] Computing e^(tensor([6979.7176], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 19:59:56.275314] Ending with PPL=32.78131922162833 (duration: 0:06:22.756729) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.759 +Monitored metric validation/losses/mmm_image_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.742 (now 3.8331172466278076). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.282 (now 0.5999723672866821). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 00:30:55.706147] Computing e^(tensor([7032.7198], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 00:30:55.711662] Ending with PPL=33.66167404475378 (duration: 0:06:33.164480) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.764 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 3.742 (now 3.8711979389190674). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.282 (now 0.5463849902153015). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 3.896 (now 4.410231590270996). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 07:20:12.016705] Computing e^(tensor([6958.1806], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 07:20:12.022377] Ending with PPL=32.43020751413087 (duration: 0:04:46.851332) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.763 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 3.742 (now 3.8711979389190674). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.282 (now 0.7130016088485718). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.896 (now 4.526498317718506). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 09:46:09.666378] Computing e^(tensor([6940.4970], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 09:46:09.670775] Ending with PPL=32.14472875082368 (duration: 0:04:48.139292) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.758 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.761 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 3.742 (now 3.8711979389190674). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.282 (now 0.6384274959564209). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.896 (now 4.526498317718506). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 12:13:12.415572] Computing e^(tensor([6964.4959], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 12:13:12.421102] Ending with PPL=32.53277169746551 (duration: 0:04:21.899707) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.756 +Metric validation/losses/mmm_image_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.760 +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.282 (now 0.6384274959564209). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 3.896 (now 4.526498317718506). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 13:55:10.503967] Computing e^(tensor([7025.1590], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 13:55:10.509494] Ending with PPL=33.5346581472065 (duration: 0:04:26.784248) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.754 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.758 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.742 (now 3.8792178630828857). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.282 (now 0.6384274959564209). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 15:51:52.685989] Computing e^(tensor([7011.6221], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 15:51:52.691624] Ending with PPL=33.30844752715328 (duration: 0:06:02.562599) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.878 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 08:13:21.118223] Computing e^(tensor([7128.4387], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 08:13:21.123850] Ending with PPL=35.311876967104716 (duration: 0:04:59.067271) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.756 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.742 (now 3.9063315391540527). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 22:24:00.387725] Computing e^(tensor([7134.7755], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 22:24:00.393326] Ending with PPL=35.42393640901365 (duration: 0:06:08.020378) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.755 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 3.742 (now 4.005520820617676). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.282 (now 0.7710217833518982). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.878 (now 4.7090277671813965). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 03:14:27.427387] Computing e^(tensor([7190.6503], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 03:14:27.433072] Ending with PPL=36.427541751759954 (duration: 0:05:30.948961) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.752 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.755 (now 0.7563478350639343). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.282 (now 0.7710217833518982). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.878 (now 4.7090277671813965). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 04:46:13.751467] Computing e^(tensor([7180.2289], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 04:46:13.757061] Ending with PPL=36.23822378165339 (duration: 0:08:37.780333) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.751 +Metric validation/losses/mmm_image_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.755 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.742 (now 3.9503536224365234). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.282 (now 0.7710217833518982). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 3.878 (now 4.7090277671813965). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 06:22:09.504538] Computing e^(tensor([7184.1467], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 06:22:09.510060] Ending with PPL=36.309279127737824 (duration: 0:06:33.755027) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.750 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.755 (now 0.7553104758262634). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.742 (now 3.933837652206421). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 07:39:20.590999] Computing e^(tensor([7176.1251], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 07:39:20.596548] Ending with PPL=36.16394210026923 (duration: 0:05:58.884273) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.774 +Metric validation/losses/mim_loss improved by 0.019 >= min_delta = 0.0. New best score: 0.764 +Metric validation/losses/mmm_image_loss improved by 0.019 >= min_delta = 0.0. New best score: 0.768 +Metric validation/losses/mmm_text_loss improved by 0.066 >= min_delta = 0.0. New best score: 3.999 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.459 (now 0.7093874216079712). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 6.015 (now 6.670577526092529). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 18:28:30.759844] Computing e^(tensor([7144.0614], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 18:28:30.763417] Ending with PPL=35.58879080760348 (duration: 0:03:23.347133) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.757 +Metric validation/losses/mmm_image_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.760 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.999 (now 4.04496955871582). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.459 (now 0.7977747321128845). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 6.015 (now 6.773106575012207). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-01 02:52:56.745545] Computing e^(tensor([7118.7922], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-01 02:52:56.749482] Ending with PPL=35.14196905351216 (duration: 0:03:21.676785) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.753 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.757 +Metric validation/losses/mmm_image_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.760 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.999 (now 4.018205165863037). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.459 (now 0.7575238347053528). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 6.015 (now 6.683054447174072). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-01 09:17:55.544858] Computing e^(tensor([7177.3394], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-01 09:17:55.548423] Ending with PPL=36.185906019055025 (duration: 0:03:21.392483) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.753 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.757 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.999 (now 4.063377857208252). Wait count has been increased from 2. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.459 (now 0.7611467242240906). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 6.015 (now 6.540430545806885). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-01 15:24:09.915793] Computing e^(tensor([7270.6744], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-01 15:24:09.919496] Ending with PPL=37.91463489983936 (duration: 0:03:15.733051) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.750 +Metric validation/losses/mmm_image_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.756 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 3.999 (now 4.02327299118042). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.459 (now 0.7611467242240906). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 6.015 (now 6.540430545806885). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-01 21:34:16.945983] Computing e^(tensor([7318.8047], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-01 21:34:16.950068] Ending with PPL=38.83812347245168 (duration: 0:03:17.250214) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.748 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.754 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 3.999 (now 4.02327299118042). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-02 13:15:43.254312] Computing e^(tensor([7283.9256], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-02 13:15:43.258017] Ending with PPL=38.16667641072995 (duration: 0:03:18.316226) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.754 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.999 (now 4.14365291595459). Wait count has been increased from 2. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.459 (now 0.9704828858375549). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 6.015 (now 6.6631693840026855). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-03 22:37:06.524643] Computing e^(tensor([7325.0094], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-03 22:37:06.528397] Ending with PPL=38.95880045350186 (duration: 0:03:23.463338) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.747 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.752 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 3.999 (now 4.064579010009766). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.459 (now 0.9704828858375549). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 6.015 (now 6.6631693840026855). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-04 05:23:59.313736] Computing e^(tensor([7404.0476], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-04 05:23:59.318159] Ending with PPL=40.52924466910131 (duration: 0:03:15.521758) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00 + main() + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 169, in main + trainer.fit(model, datamodule=datamodule, ckpt_path=config.training.lightning_load_from_checkpoint) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 531, in fit + call._call_and_handle_interrupt( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 41, in _call_and_handle_interrupt + return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 91, in launch + return function(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 570, in _fit_impl + self._run(model, ckpt_path=ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 941, in _run + self._checkpoint_connector._restore_modules_and_callbacks(ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/connectors/checkpoint_connector.py", line 395, in _restore_modules_and_callbacks + self.resume_start(checkpoint_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/connectors/checkpoint_connector.py", line 82, in resume_start + loaded_checkpoint = self.trainer.strategy.load_checkpoint(checkpoint_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 348, in load_checkpoint + return self.checkpoint_io.load_checkpoint(checkpoint_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/plugins/io/torch_io.py", line 89, in load_checkpoint + return pl_load(path, map_location=map_location) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/utilities/cloud_io.py", line 51, in _load + return torch.load(f, map_location=map_location) # type: ignore[arg-type] +Traceback (most recent call last): + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 190, in + main() + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 169, in main + trainer.fit(model, datamodule=datamodule, ckpt_path=config.training.lightning_load_from_checkpoint) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 531, in fit + call._call_and_handle_interrupt( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 41, in _call_and_handle_interrupt + return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 91, in launch + return function(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 570, in _fit_impl + self._run(model, ckpt_path=ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 941, in _run + self._checkpoint_connector._restore_modules_and_callbacks(ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/connectors/checkpoint_connector.py", line 395, in _restore_modules_and_callbacks + self.resume_start(checkpoint_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/connectors/checkpoint_connector.py", line 82, in resume_start + loaded_checkpoint = self.trainer.strategy.load_checkpoint(checkpoint_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 348, in load_checkpoint + return self.checkpoint_io.load_checkpoint(checkpoint_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/plugins/io/torch_io.py", line 89, in load_checkpoint + return pl_load(path, map_location=map_location) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/utilities/cloud_io.py", line 51, in _load + return torch.load(f, map_location=map_location) # type: ignore[arg-type] + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/serialization.py", line 797, in load + with _open_zipfile_reader(opened_file) as opened_zipfile: + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/serialization.py", line 283, in __init__ + super().__init__(torch._C.PyTorchFileReader(name_or_buffer)) +RuntimeError: PytorchStreamReader failed reading zip archive: failed finding central directory + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/serialization.py", line 797, in load + with _open_zipfile_reader(opened_file) as opened_zipfile: + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/serialization.py", line 283, in __init__ + super().__init__(torch._C.PyTorchFileReader(name_or_buffer)) +RuntimeError: PytorchStreamReader failed reading zip archive: failed finding central directory +Traceback (most recent call last): + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 190, in + main() + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 169, in main + trainer.fit(model, datamodule=datamodule, ckpt_path=config.training.lightning_load_from_checkpoint) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 531, in fit + call._call_and_handle_interrupt( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 41, in _call_and_handle_interrupt + return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 91, in launch + return function(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 570, in _fit_impl + self._run(model, ckpt_path=ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 941, in _run + self._checkpoint_connector._restore_modules_and_callbacks(ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/connectors/checkpoint_connector.py", line 395, in _restore_modules_and_callbacks + self.resume_start(checkpoint_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/connectors/checkpoint_connector.py", line 82, in resume_start + loaded_checkpoint = self.trainer.strategy.load_checkpoint(checkpoint_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 348, in load_checkpoint + return self.checkpoint_io.load_checkpoint(checkpoint_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/plugins/io/torch_io.py", line 89, in load_checkpoint + return pl_load(path, map_location=map_location) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/utilities/cloud_io.py", line 51, in _load + return torch.load(f, map_location=map_location) # type: ignore[arg-type] + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/serialization.py", line 797, in load + with _open_zipfile_reader(opened_file) as opened_zipfile: + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/serialization.py", line 283, in __init__ + super().__init__(torch._C.PyTorchFileReader(name_or_buffer)) +RuntimeError: PytorchStreamReader failed reading zip archive: failed finding central directory +wandb: Waiting for W&B process to finish... (failed 1). Press Control-C to abort syncing. +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +VLDataModule: batch_size is 32, num_workers is 4 +ImageDataModule: batch_size is 32, num_workers is 4 +MLMDataModule: batch_size is 32, num_workers is 4 +Dataset sizes (un-normalized): [10, 10, 10] +Dataset sizes (normalized): [0.3333333333333333, 0.3333333333333333, 0.3333333333333333] +Sampling weights after temperature (1.0): [0.3333333333333333, 0.3333333333333333, 0.3333333333333333] +2023-07-14 09:57:06.106560 Setting up datamodule: VLDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-14 10:06:42.141495 Setting up datamodule: ImageDataModule +2023-07-14 10:06:58.695750 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Adding ImageNet zeroshot callback +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'ImageNetZeroshotCallback'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-14 10:16:48.914505 Setting up datamodule: VLDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-14 10:27:48.398445 Setting up datamodule: ImageDataModule +2023-07-14 10:28:21.001648 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +wandb: +wandb: Run summary: +wandb: epoch 0 +wandb: evaluation/blimp/anaphor_agreement 95.86 +wandb: evaluation/blimp/argument_structure 69.42 +wandb: evaluation/blimp/binding 67.16 +wandb: evaluation/blimp/control_raising 67.48 +wandb: evaluation/blimp/determiner_noun_agreement 91.08 +wandb: evaluation/blimp/ellipsis 82.62 +wandb: evaluation/blimp/filler_gap 66.23 +wandb: evaluation/blimp/irregular_forms 84.99 +wandb: evaluation/blimp/island_effects 51.23 +wandb: evaluation/blimp/npi_licensing 62.75 +wandb: evaluation/blimp/quantifiers 61.75 +wandb: evaluation/blimp/subject_verb_agreement 73.8 +wandb: evaluation/blimp_average 72.86417 +wandb: evaluation/pseudo_perplexity 6.47154 +wandb: lr-AdamW/pg1 0.001 +wandb: lr-AdamW/pg2 0.00075 +wandb: train/losses/mlm_loss 2.24865 +wandb: trainer/global_step 18249 +wandb: validation/losses/mlm_loss 2.46103 +wandb: validation/monitor/mlm 1 +wandb: +wandb: 🚀 View run polished-bush-233 at: https://wandb.ai/rycolab/alkmi-wit/runs/3f330x4u +wandb: Synced 3 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20230714_095621-3f330x4u/logs +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +VLDataModule: batch_size is 32, num_workers is 4 +ImageDataModule: batch_size is 32, num_workers is 4 +MLMDataModule: batch_size is 32, num_workers is 4 +Dataset sizes (un-normalized): [10, 10, 10] +Dataset sizes (normalized): [0.3333333333333333, 0.3333333333333333, 0.3333333333333333] +Sampling weights after temperature (1.0): [0.3333333333333333, 0.3333333333333333, 0.3333333333333333] +2023-07-14 09:36:41.607749 Setting up datamodule: VLDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-14 09:45:45.540230 Setting up datamodule: ImageDataModule +2023-07-14 09:46:02.116108 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Adding ImageNet zeroshot callback +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'ImageNetZeroshotCallback'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-14 09:56:05.002739 Preparing data for datamodule: VLDataModule +2023-07-14 09:56:05.002915 Preparing data for datamodule: ImageDataModule +2023-07-14 09:56:05.003022 Preparing data for datamodule: MLMDataModule +2023-07-14 10:16:48.914541 Setting up datamodule: VLDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-14 10:27:48.360186 Setting up datamodule: ImageDataModule +2023-07-14 10:28:21.055315 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 diff --git a/flava-wit/text10-vision0/bs4096_seed-1_bf16-mixed/singlenode_2023-07-08-19:47:59.log b/flava-wit/text10-vision0/bs4096_seed-1_bf16-mixed/singlenode_2023-07-08-19:47:59.log new file mode 100644 index 0000000000000000000000000000000000000000..e0923338e563e28c5e46171650295639c67a951e --- /dev/null +++ b/flava-wit/text10-vision0/bs4096_seed-1_bf16-mixed/singlenode_2023-07-08-19:47:59.log @@ -0,0 +1,6584 @@ +wandb: Currently logged in as: tamariucai (rycolab). Use `wandb login --relogin` to force relogin +wandb: wandb version 0.15.5 is available! To upgrade, please run: +wandb: $ pip install wandb --upgrade +wandb: Tracking run with wandb version 0.15.4 +wandb: Run data is saved locally in ./wandb/run-20230708_195252-3f330x4u +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run polished-bush-233 +wandb: ⭐️ View project at https://wandb.ai/rycolab/alkmi-wit +wandb: 🚀 View run at https://wandb.ai/rycolab/alkmi-wit/runs/3f330x4u +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +The current host is eu-a65-02 +The current cuDNN version is 8801 +[update_ckeckpoint_dir] Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed-1_bf16-mixed/ +Text is the predominant modality (10 v.s. 0 vision), will sample proportionally for optimal BLiMP performance. +Assigning HuggingFace RAM +Found 747.21GBs of RAM available, assigning 5GBs to HuggingFace datasets (currently 5000000000 bytes). +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/plugins/environments/slurm.py:165: PossibleUserWarning: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... + rank_zero_warn( +Using bfloat16 Automatic Mixed Precision (AMP) +GPU available: True (cuda), used: True +TPU available: False, using: 0 TPU cores +IPU available: False, using: 0 IPUs +HPU available: False, using: 0 HPUs +Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/2 +wandb: Currently logged in as: tamariucai (rycolab). Use `wandb login --relogin` to force relogin +wandb: wandb version 0.15.5 is available! To upgrade, please run: +wandb: $ pip install wandb --upgrade +wandb: Tracking run with wandb version 0.15.4 +wandb: Run data is saved locally in ./wandb/run-20230708_200307-1uy9ja61 +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run trim-blaze-234 +wandb: ⭐️ View project at https://wandb.ai/rycolab/alkmi-wit +wandb: 🚀 View run at https://wandb.ai/rycolab/alkmi-wit/runs/1uy9ja61 +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +The current host is eu-a65-02 +The current cuDNN version is 8801 +[update_ckeckpoint_dir] Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed-1_bf16-mixed/ +Text is the predominant modality (10 v.s. 0 vision), will sample proportionally for optimal BLiMP performance. +Assigning HuggingFace RAM +Found 747.21GBs of RAM available, assigning 5GBs to HuggingFace datasets (currently 5000000000 bytes). +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/2 +---------------------------------------------------------------------------------------------------- +distributed_backend=nccl +All distributed processes registered. Starting with 2 processes +---------------------------------------------------------------------------------------------------- + +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at theodor1289wit-test +LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1] +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1] + + | Name | Type | Params +---------------------------------------------- +0 | model | FlavaForPreTraining | 357 M +---------------------------------------------- +303 M Trainable params +53.8 M Non-trainable params +357 M Total params +1,430.581 Total estimated model params size (MB) +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +MLMDataModule: batch_size is 32, num_workers is 4 +Only one modality, sampling weight will be [1.0] +2023-07-08 20:03:25.515710 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-08 20:12:01.293578 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-08 20:19:22.611232 Building train dataloader for datamodule: MLMDataModule +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +MLMDataModule: batch_size is 32, num_workers is 4 +Only one modality, sampling weight will be [1.0] +2023-07-08 19:53:43.651828 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-08 20:02:52.820341 Preparing data for datamodule: MLMDataModule +2023-07-08 20:12:01.293608 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-08 20:19:22.611211 Building train dataloader for datamodule: MLMDataModule +2023-07-08 20:19:23.261313 Building val dataloader for datamodule: MLMDataModule +2023-07-08 20:19:23.286828 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-08 20:55:58.629219] Computing e^(tensor([17249.5583], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-08 20:55:58.641126] Ending with PPL=5567.932844395028 (duration: 0:12:10.514171) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 7.310 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-08 22:32:55.397084] Computing e^(tensor([14921.9304], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-08 22:32:55.402082] Ending with PPL=1738.825575054727 (duration: 0:12:22.393830) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 7.041 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-08 23:20:51.593955] Computing e^(tensor([14615.9554], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-08 23:20:51.599054] Ending with PPL=1492.1565352539155 (duration: 0:12:03.563133) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.850 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 00:09:01.966949] Computing e^(tensor([14271.5667], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 00:09:01.974869] Ending with PPL=1256.120584610895 (duration: 0:12:15.765082) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.681 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 00:57:19.927926] Computing e^(tensor([13913.6768], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 00:57:19.935782] Ending with PPL=1050.3076651597053 (duration: 0:12:11.946265) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.531 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 01:45:22.346250] Computing e^(tensor([13500.8000], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 01:45:22.353357] Ending with PPL=854.4004354000439 (duration: 0:12:08.114064) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.372 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 02:33:39.599225] Computing e^(tensor([12982.0281], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 02:33:39.606625] Ending with PPL=659.1914665281772 (duration: 0:12:15.579035) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.217 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 03:21:53.491986] Computing e^(tensor([12082.1042], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 03:21:53.505500] Ending with PPL=420.3350446159581 (duration: 0:12:07.848984) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 5.944 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 04:10:05.039489] Computing e^(tensor([10839.7542], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 04:10:05.046478] Ending with PPL=225.8513689521093 (duration: 0:12:02.997035) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 5.483 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 04:58:31.480383] Computing e^(tensor([10042.4898], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 04:58:31.487289] Ending with PPL=151.59991200649512 (duration: 0:12:24.066639) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 5.121 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 05:46:32.133057] Computing e^(tensor([9481.7843], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 05:46:32.140247] Ending with PPL=114.53634236497359 (duration: 0:12:13.226237) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.899 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 06:35:12.930305] Computing e^(tensor([8946.0297], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 06:35:12.937814] Ending with PPL=87.62048745103512 (duration: 0:12:22.565265) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.694 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 07:23:04.226030] Computing e^(tensor([8486.3893], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 07:23:04.232831] Ending with PPL=69.62994093145801 (duration: 0:12:08.686397) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.524 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 08:11:02.654602] Computing e^(tensor([7978.3606], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 08:11:02.660130] Ending with PPL=54.01059995473263 (duration: 0:12:14.819177) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.331 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 08:59:19.752173] Computing e^(tensor([7589.1388], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 08:59:19.759187] Ending with PPL=44.459087820122065 (duration: 0:12:18.683346) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.180 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 09:47:19.468730] Computing e^(tensor([7282.9434], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 09:47:19.476122] Ending with PPL=38.14793693410445 (duration: 0:12:14.998329) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.058 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 10:35:23.226974] Computing e^(tensor([7000.1082], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 10:35:23.240710] Ending with PPL=33.11724392319859 (duration: 0:12:17.548777) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.939 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 11:23:24.122283] Computing e^(tensor([6780.3280], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 11:23:24.128582] Ending with PPL=29.670818478983115 (duration: 0:12:29.254737) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.844 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 12:11:42.420226] Computing e^(tensor([6554.6028], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 12:11:42.428200] Ending with PPL=26.50415136958052 (duration: 0:12:15.778615) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.757 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 13:00:14.024754] Computing e^(tensor([6368.3644], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 13:00:14.038217] Ending with PPL=24.14753207034599 (duration: 0:12:15.390007) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.676 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 13:48:55.001397] Computing e^(tensor([6207.7096], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 13:48:55.008359] Ending with PPL=22.283685532563975 (duration: 0:12:33.489080) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.604 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 14:37:33.547944] Computing e^(tensor([6059.1830], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 14:37:33.561379] Ending with PPL=20.68877992858524 (duration: 0:12:37.357765) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.534 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 15:25:40.895840] Computing e^(tensor([5930.8095], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 15:25:40.903196] Ending with PPL=19.402554427013943 (duration: 0:12:29.903589) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.471 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 16:14:28.328016] Computing e^(tensor([5796.5131], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 16:14:28.336947] Ending with PPL=18.142486921246626 (duration: 0:12:31.610237) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.417 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 17:02:39.712964] Computing e^(tensor([5685.0370], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 17:02:39.722602] Ending with PPL=17.15892558722315 (duration: 0:12:18.612702) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.364 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 17:51:19.371350] Computing e^(tensor([5591.6985], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 17:51:19.378889] Ending with PPL=16.376530921746635 (duration: 0:12:30.907747) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.322 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 18:39:39.547892] Computing e^(tensor([5505.3000], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 18:39:39.559643] Ending with PPL=15.684139817703882 (duration: 0:12:27.921424) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.281 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 19:28:01.361421] Computing e^(tensor([5408.8042], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 19:28:01.373331] Ending with PPL=14.945378423162644 (duration: 0:12:10.058967) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.245 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 20:23:38.599999] Computing e^(tensor([5344.2765], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 20:23:38.605841] Ending with PPL=14.470878224787699 (duration: 0:19:36.354109) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.205 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 21:13:39.680481] Computing e^(tensor([5263.8213], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 21:13:39.686079] Ending with PPL=13.900303101477546 (duration: 0:12:26.910442) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.182 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 22:05:05.409873] Computing e^(tensor([5203.6566], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 22:05:05.417600] Ending with PPL=13.488375971109704 (duration: 0:12:30.693717) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.148 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 22:54:07.959495] Computing e^(tensor([5156.7348], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 22:54:07.971924] Ending with PPL=13.175609842197684 (duration: 0:12:36.529968) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.127 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 23:42:47.993065] Computing e^(tensor([5101.5048], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 23:42:48.000111] Ending with PPL=12.816743179521133 (duration: 0:12:33.560897) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.096 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 00:33:28.256277] Computing e^(tensor([5035.8964], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 00:33:28.263506] Ending with PPL=12.403122026211753 (duration: 0:14:33.649899) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.072 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 01:30:38.693519] Computing e^(tensor([5014.1799], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 01:30:38.701167] Ending with PPL=12.269174305016644 (duration: 0:20:33.594440) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.059 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 02:28:44.569286] Computing e^(tensor([4961.9745], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 02:28:44.576639] Ending with PPL=11.953059200606642 (duration: 0:21:37.766661) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.039 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 03:22:03.655042] Computing e^(tensor([4906.7040], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 03:22:03.662939] Ending with PPL=11.627255756167456 (duration: 0:15:47.243795) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.016 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 04:13:33.769296] Computing e^(tensor([4872.9362], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 04:13:33.776750] Ending with PPL=11.432590367217314 (duration: 0:12:47.416284) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.995 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 05:03:36.453090] Computing e^(tensor([4849.8353], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 05:03:36.460600] Ending with PPL=11.301298851238705 (duration: 0:12:28.740747) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.985 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 05:52:47.210866] Computing e^(tensor([4810.4995], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 05:52:47.218429] Ending with PPL=11.081197609131797 (duration: 0:12:45.215774) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.961 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 06:41:34.414080] Computing e^(tensor([4772.1527], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 06:41:34.421360] Ending with PPL=10.870757372146977 (duration: 0:12:42.800513) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.948 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 07:32:06.520995] Computing e^(tensor([4749.0016], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 07:32:06.532050] Ending with PPL=10.745647611800067 (duration: 0:14:10.526274) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.928 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 08:28:16.228861] Computing e^(tensor([4712.7165], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 08:28:16.236812] Ending with PPL=10.552451946975165 (duration: 0:20:04.979906) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.913 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 09:27:16.029234] Computing e^(tensor([4687.9847], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 09:27:16.036717] Ending with PPL=10.42276474034884 (duration: 0:22:18.745372) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.909 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 10:19:53.898390] Computing e^(tensor([4630.1290], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 10:19:53.911334] Ending with PPL=10.125576238461369 (duration: 0:14:55.193631) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.886 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 11:11:42.212924] Computing e^(tensor([4614.9198], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 11:11:42.220525] Ending with PPL=10.048867305670802 (duration: 0:13:00.561103) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.876 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 12:01:55.231561] Computing e^(tensor([4613.3245], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 12:01:55.242457] Ending with PPL=10.040854963026636 (duration: 0:12:57.315800) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.868 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 12:51:08.030435] Computing e^(tensor([4597.0714], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 12:51:08.039274] Ending with PPL=9.959587988377768 (duration: 0:12:51.841953) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.855 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 13:40:52.780218] Computing e^(tensor([4571.7158], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 13:40:52.787571] Ending with PPL=9.83411949750997 (duration: 0:13:18.766559) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.848 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 14:36:53.135479] Computing e^(tensor([4556.3684], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 14:36:53.149977] Ending with PPL=9.758943989796196 (duration: 0:19:32.240858) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.836 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 15:32:12.615850] Computing e^(tensor([4536.5010], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 15:32:12.623677] Ending with PPL=9.662481389847873 (duration: 0:18:32.220709) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.821 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 16:24:45.978023] Computing e^(tensor([4506.2693], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 16:24:45.985618] Ending with PPL=9.517523103703045 (duration: 0:14:15.157894) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.813 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 17:16:14.175502] Computing e^(tensor([4493.5613], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 17:16:14.184344] Ending with PPL=9.457240600935975 (duration: 0:12:37.246899) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.805 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 18:06:17.024917] Computing e^(tensor([4457.5384], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 18:06:17.032770] Ending with PPL=9.288427038549765 (duration: 0:12:46.977371) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.796 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 18:55:19.139000] Computing e^(tensor([4443.9123], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 18:55:19.146539] Ending with PPL=9.225359398028457 (duration: 0:12:56.524182) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.790 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 19:45:14.212181] Computing e^(tensor([4438.9188], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 19:45:14.219206] Ending with PPL=9.202354701376912 (duration: 0:13:18.007759) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.783 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 20:39:52.426135] Computing e^(tensor([4406.3874], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 20:39:52.433675] Ending with PPL=9.053882551533608 (duration: 0:18:22.191391) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.767 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 21:38:42.800491] Computing e^(tensor([4407.3529], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 21:38:42.807610] Ending with PPL=9.058254433453772 (duration: 0:22:24.001889) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.756 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 23:23:31.417676] Computing e^(tensor([4380.0721], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 23:23:31.425440] Ending with PPL=8.935535402153082 (duration: 0:12:30.323392) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.746 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 00:13:48.902308] Computing e^(tensor([4362.3174], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 00:13:48.910146] Ending with PPL=8.8565625615905 (duration: 0:12:17.671577) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.739 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 01:02:05.300286] Computing e^(tensor([4339.4925], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 01:02:05.307898] Ending with PPL=8.75606210888855 (duration: 0:12:40.898486) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.730 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 01:50:19.786649] Computing e^(tensor([4330.4738], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 01:50:19.797388] Ending with PPL=8.716666789964737 (duration: 0:12:03.859681) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.724 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 03:26:38.033189] Computing e^(tensor([4320.7095], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 03:26:38.040822] Ending with PPL=8.674214463754534 (duration: 0:12:08.859444) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.715 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 04:14:30.193616] Computing e^(tensor([4301.5588], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 04:14:30.201206] Ending with PPL=8.591552079902726 (duration: 0:12:15.004056) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.711 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 05:02:39.460147] Computing e^(tensor([4270.1318], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 05:02:39.467674] Ending with PPL=8.457603885057654 (duration: 0:12:13.692534) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.703 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 05:51:06.547817] Computing e^(tensor([4259.3308], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 05:51:06.555428] Ending with PPL=8.412051673793849 (duration: 0:12:17.218983) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.698 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 06:39:21.908041] Computing e^(tensor([4249.4251], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 06:39:21.914784] Ending with PPL=8.370491161491671 (duration: 0:12:04.970573) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.696 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 07:27:38.876051] Computing e^(tensor([4260.9231], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 07:27:38.883461] Ending with PPL=8.418751621899156 (duration: 0:12:13.875086) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.696 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 08:15:50.888808] Computing e^(tensor([4251.8346], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 08:15:50.901011] Ending with PPL=8.380581483896362 (duration: 0:12:13.886777) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.681 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 09:04:05.828924] Computing e^(tensor([4234.9578], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 09:04:05.841978] Ending with PPL=8.31016018709319 (duration: 0:12:06.343734) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.678 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 09:51:58.279068] Computing e^(tensor([4234.1367], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 09:51:58.286540] Ending with PPL=8.306749169209297 (duration: 0:12:14.276984) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.673 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 11:28:24.529567] Computing e^(tensor([4224.4981], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 11:28:24.536947] Ending with PPL=8.266812858808299 (duration: 0:12:13.456074) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.666 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 12:16:37.956651] Computing e^(tensor([4202.3116], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 12:16:37.964104] Ending with PPL=8.17561382385982 (duration: 0:12:14.355283) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.662 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 13:05:19.192421] Computing e^(tensor([4207.6124], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 13:05:19.199409] Ending with PPL=8.197311373232127 (duration: 0:12:14.658362) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.651 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 13:53:26.497783] Computing e^(tensor([4195.2230], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 13:53:26.505430] Ending with PPL=8.14668822673366 (duration: 0:12:08.809014) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.646 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 15:29:46.142583] Computing e^(tensor([4164.6183], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 15:29:46.152398] Ending with PPL=8.02297359302724 (duration: 0:12:16.381448) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.646 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 16:18:08.710007] Computing e^(tensor([4159.0895], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 16:18:08.722066] Ending with PPL=8.000825774233007 (duration: 0:12:12.412225) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.632 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 17:06:13.581484] Computing e^(tensor([4150.4837], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 17:06:13.589436] Ending with PPL=7.966472770724571 (duration: 0:12:20.585151) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.632 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 17:54:29.020631] Computing e^(tensor([4141.0250], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 17:54:29.027547] Ending with PPL=7.928885827024866 (duration: 0:12:02.602354) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.625 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 18:42:13.739646] Computing e^(tensor([4121.5067], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 18:42:13.751908] Ending with PPL=7.851882712407801 (duration: 0:12:03.974283) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.620 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 19:30:40.014070] Computing e^(tensor([4126.7229], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 19:30:40.021350] Ending with PPL=7.872387986732858 (duration: 0:12:00.930008) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.613 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 20:18:55.270506] Computing e^(tensor([4101.6211], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 20:18:55.282096] Ending with PPL=7.7742000902003845 (duration: 0:12:03.884120) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.612 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 21:07:14.260098] Computing e^(tensor([4106.6034], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 21:07:14.267618] Ending with PPL=7.79359060612526 (duration: 0:12:11.736070) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.605 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 21:55:02.261940] Computing e^(tensor([4087.6527], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 21:55:02.268991] Ending with PPL=7.720092581524722 (duration: 0:12:06.463486) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.601 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 22:42:53.034849] Computing e^(tensor([4074.1805], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 22:42:53.042390] Ending with PPL=7.668263899348446 (duration: 0:11:59.477364) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.594 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 23:31:02.588505] Computing e^(tensor([4077.5496], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 23:31:02.597981] Ending with PPL=7.681192379589729 (duration: 0:12:18.476999) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.594 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 00:19:10.106500] Computing e^(tensor([4067.4690], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 00:19:10.117289] Ending with PPL=7.6425744248831355 (duration: 0:12:02.292261) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.585 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 01:07:34.443456] Computing e^(tensor([4049.3429], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 01:07:34.450584] Ending with PPL=7.573622222437391 (duration: 0:12:11.900913) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.584 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 01:55:36.934821] Computing e^(tensor([4043.2014], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 01:55:36.942031] Ending with PPL=7.550401139737048 (duration: 0:12:04.200852) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.579 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 02:43:28.334179] Computing e^(tensor([4060.3050], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 02:43:28.344069] Ending with PPL=7.615247742568839 (duration: 0:12:18.190649) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.572 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 04:20:04.758415] Computing e^(tensor([4037.3488], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 04:20:04.765540] Ending with PPL=7.528338922666562 (duration: 0:12:16.549751) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.569 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 05:07:59.499470] Computing e^(tensor([4021.3715], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 05:07:59.507377] Ending with PPL=7.468437242855599 (duration: 0:12:07.394460) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.564 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 05:55:52.807551] Computing e^(tensor([4027.3495], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 05:55:52.814933] Ending with PPL=7.490793546424062 (duration: 0:12:00.369946) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.558 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 07:32:10.759343] Computing e^(tensor([3997.8635], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 07:32:10.767139] Ending with PPL=7.381167045592813 (duration: 0:12:06.689168) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.552 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 08:20:22.887673] Computing e^(tensor([3999.0748], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 08:20:22.895333] Ending with PPL=7.385638793842275 (duration: 0:12:16.208603) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.550 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 09:08:25.219052] Computing e^(tensor([3999.3687], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 09:08:25.226897] Ending with PPL=7.386724094718702 (duration: 0:12:16.262268) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.548 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 09:56:39.572064] Computing e^(tensor([3990.0002], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 09:56:39.579280] Ending with PPL=7.352203781421635 (duration: 0:12:09.012758) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.545 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 10:44:54.658911] Computing e^(tensor([3971.3229], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 10:44:54.666841] Ending with PPL=7.28386368319936 (duration: 0:12:19.819397) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.542 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 11:33:34.048254] Computing e^(tensor([3969.5364], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 11:33:34.056149] Ending with PPL=7.277360166792786 (duration: 0:12:11.710481) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.538 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 13:10:43.633493] Computing e^(tensor([3972.8879], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 13:10:43.641379] Ending with PPL=7.289565529731023 (duration: 0:12:14.204447) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.533 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 13:59:08.278301] Computing e^(tensor([3969.0826], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 13:59:08.291039] Ending with PPL=7.275709261729032 (duration: 0:12:18.141208) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.527 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 14:47:29.505807] Computing e^(tensor([3988.9775], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 14:47:29.512884] Ending with PPL=7.348445164350344 (duration: 0:12:03.239202) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.522 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 17:13:10.076944] Computing e^(tensor([3954.0358], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 17:13:10.090680] Ending with PPL=7.2211764817328685 (duration: 0:12:14.209245) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.516 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 18:01:44.535172] Computing e^(tensor([3946.2180], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 18:01:44.541049] Ending with PPL=7.193004929367937 (duration: 0:12:08.560631) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.512 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 18:50:16.705145] Computing e^(tensor([3946.5624], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 18:50:16.717700] Ending with PPL=7.194243577325236 (duration: 0:12:21.343089) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.508 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 21:15:20.833705] Computing e^(tensor([3926.5180], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 21:15:20.843765] Ending with PPL=7.122501385894381 (duration: 0:12:20.317180) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.506 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 22:03:50.840787] Computing e^(tensor([3935.8759], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 22:03:50.847923] Ending with PPL=7.155905278956528 (duration: 0:12:17.398003) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.505 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 23:40:00.886456] Computing e^(tensor([3925.2171], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 23:40:00.900472] Ending with PPL=7.117870043687964 (duration: 0:12:17.275240) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.503 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-13 00:28:18.941868] Computing e^(tensor([3923.0800], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-13 00:28:18.949069] Ending with PPL=7.110268575700691 (duration: 0:12:20.625387) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.499 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-13 01:16:21.978302] Computing e^(tensor([3924.9572], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-13 01:16:21.985953] Ending with PPL=7.116945136307993 (duration: 0:12:20.308677) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.494 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-13 02:53:35.396565] Computing e^(tensor([3912.8914], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-13 02:53:35.410312] Ending with PPL=7.074138745358249 (duration: 0:12:06.783833) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.491 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-13 04:30:21.336530] Computing e^(tensor([3914.0611], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-13 04:30:21.341991] Ending with PPL=7.078277371808025 (duration: 0:12:21.782501) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.487 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-13 05:18:32.284921] Computing e^(tensor([3907.6352], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-13 05:18:32.297779] Ending with PPL=7.055571754372353 (duration: 0:12:08.384408) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.481 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-13 06:55:05.927320] Computing e^(tensor([3888.6073], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-13 06:55:05.932375] Ending with PPL=6.9887633917851355 (duration: 0:12:17.007256) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.479 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-13 08:31:38.258098] Computing e^(tensor([3873.2031], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-13 08:31:38.268805] Ending with PPL=6.935141984723177 (duration: 0:12:18.966151) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.474 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-13 10:56:48.685317] Computing e^(tensor([3869.2835], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-13 10:56:48.692265] Ending with PPL=6.92156405349131 (duration: 0:12:12.879490) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.474 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-13 12:33:45.495191] Computing e^(tensor([3872.8608], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-13 12:33:45.502200] Ending with PPL=6.9339553469849795 (duration: 0:12:23.742258) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.468 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-13 13:21:54.922825] Computing e^(tensor([3857.8801], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-13 13:21:54.930390] Ending with PPL=6.8822114672484505 (duration: 0:12:07.782220) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.466 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-13 14:58:48.529988] Computing e^(tensor([3868.5651], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-13 14:58:48.541968] Ending with PPL=6.919078323120883 (duration: 0:12:10.579553) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.465 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-13 15:47:07.973797] Computing e^(tensor([3867.0957], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-13 15:47:07.982131] Ending with PPL=6.913996619346045 (duration: 0:12:14.605560) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.463 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-13 16:35:25.179324] Computing e^(tensor([3862.4653], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-13 16:35:25.188389] Ending with PPL=6.898007813483982 (duration: 0:12:21.456286) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.462 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-13 18:12:48.683118] Computing e^(tensor([3870.6206], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-13 18:12:48.690623] Ending with PPL=6.926192855106465 (duration: 0:12:12.573196) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.459 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-13 19:01:14.954042] Computing e^(tensor([3850.9968], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-13 19:01:14.962257] Ending with PPL=6.85856623256396 (duration: 0:12:09.682087) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00 Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-15 15:14:27.111917 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-15 15:24:15.191792 Building train dataloader for datamodule: MLMDataModule +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +MLMDataModule: batch_size is 32, num_workers is 4 +Only one modality, sampling weight will be [1.0] +2023-07-15 14:55:34.333474 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-15 15:04:53.047160 Preparing data for datamodule: MLMDataModule +2023-07-15 15:14:27.111924 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-15 15:24:15.191780 Building train dataloader for datamodule: MLMDataModule +2023-07-15 15:24:15.859659 Building val dataloader for datamodule: MLMDataModule +2023-07-15 15:24:15.898996 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 15:43:58.258629] Computing e^(tensor([18198.7230], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 15:43:58.267622] Ending with PPL=8949.576764285066 (duration: 0:03:37.475600) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 7.767 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 16:45:23.163932] Computing e^(tensor([15171.8750], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 16:45:23.168173] Ending with PPL=1970.2929001119528 (duration: 0:04:23.462371) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 7.279 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 17:16:40.606277] Computing e^(tensor([14961.3306], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 17:16:40.612243] Ending with PPL=1773.4202318013438 (duration: 0:04:30.525598) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 7.103 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 17:46:50.008843] Computing e^(tensor([14683.7790], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 17:46:50.014750] Ending with PPL=1543.6260674350667 (duration: 0:04:33.371736) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.929 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 18:16:04.995126] Computing e^(tensor([14353.1512], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 18:16:05.000953] Ending with PPL=1308.4200100330618 (duration: 0:04:32.925496) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.767 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 18:45:16.067381] Computing e^(tensor([14033.6852], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 18:45:16.073430] Ending with PPL=1115.259721055273 (duration: 0:04:26.175273) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.632 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 19:14:19.607333] Computing e^(tensor([13645.8036], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 19:14:19.613238] Ending with PPL=918.6468610847395 (duration: 0:04:26.792218) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.484 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 19:43:03.460094] Computing e^(tensor([13036.1810], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 19:43:03.466122] Ending with PPL=677.2838681627456 (duration: 0:04:26.057432) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.291 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 20:12:07.475569] Computing e^(tensor([12160.9236], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 20:12:07.481431] Ending with PPL=437.23106155521924 (duration: 0:04:30.797897) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.020 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 20:41:21.870462] Computing e^(tensor([11125.7410], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 20:41:21.876388] Ending with PPL=260.56972649813713 (duration: 0:04:25.005556) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 5.640 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 21:10:26.920476] Computing e^(tensor([10383.4267], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 21:10:26.926330] Ending with PPL=179.77631203430207 (duration: 0:04:27.646721) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 5.344 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 21:39:31.206254] Computing e^(tensor([9884.2604], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 21:39:31.212093] Ending with PPL=140.06830798835455 (duration: 0:04:24.519451) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 5.080 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 22:08:29.431846] Computing e^(tensor([9447.1785], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 22:08:29.437751] Ending with PPL=112.57157386281828 (duration: 0:04:28.386231) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.898 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 22:37:07.924386] Computing e^(tensor([9034.7434], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 22:37:07.930240] Ending with PPL=91.59454256987006 (duration: 0:04:27.213414) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.734 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 23:06:26.509335] Computing e^(tensor([8658.7530], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 23:06:26.515239] Ending with PPL=75.89694963284138 (duration: 0:04:30.577307) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.597 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 23:35:04.185686] Computing e^(tensor([8305.3769], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 23:35:04.191606] Ending with PPL=63.60476857729084 (duration: 0:04:28.573388) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.470 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 00:04:27.004369] Computing e^(tensor([7948.0357], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 00:04:27.010232] Ending with PPL=53.19784272538342 (duration: 0:04:27.583555) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.338 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 00:33:54.339574] Computing e^(tensor([7709.1107], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 00:33:54.345496] Ending with PPL=47.20762248112026 (duration: 0:04:33.146525) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.241 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 01:03:08.142832] Computing e^(tensor([7444.3384], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 01:03:08.148665] Ending with PPL=41.35400219614 (duration: 0:04:27.420070) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.133 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 01:32:11.835052] Computing e^(tensor([7240.7912], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 01:32:11.840924] Ending with PPL=37.35234202993724 (duration: 0:04:27.310624) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.061 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 02:01:11.298576] Computing e^(tensor([7043.3975], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 02:01:11.304490] Ending with PPL=33.84186857779526 (duration: 0:04:30.381763) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.971 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 02:30:30.500220] Computing e^(tensor([6879.8352], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 02:30:30.506301] Ending with PPL=31.184388183696633 (duration: 0:04:29.167222) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.914 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 02:59:06.430469] Computing e^(tensor([6743.2031], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 02:59:06.436303] Ending with PPL=29.12513569270772 (duration: 0:04:28.324044) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.851 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 03:27:50.673395] Computing e^(tensor([6595.8696], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 03:27:50.679266] Ending with PPL=27.056703510429635 (duration: 0:04:28.356198) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.787 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 03:57:02.936738] Computing e^(tensor([6457.0074], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 03:57:02.942634] Ending with PPL=25.241859019263856 (duration: 0:04:26.746235) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.730 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 04:25:49.133844] Computing e^(tensor([6345.4444], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 04:25:49.139721] Ending with PPL=23.872381093441824 (duration: 0:04:26.773195) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.686 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 04:54:38.837946] Computing e^(tensor([6248.6998], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 04:54:38.843844] Ending with PPL=22.745103298149214 (duration: 0:04:28.832520) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.646 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 05:23:17.367744] Computing e^(tensor([6156.7967], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 05:23:17.373661] Ending with PPL=21.72358045821424 (duration: 0:04:25.469121) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.598 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 05:52:03.523627] Computing e^(tensor([6054.8018], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 05:52:03.529548] Ending with PPL=20.643508290292353 (duration: 0:04:26.889917) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.548 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 06:20:51.596746] Computing e^(tensor([5977.3399], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 06:20:51.602731] Ending with PPL=19.859251380605244 (duration: 0:04:26.320139) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.517 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 06:49:55.042338] Computing e^(tensor([5876.9926], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 06:49:55.048195] Ending with PPL=18.887423952041747 (duration: 0:04:29.540805) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.477 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 07:18:54.437383] Computing e^(tensor([5808.1504], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 07:18:54.443325] Ending with PPL=18.248359714363485 (duration: 0:04:29.500944) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.452 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 07:47:26.495216] Computing e^(tensor([5746.1795], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 07:47:26.501222] Ending with PPL=17.691596456092622 (duration: 0:04:32.212179) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.421 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 08:15:54.847478] Computing e^(tensor([5666.5516], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 08:15:54.858248] Ending with PPL=17.001061789979627 (duration: 0:04:28.678603) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.389 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 08:44:11.042400] Computing e^(tensor([5618.5793], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 08:44:11.048203] Ending with PPL=16.598123331200654 (duration: 0:04:27.251566) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.365 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 09:13:21.752902] Computing e^(tensor([5551.8504], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 09:13:21.757133] Ending with PPL=16.05347298823114 (duration: 0:04:26.218870) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.337 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 09:41:56.072329] Computing e^(tensor([5502.7139], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 09:41:56.078190] Ending with PPL=15.663872184399612 (duration: 0:04:27.508760) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.315 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 10:11:05.427714] Computing e^(tensor([5457.0473], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 10:11:05.433643] Ending with PPL=15.31026725906481 (duration: 0:04:25.432181) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.289 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 10:40:11.516306] Computing e^(tensor([5405.1915], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 10:40:11.522199] Ending with PPL=14.918406170908705 (duration: 0:04:26.359543) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.273 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 11:09:24.733229] Computing e^(tensor([5357.3875], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 11:09:24.739119] Ending with PPL=14.56605403169181 (duration: 0:04:27.031852) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.245 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 11:38:10.470035] Computing e^(tensor([5313.3528], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 11:38:10.475943] Ending with PPL=14.248853044749152 (duration: 0:04:30.380707) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.240 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 12:07:33.821672] Computing e^(tensor([5273.5210], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 12:07:33.827562] Ending with PPL=13.96788136642895 (duration: 0:04:27.687020) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.208 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 12:36:38.525981] Computing e^(tensor([5225.1351], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 12:36:38.531849] Ending with PPL=13.634011931013045 (duration: 0:04:29.934031) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.195 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 13:05:42.223022] Computing e^(tensor([5206.2418], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 13:05:42.227209] Ending with PPL=13.505822699054152 (duration: 0:04:25.671888) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.178 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 13:34:40.668161] Computing e^(tensor([5169.3175], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 13:34:40.674116] Ending with PPL=13.258763846086557 (duration: 0:04:26.001906) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.163 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 14:03:52.863248] Computing e^(tensor([5150.1730], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 14:03:52.869171] Ending with PPL=13.132452944542315 (duration: 0:04:28.169705) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.152 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 14:32:50.228142] Computing e^(tensor([5103.8791], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 14:32:50.234359] Ending with PPL=12.831967646730009 (duration: 0:04:26.165227) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.138 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 15:01:58.429395] Computing e^(tensor([5077.8617], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 15:01:58.435229] Ending with PPL=12.666121988683631 (duration: 0:04:27.506650) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.120 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 15:30:20.165305] Computing e^(tensor([5067.5118], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 15:30:20.171172] Ending with PPL=12.600744683631184 (duration: 0:04:29.124974) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.113 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 15:59:14.357562] Computing e^(tensor([5037.6251], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 15:59:14.363518] Ending with PPL=12.413847221297788 (duration: 0:04:29.979406) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.106 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 16:27:46.556565] Computing e^(tensor([5007.0077], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 16:27:46.562684] Ending with PPL=12.225254353515147 (duration: 0:04:25.531321) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.094 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 16:56:34.281936] Computing e^(tensor([4973.9446], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 16:56:34.287857] Ending with PPL=12.024813663359254 (duration: 0:04:28.984764) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.084 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 17:25:02.403315] Computing e^(tensor([4963.2838], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 17:25:02.409209] Ending with PPL=11.960886763836353 (duration: 0:04:26.600698) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.067 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 17:54:10.021137] Computing e^(tensor([4946.1417], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 17:54:10.026979] Ending with PPL=11.858807461744098 (duration: 0:04:28.246337) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.060 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 18:23:04.952457] Computing e^(tensor([4919.1191], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 18:23:04.956591] Ending with PPL=11.699657437451537 (duration: 0:04:25.916474) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.054 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 18:51:47.602208] Computing e^(tensor([4926.3656], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 18:51:47.608090] Ending with PPL=11.742124835684844 (duration: 0:04:26.261049) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.049 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 19:20:40.184863] Computing e^(tensor([4897.3593], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 19:20:40.190765] Ending with PPL=11.573056104859665 (duration: 0:04:25.706552) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.036 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 19:49:49.403140] Computing e^(tensor([4876.0028], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 19:49:49.409024] Ending with PPL=11.450133766690342 (duration: 0:04:29.431400) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.030 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 20:18:46.315763] Computing e^(tensor([4857.5678], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 20:18:46.321648] Ending with PPL=11.345077068453913 (duration: 0:04:27.919429) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.023 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 20:47:35.770014] Computing e^(tensor([4852.8551], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 20:47:35.776055] Ending with PPL=11.318375331463729 (duration: 0:04:34.353680) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.014 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 21:16:06.374893] Computing e^(tensor([4825.6383], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 21:16:06.380811] Ending with PPL=11.165393995013902 (duration: 0:04:28.718484) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.010 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 21:44:28.790839] Computing e^(tensor([4820.9136], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 21:44:28.796835] Ending with PPL=11.139048556322058 (duration: 0:04:29.495541) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.000 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 22:13:26.204522] Computing e^(tensor([4816.9770], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 22:13:26.210432] Ending with PPL=11.117144622920502 (duration: 0:04:26.051912) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.992 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 22:42:21.273280] Computing e^(tensor([4797.7148], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 22:42:21.279172] Ending with PPL=11.010588722597928 (duration: 0:04:28.159851) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.984 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 23:39:15.830703] Computing e^(tensor([4764.0196], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 23:39:15.836588] Ending with PPL=10.82664017491173 (duration: 0:04:27.337466) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.978 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 00:07:58.305315] Computing e^(tensor([4775.3089], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 00:07:58.311246] Ending with PPL=10.887925763238659 (duration: 0:04:27.120531) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.974 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 01:05:48.358510] Computing e^(tensor([4735.7631], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 01:05:48.364427] Ending with PPL=10.674754193310994 (duration: 0:04:27.679686) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.969 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 01:34:18.074247] Computing e^(tensor([4733.9865], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 01:34:18.080246] Ending with PPL=10.66527619893915 (duration: 0:04:26.217009) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.961 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 02:03:10.651395] Computing e^(tensor([4711.7923], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 02:03:10.657358] Ending with PPL=10.547577032996044 (duration: 0:04:28.339202) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.958 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 02:32:19.575954] Computing e^(tensor([4714.1505], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 02:32:19.581937] Ending with PPL=10.560021050930258 (duration: 0:04:26.459877) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.951 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 03:01:08.296266] Computing e^(tensor([4697.4575], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 03:01:08.302152] Ending with PPL=10.472248399116035 (duration: 0:04:27.961590) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.945 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 03:29:33.397167] Computing e^(tensor([4681.7004], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 03:29:33.403067] Ending with PPL=10.39006636954924 (duration: 0:04:32.491993) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.945 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 03:57:57.201946] Computing e^(tensor([4686.7954], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 03:57:57.207915] Ending with PPL=10.41656877958814 (duration: 0:04:32.429077) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.939 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 04:26:45.485899] Computing e^(tensor([4667.2989], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 04:26:45.491763] Ending with PPL=10.315518680169616 (duration: 0:04:25.981308) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.937 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 04:55:28.703567] Computing e^(tensor([4666.7955], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 04:55:28.709364] Ending with PPL=10.312922676274159 (duration: 0:04:26.409130) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.919 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 05:52:33.423407] Computing e^(tensor([4675.2809], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 05:52:33.429290] Ending with PPL=10.356770434950128 (duration: 0:04:26.471126) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.916 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 06:49:19.646067] Computing e^(tensor([4619.3353], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 06:49:19.652006] Ending with PPL=10.071076770495424 (duration: 0:04:25.523764) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.912 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 07:17:44.534566] Computing e^(tensor([4622.3392], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 07:17:44.540558] Ending with PPL=10.08621471016667 (duration: 0:04:29.802272) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.907 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 07:46:31.324528] Computing e^(tensor([4606.5501], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 07:46:31.330486] Ending with PPL=10.006902018501924 (duration: 0:04:27.802890) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.904 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 08:14:32.929588] Computing e^(tensor([4591.4353], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 08:14:32.933857] Ending with PPL=9.93156088201508 (duration: 0:04:28.334626) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.893 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 08:42:53.328672] Computing e^(tensor([4587.6211], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 08:42:53.334586] Ending with PPL=9.912638554656146 (duration: 0:04:29.040264) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.887 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 10:08:53.028035] Computing e^(tensor([4561.8768], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 10:08:53.033911] Ending with PPL=9.785859037888056 (duration: 0:04:25.752984) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.883 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 10:38:01.665532] Computing e^(tensor([4554.2365], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 10:38:01.671475] Ending with PPL=9.748546912882718 (duration: 0:04:27.335066) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.880 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 11:06:59.912196] Computing e^(tensor([4551.0314], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 11:06:59.918123] Ending with PPL=9.73293694817548 (duration: 0:04:27.728999) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.875 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 11:36:23.344084] Computing e^(tensor([4536.6500], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 11:36:23.350023] Ending with PPL=9.663201354654348 (duration: 0:04:30.287239) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.875 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 12:05:03.317202] Computing e^(tensor([4535.0634], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 12:05:03.323169] Ending with PPL=9.655538376149105 (duration: 0:04:27.208052) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.862 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 12:33:46.720866] Computing e^(tensor([4527.2102], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 12:33:46.726751] Ending with PPL=9.617699743889112 (duration: 0:04:27.103630) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.850 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 13:31:44.906087] Computing e^(tensor([4510.9435], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 13:31:44.912020] Ending with PPL=9.539792488319131 (duration: 0:04:26.848944) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00 Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-17 14:49:34.492158 Building val dataloader for datamodule: MLMDataModule +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 15:43:58.941471] Computing e^(tensor([18128.8627], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 15:43:58.951840] Ending with PPL=8642.363086006568 (duration: 0:03:38.169294) +[rank: 1] Metric validation/losses/mlm_loss improved. New best score: 9.020 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 16:14:45.772336] Computing e^(tensor([15762.9746], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 16:14:45.778191] Ending with PPL=2647.8077262700954 (duration: 0:04:27.604675) +[rank: 1] Metric validation/losses/mlm_loss improved by 1.253 >= min_delta = 0.0. New best score: 7.767 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 16:45:27.206333] Computing e^(tensor([15046.8851], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 16:45:27.212233] Ending with PPL=1850.9282330081776 (duration: 0:04:27.514420) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.488 >= min_delta = 0.0. New best score: 7.279 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 17:16:43.169922] Computing e^(tensor([14835.2436], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 17:16:43.175823] Ending with PPL=1665.0689493564334 (duration: 0:04:33.051131) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.176 >= min_delta = 0.0. New best score: 7.103 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 17:46:57.533155] Computing e^(tensor([14554.7926], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 17:46:57.539084] Ending with PPL=1447.2149834603997 (duration: 0:04:40.893253) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.174 >= min_delta = 0.0. New best score: 6.929 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 18:16:17.048021] Computing e^(tensor([14201.6934], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 18:16:17.053817] Ending with PPL=1212.9936660381966 (duration: 0:04:44.953646) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.162 >= min_delta = 0.0. New best score: 6.767 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 18:45:35.192575] Computing e^(tensor([13873.9739], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 18:45:35.196995] Ending with PPL=1029.6630898570393 (duration: 0:04:45.310888) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.136 >= min_delta = 0.0. New best score: 6.632 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 19:14:39.424503] Computing e^(tensor([13485.3226], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 19:14:39.430480] Ending with PPL=847.814032289094 (duration: 0:04:46.629442) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.147 >= min_delta = 0.0. New best score: 6.484 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 19:43:23.973721] Computing e^(tensor([12882.7859], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 19:43:23.979648] Ending with PPL=627.2799675496185 (duration: 0:04:46.522495) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.194 >= min_delta = 0.0. New best score: 6.291 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 20:12:26.015388] Computing e^(tensor([11995.2710], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 20:12:26.021241] Ending with PPL=402.47602219669955 (duration: 0:04:49.367632) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.271 >= min_delta = 0.0. New best score: 6.020 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 20:41:43.205756] Computing e^(tensor([10928.6739], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 20:41:43.211732] Ending with PPL=236.1192474968314 (duration: 0:04:46.352601) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.379 >= min_delta = 0.0. New best score: 5.640 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 21:10:46.451637] Computing e^(tensor([10160.9040], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 21:10:46.457584] Ending with PPL=160.84674397615396 (duration: 0:04:47.185416) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.296 >= min_delta = 0.0. New best score: 5.344 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 21:39:51.707781] Computing e^(tensor([9650.1904], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 21:39:51.713761] Ending with PPL=124.59832956870278 (duration: 0:04:45.016410) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.265 >= min_delta = 0.0. New best score: 5.080 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 22:08:48.430538] Computing e^(tensor([9198.0792], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 22:08:48.436377] Ending with PPL=99.38881576102021 (duration: 0:04:47.401169) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.182 >= min_delta = 0.0. New best score: 4.898 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 22:37:27.125266] Computing e^(tensor([8779.2553], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 22:37:27.131142] Ending with PPL=80.61039641208455 (duration: 0:04:46.410817) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.163 >= min_delta = 0.0. New best score: 4.734 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 23:06:44.470686] Computing e^(tensor([8387.0533], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 23:06:44.476706] Ending with PPL=66.25604014653881 (duration: 0:04:48.549990) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.137 >= min_delta = 0.0. New best score: 4.597 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 23:35:22.781895] Computing e^(tensor([8023.4376], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 23:35:22.787855] Ending with PPL=55.24173930062129 (duration: 0:04:47.180231) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.127 >= min_delta = 0.0. New best score: 4.470 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 00:04:47.228612] Computing e^(tensor([7664.5105], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 00:04:47.234915] Ending with PPL=46.16653705491442 (duration: 0:04:47.816792) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.133 >= min_delta = 0.0. New best score: 4.338 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 00:34:10.853858] Computing e^(tensor([7415.2122], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 00:34:10.859861] Ending with PPL=40.756123940417915 (duration: 0:04:49.638631) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.096 >= min_delta = 0.0. New best score: 4.241 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 01:03:28.567176] Computing e^(tensor([7152.9954], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 01:03:28.573144] Ending with PPL=35.74812077822303 (duration: 0:04:47.814016) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.109 >= min_delta = 0.0. New best score: 4.133 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 01:32:30.867274] Computing e^(tensor([6946.8900], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 01:32:30.873142] Ending with PPL=32.2476447240421 (duration: 0:04:46.349277) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.072 >= min_delta = 0.0. New best score: 4.061 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 02:01:27.970741] Computing e^(tensor([6752.8313], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 02:01:27.976787] Ending with PPL=29.265684106247914 (duration: 0:04:47.053383) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.089 >= min_delta = 0.0. New best score: 3.971 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 02:30:48.051779] Computing e^(tensor([6585.4508], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 02:30:48.057663] Ending with PPL=26.9161210335845 (duration: 0:04:46.726145) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.057 >= min_delta = 0.0. New best score: 3.914 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 02:59:25.076314] Computing e^(tensor([6429.5570], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 02:59:25.082276] Ending with PPL=24.89777632066803 (duration: 0:04:47.006310) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.063 >= min_delta = 0.0. New best score: 3.851 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 03:28:09.701670] Computing e^(tensor([6285.3447], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 03:28:09.707704] Ending with PPL=23.165691597254476 (duration: 0:04:47.331769) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.064 >= min_delta = 0.0. New best score: 3.787 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 03:57:22.245455] Computing e^(tensor([6159.9203], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 03:57:22.251353] Ending with PPL=21.757535509114046 (duration: 0:04:46.044102) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.058 >= min_delta = 0.0. New best score: 3.730 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 04:26:09.104004] Computing e^(tensor([6051.8718], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 04:26:09.110037] Ending with PPL=20.61328764142369 (duration: 0:04:46.786215) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.043 >= min_delta = 0.0. New best score: 3.686 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 04:54:56.608467] Computing e^(tensor([5948.6026], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 04:54:56.614500] Ending with PPL=19.575941148631916 (duration: 0:04:46.624935) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.040 >= min_delta = 0.0. New best score: 3.646 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 05:23:38.641067] Computing e^(tensor([5857.0348], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 05:23:38.645282] Ending with PPL=18.69988517569754 (duration: 0:04:46.736199) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.048 >= min_delta = 0.0. New best score: 3.598 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 05:52:22.735374] Computing e^(tensor([5759.8762], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 05:52:22.741280] Ending with PPL=17.813170564494044 (duration: 0:04:46.127134) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.050 >= min_delta = 0.0. New best score: 3.548 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 06:21:10.700512] Computing e^(tensor([5670.8995], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 06:21:10.706473] Ending with PPL=17.038061577157745 (duration: 0:04:45.445812) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.031 >= min_delta = 0.0. New best score: 3.517 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 06:50:12.976218] Computing e^(tensor([5580.0137], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 06:50:12.982174] Ending with PPL=16.28113097453333 (duration: 0:04:47.426949) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.041 >= min_delta = 0.0. New best score: 3.477 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 07:19:12.400615] Computing e^(tensor([5511.9229], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 07:19:12.404881] Ending with PPL=15.736162975944248 (duration: 0:04:47.471378) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.024 >= min_delta = 0.0. New best score: 3.452 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 07:47:44.186599] Computing e^(tensor([5441.9484], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 07:47:44.192618] Ending with PPL=15.195117960293896 (duration: 0:04:49.901273) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.032 >= min_delta = 0.0. New best score: 3.421 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 08:16:12.262776] Computing e^(tensor([5380.8338], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 08:16:12.268619] Ending with PPL=14.73781902289691 (duration: 0:04:46.089940) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.031 >= min_delta = 0.0. New best score: 3.389 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 08:44:31.584832] Computing e^(tensor([5328.4884], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 08:44:31.589229] Ending with PPL=14.357094144979621 (duration: 0:04:47.776748) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.024 >= min_delta = 0.0. New best score: 3.365 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 09:13:42.014246] Computing e^(tensor([5263.8137], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 09:13:42.020191] Ending with PPL=13.900250181235911 (duration: 0:04:46.533472) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.028 >= min_delta = 0.0. New best score: 3.337 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 09:42:14.199867] Computing e^(tensor([5211.7226], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 09:42:14.205825] Ending with PPL=13.54288507197612 (duration: 0:04:45.674707) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.022 >= min_delta = 0.0. New best score: 3.315 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 10:11:26.149818] Computing e^(tensor([5174.7102], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 10:11:26.155709] Ending with PPL=13.294562427057052 (duration: 0:04:46.186734) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.026 >= min_delta = 0.0. New best score: 3.289 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 10:40:31.886069] Computing e^(tensor([5130.9285], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 10:40:31.892052] Ending with PPL=13.006695110026035 (duration: 0:04:46.760813) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.017 >= min_delta = 0.0. New best score: 3.273 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 11:09:44.581921] Computing e^(tensor([5073.9001], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 11:09:44.587813] Ending with PPL=12.641057324600881 (duration: 0:04:46.888159) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.027 >= min_delta = 0.0. New best score: 3.245 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 11:38:27.557322] Computing e^(tensor([5050.5835], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 11:38:27.563117] Ending with PPL=12.494540194140688 (duration: 0:04:47.504389) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 3.240 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 12:07:52.545272] Computing e^(tensor([5003.6941], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 12:07:52.551190] Ending with PPL=12.205016609849677 (duration: 0:04:46.422213) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.032 >= min_delta = 0.0. New best score: 3.208 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 12:36:56.804013] Computing e^(tensor([4971.1588], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 12:36:56.809883] Ending with PPL=12.008075642849562 (duration: 0:04:48.240053) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.013 >= min_delta = 0.0. New best score: 3.195 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 13:06:03.299901] Computing e^(tensor([4930.3460], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 13:06:03.305808] Ending with PPL=11.765517594615874 (duration: 0:04:46.722271) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.017 >= min_delta = 0.0. New best score: 3.178 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 13:35:00.409835] Computing e^(tensor([4915.9020], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 13:35:00.415912] Ending with PPL=11.680853073031962 (duration: 0:04:45.737914) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.015 >= min_delta = 0.0. New best score: 3.163 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 14:04:11.430911] Computing e^(tensor([4877.5159], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 14:04:11.436821] Ending with PPL=11.458799503123318 (duration: 0:04:46.760586) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.011 >= min_delta = 0.0. New best score: 3.152 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 14:33:10.086462] Computing e^(tensor([4853.7953], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 14:33:10.092410] Ending with PPL=11.323697536555365 (duration: 0:04:46.031920) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.013 >= min_delta = 0.0. New best score: 3.138 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 15:02:18.130831] Computing e^(tensor([4820.3096], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 15:02:18.136778] Ending with PPL=11.135685082945127 (duration: 0:04:47.234756) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.019 >= min_delta = 0.0. New best score: 3.120 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 15:30:37.668291] Computing e^(tensor([4797.3102], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 15:30:37.674453] Ending with PPL=11.008361378471248 (duration: 0:04:46.631957) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.007 >= min_delta = 0.0. New best score: 3.113 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 15:59:31.467150] Computing e^(tensor([4781.3048], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 15:59:31.473051] Ending with PPL=10.920616116735738 (duration: 0:04:47.109517) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.007 >= min_delta = 0.0. New best score: 3.106 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 16:28:06.569099] Computing e^(tensor([4769.7222], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 16:28:06.574992] Ending with PPL=10.857554265386186 (duration: 0:04:45.628490) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.013 >= min_delta = 0.0. New best score: 3.094 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 16:56:53.195408] Computing e^(tensor([4729.4412], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 16:56:53.201238] Ending with PPL=10.641065400225221 (duration: 0:04:47.913909) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.009 >= min_delta = 0.0. New best score: 3.084 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 17:25:22.311551] Computing e^(tensor([4721.0314], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 17:25:22.315909] Ending with PPL=10.596414500522934 (duration: 0:04:46.562494) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.018 >= min_delta = 0.0. New best score: 3.067 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 17:54:28.529093] Computing e^(tensor([4707.5161], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 17:54:28.534998] Ending with PPL=10.525048960404906 (duration: 0:04:46.808524) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.007 >= min_delta = 0.0. New best score: 3.060 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 18:23:26.471677] Computing e^(tensor([4689.1851], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 18:23:26.477585] Ending with PPL=10.429022539097495 (duration: 0:04:47.428427) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.006 >= min_delta = 0.0. New best score: 3.054 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 18:52:08.047112] Computing e^(tensor([4684.4177], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 18:52:08.052982] Ending with PPL=10.404192736893805 (duration: 0:04:46.740314) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.004 >= min_delta = 0.0. New best score: 3.049 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 19:20:59.777639] Computing e^(tensor([4655.4140], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 19:20:59.783623] Ending with PPL=10.254401436682445 (duration: 0:04:45.356667) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.014 >= min_delta = 0.0. New best score: 3.036 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 19:50:07.001432] Computing e^(tensor([4640.3968], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 19:50:07.007376] Ending with PPL=10.177693130608489 (duration: 0:04:47.045830) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 3.030 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 20:19:06.889756] Computing e^(tensor([4632.4936], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 20:19:06.895717] Ending with PPL=10.137554376186241 (duration: 0:04:48.482978) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.008 >= min_delta = 0.0. New best score: 3.023 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 20:47:52.284357] Computing e^(tensor([4617.2953], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 20:47:52.290292] Ending with PPL=10.06080975073384 (duration: 0:04:50.862570) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.008 >= min_delta = 0.0. New best score: 3.014 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 21:16:24.923543] Computing e^(tensor([4595.8273], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 21:16:24.929417] Ending with PPL=9.953394280966032 (duration: 0:04:47.278002) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 3.010 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 21:44:47.138914] Computing e^(tensor([4595.7111], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 21:44:47.144847] Ending with PPL=9.952816466189493 (duration: 0:04:47.860061) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.010 >= min_delta = 0.0. New best score: 3.000 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 22:13:46.873652] Computing e^(tensor([4587.9910], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 22:13:46.879597] Ending with PPL=9.914471884207792 (duration: 0:04:46.746238) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.008 >= min_delta = 0.0. New best score: 2.992 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 22:42:41.061863] Computing e^(tensor([4571.0366], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 22:42:41.067794] Ending with PPL=9.830780043296569 (duration: 0:04:47.958045) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.992. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 23:11:21.972809] Computing e^(tensor([4563.4925], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 23:11:21.977072] Ending with PPL=9.793767621922237 (duration: 0:04:47.043184) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.008 >= min_delta = 0.0. New best score: 2.984 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 23:39:35.128172] Computing e^(tensor([4540.2056], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 23:39:35.134025] Ending with PPL=9.68039598283129 (duration: 0:04:46.664270) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 2.978 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 00:08:18.188354] Computing e^(tensor([4551.6806], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 00:08:18.194252] Ending with PPL=9.73609659240819 (duration: 0:04:47.023224) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.978. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 00:37:24.212146] Computing e^(tensor([4534.2147], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 00:37:24.218165] Ending with PPL=9.651441995804815 (duration: 0:04:48.422990) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 2.974 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 01:06:08.612493] Computing e^(tensor([4523.8994], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 01:06:08.618336] Ending with PPL=9.601791627772737 (duration: 0:04:47.993536) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 2.969 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 01:34:38.108523] Computing e^(tensor([4505.9601], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 01:34:38.114457] Ending with PPL=9.516051735346938 (duration: 0:04:46.247643) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.008 >= min_delta = 0.0. New best score: 2.961 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 02:03:29.293555] Computing e^(tensor([4493.2216], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 02:03:29.299465] Ending with PPL=9.455634593611125 (duration: 0:04:46.970981) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.003 >= min_delta = 0.0. New best score: 2.958 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 02:32:39.881714] Computing e^(tensor([4495.5272], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 02:32:39.887582] Ending with PPL=9.46654119517723 (duration: 0:04:46.790832) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.007 >= min_delta = 0.0. New best score: 2.951 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 03:01:27.317183] Computing e^(tensor([4479.7232], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 03:01:27.323209] Ending with PPL=9.39203142613244 (duration: 0:04:46.967158) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.006 >= min_delta = 0.0. New best score: 2.945 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 03:29:51.136837] Computing e^(tensor([4474.4659], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 03:29:51.142884] Ending with PPL=9.367375602347492 (duration: 0:04:50.218048) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.001 >= min_delta = 0.0. New best score: 2.945 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 03:58:14.633949] Computing e^(tensor([4462.8632], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 03:58:14.639903] Ending with PPL=9.313189259079865 (duration: 0:04:49.882672) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 2.939 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 04:27:05.936155] Computing e^(tensor([4467.5276], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 04:27:05.942156] Ending with PPL=9.334935002896673 (duration: 0:04:46.453959) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.002 >= min_delta = 0.0. New best score: 2.937 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 04:55:49.551876] Computing e^(tensor([4449.5206], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 04:55:49.557841] Ending with PPL=9.251265090146774 (duration: 0:04:47.252906) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.937. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 05:24:18.345109] Computing e^(tensor([4442.9353], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 05:24:18.351303] Ending with PPL=9.220853957440234 (duration: 0:04:47.400336) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.018 >= min_delta = 0.0. New best score: 2.919 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 05:52:54.481943] Computing e^(tensor([4437.3796], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 05:52:54.487824] Ending with PPL=9.195275460672004 (duration: 0:04:47.507264) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.919. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 06:21:21.292257] Computing e^(tensor([4427.6124], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 06:21:21.298186] Ending with PPL=9.150478889732927 (duration: 0:04:46.230869) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.003 >= min_delta = 0.0. New best score: 2.916 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 06:49:39.461046] Computing e^(tensor([4410.7150], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 06:49:39.467117] Ending with PPL=9.073494811355625 (duration: 0:04:45.388488) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.004 >= min_delta = 0.0. New best score: 2.912 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 07:18:02.418934] Computing e^(tensor([4416.4203], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 07:18:02.424846] Ending with PPL=9.099415395279102 (duration: 0:04:47.684884) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.004 >= min_delta = 0.0. New best score: 2.907 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 07:46:50.136141] Computing e^(tensor([4403.1098], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 07:46:50.141981] Ending with PPL=9.039057483848131 (duration: 0:04:46.618473) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.003 >= min_delta = 0.0. New best score: 2.904 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 08:14:52.515334] Computing e^(tensor([4388.6366], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 08:14:52.519635] Ending with PPL=8.97388160946113 (duration: 0:04:47.968447) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.011 >= min_delta = 0.0. New best score: 2.893 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 08:43:11.687443] Computing e^(tensor([4385.7451], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 08:43:11.693373] Ending with PPL=8.960916638488538 (duration: 0:04:47.424451) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.893. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 09:12:01.697441] Computing e^(tensor([4382.9727], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 09:12:01.703398] Ending with PPL=8.948504003269461 (duration: 0:04:45.961961) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 2 records. Best score: 2.893. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 09:40:59.659788] Computing e^(tensor([4367.8975], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 09:40:59.665686] Ending with PPL=8.881307214273752 (duration: 0:04:45.997049) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.006 >= min_delta = 0.0. New best score: 2.887 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 10:09:14.152844] Computing e^(tensor([4362.3652], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 10:09:14.158719] Ending with PPL=8.856774166791382 (duration: 0:04:46.884546) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.004 >= min_delta = 0.0. New best score: 2.883 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 10:38:19.775063] Computing e^(tensor([4356.9082], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 10:38:19.780948] Ending with PPL=8.832641327470064 (duration: 0:04:45.466491) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.004 >= min_delta = 0.0. New best score: 2.880 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 11:07:18.405356] Computing e^(tensor([4343.8260], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 11:07:18.411231] Ending with PPL=8.775054827400558 (duration: 0:04:46.199824) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.004 >= min_delta = 0.0. New best score: 2.875 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 11:36:41.360670] Computing e^(tensor([4334.8215], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 11:36:41.366667] Ending with PPL=8.735636203688317 (duration: 0:04:48.341376) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.001 >= min_delta = 0.0. New best score: 2.875 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 12:05:23.041003] Computing e^(tensor([4334.9122], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 12:05:23.046890] Ending with PPL=8.7360321244367 (duration: 0:04:46.943999) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.013 >= min_delta = 0.0. New best score: 2.862 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 12:34:05.479636] Computing e^(tensor([4344.7670], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 12:34:05.485515] Ending with PPL=8.779184129092808 (duration: 0:04:45.895145) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.862. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 13:03:05.017573] Computing e^(tensor([4324.7437], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 13:03:05.023580] Ending with PPL=8.691728541912672 (duration: 0:04:47.354836) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.011 >= min_delta = 0.0. New best score: 2.850 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 13:32:04.791948] Computing e^(tensor([4311.9507], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 13:32:04.797850] Ending with PPL=8.636309428542972 (duration: 0:04:46.720328) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.850. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 14:00:48.255978] Computing e^(tensor([4309.0015], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 14:00:48.261997] Ending with PPL=8.623583962263979 (duration: 0:04:45.747965) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 2 records. Best score: 2.850. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 14:29:19.467207] Computing e^(tensor([4302.6011], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 14:29:19.473094] Ending with PPL=8.596030541392796 (duration: 0:04:45.991208) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve in the last 3 records. Best score: 2.850. Signaling model to ignore task. +Starting validation +2023-07-17 14:38:43.603026 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-17 14:49:34.492129 Building val dataloader for datamodule: MLMDataModule +huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... +To disable this warning, you can either: + - Avoid using `tokenizers` before the fork if possible + - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) +huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... +To disable this warning, you can either: + - Avoid using `tokenizers` before the fork if possible + - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) +huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... +To disable this warning, you can either: + - Avoid using `tokenizers` before the fork if possible + - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) +huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... +To disable this warning, you can either: + - Avoid using `tokenizers` before the fork if possible + - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) +huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... +To disable this warning, you can either: + - Avoid using `tokenizers` before the fork if possible + - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) +huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... +To disable this warning, you can either: + - Avoid using `tokenizers` before the fork if possible + - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) +huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... +To disable this warning, you can either: + - Avoid using `tokenizers` before the fork if possible + - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) +huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... +To disable this warning, you can either: + - Avoid using `tokenizers` before the fork if possible + - Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 14:52:23.895845] Computing e^(tensor([4510.7124], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 14:52:23.902462] Ending with PPL=9.538690242344638 (duration: 0:02:48.452396) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 8.404 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-15 07:10:03.413424] Computing e^(tensor([15517.9897], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-15 07:10:03.419578] Ending with PPL=2342.548824584714 (duration: 0:04:07.021990) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 190.22it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 29.4 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.79it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.26it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 58.97 +evaluation/blimp/binding: 65.45 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 114.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 55.77 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 125.82it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 50.08 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 129.15it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 40.82 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 60.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 187.49it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 71.06 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 173.29it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 58.12 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 53.92 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 107.51it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 49.32 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 113.67it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 58.06 +evaluation/blimp/subject_verb_agreement: 48.74 +evaluation/blimp_average: 53.30916666666667 +Ending LM Evaluation Harness (duration: 0:07:17.071195) +Metric validation/losses/mlm_loss improved by 0.858 >= min_delta = 0.0. New best score: 7.546 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 136.25it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-15 07:51:52.398253] Computing e^(tensor([15183.8934], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-15 07:51:52.407179] Ending with PPL=1982.1684218441505 (duration: 0:04:08.457254) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 57.80it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 31.49 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.01it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 60.44 +evaluation/blimp/binding: 66.19 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.39it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 57.62 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.79it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 50.13 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 169.45it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 42.03 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.23it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 71.06 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 174.53it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 57.1 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.49it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 51.31 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 83.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 49.41 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 126.39it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 44.41 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 122.75it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 50.84 +evaluation/blimp_average: 52.66916666666666 +Ending LM Evaluation Harness (duration: 0:07:22.002401) +Metric validation/losses/mlm_loss improved by 0.284 >= min_delta = 0.0. New best score: 7.262 +Epoch 0, global step 424: 'evaluation/pseudo_perplexity' reached 1982.16846 (best 1982.16846), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=424.ckpt" as top 1 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-15 08:33:50.456437] Computing e^(tensor([15029.7934], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-15 08:33:50.462474] Ending with PPL=1835.1779007652158 (duration: 0:04:01.333755) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 99.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 34.82 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 57.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 61.23 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 79.14it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 65.64 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 118.02it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 71.20it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 57.53 +evaluation/blimp/determiner_noun_agreement: 50.09 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 156.15it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 44.34 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 58.80it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 196.33it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 71.06 +evaluation/blimp/irregular_forms: 55.47 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 126.48it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 52.32 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 75.05it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 46.3 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 74.95it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 37.87 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 109.12it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 50.62 +evaluation/blimp_average: 52.274166666666666 +Ending LM Evaluation Harness (duration: 0:07:09.753429) +Metric validation/losses/mlm_loss improved by 0.148 >= min_delta = 0.0. New best score: 7.114 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 184.47it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-15 09:15:42.685948] Computing e^(tensor([14834.3044], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-15 09:15:42.692787] Ending with PPL=1664.287168196666 (duration: 0:04:07.347716) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.88it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 32.0 +evaluation/blimp/argument_structure: 61.66 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 114.12it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 63.95 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 116.07it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 57.78 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 75.20it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 49.81 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 144.74it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 43.42 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 69.98 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 178.88it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 155.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 55.17 +evaluation/blimp/island_effects: 56.73 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.50it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 108.73it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 33.53 +evaluation/blimp/quantifiers: 35.01 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 120.53it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 50.62 +evaluation/blimp_average: 50.80500000000001 +Ending LM Evaluation Harness (duration: 0:07:19.940875) +Metric validation/losses/mlm_loss improved by 0.136 >= min_delta = 0.0. New best score: 6.978 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.72it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-15 09:57:50.400909] Computing e^(tensor([14558.7226], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-15 09:57:50.407263] Ending with PPL=1450.0615621523004 (duration: 0:04:05.409486) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 87.81it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 34.36 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.93it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 62.05 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.11it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 66.15 +evaluation/blimp/control_raising: 58.86 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 75.45it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 49.7 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 124.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 43.48 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 77.10it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 67.99 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 162.36it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 55.22 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 173.75it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 54.33 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 102.08it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 37.34 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 119.06it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 31.2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.67it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 50.84 +evaluation/blimp_average: 50.96000000000001 +Ending LM Evaluation Harness (duration: 0:07:28.156470) +Metric validation/losses/mlm_loss improved by 0.148 >= min_delta = 0.0. New best score: 6.830 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-15 10:39:45.987509] Computing e^(tensor([14309.9073], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-15 10:39:45.995940] Ending with PPL=1280.433067334603 (duration: 0:04:01.670267) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.36it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 38.75 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 74.04it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 62.21 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.68it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 64.74 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 131.16it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 59.04 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.83it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 50.36 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 155.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 41.86 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 89.70it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 67.94 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 172.48it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 60.66 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 166.60it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 57.55 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.28it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 34.62 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 129.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 31.45 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 124.17it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 50.62 +evaluation/blimp_average: 51.65 +Ending LM Evaluation Harness (duration: 0:07:12.450270) +Metric validation/losses/mlm_loss improved by 0.118 >= min_delta = 0.0. New best score: 6.711 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 172.01it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-15 11:21:18.445247] Computing e^(tensor([14058.2260], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-15 11:21:18.450939] Ending with PPL=1129.0287240750395 (duration: 0:04:01.354119) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 70.63it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 47.9 +evaluation/blimp/argument_structure: 62.28 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 133.56it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 64.08 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 67.36it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 59.26 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 142.31it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 51.37 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.24it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 39.78 +evaluation/blimp/filler_gap: 69.37 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 127.37it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 177.18it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +evaluation/blimp/irregular_forms: 64.17 +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 55.72 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 92.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 146.50it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 39.02 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 127.80it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 36.3 +evaluation/blimp/subject_verb_agreement: 50.42 +evaluation/blimp_average: 53.30583333333332 +Ending LM Evaluation Harness (duration: 0:07:23.927166) +Metric validation/losses/mlm_loss improved by 0.116 >= min_delta = 0.0. New best score: 6.596 +Epoch 0, global step 848: 'evaluation/pseudo_perplexity' reached 1129.02869 (best 1129.02869), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=848.ckpt" as top 2 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-15 12:05:42.208819] Computing e^(tensor([13774.6769], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-15 12:05:42.216108] Ending with PPL=979.7902026941456 (duration: 0:04:08.693786) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 117.58it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 46.78 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 87.06it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 62.33 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 74.55it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 64.25 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 132.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 59.61 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 66.91it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 52.19 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 140.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 38.86 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.56it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 67.93 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 153.28it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 71.65 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 51.36it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 61.55 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.37it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 40.02 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 87.96it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 38.2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.38it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 50.64 +evaluation/blimp_average: 54.50083333333333 +Ending LM Evaluation Harness (duration: 0:07:26.852047) +Metric validation/losses/mlm_loss improved by 0.106 >= min_delta = 0.0. New best score: 6.490 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-15 12:48:47.074748] Computing e^(tensor([13485.2132], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-15 12:48:47.080345] Ending with PPL=847.7676654929912 (duration: 0:04:05.467859) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 181.74it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 48.88 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 84.65it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.70it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 61.8 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 116.72it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 61.72 +evaluation/blimp/control_raising: 59.88 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.37it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 52.84 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 143.38it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.10it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 40.01 +evaluation/blimp/filler_gap: 66.88 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 118.83it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 161.99it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 73.03 +evaluation/blimp/island_effects: 58.18 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.07it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 108.56it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 38.51 +evaluation/blimp/quantifiers: 42.84 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 63.45it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 50.59 +evaluation/blimp_average: 54.596666666666664 +Ending LM Evaluation Harness (duration: 0:07:19.893412) +Metric validation/losses/mlm_loss improved by 0.104 >= min_delta = 0.0. New best score: 6.386 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-15 13:32:27.380674] Computing e^(tensor([13160.2142], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-15 13:32:27.387334] Ending with PPL=720.6164857125096 (duration: 0:04:08.810827) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 109.30it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 48.67 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 64.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 80.08it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 62.16 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 113.89it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 62.45 +evaluation/blimp/control_raising: 59.26 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.09it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 214.35it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 53.42 +evaluation/blimp/ellipsis: 34.87 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.83it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 65.7 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 135.49it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 72.16 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 152.88it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 50.64 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.00it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 45.48 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 137.63it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 46.14 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 50.7 +evaluation/blimp_average: 54.30416666666667 +Ending LM Evaluation Harness (duration: 0:07:25.600183) +Metric validation/losses/mlm_loss improved by 0.097 >= min_delta = 0.0. New best score: 6.289 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-15 14:18:22.759461] Computing e^(tensor([12542.4430], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-15 14:18:22.765554] Ending with PPL=529.123311788878 (duration: 0:04:04.074419) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 131.92it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 48.36 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.90it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 61.77 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.06it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 62.36 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 61.66it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 59.68 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.34it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 53.43 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 166.24it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 34.7 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.40it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 62.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 142.96it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 71.7 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 169.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 53.59 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.23it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 43.24 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 113.26it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 50.77 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.67it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 51.18 +evaluation/blimp_average: 54.47333333333333 +Ending LM Evaluation Harness (duration: 0:07:24.864482) +Metric validation/losses/mlm_loss improved by 0.178 >= min_delta = 0.0. New best score: 6.111 +Epoch 0, global step 1230: 'evaluation/pseudo_perplexity' reached 529.12329 (best 529.12329), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=1230.ckpt" as top 3 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-15 15:06:04.095271] Computing e^(tensor([11841.5728], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-15 15:06:04.101753] Ending with PPL=372.7046918628211 (duration: 0:04:08.427759) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 142.63it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 50.15 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 90.92it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 62.2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 99.32it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 92.14it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 58.58 +evaluation/blimp/control_raising: 59.32 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 82.12it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 54.42 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 162.51it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 78.55it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 36.32 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 151.44it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 62.46 +evaluation/blimp/irregular_forms: 69.52 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 130.85it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 46.3 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.06it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 132.74it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 48.69 +evaluation/blimp/quantifiers: 54.51 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 114.52it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 52.36 +evaluation/blimp_average: 54.56916666666667 +Ending LM Evaluation Harness (duration: 0:08:30.975696) +Metric validation/losses/mlm_loss improved by 0.224 >= min_delta = 0.0. New best score: 5.887 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-15 15:50:56.499502] Computing e^(tensor([10999.3870], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-15 15:50:56.506284] Ending with PPL=244.6169499323963 (duration: 0:04:07.841358) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 137.11it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 47.19 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 84.77it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 62.88 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 83.61it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 59.6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 120.10it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 58.2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 59.12it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 55.14 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 140.39it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 38.45 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.54it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 62.46 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 124.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 74.05 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 156.40it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 46.97 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.76it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 54.75 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 149.16it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 62.24 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 108.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 51.42 +evaluation/blimp_average: 56.11249999999999 +Ending LM Evaluation Harness (duration: 0:08:40.217230) +Metric validation/losses/mlm_loss improved by 0.317 >= min_delta = 0.0. New best score: 5.570 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 179.01it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-15 16:41:29.030228] Computing e^(tensor([10439.3241], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-15 16:41:29.037679] Ending with PPL=184.8716995071925 (duration: 0:04:11.380126) +Starting LM Evaluation Harness +Running on blimp... +evaluation/blimp/anaphor_agreement: 47.09 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.00it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 81.73it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 62.44 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.70it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 58.28 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 76.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 58.55 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 168.11it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 57.08 +evaluation/blimp/ellipsis: 40.99 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 87.88it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 178.62it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 64.94 +evaluation/blimp/irregular_forms: 74.66 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 159.66it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 39.65 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.17it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 107.62it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 56.1 +evaluation/blimp/quantifiers: 70.02 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 89.92it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 51.18 +evaluation/blimp_average: 56.74833333333333 +Ending LM Evaluation Harness (duration: 0:07:42.990585) +Metric validation/losses/mlm_loss improved by 0.240 >= min_delta = 0.0. New best score: 5.330 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-15 17:30:57.424881] Computing e^(tensor([10046.2755], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-15 17:30:57.437344] Ending with PPL=151.88714097712804 (duration: 0:04:07.634399) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.69it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 91.59it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 52.04 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 84.96it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 63.31 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.93it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 58.0 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.07it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 58.77 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 147.78it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 58.68 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 44.28 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 159.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 65.14 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 115.93it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 79.69 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.76it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 39.61 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 147.80it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 55.77 +evaluation/blimp/quantifiers: 73.26 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 118.61it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 52.09 +evaluation/blimp_average: 58.38666666666666 +Ending LM Evaluation Harness (duration: 0:08:31.798751) +Metric validation/losses/mlm_loss improved by 0.206 >= min_delta = 0.0. New best score: 5.124 +Epoch 0, global step 1601: 'evaluation/pseudo_perplexity' reached 151.88715 (best 151.88715), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=1601.ckpt" as top 4 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-15 18:17:54.517708] Computing e^(tensor([9668.0885], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-15 18:17:54.524079] Ending with PPL=125.71836902948097 (duration: 0:04:04.400466) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 129.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 54.75 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.94it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 64.03 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 80.65it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 57.57 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 125.26it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.58it/s] +evaluation/blimp/control_raising: 58.75 +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 60.97 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 142.99it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.44it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 46.82 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.55it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 65.06 +evaluation/blimp/irregular_forms: 79.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 168.21it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 90.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 40.06 +evaluation/blimp/npi_licensing: 54.31 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 109.67it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.76it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 66.82 +evaluation/blimp/subject_verb_agreement: 52.68 +evaluation/blimp_average: 58.47666666666666 +Ending LM Evaluation Harness (duration: 0:07:29.585386) +Metric validation/losses/mlm_loss improved by 0.154 >= min_delta = 0.0. New best score: 4.970 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-15 19:03:29.232323] Computing e^(tensor([9333.2730], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-15 19:03:29.239107] Ending with PPL=106.33946864531009 (duration: 0:04:10.112003) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 149.61it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 120.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 59.51 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 120.23it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 64.08 +evaluation/blimp/binding: 56.04 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 130.73it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 59.21 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.73it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 157.58it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 62.05 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.57it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 49.88 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 187.39it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 65.75 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 149.42it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 80.2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.40it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 37.59 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 136.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 52.32 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 119.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 67.44 +evaluation/blimp/subject_verb_agreement: 52.77 +evaluation/blimp_average: 58.90333333333333 +Ending LM Evaluation Harness (duration: 0:07:30.059215) +Metric validation/losses/mlm_loss improved by 0.129 >= min_delta = 0.0. New best score: 4.841 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-15 19:48:02.870179] Computing e^(tensor([9013.8351], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-15 19:48:02.876151] Ending with PPL=90.64198718021946 (duration: 0:04:03.784341) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 133.34it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 60.94 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 75.44it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 63.93 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 120.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 57.91 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 125.88it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 59.04 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 64.35 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 140.50it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 52.54 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 84.12it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 65.55 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 179.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 81.73 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 107.52it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 35.8 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.66it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 51.26 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 151.02it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 69.6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.74it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 53.21 +evaluation/blimp_average: 59.655 +Ending LM Evaluation Harness (duration: 0:07:26.429207) +Metric validation/losses/mlm_loss improved by 0.128 >= min_delta = 0.0. New best score: 4.713 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-15 20:37:44.425915] Computing e^(tensor([8719.8285], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-15 20:37:44.431731] Ending with PPL=78.25042567511349 (duration: 0:04:09.130903) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 132.92it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 65.64 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.54it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 109.10it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 64.55 +evaluation/blimp/binding: 57.23 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 109.77it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 58.35 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.85it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 67.54 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 44.20it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 52.37 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 121.97it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 64.83 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 169.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 77.51 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 51.07it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.24it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 36.88 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.01it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 53.16 +evaluation/blimp/quantifiers: 67.31 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 117.01it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 54.06 +evaluation/blimp_average: 59.952499999999986 +Ending LM Evaluation Harness (duration: 0:07:53.050812) +Metric validation/losses/mlm_loss improved by 0.104 >= min_delta = 0.0. New best score: 4.609 +Epoch 0, global step 1977: 'evaluation/pseudo_perplexity' reached 78.25043 (best 78.25043), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=1977.ckpt" as top 5 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 133.55it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-15 21:25:36.936907] Computing e^(tensor([8451.1970], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-15 21:25:36.942553] Ending with PPL=68.41543821757394 (duration: 0:04:04.028442) +Starting LM Evaluation Harness +Running on blimp... +evaluation/blimp/anaphor_agreement: 67.28 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 91.75it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 64.1 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 90.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 57.94 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.57it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 58.42 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 83.04it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 67.41 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 118.38it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 54.62 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 79.94it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 63.83 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 162.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 79.49 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 125.51it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 36.02 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 72.80it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 55.19 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 130.36it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 65.48 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 69.97it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 54.04 +evaluation/blimp_average: 60.31833333333333 +Ending LM Evaluation Harness (duration: 0:08:24.712159) +Metric validation/losses/mlm_loss improved by 0.095 >= min_delta = 0.0. New best score: 4.514 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-15 22:14:24.790723] Computing e^(tensor([8182.1310], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-15 22:14:24.797425] Ending with PPL=59.80357946961131 (duration: 0:04:13.285866) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 121.01it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 69.73 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 74.90it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 64.55 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 80.64it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 58.0 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.62it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 59.08 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.64it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 72.35 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.57it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 52.71 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 81.24it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 64.92 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 186.57it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 78.07 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 154.13it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 37.0 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.83it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 53.43 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.18it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 63.83 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 148.31it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 54.45 +evaluation/blimp_average: 60.67666666666668 +Ending LM Evaluation Harness (duration: 0:07:45.947494) +Metric validation/losses/mlm_loss improved by 0.104 >= min_delta = 0.0. New best score: 4.410 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-15 22:58:03.278809] Computing e^(tensor([7960.1939], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-15 22:58:03.284373] Ending with PPL=53.52222407613517 (duration: 0:04:05.482174) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 187.51it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 64.98 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 119.20it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 64.45 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 107.02it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 59.41 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.17it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 113.53it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 58.37 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 117.67it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 72.35 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 107.47it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 54.39 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 178.51it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 64.57 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 182.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 77.76 +evaluation/blimp/island_effects: 34.19 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 108.79it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 55.53 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 122.09it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 62.88 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 108.10it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 54.72 +evaluation/blimp_average: 60.300000000000004 +Ending LM Evaluation Harness (duration: 0:07:30.463363) +Metric validation/losses/mlm_loss improved by 0.089 >= min_delta = 0.0. New best score: 4.321 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 208.09it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-15 23:50:22.140454] Computing e^(tensor([7727.3190], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-15 23:50:22.146156] Ending with PPL=47.639369039433866 (duration: 0:04:03.480348) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.12it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 71.06 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 116.53it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 64.33 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 128.47it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 57.52 +evaluation/blimp/control_raising: 58.22 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.44it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 75.05 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 209.11it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 55.77 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.23it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 64.38 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 194.25it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 80.0 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 178.80it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 34.98 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 109.31it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 55.25 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 159.38it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 64.01 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 122.75it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 55.43 +evaluation/blimp_average: 61.33333333333332 +Ending LM Evaluation Harness (duration: 0:07:46.335496) +Metric validation/losses/mlm_loss improved by 0.085 >= min_delta = 0.0. New best score: 4.237 +Epoch 0, global step 2353: 'evaluation/pseudo_perplexity' reached 47.63937 (best 47.63937), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=2353.ckpt" as top 6 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 165.63it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-16 00:39:16.113285] Computing e^(tensor([7533.7871], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 00:39:16.120134] Ending with PPL=43.24551674320688 (duration: 0:04:13.604965) +Starting LM Evaluation Harness +Running on blimp... +evaluation/blimp/anaphor_agreement: 71.73 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.66it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 64.44 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 102.09it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 58.19 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 133.21it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 58.26 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 72.58it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 158.77it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 76.36 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.50it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 57.22 +evaluation/blimp/filler_gap: 64.91 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.13it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 78.52 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.20it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 39.31 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.47it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 56.51 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 140.81it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 59.53 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 114.39it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 55.83 +evaluation/blimp_average: 61.734166666666674 +Ending LM Evaluation Harness (duration: 0:08:09.386411) +Metric validation/losses/mlm_loss improved by 0.075 >= min_delta = 0.0. New best score: 4.162 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 01:26:41.495618] Computing e^(tensor([7384.7401], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 01:26:41.502092] Ending with PPL=40.13986804838206 (duration: 0:04:09.577268) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 175.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 75.51 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.48it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 64.48 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 81.11it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 57.81 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 133.81it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 58.24 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.83it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 76.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 140.25it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 57.33 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.25it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 64.8 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 181.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 78.12 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 152.12it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 38.23 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.61it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 61.66 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 151.02it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 61.1 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.27it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 56.26 +evaluation/blimp_average: 62.53666666666667 +Ending LM Evaluation Harness (duration: 0:08:16.703449) +Metric validation/losses/mlm_loss improved by 0.065 >= min_delta = 0.0. New best score: 4.097 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 02:13:35.030631] Computing e^(tensor([7233.2265], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 02:13:35.037312] Ending with PPL=37.21132790168676 (duration: 0:04:11.495473) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 149.14it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 74.74 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.78it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 64.27 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 121.21it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 57.72 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 115.43it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 58.68 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 77.59 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 160.51it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 59.41 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 91.74it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 64.1 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 174.62it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 79.13 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 186.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 36.51 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.50it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 164.30it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 58.05 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.18it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +evaluation/blimp/quantifiers: 59.97 +evaluation/blimp/subject_verb_agreement: 56.8 +evaluation/blimp_average: 62.247499999999995 +Ending LM Evaluation Harness (duration: 0:09:11.762479) +Metric validation/losses/mlm_loss improved by 0.060 >= min_delta = 0.0. New best score: 4.037 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 02:58:25.797900] Computing e^(tensor([7072.5265], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 02:58:25.803428] Ending with PPL=34.338365908258204 (duration: 0:04:03.643674) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 156.97it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.89it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 77.66 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.12it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 64.76 +evaluation/blimp/binding: 57.36 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 127.07it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 58.44 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.85it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 78.28 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 176.75it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 60.62 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.81it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 63.68 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 189.55it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 187.45it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 78.98 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 116.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 39.87 +evaluation/blimp/npi_licensing: 61.69 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.43it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 59.51 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 120.27it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 58.03 +evaluation/blimp_average: 63.23999999999999 +Ending LM Evaluation Harness (duration: 0:08:42.645409) +Metric validation/losses/mlm_loss improved by 0.064 >= min_delta = 0.0. New best score: 3.973 +Epoch 0, global step 2730: 'evaluation/pseudo_perplexity' reached 34.33837 (best 34.33837), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=2730.ckpt" as top 7 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 03:49:39.459864] Computing e^(tensor([6959.6830], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 03:49:39.465640] Ending with PPL=32.45457788624263 (duration: 0:04:07.411940) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 178.28it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 81.44 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.35it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 64.73 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 90.38it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 57.27 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 116.49it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 58.88 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 83.34it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 78.71 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 146.27it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 60.16 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.87it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 64.18 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 247.93it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 76.79 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 120.89it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 40.43 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.90it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 58.06 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 155.27it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 59.3 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.82it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 57.72 +evaluation/blimp_average: 63.139166666666675 +Ending LM Evaluation Harness (duration: 0:07:25.567907) +Metric validation/losses/mlm_loss improved by 0.048 >= min_delta = 0.0. New best score: 3.925 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 04:37:41.175359] Computing e^(tensor([6846.0689], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 04:37:41.181950] Ending with PPL=30.662316815249593 (duration: 0:04:14.065984) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 139.72it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 80.16 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 59.58it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 65.05 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.07it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 92.06it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 56.38 +evaluation/blimp/control_raising: 60.14 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 74.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 121.23it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 79.69 +evaluation/blimp/ellipsis: 59.64 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 89.31it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 63.18 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 122.08it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 154.36it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +evaluation/blimp/irregular_forms: 80.1 +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 38.57 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.91it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 59.84 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 149.05it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 59.02 +evaluation/blimp/subject_verb_agreement: 57.92 +evaluation/blimp_average: 63.3075 +Ending LM Evaluation Harness (duration: 0:07:38.160391) +Metric validation/losses/mlm_loss improved by 0.042 >= min_delta = 0.0. New best score: 3.884 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 05:23:23.109366] Computing e^(tensor([6741.1826], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 05:23:23.115348] Ending with PPL=29.095726527936637 (duration: 0:04:08.268909) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 164.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 29.14it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 81.6 +evaluation/blimp/argument_structure: 64.26 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.07it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 128.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 58.1 +evaluation/blimp/control_raising: 59.79 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.40it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 79.73 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 154.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 62.76 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.72it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 64.16 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 177.64it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 172.48it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 78.83 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.43it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 38.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 113.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 60.28 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 116.18it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 60.59 +evaluation/blimp/subject_verb_agreement: 57.56 +evaluation/blimp_average: 63.879999999999995 +Ending LM Evaluation Harness (duration: 0:07:25.219126) +Metric validation/losses/mlm_loss improved by 0.054 >= min_delta = 0.0. New best score: 3.830 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 06:11:09.909721] Computing e^(tensor([6624.6357], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 06:11:09.916384] Ending with PPL=27.448673413792235 (duration: 0:04:11.314131) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.69it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 81.13 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 82.85it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 65.25 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 117.21it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 57.75 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 120.48it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 59.46 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.67it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 80.81 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 178.16it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 61.32 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.40it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 62.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 185.82it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 79.08 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 157.44it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 42.0 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 116.17it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 58.59 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 145.28it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 57.19 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 109.67it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 58.79 +evaluation/blimp_average: 63.689166666666665 +Ending LM Evaluation Harness (duration: 0:07:38.410967) +Metric validation/losses/mlm_loss improved by 0.039 >= min_delta = 0.0. New best score: 3.791 +Epoch 0, global step 3106: 'evaluation/pseudo_perplexity' reached 27.44867 (best 27.44867), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=3106.ckpt" as top 8 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 06:55:31.352561] Computing e^(tensor([6530.9087], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 06:55:31.358167] Ending with PPL=26.192008056244198 (duration: 0:04:09.732838) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 133.06it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 79.45 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.82it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 132.45it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 65.14 +evaluation/blimp/binding: 60.63 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 133.76it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 60.54 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.34it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 178.16it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 81.87 +evaluation/blimp/ellipsis: 63.34 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 102.77it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 63.68 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 164.78it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 82.54 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 167.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.91it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +evaluation/blimp/island_effects: 43.12 +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 59.29 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.30it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 58.29 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 124.53it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 59.06 +evaluation/blimp_average: 64.74583333333334 +Ending LM Evaluation Harness (duration: 0:07:42.867518) +Metric validation/losses/mlm_loss improved by 0.043 >= min_delta = 0.0. New best score: 3.748 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 07:40:12.172187] Computing e^(tensor([6430.8138], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 07:40:12.177795] Ending with PPL=24.913426819763195 (duration: 0:04:07.851895) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 150.11it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.38it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 86.71 +evaluation/blimp/argument_structure: 63.99 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.50it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 59.26 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 132.68it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 61.27 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.59it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 137.00it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 82.07 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.57it/s] +evaluation/blimp/ellipsis: 65.24 +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 140.40it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 63.85 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 145.94it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 79.03 +evaluation/blimp/island_effects: 40.77 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.96it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 57.94 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.36it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.59it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 57.99 +evaluation/blimp/subject_verb_agreement: 59.69 +evaluation/blimp_average: 64.8175 +Ending LM Evaluation Harness (duration: 0:08:24.717695) +Metric validation/losses/mlm_loss improved by 0.046 >= min_delta = 0.0. New best score: 3.702 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 08:23:36.810806] Computing e^(tensor([6347.0983], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 08:23:36.817431] Ending with PPL=23.892130708323716 (duration: 0:04:06.700886) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 143.11it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 83.44 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 108.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 64.68 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.06it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 60.11 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.10it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 61.49 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.25it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +evaluation/blimp/determiner_noun_agreement: 83.15 +100%|██████████| 1/1 [00:00<00:00, 117.37it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.42it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 64.49 +evaluation/blimp/filler_gap: 64.72 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 157.57it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 80.15 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 115.83it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 84.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 43.95 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.56it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 56.03 +evaluation/blimp/quantifiers: 57.55 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 59.82 +evaluation/blimp_average: 64.965 +Ending LM Evaluation Harness (duration: 0:08:34.056913) +Metric validation/losses/mlm_loss improved by 0.041 >= min_delta = 0.0. New best score: 3.661 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 09:10:42.627537] Computing e^(tensor([6264.0500], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 09:10:42.633424] Ending with PPL=22.920346669448193 (duration: 0:04:05.959159) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 83.54 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.25it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 64.29 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 115.39it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 58.85 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 128.91it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 61.51 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.91it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 83.85 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 150.93it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 63.86 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.15it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 183.25it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 62.26 +evaluation/blimp/irregular_forms: 86.16 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 175.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 42.64 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.67it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 151.87it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 58.53 +evaluation/blimp/quantifiers: 55.49 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 59.28 +evaluation/blimp_average: 65.02166666666666 +Ending LM Evaluation Harness (duration: 0:07:25.191484) +Metric validation/losses/mlm_loss improved by 0.030 >= min_delta = 0.0. New best score: 3.631 +Epoch 0, global step 3484: 'evaluation/pseudo_perplexity' reached 22.92035 (best 22.92035), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=3484.ckpt" as top 9 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 09:53:15.653880] Computing e^(tensor([6189.7857], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 09:53:15.659408] Ending with PPL=22.084872241338108 (duration: 0:04:06.376400) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 193.05it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 85.79 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 92.14it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 64.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 77.31it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 59.63 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.26it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 62.77 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 82.75it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 82.95 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 169.36it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 65.7 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 109.28it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 63.6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 178.54it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 81.98 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 171.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 44.13 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 77.07it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 58.44 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.59it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 56.44 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 68.47it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 59.8 +evaluation/blimp_average: 65.51083333333332 +Ending LM Evaluation Harness (duration: 0:08:22.240426) +Metric validation/losses/mlm_loss improved by 0.028 >= min_delta = 0.0. New best score: 3.602 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 10:37:11.973278] Computing e^(tensor([6129.7015], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 10:37:11.978918] Ending with PPL=21.431262740033013 (duration: 0:04:04.333132) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 122.38it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 88.5 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 84.57it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 65.39 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 107.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 59.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.72it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 61.64 +evaluation/blimp/determiner_noun_agreement: 83.66 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 121.12it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 67.73 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 83.02it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 132.27it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 63.46 +evaluation/blimp/irregular_forms: 84.43 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 153.00it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 41.59 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 108.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 169.00it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 56.5 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 108.68it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 56.75 +evaluation/blimp/subject_verb_agreement: 60.67 +evaluation/blimp_average: 65.85166666666667 +Ending LM Evaluation Harness (duration: 0:09:08.595910) +Metric validation/losses/mlm_loss improved by 0.031 >= min_delta = 0.0. New best score: 3.572 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 11:23:34.013147] Computing e^(tensor([6046.1688], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 11:23:34.021582] Ending with PPL=20.55459244800673 (duration: 0:04:14.454057) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 163.01it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 89.83 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.38it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 65.06 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.83it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 61.25 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.94it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 62.15 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 113.16it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 83.56 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 148.34it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 68.19 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.42it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 62.17 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 165.36it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 81.68 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 135.40it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 42.04 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 99.96it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 55.12 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 138.12it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 56.62 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.63it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 61.01 +evaluation/blimp_average: 65.72333333333333 +Ending LM Evaluation Harness (duration: 0:08:51.358876) +Metric validation/losses/mlm_loss improved by 0.035 >= min_delta = 0.0. New best score: 3.537 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 12:08:50.816762] Computing e^(tensor([5983.1153], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 12:08:50.822868] Ending with PPL=19.916681276715785 (duration: 0:04:11.814874) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 166.94it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.507 +Epoch 0, global step 3859: 'evaluation/pseudo_perplexity' reached 19.91668 (best 19.91668), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=3859.ckpt" as top 10 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 12:57:21.245292] Computing e^(tensor([5943.5849], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 12:57:21.251509] Ending with PPL=19.526889113621827 (duration: 0:04:08.841783) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 125.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 89.16 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 87.97it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 65.35 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 82.96it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 62.05 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 120.97it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 62.2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.48it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 84.5 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 129.27it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 70.79 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.53 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 174.55it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 79.64 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 172.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 42.79 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 87.56it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 52.43 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 136.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 54.61 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 113.81it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 61.75 +evaluation/blimp_average: 65.56666666666666 +Ending LM Evaluation Harness (duration: 0:09:04.777771) +Metric validation/losses/mlm_loss improved by 0.023 >= min_delta = 0.0. New best score: 3.484 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 119.14it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-16 13:40:18.870662] Computing e^(tensor([5882.5389], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 13:40:18.876553] Ending with PPL=18.93987451562711 (duration: 0:04:08.696886) +Starting LM Evaluation Harness +Running on blimp... +evaluation/blimp/anaphor_agreement: 90.7 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 113.44it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 66.49 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.97it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 61.84 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 124.81it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 62.73 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 87.28it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 85.34 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 155.18it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 67.55 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.76it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 62.64 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 177.28it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 83.97 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 178.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 43.91 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 78.57it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 53.13 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 118.89it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 53.25 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 116.44it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 61.88 +evaluation/blimp_average: 66.11916666666666 +Ending LM Evaluation Harness (duration: 0:07:45.980794) +Metric validation/losses/mlm_loss improved by 0.026 >= min_delta = 0.0. New best score: 3.458 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 162.24it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-16 14:23:58.325250] Computing e^(tensor([5809.7066], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 14:23:58.331170] Ending with PPL=18.262564719804246 (duration: 0:04:14.126006) +Starting LM Evaluation Harness +Running on blimp... +evaluation/blimp/anaphor_agreement: 90.18 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 71.26it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 65.76 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.88it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 119.39it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 63.73 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.26it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 63.17 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 157.29it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 86.18 +evaluation/blimp/ellipsis: 69.98 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 102.11it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 181.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 62.2 +evaluation/blimp/irregular_forms: 83.97 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 124.89it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.28it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 43.8 +evaluation/blimp/npi_licensing: 53.29 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 102.83it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 56.29 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 115.67it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 62.13 +evaluation/blimp_average: 66.72333333333331 +Ending LM Evaluation Harness (duration: 0:07:44.392493) +Metric validation/losses/mlm_loss improved by 0.025 >= min_delta = 0.0. New best score: 3.433 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 15:10:52.265410] Computing e^(tensor([5785.0617], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 15:10:52.270118] Ending with PPL=18.03890528435214 (duration: 0:04:12.714497) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 173.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 88.8 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.12it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 65.66 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.14it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 64.01 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 120.66it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 62.81 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 66.15it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 85.93 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 146.08it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 70.38 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.60it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.75 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 119.29it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 83.87 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 154.42it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 41.97 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.27it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 55.34 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.83it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 53.5 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.92it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 62.53 +evaluation/blimp_average: 66.37916666666668 +Ending LM Evaluation Harness (duration: 0:07:56.980595) +Metric validation/losses/mlm_loss improved by 0.023 >= min_delta = 0.0. New best score: 3.410 +Epoch 0, global step 4236: 'evaluation/pseudo_perplexity' reached 18.03891 (best 18.03891), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=4236.ckpt" as top 11 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 137.28it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-16 15:54:29.591490] Computing e^(tensor([5732.0940], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 15:54:29.597490] Ending with PPL=17.567436546791836 (duration: 0:04:11.835059) +Starting LM Evaluation Harness +Running on blimp... +evaluation/blimp/anaphor_agreement: 90.13 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 91.05it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 66.57 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 80.77it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 63.59 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.50it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 64.63 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 91.15it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 85.64 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.48it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 70.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 83.94it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 62.93 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 173.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 165.54it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 83.41 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.92it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 42.15 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 59.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 52.52 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.69it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 57.24 +evaluation/blimp/subject_verb_agreement: 62.67 +evaluation/blimp_average: 66.865 +Ending LM Evaluation Harness (duration: 0:07:55.858355) +Metric validation/losses/mlm_loss improved by 0.023 >= min_delta = 0.0. New best score: 3.386 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 16:37:15.450899] Computing e^(tensor([5696.6887], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 16:37:15.457075] Ending with PPL=17.25918271799377 (duration: 0:04:08.495690) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.59it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 90.8 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 77.77it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 65.85 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.48it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 62.93 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 113.69it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 62.64 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.77it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 84.51 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 130.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 71.59 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 57.51it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 59.82 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 144.06it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 80.87 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 152.53it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 44.69 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 107.52it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 53.26 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 141.53it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 59.12 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 62.13 +evaluation/blimp_average: 66.5175 +Ending LM Evaluation Harness (duration: 0:07:55.063079) +Metric validation/losses/mlm_loss improved by 0.019 >= min_delta = 0.0. New best score: 3.368 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 17:20:55.656423] Computing e^(tensor([5636.4626], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 17:20:55.662304] Ending with PPL=16.74720367173347 (duration: 0:04:10.158941) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 144.00it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 89.11 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 82.33it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 66.08 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.68it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 64.51 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 121.15it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 61.84 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.35it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 85.15 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 162.44it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 69.8 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 82.00it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.17 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 163.89it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 83.1 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 151.45it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 43.09 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 89.58it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 54.19 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 131.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.89it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 54.04 +evaluation/blimp/subject_verb_agreement: 63.34 +evaluation/blimp_average: 66.285 +Ending LM Evaluation Harness (duration: 0:08:01.987958) +Metric validation/losses/mlm_loss improved by 0.018 >= min_delta = 0.0. New best score: 3.350 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 124.95it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-16 18:06:32.276039] Computing e^(tensor([5598.0970], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 18:06:32.281823] Ending with PPL=16.429007382510346 (duration: 0:04:07.616566) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 99.13it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 87.99 +evaluation/blimp/argument_structure: 65.91 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.22it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 119.63it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 66.47 +evaluation/blimp/control_raising: 65.13 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 89.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 85.2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 149.60it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 68.19 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.38it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.62 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 161.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 135.58it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 82.65 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 66.65it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 45.96 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.37it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 54.27 +evaluation/blimp/quantifiers: 56.8 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 76.66it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 62.75 +evaluation/blimp_average: 66.91166666666666 +Ending LM Evaluation Harness (duration: 0:07:39.842973) +Metric validation/losses/mlm_loss improved by 0.023 >= min_delta = 0.0. New best score: 3.327 +Epoch 0, global step 4622: 'evaluation/pseudo_perplexity' reached 16.42901 (best 16.42901), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=4622.ckpt" as top 12 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 18:51:27.195584] Computing e^(tensor([5573.8790], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 18:51:27.201527] Ending with PPL=16.231267921544134 (duration: 0:04:13.714767) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 150.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 90.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 72.94it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 66.15 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.30it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 63.64 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.54it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 64.85 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 78.52it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 85.84 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 157.21it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 74.83 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.47it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.62 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 142.27it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 80.41 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 166.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 41.89 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 126.77it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 136.02it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 52.17 +evaluation/blimp/quantifiers: 54.59 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 138.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 62.94 +evaluation/blimp_average: 66.65249999999999 +Ending LM Evaluation Harness (duration: 0:07:45.714942) +Metric validation/losses/mlm_loss improved by 0.012 >= min_delta = 0.0. New best score: 3.315 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 19:34:14.759251] Computing e^(tensor([5536.6371], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 19:34:14.765183] Ending with PPL=15.931822715175306 (duration: 0:04:14.003734) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 114.29it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 91.67 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.22it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 66.29 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.77it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 65.7 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 64.54 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 102.42it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 85.6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 150.83it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 78.09it/s] +evaluation/blimp/ellipsis: 71.82 +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 168.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 60.85 +evaluation/blimp/irregular_forms: 83.66 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 147.78it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 43.46 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.27it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 50.85 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 129.02it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.92it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 55.26 +evaluation/blimp/subject_verb_agreement: 63.61 +evaluation/blimp_average: 66.94250000000001 +Ending LM Evaluation Harness (duration: 0:08:08.241527) +Metric validation/losses/mlm_loss improved by 0.013 >= min_delta = 0.0. New best score: 3.302 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 162.78it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-16 20:20:57.251294] Computing e^(tensor([5497.2101], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 20:20:57.257221] Ending with PPL=15.620826734154026 (duration: 0:04:07.367019) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 83.07it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +evaluation/blimp/anaphor_agreement: 90.85 +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 83.31it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 65.96 +evaluation/blimp/binding: 65.91 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 122.81it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 64.05 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 77.81it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 85.14 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 152.75it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 72.34 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 75.35it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.1 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 166.04it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 83.82 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 146.76it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 45.33 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 91.09it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 53.14 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 131.33it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 58.27 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 92.34it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 63.47 +evaluation/blimp_average: 67.44833333333334 +Ending LM Evaluation Harness (duration: 0:08:01.020016) +Metric validation/losses/mlm_loss improved by 0.015 >= min_delta = 0.0. New best score: 3.287 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 21:04:26.918492] Computing e^(tensor([5450.5185], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 21:04:26.924975] Ending with PPL=15.26036974660436 (duration: 0:04:13.719720) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 143.42it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 90.03 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 74.97it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 65.66 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 83.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.16 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 114.65it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 64.47 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 84.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 86.16 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 153.16it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 74.77 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 91.89it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 60.36 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 169.38it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 83.31 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 154.61it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 45.07 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.93it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 56.16 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 144.89it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 58.29 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 75.25it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 64.1 +evaluation/blimp_average: 67.96166666666667 +Ending LM Evaluation Harness (duration: 0:07:56.601086) +Metric validation/losses/mlm_loss improved by 0.023 >= min_delta = 0.0. New best score: 3.264 +Epoch 0, global step 5003: 'evaluation/pseudo_perplexity' reached 15.26037 (best 15.26037), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=5003.ckpt" as top 13 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 176.66it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-16 21:49:03.900854] Computing e^(tensor([5417.0593], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 21:49:03.906787] Ending with PPL=15.007193625200564 (duration: 0:04:13.140161) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 74.21it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 92.59 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.25it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 66.36 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.80it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 66.1 +evaluation/blimp/control_raising: 64.34 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.16it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 86.01 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 160.42it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 74.42 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.17it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 239.99it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 60.71 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 198.79it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 81.37 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 62.42it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 45.74 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 124.42it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 55.48 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.44it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 57.6 +evaluation/blimp/subject_verb_agreement: 64.82 +evaluation/blimp_average: 67.96166666666666 +Ending LM Evaluation Harness (duration: 0:07:42.462508) +Metric validation/losses/mlm_loss improved by 0.021 >= min_delta = 0.0. New best score: 3.243 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 22:33:14.472913] Computing e^(tensor([5378.6980], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 22:33:14.479924] Ending with PPL=14.722088614314268 (duration: 0:04:09.997834) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 147.78it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 92.64 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.62it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 66.99 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.45it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 68.66 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.94it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 64.89 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.45it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 86.21 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 154.23it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 72.52 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 92.34it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 60.85 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 165.16it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 82.34 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 139.77it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 43.5 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.00it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 52.73 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 116.95it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 58.71 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 71.85it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 64.59 +evaluation/blimp_average: 67.88583333333334 +Ending LM Evaluation Harness (duration: 0:08:26.897377) +Metric validation/losses/mlm_loss improved by 0.013 >= min_delta = 0.0. New best score: 3.230 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-16 23:16:41.985849] Computing e^(tensor([5349.1045], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-16 23:16:41.991736] Ending with PPL=14.505853131359492 (duration: 0:04:10.992808) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:03<00:00, 3.77s/it] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 84.34it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 90.13 +evaluation/blimp/argument_structure: 67.1 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.62it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.97it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 66.47 +evaluation/blimp/control_raising: 64.94 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 79.56it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 87.72 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 156.13it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.36it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 73.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 163.06it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 61.17 +evaluation/blimp/irregular_forms: 81.48 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 135.33it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 41.82 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.92it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 52.84 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 130.18it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 58.58 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.42it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 63.85 +evaluation/blimp_average: 67.50000000000001 +Ending LM Evaluation Harness (duration: 0:08:54.901619) +Metric validation/losses/mlm_loss improved by 0.006 >= min_delta = 0.0. New best score: 3.224 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 00:02:49.752187] Computing e^(tensor([5322.4141], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 00:02:49.758485] Ending with PPL=14.313556015133052 (duration: 0:04:11.014374) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 134.31it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 92.48 +evaluation/blimp/argument_structure: 66.05 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 81.93it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.29it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 67.11 +evaluation/blimp/control_raising: 65.91 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.49it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +evaluation/blimp/determiner_noun_agreement: 84.5 +100%|██████████| 1/1 [00:00<00:00, 129.36it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.32it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 72.98 +evaluation/blimp/filler_gap: 62.14 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 148.67it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 79.85 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 151.64it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 89.02it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 43.12 +evaluation/blimp/npi_licensing: 53.39 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 122.15it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 60.3 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.52it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 63.4 +evaluation/blimp_average: 67.60249999999999 +Ending LM Evaluation Harness (duration: 0:08:09.800967) +Metric validation/losses/mlm_loss improved by 0.012 >= min_delta = 0.0. New best score: 3.212 +Epoch 0, global step 5387: 'evaluation/pseudo_perplexity' reached 14.31356 (best 14.31356), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=5387.ckpt" as top 14 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 00:45:47.731967] Computing e^(tensor([5298.3646], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 00:45:47.737815] Ending with PPL=14.142469326762427 (duration: 0:04:05.763571) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 128.35it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 93.46 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 91.99it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 66.44 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 117.54it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 68.06 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.95it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 65.95 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 72.13it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 142.04it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 87.51 +evaluation/blimp/ellipsis: 72.58 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.60it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.3 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 134.00it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 81.93 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 166.17it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 99.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 43.91 +evaluation/blimp/npi_licensing: 52.35 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 142.21it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 56.67 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 91.47it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 64.37 +evaluation/blimp_average: 67.8775 +Ending LM Evaluation Harness (duration: 0:08:26.076356) +Metric validation/losses/mlm_loss improved by 0.015 >= min_delta = 0.0. New best score: 3.197 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 136.11it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-17 01:30:51.627397] Computing e^(tensor([5273.3543], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 01:30:51.633516] Ending with PPL=13.966717373581037 (duration: 0:04:06.465869) +Starting LM Evaluation Harness +Running on blimp... +evaluation/blimp/anaphor_agreement: 92.33 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 99.00it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 66.25 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 68.75it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.91it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 67.48 +evaluation/blimp/control_raising: 64.89 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 99.79it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 145.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 87.51 +evaluation/blimp/ellipsis: 75.23 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.42it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 144.34it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 62.18 +evaluation/blimp/irregular_forms: 82.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 165.33it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 89.63it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 38.83 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 167.93it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 54.45 +evaluation/blimp/quantifiers: 54.92 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 116.49it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 65.06 +evaluation/blimp_average: 67.66916666666667 +Ending LM Evaluation Harness (duration: 0:07:51.971240) +Metric validation/losses/mlm_loss improved by 0.015 >= min_delta = 0.0. New best score: 3.182 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 02:12:56.973361] Computing e^(tensor([5251.2442], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 02:12:56.979370] Ending with PPL=13.813164769648505 (duration: 0:04:03.391418) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 180.60it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 90.24 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 23.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 66.86 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 117.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.51 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 124.42it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 65.42 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.28it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 87.42 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 165.37it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 73.38 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 82.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 62.28 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 126.77it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 82.7 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 162.08it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 39.91 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.20it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 50.67 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 130.18it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 54.04 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 99.58it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 64.72 +evaluation/blimp_average: 67.09583333333333 +Ending LM Evaluation Harness (duration: 0:08:17.113491) +Metric validation/losses/mlm_loss improved by 0.012 >= min_delta = 0.0. New best score: 3.170 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 02:56:25.911201] Computing e^(tensor([5219.9693], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 02:56:25.916908] Ending with PPL=13.598842035166555 (duration: 0:04:04.462501) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 114.39it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 93.92 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 92.80it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 66.66 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 81.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 68.57 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.08it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.88 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.52it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 87.52 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 128.02it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 72.75 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.17it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 62.59 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 181.82it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 81.98 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 165.44it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 38.71 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 71.83it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 54.22 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 138.29it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 57.32 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 120.16it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 64.46 +evaluation/blimp_average: 67.96500000000002 +Ending LM Evaluation Harness (duration: 0:09:29.951969) +Metric validation/losses/mlm_loss improved by 0.008 >= min_delta = 0.0. New best score: 3.162 +Epoch 0, global step 5788: 'evaluation/pseudo_perplexity' reached 13.59884 (best 13.59884), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=5788.ckpt" as top 15 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 03:39:30.709932] Computing e^(tensor([5192.2134], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 03:39:30.715800] Ending with PPL=13.411421383549294 (duration: 0:04:09.709587) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 108.99it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 92.08 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.63it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 107.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 66.93 +evaluation/blimp/binding: 67.69 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.11it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 64.96 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.55it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 87.11 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 151.65it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 76.85 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 84.36it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 188.35it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 61.5 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 160.52it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 84.07 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 135.37it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 39.09 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 195.57it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 52.93 +evaluation/blimp/quantifiers: 56.11 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.59it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 65.64 +evaluation/blimp_average: 67.91333333333334 +Ending LM Evaluation Harness (duration: 0:09:23.165355) +Metric validation/losses/mlm_loss improved by 0.013 >= min_delta = 0.0. New best score: 3.149 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 04:24:52.169089] Computing e^(tensor([5178.5933], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 04:24:52.175113] Ending with PPL=13.320399627049603 (duration: 0:04:10.446159) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 188.21it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 91.16 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 87.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 66.61 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 109.62it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 66.49 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 109.22it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 64.67 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 82.01it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 87.93 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 166.73it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 72.98 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 107.65it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.34 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 141.85it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 80.92 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 144.57it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 39.24 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.76it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 49.5 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.61it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 54.48 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 113.63it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 65.09 +evaluation/blimp_average: 66.70083333333334 +Ending LM Evaluation Harness (duration: 0:09:34.258665) +Metric validation/losses/mlm_loss improved by 0.010 >= min_delta = 0.0. New best score: 3.139 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 05:11:18.553414] Computing e^(tensor([5146.7638], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 05:11:18.563852] Ending with PPL=13.110086637514087 (duration: 0:04:10.583766) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.49it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 91.41 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.24it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 66.43 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 74.21it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 66.84 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.50it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 64.78 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.25it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 87.99 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 174.78it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 75.64 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 115.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 61.73 +evaluation/blimp/irregular_forms: 82.49 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 168.30it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.66it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 41.18 +evaluation/blimp/npi_licensing: 49.64 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 113.43it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 120.95it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 55.98 +evaluation/blimp/subject_verb_agreement: 64.82 +evaluation/blimp_average: 67.41083333333334 +Ending LM Evaluation Harness (duration: 0:09:12.033112) +Metric validation/losses/mlm_loss improved by 0.017 >= min_delta = 0.0. New best score: 3.122 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 05:55:56.603677] Computing e^(tensor([5125.6397], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 05:55:56.610800] Ending with PPL=12.972345853125306 (duration: 0:04:08.671377) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.69it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 91.36 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.47it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 74.64it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 66.69 +evaluation/blimp/binding: 68.58 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.18it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 64.98 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 67.07it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 88.36 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 136.99it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 76.85 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 69.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 60.97 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 157.91it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 82.29 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 113.69it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 41.85 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 65.45it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 51.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 134.42it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 56.57 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.00it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 66.61 +evaluation/blimp_average: 68.08416666666668 +Ending LM Evaluation Harness (duration: 0:09:42.614563) +Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 3.122. Wait count has been increased from 0. +Epoch 0, global step 6155: 'evaluation/pseudo_perplexity' reached 12.97235 (best 12.97235), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=6155.ckpt" as top 16 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 06:42:19.726581] Computing e^(tensor([5106.9875], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 06:42:19.733388] Ending with PPL=12.851926620426505 (duration: 0:04:13.866544) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 126.02it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 92.59 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.56it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 66.59 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 91.70it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 66.41 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 102.67it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.46 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 78.53it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 89.05 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 147.14it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 74.13 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 90.07it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.86 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 160.85it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 77.2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 129.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 38.42 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 52.02 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 122.40it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 58.45 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.74it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 64.52 +evaluation/blimp_average: 67.30833333333334 +Ending LM Evaluation Harness (duration: 0:10:00.731463) +Metric validation/losses/mlm_loss improved by 0.011 >= min_delta = 0.0. New best score: 3.111 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 07:26:21.881184] Computing e^(tensor([5075.3494], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 07:26:21.888069] Ending with PPL=12.650221255539874 (duration: 0:04:06.494744) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 129.64it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 93.56 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 89.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 67.77 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 78.92it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.57 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.59it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.47it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 87.75 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.17it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 75.0 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 146.10it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 62.06 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 138.64it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 81.88 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 83.17it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 42.38 +evaluation/blimp/npi_licensing: 51.7 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.91it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 57.65 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.25it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 64.97 +evaluation/blimp_average: 68.2075 +Ending LM Evaluation Harness (duration: 0:09:26.497766) +Metric validation/losses/mlm_loss improved by 0.007 >= min_delta = 0.0. New best score: 3.104 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 08:11:49.584138] Computing e^(tensor([5065.6014], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 08:11:49.589800] Ending with PPL=12.588714087654349 (duration: 0:04:05.819472) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 131.18it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 92.13 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.68it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 67.52 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 40.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.5 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 137.29it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 67.45 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 131.26it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 88.37 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 179.54it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 74.13 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.40it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.8 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 204.05it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 82.7 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 156.39it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 41.78 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.67it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 52.29 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 142.70it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 57.91 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 108.78it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 64.9 +evaluation/blimp_average: 68.20666666666666 +Ending LM Evaluation Harness (duration: 0:08:05.476385) +Metric validation/losses/mlm_loss improved by 0.009 >= min_delta = 0.0. New best score: 3.095 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 08:55:41.088690] Computing e^(tensor([5065.0546], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 08:55:41.094329] Ending with PPL=12.585272806488755 (duration: 0:04:06.901871) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 176.52it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 94.07 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 77.04it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 66.99 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 114.64it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.19 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 131.77it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 67.12 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.52it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 88.42 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 163.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 75.35 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.48it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 62.23 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 188.45it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +evaluation/blimp/irregular_forms: 81.63 +100%|██████████| 1/1 [00:00<00:00, 178.50it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 42.49 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 67.51it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 55.59 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 142.16it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 56.98 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.53it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 66.38 +evaluation/blimp_average: 68.70333333333333 +Ending LM Evaluation Harness (duration: 0:08:10.734589) +Metric validation/losses/mlm_loss improved by 0.014 >= min_delta = 0.0. New best score: 3.081 +Epoch 0, global step 6536: 'evaluation/pseudo_perplexity' reached 12.58527 (best 12.58527), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=6536.ckpt" as top 17 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 09:40:34.519154] Computing e^(tensor([5030.0639], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 09:40:34.524552] Ending with PPL=12.367003901911621 (duration: 0:04:04.579895) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 133.31it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 94.07 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 89.31it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 67.26 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.57it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.47 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.18it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.08 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.09it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 88.88 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 122.18it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 75.17 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.60it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.89 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 189.58it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 82.24 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 171.34it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 39.5 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.22it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 53.4 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 131.48it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 55.92 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 124.11it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 66.65 +evaluation/blimp_average: 68.21083333333333 +Ending LM Evaluation Harness (duration: 0:08:31.194071) +Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 3.081. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 10:24:12.434249] Computing e^(tensor([5018.0815], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 10:24:12.439818] Ending with PPL=12.293132231385405 (duration: 0:04:09.690479) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 132.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 90.85it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 93.1 +evaluation/blimp/argument_structure: 67.7 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 142.36it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.41 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 121.18it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.79 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 74.14it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 87.66 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 115.42it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 90.50it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 75.29 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 171.23it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 61.69 +evaluation/blimp/irregular_forms: 81.88 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 151.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 77.35it/s] +evaluation/blimp/island_effects: 40.47 +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 50.67 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 168.55it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 55.1 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 117.90it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 65.58 +evaluation/blimp_average: 67.77833333333334 +Ending LM Evaluation Harness (duration: 0:07:32.667292) +Metric validation/losses/mlm_loss improved by 0.016 >= min_delta = 0.0. New best score: 3.065 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 11:05:18.179419] Computing e^(tensor([5005.7030], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 11:05:18.185530] Ending with PPL=12.217282191633197 (duration: 0:04:03.321321) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 128.92it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 92.28 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.32it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 67.64 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 31.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.23 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 138.01it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 68.16 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 133.28it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 166.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 88.16 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 80.10it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 74.31 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 189.63it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 61.5 +evaluation/blimp/irregular_forms: 81.48 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 231.51it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.72it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 40.62 +evaluation/blimp/npi_licensing: 52.2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 119.72it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 68.32it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 56.13 +evaluation/blimp/subject_verb_agreement: 65.19 +evaluation/blimp_average: 67.90833333333335 +Ending LM Evaluation Harness (duration: 0:07:30.639096) +Metric validation/losses/mlm_loss improved by 0.001 >= min_delta = 0.0. New best score: 3.064 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 11:49:51.010252] Computing e^(tensor([4997.2250], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 11:49:51.016204] Ending with PPL=12.165602697138869 (duration: 0:04:06.149566) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.09it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 76.74it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 93.76 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.14it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 67.59 +evaluation/blimp/binding: 66.96 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 130.14it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 99.53it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 66.55 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 152.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 87.83 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.15it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 75.4 +evaluation/blimp/filler_gap: 60.19 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 170.33it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 81.98 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 174.45it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 40.32 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 82.31it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 53.02 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 121.50it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 60.84 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.31it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 65.49 +evaluation/blimp_average: 68.3275 +Ending LM Evaluation Harness (duration: 0:07:29.883324) +Metric validation/losses/mlm_loss improved by 0.008 >= min_delta = 0.0. New best score: 3.056 +Epoch 0, global step 6949: 'evaluation/pseudo_perplexity' reached 12.16560 (best 12.16560), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=6949.ckpt" as top 18 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 180.39it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-17 12:31:12.625733] Computing e^(tensor([4991.9152], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 12:31:12.631383] Ending with PPL=12.133346685386304 (duration: 0:04:08.300852) +Starting LM Evaluation Harness +Running on blimp... +evaluation/blimp/anaphor_agreement: 92.79 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 78.76it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 80.37it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 67.73 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 68.27 +evaluation/blimp/control_raising: 67.12 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 70.82it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 203.29it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 87.63 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.83it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 74.31 +evaluation/blimp/filler_gap: 61.14 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 138.42it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 82.04 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 145.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.55it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 41.85 +evaluation/blimp/npi_licensing: 55.86 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 127.13it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.16it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 58.06 +evaluation/blimp/subject_verb_agreement: 65.67 +evaluation/blimp_average: 68.53916666666666 +Ending LM Evaluation Harness (duration: 0:07:31.958925) +Metric validation/losses/mlm_loss improved by 0.009 >= min_delta = 0.0. New best score: 3.047 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.34it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-17 13:12:48.254180] Computing e^(tensor([4961.2507], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 13:12:48.260365] Ending with PPL=11.948734053342065 (duration: 0:04:07.420851) +Starting LM Evaluation Harness +Running on blimp... +evaluation/blimp/anaphor_agreement: 92.94 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 61.89it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 73.21it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 67.69 +evaluation/blimp/binding: 68.24 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 121.13it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 65.42 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 84.13it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 139.62it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 87.74 +evaluation/blimp/ellipsis: 75.12 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.59it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 180.38it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 61.03 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 162.38it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 82.65 +evaluation/blimp/island_effects: 42.08 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 79.11it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 51.62 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 141.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 56.47 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.97it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 66.27 +evaluation/blimp_average: 68.10583333333334 +Ending LM Evaluation Harness (duration: 0:07:32.413353) +Metric validation/losses/mlm_loss improved by 0.002 >= min_delta = 0.0. New best score: 3.046 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 13:54:15.951062] Computing e^(tensor([4951.3823], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 13:54:15.956630] Ending with PPL=11.889922148409825 (duration: 0:04:10.375008) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 160.49it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 92.59 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 69.62it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.4 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 61.96it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 69.47 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 118.75it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.84 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.02it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 87.67 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 143.38it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 76.1 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 66.79it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 60.27 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 158.68it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 84.58 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 115.61it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 39.65 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.22it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 52.88 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 139.72it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 57.24 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 109.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 66.0 +evaluation/blimp_average: 68.47416666666668 +Ending LM Evaluation Harness (duration: 0:07:38.477880) +Metric validation/losses/mlm_loss improved by 0.013 >= min_delta = 0.0. New best score: 3.033 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 14:36:05.537329] Computing e^(tensor([4934.1873], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 14:36:05.542923] Ending with PPL=11.788136454292932 (duration: 0:04:05.804932) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 125.76it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 93.71 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.72it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 67.74 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.52it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 68.69 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 119.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.28 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 70.90it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 87.84 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 142.72it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 76.44 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 76.07it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 60.68 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 139.00it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 85.14 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 164.95it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 42.34 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 71.49it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 52.17 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 141.94it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 57.5 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 114.50it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 67.14 +evaluation/blimp_average: 68.80583333333333 +Ending LM Evaluation Harness (duration: 0:07:36.556993) +Metric validation/losses/mlm_loss improved by 0.006 >= min_delta = 0.0. New best score: 3.027 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 15:17:40.047536] Computing e^(tensor([4925.9323], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 15:17:40.053202] Ending with PPL=11.73958141640246 (duration: 0:04:08.766187) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 179.37it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 93.3 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.59it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 67.98 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.68it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.81 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 108.68it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.44 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.09it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 87.11 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 81.12it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 75.46 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 62.04 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 201.05it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 182.95it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 80.2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 99.20it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 42.86 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 132.42it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 49.86 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 63.47 +evaluation/blimp/subject_verb_agreement: 65.8 +evaluation/blimp_average: 68.5275 +Ending LM Evaluation Harness (duration: 0:07:38.645004) +Metric validation/losses/mlm_loss improved by 0.003 >= min_delta = 0.0. New best score: 3.024 +Epoch 0, global step 7342: 'evaluation/pseudo_perplexity' reached 11.73958 (best 11.73958), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=7342.ckpt" as top 19 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 16:00:32.056343] Computing e^(tensor([4903.0179], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 16:00:32.062019] Ending with PPL=11.605845909532732 (duration: 0:04:06.633803) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 128.92it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 94.58 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 49.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 91.08it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 67.94 +evaluation/blimp/binding: 67.14 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.10it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.37 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.16it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 146.37it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 87.46 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.87it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 74.6 +evaluation/blimp/filler_gap: 61.38 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 185.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 83.97 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 164.90it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 36.92 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 109.30it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 52.31 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 132.92it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 57.81 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 84.89it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 67.48 +evaluation/blimp_average: 68.16333333333331 +Ending LM Evaluation Harness (duration: 0:07:37.751380) +Metric validation/losses/mlm_loss improved by 0.006 >= min_delta = 0.0. New best score: 3.019 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 16:45:40.424015] Computing e^(tensor([4885.9288], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 16:45:40.434080] Ending with PPL=11.507101859662487 (duration: 0:04:12.351040) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 115.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 93.92 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 59.94it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 67.94 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.49it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.85 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 115.35it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 65.31 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 75.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 87.39 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 109.28it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 75.0 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 74.07it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.55 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 195.59it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 83.87 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.97it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 107.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 43.54 +evaluation/blimp/npi_licensing: 51.09 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.14it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.60it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 60.61 +evaluation/blimp/subject_verb_agreement: 65.56 +evaluation/blimp_average: 68.63583333333332 +Ending LM Evaluation Harness (duration: 0:07:41.624754) +Metric validation/losses/mlm_loss improved by 0.008 >= min_delta = 0.0. New best score: 3.010 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 17:33:57.059324] Computing e^(tensor([4862.2823], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 17:33:57.067871] Ending with PPL=11.37185138667793 (duration: 0:04:10.186452) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 167.20it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 71.13it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 94.33 +evaluation/blimp/argument_structure: 68.96 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 73.77it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 125.50it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 68.49 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 84.45it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 66.44 +evaluation/blimp/determiner_noun_agreement: 87.91 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 156.92it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 75.35 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 74.39it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 158.26it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 61.19 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 157.12it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 82.85 +evaluation/blimp/island_effects: 41.52 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.87it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 52.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.76it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 55.98 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 79.80it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 67.15 +evaluation/blimp_average: 68.58916666666667 +Ending LM Evaluation Harness (duration: 0:07:33.330619) +Metric validation/losses/mlm_loss improved by 0.006 >= min_delta = 0.0. New best score: 3.004 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 168.58it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-17 18:21:41.813116] Computing e^(tensor([4864.1114], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 18:21:41.819183] Ending with PPL=11.382256811349166 (duration: 0:04:12.733989) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 84.24it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 95.4 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 46.07it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 68.76 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 65.82 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 80.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 66.88 +evaluation/blimp/determiner_noun_agreement: 87.67 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 144.08it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 92.85it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 72.34 +evaluation/blimp/filler_gap: 61.33 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 143.69it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 194.11it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 85.04 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 70.08it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 41.22 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 120.47it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 54.78 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 89.45it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 58.53 +evaluation/blimp/subject_verb_agreement: 66.65 +evaluation/blimp_average: 68.70166666666667 +Ending LM Evaluation Harness (duration: 0:07:46.057105) +Metric validation/losses/mlm_loss improved by 0.006 >= min_delta = 0.0. New best score: 2.998 +Epoch 0, global step 7718: 'evaluation/pseudo_perplexity' reached 11.38226 (best 11.38226), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=7718.ckpt" as top 20 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 19:06:42.507957] Computing e^(tensor([4848.7789], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 19:06:42.513890] Ending with PPL=11.29533068039929 (duration: 0:04:03.537535) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 141.48it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 95.3 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 69.89it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.56 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 116.21it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 69.04 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 120.39it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 65.42 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 77.40it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 88.36 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 177.37it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 77.94 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 69.91it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 60.6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 170.57it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 84.33 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 144.59it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 40.81 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 54.04it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 54.11 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 72.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 58.19 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.78it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 66.92 +evaluation/blimp_average: 69.13166666666667 +Ending LM Evaluation Harness (duration: 0:07:37.233392) +Metric validation/losses/mlm_loss improved by 0.006 >= min_delta = 0.0. New best score: 2.992 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 19:51:12.086651] Computing e^(tensor([4833.3695], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 19:51:12.092997] Ending with PPL=11.208638149082729 (duration: 0:04:12.405879) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.58it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 75.17it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 94.07 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.07it/s] +evaluation/blimp/argument_structure: 67.4 +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.65 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 79.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.06 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.11it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 88.2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 140.49it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 74.34it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 74.94 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 192.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 61.27 +evaluation/blimp/irregular_forms: 82.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 175.81it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 43.12 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.44it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 53.39 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 87.21it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 59.25 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.30it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 66.96 +evaluation/blimp_average: 68.7675 +Ending LM Evaluation Harness (duration: 0:07:35.835216) +Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 2.986 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 20:38:13.331659] Computing e^(tensor([4842.5128], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 20:38:13.338086] Ending with PPL=11.259997211050157 (duration: 0:04:07.388414) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 143.54it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 93.2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 73.10it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 67.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 66.8 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 127.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 65.8 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 49.53it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 87.31 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 115.04it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 79.04 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 75.48it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 62.71 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 182.44it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 82.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 150.18it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 40.58 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 113.29it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 51.21 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 151.08it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 62.13 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.77it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 66.96 +evaluation/blimp_average: 68.87833333333334 +Ending LM Evaluation Harness (duration: 0:07:30.428143) +Metric validation/losses/mlm_loss improved by 0.001 >= min_delta = 0.0. New best score: 2.985 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 21:20:46.386418] Computing e^(tensor([4821.1111], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 21:20:46.393410] Ending with PPL=11.140148605540197 (duration: 0:04:09.544481) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 118.29it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 71.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 93.15 +evaluation/blimp/argument_structure: 68.8 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 65.55it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 68.12 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 45.58it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.16it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 66.66 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 131.91it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 88.24 +evaluation/blimp/ellipsis: 74.31 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 66.61it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.17 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 121.50it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 86.36 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 154.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 40.58 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.88it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 53.61 +evaluation/blimp/quantifiers: 59.89 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.40it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 67.9 +evaluation/blimp_average: 69.06583333333334 +Ending LM Evaluation Harness (duration: 0:07:31.845484) +Metric validation/losses/mlm_loss improved by 0.007 >= min_delta = 0.0. New best score: 2.979 +Epoch 0, global step 8097: 'evaluation/pseudo_perplexity' reached 11.14015 (best 11.14015), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=8097.ckpt" as top 21 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 22:03:56.739063] Computing e^(tensor([4819.5506], dtype=torch.float64) / 2000) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 118.11it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-17 22:03:56.745765] Ending with PPL=11.131459476730404 (duration: 0:04:08.287067) +Starting LM Evaluation Harness +Running on blimp... +evaluation/blimp/anaphor_agreement: 93.3 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 77.72it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 78.35it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 67.1 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 66.16 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.07it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 66.77 +evaluation/blimp/determiner_noun_agreement: 87.74 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 130.53it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 87.42it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 74.54 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 174.87it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 61.02 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 163.91it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 83.31 +evaluation/blimp/island_effects: 42.79 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 83.73it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 53.99 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 102.55it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 57.52 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.85it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 67.39 +evaluation/blimp_average: 68.46916666666667 +Ending LM Evaluation Harness (duration: 0:07:46.544841) +Metric validation/losses/mlm_loss improved by 0.012 >= min_delta = 0.0. New best score: 2.966 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 22:47:38.481433] Computing e^(tensor([4808.4937], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 22:47:38.487437] Ending with PPL=11.070089460053545 (duration: 0:04:13.992751) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 148.69it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 93.87 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 64.11it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 67.8 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 58.51it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 69.03 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 114.45it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 79.55it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 66.79 +evaluation/blimp/determiner_noun_agreement: 87.72 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 180.38it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 76.04 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 46.78it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.03 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 140.10it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 84.43 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 146.05it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 42.45 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 60.40it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 54.21 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.83it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 56.21 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 132.81it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 68.17 +evaluation/blimp_average: 68.97916666666667 +Ending LM Evaluation Harness (duration: 0:07:47.045268) +Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.966. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 23:31:24.243677] Computing e^(tensor([4798.1594], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 23:31:24.249684] Ending with PPL=11.013036178089148 (duration: 0:04:10.921687) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 109.57it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 95.71 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 69.73it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 67.71 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.96it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 66.8 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.10it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 67.28 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 59.70it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 87.19 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 136.40it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 71.77 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 83.29it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.84 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 181.60it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 83.41 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 158.00it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 45.18 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 83.27it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 55.56 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 173.64it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 59.99 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 56.52it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 67.77 +evaluation/blimp_average: 69.18416666666666 +Ending LM Evaluation Harness (duration: 0:07:33.427879) +Monitored metric validation/losses/mlm_loss did not improve since the last 2 records. Best score: 2.966. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-18 00:15:52.502064] Computing e^(tensor([4777.5050], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 00:15:52.508262] Ending with PPL=10.899888070293514 (duration: 0:04:13.116520) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 52.92it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 93.97 +evaluation/blimp/argument_structure: 68.09 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.29it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 89.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 69.0 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 59.91it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 66.53 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 153.64it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 87.66 +evaluation/blimp/ellipsis: 72.63 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 46.79it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 144.59it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 61.55 +evaluation/blimp/irregular_forms: 85.55 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 145.65it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 80.64it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 45.78 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.25it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 55.36 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 83.28it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 57.57 +evaluation/blimp/subject_verb_agreement: 67.61 +evaluation/blimp_average: 69.27499999999999 +Ending LM Evaluation Harness (duration: 0:07:43.791993) +Metric validation/losses/mlm_loss improved by 0.002 >= min_delta = 0.0. New best score: 2.964 +Epoch 0, global step 8483: 'evaluation/pseudo_perplexity' reached 10.89989 (best 10.89989), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=8483.ckpt" as top 22 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 01:01:17.042119] Computing e^(tensor([4796.3916], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 01:01:17.048600] Ending with PPL=11.003306125968672 (duration: 0:04:12.468429) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 125.68it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +evaluation/blimp/anaphor_agreement: 90.18 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 74.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.4 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.57it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 68.7 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.92it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 65.78 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 59.73it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 88.33 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 136.96it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 73.67 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 72.47it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.78 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 153.90it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 83.92 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 41.26 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 92.59it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 55.95 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.76it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 55.02 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.05it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 67.21 +evaluation/blimp_average: 68.35000000000001 +Ending LM Evaluation Harness (duration: 0:07:48.669376) +Metric validation/losses/mlm_loss improved by 0.004 >= min_delta = 0.0. New best score: 2.960 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 01:45:15.375265] Computing e^(tensor([4773.0673], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 01:45:15.381413] Ending with PPL=10.87572969086802 (duration: 0:04:08.187922) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.90it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 93.76 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 87.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 67.26 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 76.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 68.69 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 72.70it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 65.42 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 68.02it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 87.58 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.99it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 58.34it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 74.02 +evaluation/blimp/filler_gap: 61.02 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 137.23it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 125.64it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 87.02 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 39.87 +evaluation/blimp/npi_licensing: 52.31 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 109.33it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 57.96 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 99.35it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 67.97 +evaluation/blimp_average: 68.57333333333334 +Ending LM Evaluation Harness (duration: 0:07:42.732737) +Metric validation/losses/mlm_loss improved by 0.008 >= min_delta = 0.0. New best score: 2.952 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 117.04it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-18 02:29:19.154413] Computing e^(tensor([4770.5888], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 02:29:19.160931] Ending with PPL=10.86225984394054 (duration: 0:04:08.497020) +Starting LM Evaluation Harness +Running on blimp... +evaluation/blimp/anaphor_agreement: 92.08 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 90.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.05 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.78it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 68.02 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 83.94it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 77.35it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 141.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 88.04 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 87.68it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 74.77 +evaluation/blimp/filler_gap: 61.53 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 146.48it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 125.12it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 85.75 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.15it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 42.12 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 75.96it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 54.36 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.00it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 56.59 +evaluation/blimp/subject_verb_agreement: 67.88 +evaluation/blimp_average: 68.84083333333334 +Ending LM Evaluation Harness (duration: 0:07:45.755847) +Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.952. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 03:13:15.855006] Computing e^(tensor([4759.2429], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 03:13:15.861437] Ending with PPL=10.800813539850212 (duration: 0:04:09.950047) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.43it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 94.27 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 63.73it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.08 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 79.36it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.35 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 122.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 65.6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 87.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 87.47 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 141.89it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 74.65 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 92.62it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 60.92 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 155.14it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 86.01 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 168.20it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 45.33 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.79it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 52.46 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 89.25it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 55.33 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.11it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 67.1 +evaluation/blimp_average: 68.71416666666667 +Ending LM Evaluation Harness (duration: 0:07:44.882128) +Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 2.947 +Epoch 0, global step 8868: 'evaluation/pseudo_perplexity' reached 10.80081 (best 10.80081), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=8868.ckpt" as top 23 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.27it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-18 03:57:39.463197] Computing e^(tensor([4748.8453], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 03:57:39.469560] Ending with PPL=10.744808097587885 (duration: 0:04:12.561160) +Starting LM Evaluation Harness +Running on blimp... +evaluation/blimp/anaphor_agreement: 92.64 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 91.21it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 56.49it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 68.76 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 122.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 69.29 +evaluation/blimp/control_raising: 65.33 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.72it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 89.71 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 154.58it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 76.96 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.56it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 153.77it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 60.52 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 145.31it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 86.01 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 81.78it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 42.94 +evaluation/blimp/npi_licensing: 55.07 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 132.26it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.68it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 59.74 +evaluation/blimp/subject_verb_agreement: 68.09 +evaluation/blimp_average: 69.58833333333332 +Ending LM Evaluation Harness (duration: 0:07:45.673210) +Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.947. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 04:41:25.947410] Computing e^(tensor([4752.8208], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 04:41:25.953719] Ending with PPL=10.766186995030553 (duration: 0:04:14.760212) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 143.24it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 94.38 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 71.21it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 69.19 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 76.31it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.53it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 68.27 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 74.24it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 89.46 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 71.16it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 75.12 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.12it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 60.64 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 160.33it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 85.5 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 183.96it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 43.61 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.93it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 55.04 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 121.21it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 56.8 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 108.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 67.73 +evaluation/blimp_average: 69.47 +Ending LM Evaluation Harness (duration: 0:07:47.612712) +Metric validation/losses/mlm_loss improved by 0.010 >= min_delta = 0.0. New best score: 2.937 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 05:25:26.919310] Computing e^(tensor([4736.2605], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 05:25:26.925698] Ending with PPL=10.677409298549657 (duration: 0:04:08.631944) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 122.45it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 93.2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 59.30it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.7 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 89.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.68 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 113.09it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 65.42 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.64it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 66.75it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 87.96 +evaluation/blimp/ellipsis: 74.54 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 64.91it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.28 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 188.35it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 84.99 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 130.65it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 46.11 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.01it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 54.37 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.01it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 59.02 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 84.85it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 68.56 +evaluation/blimp_average: 69.31916666666666 +Ending LM Evaluation Harness (duration: 0:07:50.603028) +Metric validation/losses/mlm_loss improved by 0.003 >= min_delta = 0.0. New best score: 2.935 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 146.60it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-18 06:07:35.624661] Computing e^(tensor([4745.2292], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 06:07:35.630774] Ending with PPL=10.725398160395354 (duration: 0:04:09.778835) +Starting LM Evaluation Harness +Running on blimp... +evaluation/blimp/anaphor_agreement: 93.87 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 89.01it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 81.08it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 68.38 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 58.87it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 66.4 +evaluation/blimp/control_raising: 66.08 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.63it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 158.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 88.46 +evaluation/blimp/ellipsis: 76.33 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.18it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 60.61 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 169.04it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 154.24it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 85.45 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 55.13it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 44.69 +evaluation/blimp/npi_licensing: 54.48 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 99.00it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 87.36it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 57.39 +evaluation/blimp/subject_verb_agreement: 67.9 +evaluation/blimp_average: 69.17 +Ending LM Evaluation Harness (duration: 0:07:49.601274) +Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 2.930 +Epoch 0, global step 9265: 'evaluation/pseudo_perplexity' reached 10.72540 (best 10.72540), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=9265.ckpt" as top 24 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 06:51:33.531635] Computing e^(tensor([4726.2401], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 06:51:33.537755] Ending with PPL=10.624047534325747 (duration: 0:04:10.165915) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 155.65it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 75.69it/s] +evaluation/blimp/anaphor_agreement: 95.3 +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.16 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 77.25it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 68.71 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.93it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 65.8 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.15it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 88.42 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 158.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 74.83 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 76.54it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 60.43 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 144.14it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 86.92 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 130.13it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 44.62 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.52it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 52.64 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 141.37it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 56.34 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 109.16it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 68.8 +evaluation/blimp_average: 69.24749999999999 +Ending LM Evaluation Harness (duration: 0:07:53.161133) +Metric validation/losses/mlm_loss improved by 0.001 >= min_delta = 0.0. New best score: 2.929 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 07:35:27.097499] Computing e^(tensor([4722.1107], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 07:35:27.103795] Ending with PPL=10.602134591931332 (duration: 0:04:11.483554) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 118.88it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 94.79 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.57 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 74.04it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.72 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.06it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 65.91 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 80.04it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 87.75 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 113.85it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 75.75 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.56it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 60.71 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 174.52it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 83.72 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 144.91it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 44.77 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.48it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 77.10it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 54.95 +evaluation/blimp/quantifiers: 55.74 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 102.44it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 67.86 +evaluation/blimp_average: 69.02000000000001 +Ending LM Evaluation Harness (duration: 0:07:35.924147) +Metric validation/losses/mlm_loss improved by 0.003 >= min_delta = 0.0. New best score: 2.926 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.22it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-18 08:17:54.378073] Computing e^(tensor([4715.2331], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 08:17:54.384973] Ending with PPL=10.565738592773625 (duration: 0:04:13.043077) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 92.78it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 93.66 +evaluation/blimp/argument_structure: 68.87 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 91.70it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 64.57 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.99it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.04 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.59it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 148.96it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 88.49 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 76.79it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 71.3 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 188.23it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 60.18 +evaluation/blimp/irregular_forms: 85.14 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 180.49it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 76.23it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 44.99 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 190.88it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 52.13 +evaluation/blimp/quantifiers: 54.59 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 114.39it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 69.32 +evaluation/blimp_average: 68.27333333333333 +Ending LM Evaluation Harness (duration: 0:07:43.789312) +Metric validation/losses/mlm_loss improved by 0.003 >= min_delta = 0.0. New best score: 2.922 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 09:00:09.445416] Computing e^(tensor([4705.5356], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 09:00:09.452095] Ending with PPL=10.514631826741045 (duration: 0:04:11.720306) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.05it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 94.89 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 81.82it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 57.59it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 66.27 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.24it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 67.57 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.99it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 88.86 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 141.32it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 74.77 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 68.63it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.03 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 206.42it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 86.11 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 144.30it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 46.19 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 80.14it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 55.75 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 126.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 53.12 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.55it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 70.23 +evaluation/blimp_average: 69.44916666666667 +Ending LM Evaluation Harness (duration: 0:07:48.276387) +Metric validation/losses/mlm_loss improved by 0.006 >= min_delta = 0.0. New best score: 2.916 +Epoch 0, global step 9666: 'evaluation/pseudo_perplexity' reached 10.51463 (best 10.51463), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=9666.ckpt" as top 25 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 09:45:05.535408] Computing e^(tensor([4714.3102], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 09:45:05.541341] Ending with PPL=10.56086413158468 (duration: 0:04:12.082641) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.12it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 55.37it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 94.17 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 82.55it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 68.53 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.64it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 67.44 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 99.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 65.6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 143.61it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 88.72 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 63.53it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 74.36 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 155.92it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 61.33 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 151.75it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 86.82 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 82.62it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 47.27 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.56it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 53.63 +evaluation/blimp/quantifiers: 53.79 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.88it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 67.32 +evaluation/blimp_average: 69.08166666666666 +Ending LM Evaluation Harness (duration: 0:07:51.707805) +Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.916. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 128.64it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-18 10:29:06.575070] Computing e^(tensor([4701.9414], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 10:29:06.581489] Ending with PPL=10.49575320208839 (duration: 0:04:09.652528) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 79.53it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 92.94 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.74it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 69.27 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.63it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 66.89 +evaluation/blimp/control_raising: 66.22 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 53.63it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 87.05 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 189.82it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 74.25 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 82.95it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.48 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 143.72it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 88.09 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 147.31it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 44.54 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.79it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 55.38 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 127.80it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 58.27 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.55it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 67.79 +evaluation/blimp_average: 69.34749999999998 +Ending LM Evaluation Harness (duration: 0:07:46.895152) +Metric validation/losses/mlm_loss improved by 0.007 >= min_delta = 0.0. New best score: 2.909 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 11:12:12.161615] Computing e^(tensor([4683.5620], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 11:12:12.167553] Ending with PPL=10.399742278848617 (duration: 0:04:05.805651) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 158.48it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 94.12 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 73.07it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 69.33 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 81.94it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 66.22 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 122.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 65.91 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.73it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 89.67 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 147.83it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 71.88 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 83.08it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 59.94 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 174.04it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 86.26 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 152.02it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 44.28 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 83.90it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 54.95 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 135.95it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 55.44 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 84.64it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 68.67 +evaluation/blimp_average: 68.88916666666667 +Ending LM Evaluation Harness (duration: 0:07:41.741703) +Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.909. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 11:55:31.812771] Computing e^(tensor([4681.5003], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 11:55:31.820459] Ending with PPL=10.389026916269897 (duration: 0:04:09.915417) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 174.20it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 59.30it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 94.58 +evaluation/blimp/argument_structure: 69.48 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 76.02it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 113.43it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 67.63 +evaluation/blimp/control_raising: 66.68 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.00it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 152.97it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 88.74 +evaluation/blimp/ellipsis: 74.94 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 91.07it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 62.57 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 157.67it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 57.30it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 85.39 +evaluation/blimp/island_effects: 43.24 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 99.56it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 53.48 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 91.44it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 53.25 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.45it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 68.76 +evaluation/blimp_average: 69.06166666666667 +Ending LM Evaluation Harness (duration: 0:07:19.279205) +Metric validation/losses/mlm_loss improved by 0.001 >= min_delta = 0.0. New best score: 2.909 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 178.79it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-18 12:37:36.882459] Computing e^(tensor([4667.6525], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 12:37:36.888257] Ending with PPL=10.317343026693527 (duration: 0:04:07.921386) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 102.31it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 94.84 +evaluation/blimp/argument_structure: 68.37 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 54.99it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 68.45 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 129.69it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.79 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 72.75it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 167.14it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 87.99 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.69it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 75.52 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 170.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 61.76 +evaluation/blimp/irregular_forms: 85.39 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 149.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 46.94 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 68.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 55.06 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 131.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 57.26 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.61it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 69.72 +evaluation/blimp_average: 69.84083333333332 +Ending LM Evaluation Harness (duration: 0:07:22.340068) +Metric validation/losses/mlm_loss improved by 0.008 >= min_delta = 0.0. New best score: 2.901 +Epoch 0, global step 10071: 'evaluation/pseudo_perplexity' reached 10.31734 (best 10.31734), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=10071.ckpt" as top 26 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 13:21:49.852490] Computing e^(tensor([4654.5233], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 13:21:49.858363] Ending with PPL=10.249835643115683 (duration: 0:04:10.203495) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 176.16it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 95.45 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 46.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.28 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.08it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.45 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 107.54it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.99 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 102.49it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 90.55 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 173.66it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 74.13 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 115.25it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 166.40it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 83.41 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 168.16it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 46.71 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.91it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 52.73 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.18it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 116.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 56.31 +evaluation/blimp/subject_verb_agreement: 69.83 +evaluation/blimp_average: 69.42 +Ending LM Evaluation Harness (duration: 0:07:32.909794) +Metric validation/losses/mlm_loss improved by 0.006 >= min_delta = 0.0. New best score: 2.895 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 119.33it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-18 14:16:52.499759] Computing e^(tensor([4654.1825], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 14:16:52.505354] Ending with PPL=10.24808910979312 (duration: 0:04:04.926690) +Starting LM Evaluation Harness +Running on blimp... +evaluation/blimp/anaphor_agreement: 91.72 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.51it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.67it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 68.34 +evaluation/blimp/binding: 68.85 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.34it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.66 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 76.69it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 136.94it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 89.64 +evaluation/blimp/ellipsis: 74.71 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 62.07it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 177.28it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 62.87 +evaluation/blimp/irregular_forms: 81.48 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.60it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 87.28it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 49.22 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 114.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 52.95 +evaluation/blimp/quantifiers: 56.31 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 84.31it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 68.13 +evaluation/blimp_average: 69.24 +Ending LM Evaluation Harness (duration: 0:07:24.569560) +Metric validation/losses/mlm_loss improved by 0.004 >= min_delta = 0.0. New best score: 2.891 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 15:06:57.911770] Computing e^(tensor([4653.5597], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 15:06:57.920150] Ending with PPL=10.244898431756036 (duration: 0:04:02.782348) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.97it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 93.4 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.38 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.51it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.75 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.53it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.33 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.27it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 89.25 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 147.37it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 73.56 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 69.11it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 62.32 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 160.56it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 85.85 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 137.75it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 49.81 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.96it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 55.72 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 145.53it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 55.72 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 115.01it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 68.51 +evaluation/blimp_average: 69.71666666666668 +Ending LM Evaluation Harness (duration: 0:07:25.897305) +Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.891. Wait count has been increased from 0. +Epoch 0, global step 10427: 'evaluation/pseudo_perplexity' reached 10.24490 (best 10.24490), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=10427.ckpt" as top 27 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 16:00:30.830565] Computing e^(tensor([4629.4409], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 16:00:30.836135] Ending with PPL=10.122092866739091 (duration: 0:04:05.460064) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 132.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 92.28 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.39 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.36 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.45it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.91it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 66.64 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 139.75it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 89.19 +evaluation/blimp/ellipsis: 72.17 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 107.22it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.24 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 176.81it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 84.53 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 150.77it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 91.13it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 42.38 +evaluation/blimp/npi_licensing: 53.69 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 116.32it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 54.71 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 107.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 68.35 +evaluation/blimp_average: 68.41083333333333 +Ending LM Evaluation Harness (duration: 0:07:30.691885) +Metric validation/losses/mlm_loss improved by 0.010 >= min_delta = 0.0. New best score: 2.881 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 16:47:25.208788] Computing e^(tensor([4626.5760], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 16:47:25.214497] Ending with PPL=10.107604041367654 (duration: 0:04:10.361469) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 145.20it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 81.85it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 91.46 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 87.66it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 68.77 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.59it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 69.5 +evaluation/blimp/control_raising: 66.77 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 115.16it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 88.89 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 148.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 92.24it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 74.6 +evaluation/blimp/filler_gap: 61.7 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 163.50it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 85.19 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 145.40it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 40.58 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 89.26it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 55.0 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 72.06it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 55.67 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.30it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 70.8 +evaluation/blimp_average: 69.0775 +Ending LM Evaluation Harness (duration: 0:07:30.815437) +Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.881. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 17:39:55.358083] Computing e^(tensor([4637.2732], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 17:39:55.364060] Ending with PPL=10.161809997462605 (duration: 0:04:13.121510) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 147.44it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 93.05 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 91.62it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 67.87 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 66.20it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.3 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.74it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.31 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 89.20it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 88.77 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 143.87it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 71.02 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 99.08it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 62.03 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 215.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 83.87 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 181.68it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 45.29 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 92.15it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 51.4 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 116.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 58.17 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 117.80it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +evaluation/blimp/subject_verb_agreement: 67.99 +evaluation/blimp_average: 68.58916666666666 +Ending LM Evaluation Harness (duration: 0:07:35.421525) +Metric validation/losses/mlm_loss improved by 0.002 >= min_delta = 0.0. New best score: 2.879 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 129.01it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-18 18:29:15.522991] Computing e^(tensor([4630.6410], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 18:29:15.529554] Ending with PPL=10.128168268480621 (duration: 0:04:07.560103) +Starting LM Evaluation Harness +Running on blimp... +evaluation/blimp/anaphor_agreement: 93.97 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 68.85it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.05 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 57.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 89.70it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 67.78 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.43it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 66.24 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.06it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 87.75 +evaluation/blimp/ellipsis: 72.75 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.25it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 60.85 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 156.15it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 84.43 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.97it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.82it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 47.12 +evaluation/blimp/npi_licensing: 55.85 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.83it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.30it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 57.68 +evaluation/blimp/subject_verb_agreement: 69.92 +evaluation/blimp_average: 69.36583333333333 +Ending LM Evaluation Harness (duration: 0:07:40.696162) +Metric validation/losses/mlm_loss improved by 0.002 >= min_delta = 0.0. New best score: 2.877 +Epoch 0, global step 10744: 'evaluation/pseudo_perplexity' reached 10.12817 (best 10.12817), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=10744.ckpt" as top 28 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 19:22:12.690997] Computing e^(tensor([4595.7535], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 19:22:12.696815] Ending with PPL=9.953027426759304 (duration: 0:04:08.871664) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 126.39it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 95.25 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 78.92it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.67 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 75.22it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 107.40it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 68.09 +evaluation/blimp/control_raising: 66.55 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 151.94it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 89.39 +evaluation/blimp/ellipsis: 74.71 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 91.31it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 171.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 61.45 +evaluation/blimp/irregular_forms: 84.07 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 124.24it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.868 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 20:17:17.341406] Computing e^(tensor([4608.2097], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 20:17:17.347280] Ending with PPL=10.015209071679935 (duration: 0:04:08.428792) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 146.30it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 94.73 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 91.43it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.71 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 66.98 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 116.82it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.13it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 67.57 +evaluation/blimp/determiner_noun_agreement: 89.21 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 172.59it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 71.82 +evaluation/blimp/filler_gap: 60.44 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 267.78it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 133.28it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 82.04 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 71.40it/s] +evaluation/blimp/island_effects: 47.38 +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 54.8 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 129.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 90.05it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 55.74 +evaluation/blimp/subject_verb_agreement: 68.74 +evaluation/blimp_average: 69.01333333333334 +Ending LM Evaluation Harness (duration: 0:07:38.678190) +Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.868. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 21:09:26.010804] Computing e^(tensor([4602.1549], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 21:09:26.016517] Ending with PPL=9.984935109159096 (duration: 0:04:08.889338) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 136.33it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 93.76 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.06it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.37 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 115.89it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.3 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.11it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.79 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 89.96 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 143.99it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 75.81 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 73.96it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 59.96 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 183.45it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 82.6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 148.54it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 46.19 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 72.06it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 53.55 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 144.38it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 55.26 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.90it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 70.62 +evaluation/blimp_average: 69.18083333333333 +Ending LM Evaluation Harness (duration: 0:07:35.784329) +Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 2.864 +Epoch 0, global step 11075: 'evaluation/pseudo_perplexity' reached 9.98493 (best 9.98493), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=11075.ckpt" as top 29 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 22:03:46.712246] Computing e^(tensor([4586.4753], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 22:03:46.717812] Ending with PPL=9.906961190921438 (duration: 0:04:05.930157) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 140.87it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.861 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 22:52:45.389198] Computing e^(tensor([4580.7994], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 22:52:45.394578] Ending with PPL=9.878885561214299 (duration: 0:04:09.061697) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 190.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.89it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 94.63 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 69.12 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 128.70it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 67.93 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.95it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 66.15 +evaluation/blimp/determiner_noun_agreement: 89.17 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 154.20it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 70.85it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 77.48 +evaluation/blimp/filler_gap: 60.19 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 168.52it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 179.95it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 84.43 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 102.68it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 44.28 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 142.69it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 53.98 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 119.26it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 55.67 +evaluation/blimp/subject_verb_agreement: 69.18 +evaluation/blimp_average: 69.35083333333334 +Ending LM Evaluation Harness (duration: 0:07:29.271135) +Metric validation/losses/mlm_loss improved by 0.002 >= min_delta = 0.0. New best score: 2.860 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 23:39:27.741599] Computing e^(tensor([4588.9716], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 23:39:27.747198] Ending with PPL=9.91933427993529 (duration: 0:04:10.583957) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 137.42it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 94.27 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 87.25it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.66 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.65it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 65.82 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 125.74it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 68.14 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 87.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 88.98 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 118.61it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 76.96 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 76.85it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.89 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 142.90it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 83.1 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 124.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 43.65 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.36it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 55.44 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 148.81it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 58.14 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.48it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 68.49 +evaluation/blimp_average: 69.46166666666667 +Ending LM Evaluation Harness (duration: 0:07:27.272922) +Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 2.855 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 00:24:56.029223] Computing e^(tensor([4582.2025], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 00:24:56.034768] Ending with PPL=9.885818531923595 (duration: 0:04:07.810418) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 125.22it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 93.66 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 82.75it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.7 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 77.99it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 68.12 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 133.70it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.42 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 102.14it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 89.6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 169.95it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 74.02 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 80.31it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 62.99 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 171.79it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 85.14 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 181.23it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 48.95 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.52it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 55.98 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 127.92it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 55.85 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 114.51it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 68.49 +evaluation/blimp_average: 69.82666666666667 +Ending LM Evaluation Harness (duration: 0:07:27.681245) +Metric validation/losses/mlm_loss improved by 0.001 >= min_delta = 0.0. New best score: 2.853 +Epoch 0, global step 11418: 'evaluation/pseudo_perplexity' reached 9.88582 (best 9.88582), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=11418.ckpt" as top 30 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 124.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-19 01:10:29.420140] Computing e^(tensor([4572.4202], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 01:10:29.425740] Ending with PPL=9.837583579576396 (duration: 0:04:07.461808) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 90.18it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 94.07 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.51it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 69.14 +evaluation/blimp/binding: 66.93 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.69it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 67.28 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 128.96it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 89.58 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 171.82it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 75.4 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 107.85it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.27 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 194.69it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 82.39 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 47.09 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 55.06 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 117.29it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 58.81 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.12it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 69.18 +evaluation/blimp_average: 69.68333333333334 +Ending LM Evaluation Harness (duration: 0:07:29.032038) +Metric validation/losses/mlm_loss improved by 0.007 >= min_delta = 0.0. New best score: 2.846 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 01:52:05.957843] Computing e^(tensor([4578.9960], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 01:52:05.965290] Ending with PPL=9.86998152796425 (duration: 0:04:06.171113) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 165.54it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 94.58 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.63it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.26 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.27it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 68.39 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.05it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 67.12 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.38it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 88.66 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 172.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 74.65 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 82.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 62.56 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 177.96it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 82.04 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 175.51it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 47.94 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.99it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 55.45 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 133.36it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 61.82 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.30it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 68.18 +evaluation/blimp_average: 69.97083333333335 +Ending LM Evaluation Harness (duration: 0:07:29.260428) +Metric validation/losses/mlm_loss improved by 0.001 >= min_delta = 0.0. New best score: 2.844 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 116.61it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-19 02:38:32.215177] Computing e^(tensor([4563.5285], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 02:38:32.220694] Ending with PPL=9.793944329389642 (duration: 0:04:06.438957) +Starting LM Evaluation Harness +Running on blimp... +evaluation/blimp/anaphor_agreement: 93.81 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 71.01it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.83 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 91.31it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.94 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.15it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 91.35it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +evaluation/blimp/control_raising: 66.55 +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 89.82 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.73it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 73.73 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 83.20it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 62.62 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 162.27it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 163.80it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 84.02 +evaluation/blimp/island_effects: 49.18 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.72it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 136.13it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 54.83 +evaluation/blimp/quantifiers: 61.28 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 108.93it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 67.9 +evaluation/blimp_average: 70.04249999999999 +Ending LM Evaluation Harness (duration: 0:07:35.350649) +Metric validation/losses/mlm_loss improved by 0.002 >= min_delta = 0.0. New best score: 2.842 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 03:23:00.500029] Computing e^(tensor([4548.7354], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 03:23:00.505598] Ending with PPL=9.721769790862401 (duration: 0:04:11.324837) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 120.12it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 95.09 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 89.66it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.49 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 102.80it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 68.61 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 132.17it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 80.13it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 67.59 +evaluation/blimp/determiner_noun_agreement: 88.29 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 158.68it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.35it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 74.77 +evaluation/blimp/filler_gap: 61.3 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 155.58it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 157.87it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 83.92 +evaluation/blimp/island_effects: 46.64 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 78.51it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 53.22 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 150.94it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 121.30it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 60.02 +evaluation/blimp/subject_verb_agreement: 69.74 +evaluation/blimp_average: 69.80666666666666 +Ending LM Evaluation Harness (duration: 0:07:31.084793) +Metric validation/losses/mlm_loss improved by 0.002 >= min_delta = 0.0. New best score: 2.840 +Epoch 0, global step 11804: 'evaluation/pseudo_perplexity' reached 9.72177 (best 9.72177), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=11804.ckpt" as top 31 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 04:07:41.939804] Computing e^(tensor([4538.9283], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 04:07:41.945437] Ending with PPL=9.674215430681901 (duration: 0:04:09.678245) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 166.22it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 94.84 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 79.42it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.08 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.85it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 68.91 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.68it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 67.3 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.02it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 88.19 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 126.13it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 71.94 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.47it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.39 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 170.12it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 83.46 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 166.88it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 45.48 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.05it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 54.24 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.08it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 58.24 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.96it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 68.6 +evaluation/blimp_average: 69.22250000000001 +Ending LM Evaluation Harness (duration: 0:07:34.324368) +Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 2.836 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 04:52:56.159067] Computing e^(tensor([4536.3292], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 04:52:56.164827] Ending with PPL=9.661651511926292 (duration: 0:04:10.479728) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 114.32it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 80.21it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 93.97 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.95it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 68.02 +evaluation/blimp/binding: 69.16 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 107.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 65.75 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 99.53it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 89.03 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 136.27it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.52it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 75.81 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 175.50it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 61.8 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 162.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 81.73 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 102.44it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 46.34 +evaluation/blimp/npi_licensing: 54.6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 108.15it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.32it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 60.05 +evaluation/blimp/subject_verb_agreement: 69.39 +evaluation/blimp_average: 69.6375 +Ending LM Evaluation Harness (duration: 0:07:29.637910) +Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.836. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 05:39:08.543546] Computing e^(tensor([4529.9143], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 05:39:08.549081] Ending with PPL=9.630712110978905 (duration: 0:04:02.162988) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 192.15it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 93.25 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.67 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 78.23it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.72it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 68.94 +evaluation/blimp/control_raising: 67.3 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.28it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 163.34it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 89.23 +evaluation/blimp/ellipsis: 73.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 62.94it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 188.30it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 63.52 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 175.07it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 84.83 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.832 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 176.54it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-19 06:23:39.587242] Computing e^(tensor([4541.8519], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 06:23:39.592706] Ending with PPL=9.68836746036788 (duration: 0:04:09.082115) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.70it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +evaluation/blimp/anaphor_agreement: 94.68 +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.60it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 69.13 +evaluation/blimp/binding: 69.49 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 129.83it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 67.01 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 91.94it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 88.24 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 180.40it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 73.38 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.45it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.56 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 137.18it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 84.89 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 171.53it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 49.63 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 61.37it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 58.41 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 144.52it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 61.75 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 121.48it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 68.47 +evaluation/blimp_average: 70.55333333333333 +Ending LM Evaluation Harness (duration: 0:07:28.520020) +Metric validation/losses/mlm_loss improved by 0.001 >= min_delta = 0.0. New best score: 2.831 +Epoch 0, global step 12175: 'evaluation/pseudo_perplexity' reached 9.68837 (best 9.68837), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=12175.ckpt" as top 32 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 07:08:57.845305] Computing e^(tensor([4525.3507], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 07:08:57.851319] Ending with PPL=9.608761701187566 (duration: 0:04:10.342489) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 165.89it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 93.87 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 91.04it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.72 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.32it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 126.18it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 68.86 +evaluation/blimp/control_raising: 67.54 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 78.64it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 134.61it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 88.72 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 67.70it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 74.77 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 128.96it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 60.82 +evaluation/blimp/irregular_forms: 82.39 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 122.75it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.11it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 49.29 +evaluation/blimp/npi_licensing: 58.76 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.16it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 59.12 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.21it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 69.18 +evaluation/blimp_average: 70.17 +Ending LM Evaluation Harness (duration: 0:07:33.511962) +Metric validation/losses/mlm_loss improved by 0.006 >= min_delta = 0.0. New best score: 2.825 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 153.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-19 07:52:44.540493] Computing e^(tensor([4520.1503], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 07:52:44.546049] Ending with PPL=9.583809129659985 (duration: 0:04:09.251849) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 94.68 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 81.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 68.43 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 92.58it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 68.88 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 71.21it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 67.39 +evaluation/blimp/determiner_noun_agreement: 87.8 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 173.61it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 68.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 74.25 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 108.39it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 60.97 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 172.68it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 83.0 +evaluation/blimp/island_effects: 48.32 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.61it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 56.27 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 90.78it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 57.24 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 107.28it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 69.88 +evaluation/blimp_average: 69.75916666666667 +Ending LM Evaluation Harness (duration: 0:07:27.707808) +Metric validation/losses/mlm_loss improved by 0.000 >= min_delta = 0.0. New best score: 2.825 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 08:38:21.703263] Computing e^(tensor([4508.9108], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 08:38:21.708826] Ending with PPL=9.530101755668863 (duration: 0:04:05.600710) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.58it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 94.68 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.83it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.51 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 81.34it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 68.15 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 78.45it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 67.43 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.53it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.10it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 89.34 +evaluation/blimp/ellipsis: 75.58 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 80.83it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.78 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 171.28it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 82.24 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 115.50it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 43.83 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.90it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 58.09 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.83it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 55.26 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.49it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 70.32 +evaluation/blimp_average: 69.60083333333334 +Ending LM Evaluation Harness (duration: 0:07:27.803431) +Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.825. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.817 +Epoch 0, global step 12552: 'evaluation/pseudo_perplexity' reached 9.62108 (best 9.62108), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=12552.ckpt" as top 33 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 10:08:54.322365] Computing e^(tensor([4499.5033], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 10:08:54.328150] Ending with PPL=9.485379935912023 (duration: 0:04:08.462924) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.67it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 95.76 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 46.38it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.08 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.73it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.89it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 69.98 +evaluation/blimp/control_raising: 67.48 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 125.69it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 88.84 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 153.89it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 75.58 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.54it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.25 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 183.52it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 80.56 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 139.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 46.26 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 80.58it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 57.46 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 56.47 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 127.18it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 69.39 +evaluation/blimp_average: 69.75916666666667 +Ending LM Evaluation Harness (duration: 0:07:29.115181) +Metric validation/losses/mlm_loss improved by 0.002 >= min_delta = 0.0. New best score: 2.815 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 10:51:18.579875] Computing e^(tensor([4499.0738], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 10:51:18.585494] Ending with PPL=9.483343137348157 (duration: 0:04:03.322727) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 153.80it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 95.14 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.73it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 71.89it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 69.2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 120.45it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 67.83 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.73it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 89.18 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 228.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 74.71 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.88it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 62.14 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 186.12it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 81.98 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 156.52it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 48.09 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.01it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 59.02 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 109.96it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 56.65 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 83.70it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 70.61 +evaluation/blimp_average: 70.28750000000001 +Ending LM Evaluation Harness (duration: 0:07:28.285213) +Metric validation/losses/mlm_loss improved by 0.000 >= min_delta = 0.0. New best score: 2.815 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 11:36:23.589527] Computing e^(tensor([4490.0469], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 11:36:23.596062] Ending with PPL=9.440637151800408 (duration: 0:04:09.389467) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 126.60it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 94.17 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 84.91it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 69.17 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.63it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 69.4 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.96it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 65.91 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.17it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 153.56it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 89.11 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 81.02it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 74.94 +evaluation/blimp/filler_gap: 62.93 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 194.01it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 136.24it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 80.15 +evaluation/blimp/island_effects: 48.32 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.82it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 142.83it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 58.78 +evaluation/blimp/quantifiers: 57.52 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.35it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 69.41 +evaluation/blimp_average: 69.98416666666667 +Ending LM Evaluation Harness (duration: 0:07:28.658041) +Metric validation/losses/mlm_loss improved by 0.003 >= min_delta = 0.0. New best score: 2.811 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 145.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-19 12:21:07.455908] Computing e^(tensor([4492.3433], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 12:21:07.461676] Ending with PPL=9.451483122701461 (duration: 0:04:05.136632) +Starting LM Evaluation Harness +Running on blimp... +evaluation/blimp/anaphor_agreement: 93.87 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 61.36it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.76 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 66.58it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 68.74 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.15it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 74.60it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 66.44 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 144.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 88.29 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 92.06it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 75.12 +evaluation/blimp/filler_gap: 61.14 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 183.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 81.68 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 160.25it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 108.39it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 47.46 +evaluation/blimp/npi_licensing: 58.34 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 171.11it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 58.4 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 89.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 69.23 +evaluation/blimp_average: 69.78916666666667 +Ending LM Evaluation Harness (duration: 0:07:29.288201) +Metric validation/losses/mlm_loss improved by 0.001 >= min_delta = 0.0. New best score: 2.810 +Epoch 0, global step 12942: 'evaluation/pseudo_perplexity' reached 9.45148 (best 9.45148), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=12942.ckpt" as top 34 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 13:05:09.849589] Computing e^(tensor([4484.3374], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 13:05:09.855182] Ending with PPL=9.413724755902152 (duration: 0:04:06.268996) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 152.54it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 93.56 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.66 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.54it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 69.35 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 99.22it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.35 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.52it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 87.38 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 116.92it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 76.1 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 80.94it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 62.01 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 145.38it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 80.51 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 164.75it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 48.06 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 60.34 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 99.93it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 59.94 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 108.17it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 70.17 +evaluation/blimp_average: 70.2025 +Ending LM Evaluation Harness (duration: 0:07:36.855198) +Metric validation/losses/mlm_loss improved by 0.006 >= min_delta = 0.0. New best score: 2.804 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 13:49:27.617640] Computing e^(tensor([4484.9494], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 13:49:27.623127] Ending with PPL=9.416605858214924 (duration: 0:04:09.674308) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 173.10it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 64.35it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 95.76 +evaluation/blimp/argument_structure: 68.48 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.42it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 117.38it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 70.18 +evaluation/blimp/control_raising: 66.13 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 71.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 89.14 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 155.78it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 74.42 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 90.30it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 62.29 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 165.31it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 81.07 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.94it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 49.14 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 119.43it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 60.39 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 140.29it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 59.43 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 74.48it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 70.42 +evaluation/blimp_average: 70.57083333333333 +Ending LM Evaluation Harness (duration: 0:07:29.209875) +Metric validation/losses/mlm_loss improved by 0.001 >= min_delta = 0.0. New best score: 2.803 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 14:34:40.832283] Computing e^(tensor([4458.3363], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 14:34:40.837809] Ending with PPL=9.292133115926472 (duration: 0:04:04.722974) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 185.65it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 75.32it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 94.99 +evaluation/blimp/argument_structure: 68.13 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 79.68it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 69.53 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 108.95it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 67.15 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.40it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 89.35 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 141.43it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 76.5 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 67.62it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 63.29 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 134.35it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 79.85 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 56.31it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.60it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 49.59 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.13it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 55.57 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.96it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 57.44 +evaluation/blimp/subject_verb_agreement: 68.58 +evaluation/blimp_average: 69.99750000000002 +Ending LM Evaluation Harness (duration: 0:07:37.012316) +Metric validation/losses/mlm_loss improved by 0.001 >= min_delta = 0.0. New best score: 2.801 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 15:16:50.702585] Computing e^(tensor([4453.6227], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 15:16:50.708754] Ending with PPL=9.270259323341012 (duration: 0:04:09.548592) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.32it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 76.30it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 95.04 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 51.04it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 68.11 +evaluation/blimp/binding: 69.16 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 118.57it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.13 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 70.32it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 89.39 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 141.64it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 76.27 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 84.69it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 61.75 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 153.54it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 78.63 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 118.18it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 52.39 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 49.22it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 58.59 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 120.96it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 56.93 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 68.36 +evaluation/blimp_average: 70.06249999999999 +Ending LM Evaluation Harness (duration: 0:07:35.592312) +Metric validation/losses/mlm_loss improved by 0.003 >= min_delta = 0.0. New best score: 2.799 +Epoch 0, global step 13342: 'evaluation/pseudo_perplexity' reached 9.27026 (best 9.27026), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=13342.ckpt" as top 35 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 15:59:09.505947] Computing e^(tensor([4469.0586], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 15:59:09.512013] Ending with PPL=9.342083318975396 (duration: 0:04:11.261999) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 124.47it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 95.35 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 81.54it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 69.37 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 51.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 68.31 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 94.20it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 92.80it/s] +evaluation/blimp/control_raising: 68.16 +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 88.74 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 130.80it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 74.77 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 81.40it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 60.3 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 122.78it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 79.69 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 114.32it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 47.72 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 78.75it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 57.91 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 145.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 57.16 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 72.32it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 69.65 +evaluation/blimp_average: 69.76083333333334 +Ending LM Evaluation Harness (duration: 0:07:39.940841) +Metric validation/losses/mlm_loss improved by 0.003 >= min_delta = 0.0. New best score: 2.795 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.792 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 17:24:30.849465] Computing e^(tensor([4466.2021], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 17:24:30.855421] Ending with PPL=9.3287503050264 (duration: 0:04:12.536747) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 124.50it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 93.25 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 69.75it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 69.08 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.06it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 71.22 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 125.25it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 68.38 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 89.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 90.44 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 143.74it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 72.69 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 75.13it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 62.26 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 181.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 80.15 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 163.72it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 48.02 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.28it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 57.96 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 137.78it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 58.42 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 121.42it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 70.44 +evaluation/blimp_average: 70.1925 +Ending LM Evaluation Harness (duration: 0:07:33.886598) +Metric validation/losses/mlm_loss improved by 0.003 >= min_delta = 0.0. New best score: 2.790 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 18:11:23.412889] Computing e^(tensor([4459.3465], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 18:11:23.419283] Ending with PPL=9.296828018037054 (duration: 0:04:12.076699) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 162.09it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 94.89 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 76.98it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 69.12 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.63it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 69.25 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 128.25it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 65.86 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.79it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 90.03 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 141.04it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 73.79 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 80.35it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 63.57 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 117.07it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 80.71 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 218.28it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 47.38 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.16it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 58.49 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 126.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 56.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 109.53it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 70.95 +evaluation/blimp_average: 70.07833333333333 +Ending LM Evaluation Harness (duration: 0:07:27.490763) +Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.790. Wait count has been increased from 0. +Epoch 0, global step 13728: 'evaluation/pseudo_perplexity' reached 9.29683 (best 9.27026), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=13728.ckpt" as top 36 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 18:56:41.155748] Computing e^(tensor([4446.7252], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 18:56:41.161511] Ending with PPL=9.238343771465441 (duration: 0:04:15.175389) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 154.78it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 63.02it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 95.81 +evaluation/blimp/argument_structure: 69.6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 68.00it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 70.41 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 63.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.11 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 83.54it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 89.57 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 158.92it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 74.71 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.00it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 64.13 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 184.10it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 82.29 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 152.79it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 109.02it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 48.13 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 87.17it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 57.67 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.54it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 55.82 +evaluation/blimp/subject_verb_agreement: 69.41 +evaluation/blimp_average: 70.30499999999999 +Ending LM Evaluation Harness (duration: 0:07:24.692097) +Metric validation/losses/mlm_loss improved by 0.002 >= min_delta = 0.0. New best score: 2.788 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 158.38it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-19 19:41:21.967430] Computing e^(tensor([4458.9599], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 19:41:21.973768] Ending with PPL=9.295031050993542 (duration: 0:04:09.287578) +Starting LM Evaluation Harness +Running on blimp... +evaluation/blimp/anaphor_agreement: 94.12 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 67.89it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.55 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 78.97it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 126.69it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 70.87 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 63.51it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 68.05 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 164.59it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 90.03 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.85it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 72.58 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 242.47it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 63.94 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 121.90it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 83.31 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 51.31 +evaluation/blimp/npi_licensing: 57.96 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 152.59it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 57.6 +evaluation/blimp/subject_verb_agreement: 70.61 +evaluation/blimp_average: 70.74416666666667 +Ending LM Evaluation Harness (duration: 0:07:29.355426) +Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.788. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 20:24:47.979606] Computing e^(tensor([4442.2602], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 20:24:47.985110] Ending with PPL=9.217741774597432 (duration: 0:04:17.320592) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 189.27it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 94.17 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 90.81it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.22it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 68.67 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 122.24it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.84 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.69it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 88.65 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 155.50it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 74.48 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.93it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 63.15 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 179.21it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 79.85 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 189.03it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 48.73 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 92.51it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 57.83 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 145.47it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 58.55 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.39it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 68.6 +evaluation/blimp_average: 69.84333333333333 +Ending LM Evaluation Harness (duration: 0:07:27.349326) +Metric validation/losses/mlm_loss improved by 0.002 >= min_delta = 0.0. New best score: 2.785 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 21:09:30.709954] Computing e^(tensor([4443.4388], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 21:09:30.715594] Ending with PPL=9.223175465487786 (duration: 0:04:14.879297) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 109.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 94.53 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 72.82it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 69.05 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.06it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 68.37 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 119.38it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.99 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 69.33it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 153.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 90.03 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 70.15it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +evaluation/blimp/ellipsis: 72.81 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 175.61it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 62.87 +evaluation/blimp/irregular_forms: 79.08 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 156.78it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.60it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 49.55 +evaluation/blimp/npi_licensing: 57.74 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 125.99it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 57.65 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 113.76it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 68.51 +evaluation/blimp_average: 69.765 +Ending LM Evaluation Harness (duration: 0:07:29.746671) +Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.785. Wait count has been increased from 0. +Epoch 0, global step 14114: 'evaluation/pseudo_perplexity' reached 9.22318 (best 9.22318), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=14114.ckpt" as top 37 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 120.23it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-19 21:54:17.920478] Computing e^(tensor([4449.0694], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 21:54:17.926939] Ending with PPL=9.249177952884994 (duration: 0:04:13.426976) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 95.19 +evaluation/blimp/argument_structure: 69.4 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.24it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.88 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.35it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.46 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.88it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 118.16it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 89.57 +evaluation/blimp/ellipsis: 70.38 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.66it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 63.46 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 173.00it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 80.92 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 160.05it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 49.51 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 106.58it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 57.8 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 113.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 61.46 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.62it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 68.93 +evaluation/blimp_average: 70.08 +Ending LM Evaluation Harness (duration: 0:07:28.415695) +Monitored metric validation/losses/mlm_loss did not improve since the last 2 records. Best score: 2.785. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 22:37:22.841407] Computing e^(tensor([4427.3173], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 22:37:22.846832] Ending with PPL=9.149128596748701 (duration: 0:04:14.089071) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 182.83it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 94.84 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 84.28it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 113.57it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 68.8 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.37it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 66.86 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.05it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 89.74 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 132.35it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 72.34 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 90.62it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 63.31 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 127.05it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 80.05 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 149.35it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 51.46 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 55.94 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 155.14it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 60.54 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 121.12it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 68.44 +evaluation/blimp_average: 70.10166666666667 +Ending LM Evaluation Harness (duration: 0:07:27.189532) +Metric validation/losses/mlm_loss improved by 0.010 >= min_delta = 0.0. New best score: 2.775 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 23:20:32.167632] Computing e^(tensor([4424.8852], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 23:20:32.173283] Ending with PPL=9.138009663760046 (duration: 0:04:20.572135) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 83.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 89.79it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 94.48 +evaluation/blimp/argument_structure: 69.05 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.13it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 125.14it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 67.99 +evaluation/blimp/control_raising: 66.86 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.23it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 89.94 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 138.97it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 78.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/ellipsis: 74.13 +evaluation/blimp/filler_gap: 61.7 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 165.77it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 167.95it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 83.41 +evaluation/blimp/island_effects: 48.43 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 83.43it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 57.55 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 139.38it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 58.71 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 108.89it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 68.67 +evaluation/blimp_average: 70.07666666666665 +Ending LM Evaluation Harness (duration: 0:07:32.067629) +Metric validation/losses/mlm_loss improved by 0.001 >= min_delta = 0.0. New best score: 2.774 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 00:03:20.201752] Computing e^(tensor([4425.5130], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 00:03:20.207352] Ending with PPL=9.140878567918556 (duration: 0:04:11.914492) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.70it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 84.32it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 94.02 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 85.75it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 68.08 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 101.67it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/binding: 69.41 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 70.00it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 67.3 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.91it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 89.43 +evaluation/blimp/ellipsis: 74.42 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 71.80it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 174.30it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/filler_gap: 62.53 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 16.79it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 83.36 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/island_effects: 49.07 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 127.89it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 56.68 +evaluation/blimp/quantifiers: 59.99 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 116.12it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 70.95 +evaluation/blimp_average: 70.43666666666668 +Ending LM Evaluation Harness (duration: 0:07:29.835173) +Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.774. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 00:48:11.370807] Computing e^(tensor([4409.0579], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 00:48:11.376450] Ending with PPL=9.065980196491314 (duration: 0:04:17.001096) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 189.50it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 94.22 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 89.66it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.65 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.93it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.81 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 124.21it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 68.05 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 95.87it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 89.39 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 162.55it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 70.03 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 76.56it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 64.89 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 142.15it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 80.76 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 185.00it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 51.98 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.19it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 57.71 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 118.09it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 57.47 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 111.05it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 69.47 +evaluation/blimp_average: 70.03583333333334 +Ending LM Evaluation Harness (duration: 0:07:27.536320) +Metric validation/losses/mlm_loss improved by 0.008 >= min_delta = 0.0. New best score: 2.766 +Epoch 0, global step 14495: 'evaluation/pseudo_perplexity' reached 9.06598 (best 9.06598), saving model to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(05:10:09)/seed(5501650)-text_perc(10)-vision_perc(0)-magic({'enable': True}/flava-epoch=00-step=14495.ckpt" as top 38 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 01:32:00.127945] Computing e^(tensor([4423.2741], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 01:32:00.134025] Ending with PPL=9.130651668014307 (duration: 0:04:08.469526) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.70it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 94.94 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 58.96it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.88 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.66it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 67.84 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.81it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 86.45it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 66.77 +evaluation/blimp/determiner_noun_agreement: 89.31 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 130.49it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 73.04 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.39it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 63.73 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 155.68it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 138.14it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 82.04 +evaluation/blimp/island_effects: 47.83 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 99.07it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 136.68it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/npi_licensing: 56.88 +evaluation/blimp/quantifiers: 60.02 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 116.95it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 69.2 +evaluation/blimp_average: 70.04 +Ending LM Evaluation Harness (duration: 0:07:28.571702) +Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.766. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 165.07it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +[PPL Evaluation 2023-08-20 02:15:54.599215] Computing e^(tensor([4408.9413], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 02:15:54.606550] Ending with PPL=9.06545167459758 (duration: 0:04:18.327971) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 80.95it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/anaphor_agreement: 94.53 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.86it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/argument_structure: 68.66 +evaluation/blimp/binding: 67.72 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.00it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 65.78it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/control_raising: 67.06 +evaluation/blimp/determiner_noun_agreement: 90.02 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 97.84it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 72.34 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 100.16it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 63.24 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 155.10it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 80.81 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 149.47it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 51.05 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 103.67it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 57.77 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 140.69it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 57.7 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 123.44it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 70.26 +evaluation/blimp_average: 70.09666666666668 +Ending LM Evaluation Harness (duration: 0:07:32.620256) +Monitored metric validation/losses/mlm_loss did not improve since the last 2 records. Best score: 2.766. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 02:59:18.039870] Computing e^(tensor([4405.1918], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 02:59:18.046693] Ending with PPL=9.04847186594535 (duration: 0:04:42.163144) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 146.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 94.94 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 96.77it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.96 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.79it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 68.63 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 135.53it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 67.19 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 82.79it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +evaluation/blimp/determiner_noun_agreement: 89.37 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 162.55it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/ellipsis: 76.15 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 98.67it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 63.83 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 183.49it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 161.38it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/irregular_forms: 76.03 +evaluation/blimp/island_effects: 50.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 88.02it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 59.14 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 131.35it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +evaluation/blimp/quantifiers: 62.29 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 104.97it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/subject_verb_agreement: 68.69 +evaluation/blimp_average: 70.50999999999999 +Ending LM Evaluation Harness (duration: 0:08:02.434825) +Monitored metric validation/losses/mlm_loss did not improve in the last 3 records. Best score: 2.766. Signaling model to ignore task. +Starting validation +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/plugins/environments/slurm.py:165: PossibleUserWarning: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... + rank_zero_warn( +2023-08-20 03:11:20.987058 Preparing data for datamodule: MLMDataModule +2023-08-20 03:11:20.987849 Setting up datamodule: MLMDataModule +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0] +2023-08-20 03:13:59.677779 Building val dataloader for datamodule: MLMDataModule +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 03:17:27.175390] Computing e^(tensor([4405.1918], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 03:17:27.186748] Ending with PPL=9.04847186594535 (duration: 0:03:26.685564) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 176.46it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-4e8ab6eecd031c53.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e8d7c1deeaa5e1aa.arrow +evaluation/blimp/anaphor_agreement: 94.94 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 82.44it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-22c535cfbac2ceaf.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-cbaea4d80f9d3edf/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f25663a5769fab9.arrow +evaluation/blimp/argument_structure: 68.96 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 109.90it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3505f2598f1873ba.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d69fd047613d4a23/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e3db5bbc087df317.arrow +evaluation/blimp/binding: 68.63 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 93.48it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-3e4a6e91e63c1097.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-3e7d5c6dd88a69e8/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9f46b965c1d14b10.arrow +evaluation/blimp/control_raising: 67.19 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 65.41it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-78caa3d652b7860e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d3f405b92da7a3b1/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-824c85e05a0f97d7.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 154.29it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8e1e706afe6eacae.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-793a32c37dbe640b/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6cebe459d96cbd46.arrow +evaluation/blimp/determiner_noun_agreement: 89.37 +evaluation/blimp/ellipsis: 76.15 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 78.71it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-8fea8e440d05648e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-548406f41817d8d2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-e34293eba72a1bba.arrow +evaluation/blimp/filler_gap: 63.83 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 142.20it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-380c808e85612df8.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-9fb5f1b9d0c05a98/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0b8adef642b706b3.arrow +evaluation/blimp/irregular_forms: 76.03 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 113.57it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-0405bf166d8ed47e.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-74572f7735a9d8f0/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-647c8d9e408688df.arrow +evaluation/blimp/island_effects: 50.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 108.29it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c3fe6b7eb710de6f.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4315c9ef29b23044/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-b56414c71f2efbd6.arrow +evaluation/blimp/npi_licensing: 59.14 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 135.54it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-636039eede55c884.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98bfa98da185eb25/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-89911fa93aa74ac8.arrow +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 113.64it/s] +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-c2f072d13456fd82.arrow +Loading cached processed dataset at /cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-722a146358509ecd/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-9e57e2320bafe3ec.arrow +evaluation/blimp/quantifiers: 62.29 +evaluation/blimp/subject_verb_agreement: 68.69 +evaluation/blimp_average: 70.50999999999999 +Ending LM Evaluation Harness (duration: 0:07:21.460634) +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + Validate metric DataLoader 0 +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + evaluation/blimp/anaphor_agreement 94.94000244140625 + evaluation/blimp/argument_structure 68.95999908447266 + evaluation/blimp/binding 68.62999725341797 + evaluation/blimp/control_raising 67.19000244140625 +evaluation/blimp/determiner_noun_agreement 89.37000274658203 + evaluation/blimp/ellipsis 76.1500015258789 + evaluation/blimp/filler_gap 63.83000183105469 + evaluation/blimp/irregular_forms 76.02999877929688 + evaluation/blimp/island_effects 50.900001525878906 + evaluation/blimp/npi_licensing 59.13999938964844 + evaluation/blimp/quantifiers 62.290000915527344 + evaluation/blimp/subject_verb_agreement 68.69000244140625 + evaluation/blimp_average 70.51000213623047 + evaluation/pseudo_perplexity 9.048471450805664 + validation/losses/mlm_loss 2.768446683883667 +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── \ No newline at end of file diff --git a/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..5322c985f55f8b90c9541f57ee223c2055aeec34 --- /dev/null +++ b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d96794cfa85197b0e660f59896df2ebb44a72a1b1593dddc883e7a4ca69403c +size 1323061577 diff --git a/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..54ea260353035149404013b0127f72767ad0555a --- /dev/null +++ b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35ee1f6c46eda5c4c18be0ad2d2c7b636742b82df276661b50b620f634f58bbf +size 1323061895 diff --git a/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..5185da02903b5afcaa16ca6f058a4326ffec971c --- /dev/null +++ b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcc629765c6e21950ca5442abf0417f07fe04310a3a8ca52fd14af60f4c10880 +size 1323062341 diff --git a/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f9e9fcabb97670b0fc294004bc2eb698ceddd886 --- /dev/null +++ b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fdb061037f5c5eb51bc02aedd28a9c25a5d6f4b6b442b8051b5f32a76f0bc0f +size 1323056991 diff --git a/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..e195ec9716e8b5b27226de640cfcba749993ce6d --- /dev/null +++ b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54bfa32a8db7253f8659143b442a6cae14036b302dfd1f0b7faff07c53706f01 +size 1323057373 diff --git a/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..da6b92244a416c46af5cac407eab06dbe7ad4e65 --- /dev/null +++ b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87c36c9e05527141578ccb8cf8a65d4bea6d26e62fce837eb18b9ebd1fc39ec0 +size 1323057755 diff --git a/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..4b1c0b97267e5b91a442e69cc7a104536f570024 --- /dev/null +++ b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8e471f7aa5e504561560a1a2a4510f6239a67dd56e1bddb44932b15c87a4877 +size 1323058137 diff --git a/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..3698491951273e64eac722e8b59982459b6514c6 --- /dev/null +++ b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:664814a7402c26ca0895fe55d0edcfd47ebb670c4e14483513d20b6977673368 +size 1323058647 diff --git a/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..a805a4855d90b4358f43a4db48b4b4385828e812 --- /dev/null +++ b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3fa71e6cfba5c4cb61ad22923b7b7ed4e7ee1ae43c8485d3a61d3e4d09a5219 +size 1323059093 diff --git a/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..52a4e770421847d016bbbcfdcc321ff5cd9af348 --- /dev/null +++ b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4e35f29935c84e5c133fef088639137abf0b62726e4e7da192e519e13a3cda0 +size 1323059475 diff --git a/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..7da0d91029ba0180d3b4944f9506e54f00f69b03 --- /dev/null +++ b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0b3ec7812b83b78029889231c381cbc486a301229dd2b5140ca6c3ac6271051 +size 1323059921 diff --git a/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=751.ckpt b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=751.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..0ff1f5de3304f965ce570b56487df91b03891f0b --- /dev/null +++ b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=751.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a5389eb4b608826509ce993ad4ce8739cce59506e196e7f9e3a4935ac1485d8 +size 1323056481 diff --git a/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..63d7598aa53da6ad7f23f9fd41a10e48c851dee7 --- /dev/null +++ b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2d8af0dc68b21645dd4fb8c97094f499b0bfa89349a50768b88304fde81a76c +size 1323060303 diff --git a/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..07024aa600916072a4978366f4c857683c84cf4a --- /dev/null +++ b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff53a559d9c5e48c2e27e887d0840b8c8b76a7eda003c73415e69bde3fe06914 +size 1323060749 diff --git a/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..bdb066981115b7599ad93febc2c5c1c872715d40 --- /dev/null +++ b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae524871998cbb0bdbd4f1cfd5b0c768b9c2a9bab91764a5635eb90ae0380eb5 +size 1323061131 diff --git a/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/last.ckpt b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/last.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..17ba8e9d23098be43cc52a24a0656359c9f4c368 --- /dev/null +++ b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/last.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33532fb0f366bb82d6fe40758ca0e55516e8037b9727e2c1391ffc73faae7274 +size 1323062341 diff --git a/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/main_log.log b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/main_log.log new file mode 100644 index 0000000000000000000000000000000000000000..2f54a7588f9146ba17ae4e49d23d73219b476662 --- /dev/null +++ b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/main_log.log @@ -0,0 +1,3773 @@ +[update_ckeckpoint_dir] Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/ +Text is the predominant modality (10 v.s. 0 vision), will sample proportionally for optimal BLiMP performance. +Assigning HuggingFace RAM +Found 747.32GBs of RAM available, assigning 5GBs to HuggingFace datasets (currently 5000000000 bytes). +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[rank: 0] Global seed set to 5501650 +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +MLMDataModule: batch_size is 32, num_workers is 4 +Only one modality, sampling weight will be [1.0] +2023-07-15 14:55:34.333474 Setting up datamodule: MLMDataModule +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-15 15:04:53.047160 Preparing data for datamodule: MLMDataModule +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/plugins/environments/slurm.py:165: PossibleUserWarning: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... + rank_zero_warn( +Using bfloat16 Automatic Mixed Precision (AMP) +GPU available: True (cuda), used: True +TPU available: False, using: 0 TPU cores +IPU available: False, using: 0 IPUs +HPU available: False, using: 0 HPUs +[rank: 0] Global seed set to 5501650 +Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/2 +---------------------------------------------------------------------------------------------------- +distributed_backend=nccl +All distributed processes registered. Starting with 2 processes +---------------------------------------------------------------------------------------------------- +2023-07-15 15:14:27.111924 Setting up datamodule: MLMDataModule +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed exists and is not empty. + rank_zero_warn(f"Checkpoint directory {dirpath} exists and is not empty.") +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1] + | Name | Type | Params +---------------------------------------------- +0 | model | FlavaForPreTraining | 215 M +---------------------------------------------- +161 M Trainable params +53.8 M Non-trainable params +215 M Total params +862.525 Total estimated model params size (MB) +2023-07-15 15:24:15.191780 Building train dataloader for datamodule: MLMDataModule +2023-07-15 15:24:15.898996 Building val dataloader for datamodule: MLMDataModule +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 15:43:58.258629] Computing e^(tensor([18198.7230], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 15:43:58.267622] Ending with PPL=8949.576764285066 (duration: 0:03:37.475600) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 140.09it/s] +evaluation/blimp/anaphor_agreement: 39.83 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 121.61it/s] +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 112.70it/s] +Map: 30%|██▉ | 2000/6738 [00:00<00:00, 7712.71 examples/s] + +evaluation/blimp/argument_structure: 59.44 +evaluation/blimp/binding: 64.68 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 136.21it/s] +evaluation/blimp/control_raising: 56.65 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 129.88it/s] + +Map: 80%|████████ | 6043/7542 [00:00<00:00, 7984.88 examples/s] +evaluation/blimp/determiner_noun_agreement: 50.05 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 161.95it/s] +evaluation/blimp/ellipsis: 39.2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 108.30it/s] +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 180.04it/s] +Map: 27%|██▋ | 537/1965 [00:00<00:00, 5273.06 examples/s] + +evaluation/blimp/filler_gap: 50.87 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 147.59it/s] +Map: 0%| | 0/2676 [00:00= min_delta = 0.0. New best score: 7.767 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 16:45:23.163932] Computing e^(tensor([15171.8750], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 16:45:23.168173] Ending with PPL=1970.2929001119528 (duration: 0:04:23.462371) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 139.83it/s] +evaluation/blimp/anaphor_agreement: 36.66 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.05it/s] +evaluation/blimp/argument_structure: 61.32 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 134.33it/s] +evaluation/blimp/binding: 62.82 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 131.41it/s] +evaluation/blimp/control_raising: 57.84 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 122.66it/s] +evaluation/blimp/determiner_noun_agreement: 50.17 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 168.34it/s] +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 128.92it/s] +Map: 33%|███▎ | 2139/6426 [00:00<00:00, 6361.64 examples/s] + +evaluation/blimp/ellipsis: 42.15 +evaluation/blimp/filler_gap: 71.02 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 155.50it/s] +evaluation/blimp/irregular_forms: 56.44 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 182.77it/s] +evaluation/blimp/island_effects: 49.29 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 134.93it/s] + +Map: 60%|██████ | 3981/6586 [00:00<00:00, 7986.87 examples/s] +evaluation/blimp/npi_licensing: 54.54 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 160.11it/s] +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.05it/s]mariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.05it/s]mariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +evaluation/blimp/subject_verb_agreement: 49.54 +evaluation/blimp_average: 53.60666666666666 +Ending LM Evaluation Harness (duration: 0:07:37.842020) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.488 >= min_delta = 0.0. New best score: 7.279 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 17:16:40.606277] Computing e^(tensor([14961.3306], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 17:16:40.612243] Ending with PPL=1773.4202318013438 (duration: 0:04:30.525598) +Starting LM Evaluation Harness + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 160.88it/s]mariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 160.88it/s]mariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 160.88it/s]mariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Map: 44%|████▍ | 2000/4526 [00:00<00:00, 7803.50 examples/s]eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 151.45it/s]mariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 151.45it/s]mariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 160.33it/s]mariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 160.33it/s]mariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +evaluation/blimp/irregular_forms: 49.26 +Map: 12%|█▏ | 813/6586 [00:00<00:00, 8005.99 examples/s]ceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Map: 12%|█▏ | 813/6586 [00:00<00:00, 8005.99 examples/s]ceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 128.96it/s]mariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 128.96it/s]mariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +evaluation/blimp/subject_verb_agreement: 48.42 +evaluation/blimp_average: 52.23083333333333 +Ending LM Evaluation Harness (duration: 0:07:39.419075) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.176 >= min_delta = 0.0. New best score: 7.103 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 17:46:50.008843] Computing e^(tensor([14683.7790], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 17:46:50.014750] Ending with PPL=1543.6260674350667 (duration: 0:04:33.371736) +Starting LM Evaluation Harness + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 132.12it/s]mariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 132.12it/s]mariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +evaluation/blimp/control_raising: 58.9 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 151.11it/s]mariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 151.11it/s]mariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 170.92it/s]mariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Map: 8%|▊ | 531/6586 [00:00<00:01, 5219.51 examples/s]ceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Map: 8%|▊ | 531/6586 [00:00<00:01, 5219.51 examples/s]ceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 118.62it/s]mariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 118.62it/s]mariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +evaluation/blimp/subject_verb_agreement: 49.56 +evaluation/blimp_average: 51.91833333333333 +Ending LM Evaluation Harness (duration: 0:07:40.100414) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.174 >= min_delta = 0.0. New best score: 6.929 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 18:16:04.995126] Computing e^(tensor([14353.1512], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 18:16:05.000953] Ending with PPL=1308.4200100330618 (duration: 0:04:32.925496) +Starting LM Evaluation Harness + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +evaluation/blimp/anaphor_agreement: 33.49 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 105.76it/s]mariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Map: 18%|█▊ | 809/4526 [00:00<00:00, 7951.24 examples/s]ceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Map: 18%|█▊ | 809/4526 [00:00<00:00, 7951.24 examples/s]ceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +evaluation/blimp/control_raising: 59.79 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 163.82it/s]mariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Map: 62%|██████▏ | 4000/6426 [00:00<00:00, 7804.22 examples/s]eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Map: 62%|██████▏ | 4000/6426 [00:00<00:00, 7804.22 examples/s]eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +evaluation/blimp/filler_gap: 68.49 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 154.69it/s]mariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 154.69it/s]mariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 139.14it/s]mariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 139.14it/s]mariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +evaluation/blimp/quantifiers: 34.98 +evaluation/blimp/subject_verb_agreement: 49.3 +evaluation/blimp_average: 52.41 +Ending LM Evaluation Harness (duration: 0:07:39.064816) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.162 >= min_delta = 0.0. New best score: 6.767 +100%|██████████| 1/1 [00:00<00:00, 139.14it/s]mariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 99.47it/s]]mariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 99.47it/s]]mariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation 2023-07-15 18:45:16.067381] Computing e^(tensor([14033.6852], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 18:45:16.073430] Ending with PPL=1115.259721055273 (duration: 0:04:26.175273) +Starting LM Evaluation Harness +Running on blimp... + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +evaluation/blimp/argument_structure: 61.45 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 135.53it/s]mariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Map: 52%|█████▏ | 3956/7542 [00:00<00:00, 7906.63 examples/s]eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Map: 52%|█████▏ | 3956/7542 [00:00<00:00, 7906.63 examples/s]eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +evaluation/blimp/determiner_noun_agreement: 49.54 +100%|██████████| 1/1 [00:00<00:00, 138.99it/s]mariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 138.99it/s]mariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +evaluation/blimp/ellipsis: 39.67 +Map: 0%| | 0/1965 [00:00= min_delta = 0.0. New best score: 6.632 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 19:14:19.607333] Computing e^(tensor([13645.8036], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 19:14:19.613238] Ending with PPL=918.6468610847395 (duration: 0:04:26.792218) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +evaluation/blimp/subject_verb_agreement: 50.23 +evaluation/blimp_average: 53.729166666666664 +Ending LM Evaluation Harness (duration: 0:07:39.999598) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.147 >= min_delta = 0.0. New best score: 6.484 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 19:43:03.460094] Computing e^(tensor([13036.1810], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 19:43:03.466122] Ending with PPL=677.2838681627456 (duration: 0:04:26.057432) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +evaluation/blimp/subject_verb_agreement: 51.2 +evaluation/blimp_average: 54.405833333333334 +Ending LM Evaluation Harness (duration: 0:07:39.988135) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.194 >= min_delta = 0.0. New best score: 6.291 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 20:12:07.475569] Computing e^(tensor([12160.9236], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 20:12:07.481431] Ending with PPL=437.23106155521924 (duration: 0:04:30.797897) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +evaluation/blimp/subject_verb_agreement: 49.77 +evaluation/blimp_average: 55.267500000000005 +Ending LM Evaluation Harness (duration: 0:07:39.420533) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.271 >= min_delta = 0.0. New best score: 6.020 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 20:41:21.870462] Computing e^(tensor([11125.7410], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 20:41:21.876388] Ending with PPL=260.56972649813713 (duration: 0:04:25.005556) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +evaluation/blimp/subject_verb_agreement: 51.15 +evaluation/blimp_average: 55.49833333333334 +Ending LM Evaluation Harness (duration: 0:07:40.623710) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.379 >= min_delta = 0.0. New best score: 5.640 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 21:10:26.920476] Computing e^(tensor([10383.4267], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 21:10:26.926330] Ending with PPL=179.77631203430207 (duration: 0:04:27.646721) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +evaluation/blimp/subject_verb_agreement: 52.16 +evaluation/blimp_average: 56.990833333333335 +Ending LM Evaluation Harness (duration: 0:07:40.875138) +Epoch 0, global step 1545: 'evaluation/pseudo_perplexity' reached 170.31152 (best 170.31152), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Epoch 0, global step 1545: 'evaluation/pseudo_perplexity' reached 170.31152 (best 170.31152), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 21:39:31.206254] Computing e^(tensor([9884.2604], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 21:39:31.212093] Ending with PPL=140.06830798835455 (duration: 0:04:24.519451) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +evaluation/blimp/subject_verb_agreement: 52.21 +evaluation/blimp_average: 58.19583333333335 +Ending LM Evaluation Harness (duration: 0:07:42.361444) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.265 >= min_delta = 0.0. New best score: 5.080 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 22:08:29.431846] Computing e^(tensor([9447.1785], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 22:08:29.437751] Ending with PPL=112.57157386281828 (duration: 0:04:28.386231) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +evaluation/blimp/subject_verb_agreement: 52.74 +evaluation/blimp_average: 58.74 +Ending LM Evaluation Harness (duration: 0:07:40.429056) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.182 >= min_delta = 0.0. New best score: 4.898 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +[PPL Evaluation 2023-07-15 22:37:07.924386] Computing e^(tensor([9034.7434], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 22:37:07.930240] Ending with PPL=91.59454256987006 (duration: 0:04:27.213414) +Starting LM Evaluation Harness + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +evaluation/blimp/filler_gap: 65.16 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +evaluation/blimp/subject_verb_agreement: 52.76 +evaluation/blimp_average: 59.46416666666667 +Ending LM Evaluation Harness (duration: 0:07:42.537379) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.163 >= min_delta = 0.0. New best score: 4.734 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 23:06:26.509335] Computing e^(tensor([8658.7530], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 23:06:26.515239] Ending with PPL=75.89694963284138 (duration: 0:04:30.577307) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +evaluation/blimp/subject_verb_agreement: 54.31 +evaluation/blimp_average: 59.75083333333333 +Ending LM Evaluation Harness (duration: 0:07:43.004718) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.137 >= min_delta = 0.0. New best score: 4.597 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-15 23:35:04.185686] Computing e^(tensor([8305.3769], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 23:35:04.191606] Ending with PPL=63.60476857729084 (duration: 0:04:28.573388) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +evaluation/blimp/ellipsis: 55.89 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +evaluation/blimp/subject_verb_agreement: 54.53 +evaluation/blimp_average: 60.89249999999999 +Ending LM Evaluation Harness (duration: 0:07:42.469456) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.127 >= min_delta = 0.0. New best score: 4.470 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 00:04:27.004369] Computing e^(tensor([7948.0357], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 00:04:27.010232] Ending with PPL=53.19784272538342 (duration: 0:04:27.583555) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)it/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1545.ckpt' as top 2 +evaluation/blimp/subject_verb_agreement: 55.28 +evaluation/blimp_average: 61.66249999999999 +Ending LM Evaluation Harness (duration: 0:07:41.649746) +Epoch 0, global step 2345: 'evaluation/pseudo_perplexity' reached 49.68219 (best 49.68219), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Epoch 0, global step 2345: 'evaluation/pseudo_perplexity' reached 49.68219 (best 49.68219), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 00:33:54.339574] Computing e^(tensor([7709.1107], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 00:33:54.345496] Ending with PPL=47.20762248112026 (duration: 0:04:33.146525) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + asets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + asets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +evaluation/blimp/npi_licensing: 60.08 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +evaluation/blimp/subject_verb_agreement: 55.3 +evaluation/blimp_average: 62.151666666666664 +Ending LM Evaluation Harness (duration: 0:07:40.236310) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.096 >= min_delta = 0.0. New best score: 4.241 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +[PPL Evaluation 2023-07-16 01:03:08.142832] Computing e^(tensor([7444.3384], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 01:03:08.148665] Ending with PPL=41.35400219614 (duration: 0:04:27.420070) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +evaluation/blimp/argument_structure: 64.97 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +evaluation/blimp/subject_verb_agreement: 56.75 +evaluation/blimp_average: 62.81583333333334 +Ending LM Evaluation Harness (duration: 0:07:40.919261) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.109 >= min_delta = 0.0. New best score: 4.133 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 01:32:11.835052] Computing e^(tensor([7240.7912], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 01:32:11.840924] Ending with PPL=37.35234202993724 (duration: 0:04:27.310624) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +evaluation/blimp/subject_verb_agreement: 57.52 +evaluation/blimp_average: 63.452499999999986 +Ending LM Evaluation Harness (duration: 0:07:41.060889) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.072 >= min_delta = 0.0. New best score: 4.061 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 02:01:11.298576] Computing e^(tensor([7043.3975], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 02:01:11.304490] Ending with PPL=33.84186857779526 (duration: 0:04:30.381763) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +evaluation/blimp/subject_verb_agreement: 57.42 +evaluation/blimp_average: 64.12916666666666 +Ending LM Evaluation Harness (duration: 0:07:41.158212) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.089 >= min_delta = 0.0. New best score: 3.971 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +[PPL Evaluation 2023-07-16 02:30:30.500220] Computing e^(tensor([6879.8352], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 02:30:30.506301] Ending with PPL=31.184388183696633 (duration: 0:04:29.167222) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +evaluation/blimp/subject_verb_agreement: 57.18 +evaluation/blimp_average: 64.25666666666667 +Ending LM Evaluation Harness (duration: 0:07:41.298206) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.057 >= min_delta = 0.0. New best score: 3.914 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 02:59:06.430469] Computing e^(tensor([6743.2031], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 02:59:06.436303] Ending with PPL=29.12513569270772 (duration: 0:04:28.324044) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +evaluation/blimp/subject_verb_agreement: 58.39 +evaluation/blimp_average: 64.60583333333334 +Ending LM Evaluation Harness (duration: 0:07:41.124936) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.063 >= min_delta = 0.0. New best score: 3.851 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +[PPL Evaluation 2023-07-16 03:27:50.673395] Computing e^(tensor([6595.8696], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 03:27:50.679266] Ending with PPL=27.056703510429635 (duration: 0:04:28.356198) +Starting LM Evaluation Harness + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +100%|██████████| 1/1 [00:00<00:00, 127.19it/s]mariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +100%|██████████| 1/1 [00:00<00:00, 127.19it/s]mariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2345.ckpt' as top 3 2 +evaluation/blimp/subject_verb_agreement: 59.39 +evaluation/blimp_average: 65.97333333333331 +Ending LM Evaluation Harness (duration: 0:07:40.399894) +Epoch 0, global step 3126: 'evaluation/pseudo_perplexity' reached 25.11120 (best 25.11120), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Epoch 0, global step 3126: 'evaluation/pseudo_perplexity' reached 25.11120 (best 25.11120), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +[PPL Evaluation 2023-07-16 03:57:02.936738] Computing e^(tensor([6457.0074], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 03:57:02.942634] Ending with PPL=25.241859019263856 (duration: 0:04:26.746235) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +evaluation/blimp/subject_verb_agreement: 59.28 +evaluation/blimp_average: 65.4375 +Ending LM Evaluation Harness (duration: 0:07:39.762646) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.058 >= min_delta = 0.0. New best score: 3.730 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 04:25:49.133844] Computing e^(tensor([6345.4444], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 04:25:49.139721] Ending with PPL=23.872381093441824 (duration: 0:04:26.773195) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +evaluation/blimp/subject_verb_agreement: 59.31 +evaluation/blimp_average: 65.3325 +Ending LM Evaluation Harness (duration: 0:07:40.419662) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.043 >= min_delta = 0.0. New best score: 3.686 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 04:54:38.837946] Computing e^(tensor([6248.6998], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 04:54:38.843844] Ending with PPL=22.745103298149214 (duration: 0:04:28.832520) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +100%|██████████| 1/1 [00:00<00:00, 132.66it/s]mariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +100%|██████████| 1/1 [00:00<00:00, 132.66it/s]mariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +evaluation/blimp/subject_verb_agreement: 59.98 +evaluation/blimp_average: 66.49583333333335 +Ending LM Evaluation Harness (duration: 0:07:41.118643) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.040 >= min_delta = 0.0. New best score: 3.646 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +[PPL Evaluation 2023-07-16 05:23:17.367744] Computing e^(tensor([6156.7967], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 05:23:17.373661] Ending with PPL=21.72358045821424 (duration: 0:04:25.469121) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +evaluation/blimp/determiner_noun_agreement: 88.04 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +evaluation/blimp/subject_verb_agreement: 61.01 +evaluation/blimp_average: 66.70416666666667 +Ending LM Evaluation Harness (duration: 0:07:41.431974) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.048 >= min_delta = 0.0. New best score: 3.598 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 05:52:03.523627] Computing e^(tensor([6054.8018], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 05:52:03.529548] Ending with PPL=20.643508290292353 (duration: 0:04:26.889917) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +evaluation/blimp/determiner_noun_agreement: 86.28 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +evaluation/blimp/subject_verb_agreement: 61.12 +evaluation/blimp_average: 66.21500000000002 +Ending LM Evaluation Harness (duration: 0:07:41.241768) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.050 >= min_delta = 0.0. New best score: 3.548 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +[PPL Evaluation 2023-07-16 06:20:51.596746] Computing e^(tensor([5977.3399], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 06:20:51.602731] Ending with PPL=19.859251380605244 (duration: 0:04:26.320139) +Starting LM Evaluation Harness + HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 + HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3126.ckpt' as top 4 2 +evaluation/blimp/subject_verb_agreement: 61.37 +evaluation/blimp_average: 66.3625 +Ending LM Evaluation Harness (duration: 0:07:41.288999) +Epoch 0, global step 3932: 'evaluation/pseudo_perplexity' reached 18.44866 (best 18.44866), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Epoch 0, global step 3932: 'evaluation/pseudo_perplexity' reached 18.44866 (best 18.44866), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 06:49:55.042338] Computing e^(tensor([5876.9926], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 06:49:55.048195] Ending with PPL=18.887423952041747 (duration: 0:04:29.540805) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +evaluation/blimp/subject_verb_agreement: 61.48 +evaluation/blimp_average: 66.545 +Ending LM Evaluation Harness (duration: 0:07:41.486862) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.041 >= min_delta = 0.0. New best score: 3.477 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 07:18:54.437383] Computing e^(tensor([5808.1504], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 07:18:54.443325] Ending with PPL=18.248359714363485 (duration: 0:04:29.500944) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +evaluation/blimp/subject_verb_agreement: 62.24 +evaluation/blimp_average: 67.00916666666667 +Ending LM Evaluation Harness (duration: 0:07:39.704652) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.024 >= min_delta = 0.0. New best score: 3.452 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +[PPL Evaluation 2023-07-16 07:47:26.495216] Computing e^(tensor([5746.1795], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 07:47:26.501222] Ending with PPL=17.691596456092622 (duration: 0:04:32.212179) +Starting LM Evaluation Harness + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + asets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + asets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +evaluation/blimp/determiner_noun_agreement: 88.45 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +evaluation/blimp/subject_verb_agreement: 61.82 +evaluation/blimp_average: 67.80833333333334 +Ending LM Evaluation Harness (duration: 0:07:41.126536) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.032 >= min_delta = 0.0. New best score: 3.421 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +[PPL Evaluation 2023-07-16 08:15:54.847478] Computing e^(tensor([5666.5516], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 08:15:54.858248] Ending with PPL=17.001061789979627 (duration: 0:04:28.678603) +Starting LM Evaluation Harness + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +evaluation/blimp/subject_verb_agreement: 62.89 +evaluation/blimp_average: 66.56500000000001 +Ending LM Evaluation Harness (duration: 0:07:40.494544) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.031 >= min_delta = 0.0. New best score: 3.389 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 08:44:11.042400] Computing e^(tensor([5618.5793], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 08:44:11.048203] Ending with PPL=16.598123331200654 (duration: 0:04:27.251566) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +evaluation/blimp/subject_verb_agreement: 63.67 +evaluation/blimp_average: 67.29916666666666 +Ending LM Evaluation Harness (duration: 0:07:42.865816) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.024 >= min_delta = 0.0. New best score: 3.365 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 09:13:21.752902] Computing e^(tensor([5551.8504], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 09:13:21.757133] Ending with PPL=16.05347298823114 (duration: 0:04:26.218870) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +evaluation/blimp/determiner_noun_agreement: 87.58 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +evaluation/blimp/npi_licensing: 54.52 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +evaluation/blimp/subject_verb_agreement: 63.14 +evaluation/blimp_average: 67.15249999999999 +Ending LM Evaluation Harness (duration: 0:07:40.026203) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.028 >= min_delta = 0.0. New best score: 3.337 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 09:41:56.072329] Computing e^(tensor([5502.7139], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 09:41:56.078190] Ending with PPL=15.663872184399612 (duration: 0:04:27.508760) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +evaluation/blimp/binding: 63.58 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3932.ckpt' as top 5 2 +evaluation/blimp/subject_verb_agreement: 63.81 +evaluation/blimp_average: 67.93083333333334 +Ending LM Evaluation Harness (duration: 0:07:40.638771) +Epoch 0, global step 4751: 'evaluation/pseudo_perplexity' reached 14.60338 (best 14.60338), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Epoch 0, global step 4751: 'evaluation/pseudo_perplexity' reached 14.60338 (best 14.60338), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 10:11:05.427714] Computing e^(tensor([5457.0473], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 10:11:05.433643] Ending with PPL=15.31026725906481 (duration: 0:04:25.432181) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +evaluation/blimp/subject_verb_agreement: 63.22 +evaluation/blimp_average: 67.07916666666667 +Ending LM Evaluation Harness (duration: 0:07:40.896381) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.026 >= min_delta = 0.0. New best score: 3.289 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 10:40:11.516306] Computing e^(tensor([5405.1915], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 10:40:11.522199] Ending with PPL=14.918406170908705 (duration: 0:04:26.359543) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +evaluation/blimp/binding: 62.56 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +evaluation/blimp/determiner_noun_agreement: 88.29 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +evaluation/blimp/subject_verb_agreement: 63.9 +evaluation/blimp_average: 67.09166666666665 +Ending LM Evaluation Harness (duration: 0:07:40.543835) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.017 >= min_delta = 0.0. New best score: 3.273 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 11:09:24.733229] Computing e^(tensor([5357.3875], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 11:09:24.739119] Ending with PPL=14.56605403169181 (duration: 0:04:27.031852) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +evaluation/blimp/subject_verb_agreement: 63.16 +evaluation/blimp_average: 67.63916666666665 +Ending LM Evaluation Harness (duration: 0:07:40.118806) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.027 >= min_delta = 0.0. New best score: 3.245 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 11:38:10.470035] Computing e^(tensor([5313.3528], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 11:38:10.475943] Ending with PPL=14.248853044749152 (duration: 0:04:30.380707) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +evaluation/blimp/determiner_noun_agreement: 87.79 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +evaluation/blimp/subject_verb_agreement: 64.17 +evaluation/blimp_average: 67.93833333333333 +Ending LM Evaluation Harness (duration: 0:07:39.673613) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 3.240 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +[PPL Evaluation 2023-07-16 12:07:33.821672] Computing e^(tensor([5273.5210], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 12:07:33.827562] Ending with PPL=13.96788136642895 (duration: 0:04:27.687020) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +evaluation/blimp/subject_verb_agreement: 64.17 +evaluation/blimp_average: 68.0775 +Ending LM Evaluation Harness (duration: 0:07:40.031550) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.032 >= min_delta = 0.0. New best score: 3.208 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +[PPL Evaluation 2023-07-16 12:36:38.525981] Computing e^(tensor([5225.1351], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 12:36:38.531849] Ending with PPL=13.634011931013045 (duration: 0:04:29.934031) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + asets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4751-v1.ckpt' as top 6 +evaluation/blimp/subject_verb_agreement: 63.67 +evaluation/blimp_average: 68.02083333333333 +Ending LM Evaluation Harness (duration: 0:07:40.445355) +Epoch 0, global step 5540: 'evaluation/pseudo_perplexity' reached 12.82104 (best 12.82104), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Epoch 0, global step 5540: 'evaluation/pseudo_perplexity' reached 12.82104 (best 12.82104), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 13:05:42.223022] Computing e^(tensor([5206.2418], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 13:05:42.227209] Ending with PPL=13.505822699054152 (duration: 0:04:25.671888) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +evaluation/blimp/subject_verb_agreement: 65.08 +evaluation/blimp_average: 69.015 +Ending LM Evaluation Harness (duration: 0:07:40.112260) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.017 >= min_delta = 0.0. New best score: 3.178 +[PPL Evaluation] Starting from index 0 to index 2000. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +[PPL Evaluation 2023-07-16 13:34:40.674116] Ending with PPL=13.258763846086557 (duration: 0:04:26.001906) +Starting LM Evaluation Harness + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +evaluation/blimp/subject_verb_agreement: 64.68 +evaluation/blimp_average: 69.21166666666669 +Ending LM Evaluation Harness (duration: 0:07:39.521461) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.015 >= min_delta = 0.0. New best score: 3.163 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 14:03:52.863248] Computing e^(tensor([5150.1730], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 14:03:52.869171] Ending with PPL=13.132452944542315 (duration: 0:04:28.169705) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +evaluation/blimp/determiner_noun_agreement: 88.69 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +evaluation/blimp/subject_verb_agreement: 65.19 +evaluation/blimp_average: 69.47166666666668 +Ending LM Evaluation Harness (duration: 0:07:39.651834) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.011 >= min_delta = 0.0. New best score: 3.152 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 14:32:50.228142] Computing e^(tensor([5103.8791], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 14:32:50.234359] Ending with PPL=12.831967646730009 (duration: 0:04:26.165227) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +100%|██████████| 1/1 [00:00<00:00, 125.87it/s]mariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +100%|██████████| 1/1 [00:00<00:00, 125.87it/s]mariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +evaluation/blimp/subject_verb_agreement: 65.96 +evaluation/blimp_average: 69.30333333333333 +Ending LM Evaluation Harness (duration: 0:07:40.339411) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.013 >= min_delta = 0.0. New best score: 3.138 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 15:01:58.429395] Computing e^(tensor([5077.8617], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 15:01:58.435229] Ending with PPL=12.666121988683631 (duration: 0:04:27.506650) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +evaluation/blimp/binding: 66.55 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +evaluation/blimp/subject_verb_agreement: 64.37 +evaluation/blimp_average: 68.44333333333333 +Ending LM Evaluation Harness (duration: 0:07:39.824465) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.019 >= min_delta = 0.0. New best score: 3.120 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +[PPL Evaluation 2023-07-16 15:30:20.165305] Computing e^(tensor([5067.5118], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 15:30:20.171172] Ending with PPL=12.600744683631184 (duration: 0:04:29.124974) +Starting LM Evaluation Harness + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +evaluation/blimp/npi_licensing: 53.96 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5540.ckpt' as top 7p 6 +evaluation/blimp/subject_verb_agreement: 65.76 +evaluation/blimp_average: 68.7475 +Ending LM Evaluation Harness (duration: 0:07:41.362127) +Epoch 0, global step 6350: 'evaluation/pseudo_perplexity' reached 11.80455 (best 11.80455), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Epoch 0, global step 6350: 'evaluation/pseudo_perplexity' reached 11.80455 (best 11.80455), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 15:59:14.357562] Computing e^(tensor([5037.6251], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 15:59:14.363518] Ending with PPL=12.413847221297788 (duration: 0:04:29.979406) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +evaluation/blimp/subject_verb_agreement: 64.53 +evaluation/blimp_average: 69.40749999999998 +Ending LM Evaluation Harness (duration: 0:07:41.051492) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.007 >= min_delta = 0.0. New best score: 3.106 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 16:27:46.556565] Computing e^(tensor([5007.0077], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 16:27:46.562684] Ending with PPL=12.225254353515147 (duration: 0:04:25.531321) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +evaluation/blimp/determiner_noun_agreement: 87.87 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +evaluation/blimp/subject_verb_agreement: 64.46 +evaluation/blimp_average: 68.45916666666668 +Ending LM Evaluation Harness (duration: 0:07:41.168851) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.013 >= min_delta = 0.0. New best score: 3.094 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +[PPL Evaluation 2023-07-16 16:56:34.281936] Computing e^(tensor([4973.9446], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 16:56:34.287857] Ending with PPL=12.024813663359254 (duration: 0:04:28.984764) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +evaluation/blimp/subject_verb_agreement: 65.13 +evaluation/blimp_average: 69.11166666666666 +Ending LM Evaluation Harness (duration: 0:07:41.026259) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.009 >= min_delta = 0.0. New best score: 3.084 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 17:25:02.403315] Computing e^(tensor([4963.2838], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 17:25:02.409209] Ending with PPL=11.960886763836353 (duration: 0:04:26.600698) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +evaluation/blimp/subject_verb_agreement: 65.09 +evaluation/blimp_average: 69.0425 +Ending LM Evaluation Harness (duration: 0:07:40.203208) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.018 >= min_delta = 0.0. New best score: 3.067 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 17:54:10.021137] Computing e^(tensor([4946.1417], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 17:54:10.026979] Ending with PPL=11.858807461744098 (duration: 0:04:28.246337) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +evaluation/blimp/subject_verb_agreement: 64.72 +evaluation/blimp_average: 69.31083333333335 +Ending LM Evaluation Harness (duration: 0:07:40.392422) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.007 >= min_delta = 0.0. New best score: 3.060 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +[PPL Evaluation 2023-07-16 18:23:04.952457] Computing e^(tensor([4919.1191], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 18:23:04.956591] Ending with PPL=11.699657437451537 (duration: 0:04:25.916474) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +evaluation/blimp/subject_verb_agreement: 64.99 +evaluation/blimp_average: 69.205 +Ending LM Evaluation Harness (duration: 0:07:41.153039) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.006 >= min_delta = 0.0. New best score: 3.054 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +[PPL Evaluation 2023-07-16 18:51:47.602208] Computing e^(tensor([4926.3656], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 18:51:47.608090] Ending with PPL=11.742124835684844 (duration: 0:04:26.261049) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6350.ckpt' as top 8p 6 +evaluation/blimp/subject_verb_agreement: 65.56 +evaluation/blimp_average: 69.63583333333334 +Ending LM Evaluation Harness (duration: 0:07:39.355346) +Epoch 0, global step 7126: 'evaluation/pseudo_perplexity' reached 11.07316 (best 11.07316), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Epoch 0, global step 7126: 'evaluation/pseudo_perplexity' reached 11.07316 (best 11.07316), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 19:20:40.184863] Computing e^(tensor([4897.3593], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 19:20:40.190765] Ending with PPL=11.573056104859665 (duration: 0:04:25.706552) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +evaluation/blimp/subject_verb_agreement: 64.16 +evaluation/blimp_average: 68.89333333333333 +Ending LM Evaluation Harness (duration: 0:07:39.498281) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.014 >= min_delta = 0.0. New best score: 3.036 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +[PPL Evaluation 2023-07-16 19:49:49.403140] Computing e^(tensor([4876.0028], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 19:49:49.409024] Ending with PPL=11.450133766690342 (duration: 0:04:29.431400) +Starting LM Evaluation Harness + HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +evaluation/blimp/subject_verb_agreement: 64.75 +evaluation/blimp_average: 69.86083333333333 +Ending LM Evaluation Harness (duration: 0:07:40.801756) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 3.030 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 20:18:46.315763] Computing e^(tensor([4857.5678], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 20:18:46.321648] Ending with PPL=11.345077068453913 (duration: 0:04:27.919429) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +evaluation/blimp/subject_verb_agreement: 65.31 +evaluation/blimp_average: 69.67999999999999 +Ending LM Evaluation Harness (duration: 0:07:40.568180) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.008 >= min_delta = 0.0. New best score: 3.023 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 20:47:35.770014] Computing e^(tensor([4852.8551], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 20:47:35.776055] Ending with PPL=11.318375331463729 (duration: 0:04:34.353680) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +evaluation/blimp/subject_verb_agreement: 64.88 +evaluation/blimp_average: 69.58333333333334 +Ending LM Evaluation Harness (duration: 0:07:40.175279) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.008 >= min_delta = 0.0. New best score: 3.014 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 21:16:06.374893] Computing e^(tensor([4825.6383], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 21:16:06.380811] Ending with PPL=11.165393995013902 (duration: 0:04:28.718484) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +evaluation/blimp/subject_verb_agreement: 64.95 +evaluation/blimp_average: 69.48333333333333 +Ending LM Evaluation Harness (duration: 0:07:40.356964) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 3.010 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +[PPL Evaluation 2023-07-16 21:44:28.790839] Computing e^(tensor([4820.9136], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 21:44:28.796835] Ending with PPL=11.139048556322058 (duration: 0:04:29.495541) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt' as top 9p 6 +evaluation/blimp/subject_verb_agreement: 65.28 +evaluation/blimp_average: 69.07833333333333 +Ending LM Evaluation Harness (duration: 0:07:38.618967) +Epoch 0, global step 7936: 'evaluation/pseudo_perplexity' reached 10.54593 (best 10.54593), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Epoch 0, global step 7936: 'evaluation/pseudo_perplexity' reached 10.54593 (best 10.54593), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +[PPL Evaluation 2023-07-16 22:13:26.204522] Computing e^(tensor([4816.9770], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 22:13:26.210432] Ending with PPL=11.117144622920502 (duration: 0:04:26.051912) +Starting LM Evaluation Harness + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +evaluation/blimp/subject_verb_agreement: 64.72 +evaluation/blimp_average: 69.265 +Ending LM Evaluation Harness (duration: 0:07:41.765669) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.008 >= min_delta = 0.0. New best score: 2.992 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 22:42:21.273280] Computing e^(tensor([4797.7148], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 22:42:21.279172] Ending with PPL=11.010588722597928 (duration: 0:04:28.159851) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +evaluation/blimp/subject_verb_agreement: 65.37 +evaluation/blimp_average: 69.32833333333333 +Ending LM Evaluation Harness (duration: 0:07:40.278679) +[rank: 0] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.992. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 23:11:02.441541] Computing e^(tensor([4796.1496], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 23:11:02.447524] Ending with PPL=11.001975201161708 (duration: 0:04:27.525387) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +evaluation/blimp/subject_verb_agreement: 65.29 +evaluation/blimp_average: 68.61083333333333 +Ending LM Evaluation Harness (duration: 0:07:41.060548) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.008 >= min_delta = 0.0. New best score: 2.984 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-16 23:39:15.830703] Computing e^(tensor([4764.0196], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 23:39:15.836588] Ending with PPL=10.82664017491173 (duration: 0:04:27.337466) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +evaluation/blimp/subject_verb_agreement: 66.31 +evaluation/blimp_average: 69.31166666666667 +Ending LM Evaluation Harness (duration: 0:07:39.915575) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 2.978 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 00:07:58.305315] Computing e^(tensor([4775.3089], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 00:07:58.311246] Ending with PPL=10.887925763238659 (duration: 0:04:27.120531) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +evaluation/blimp/subject_verb_agreement: 66.76 +evaluation/blimp_average: 69.43333333333332 +Ending LM Evaluation Harness (duration: 0:07:46.025785) +[rank: 0] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.978. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 00:37:05.177933] Computing e^(tensor([4754.5455], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 00:37:05.183951] Ending with PPL=10.775475464753637 (duration: 0:04:29.396562) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +evaluation/blimp/subject_verb_agreement: 66.23 +evaluation/blimp_average: 69.34416666666667 +Ending LM Evaluation Harness (duration: 0:07:41.474531) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 2.974 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 01:05:48.358510] Computing e^(tensor([4735.7631], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 01:05:48.364427] Ending with PPL=10.674754193310994 (duration: 0:04:27.679686) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7936.ckpt' as top 10 6 +evaluation/blimp/subject_verb_agreement: 66.41 +evaluation/blimp_average: 69.02499999999999 +Ending LM Evaluation Harness (duration: 0:07:40.654102) +Epoch 0, global step 8751: 'evaluation/pseudo_perplexity' reached 10.13827 (best 10.13827), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Epoch 0, global step 8751: 'evaluation/pseudo_perplexity' reached 10.13827 (best 10.13827), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +[PPL Evaluation 2023-07-17 01:34:18.074247] Computing e^(tensor([4733.9865], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 01:34:18.080246] Ending with PPL=10.66527619893915 (duration: 0:04:26.217009) +Starting LM Evaluation Harness + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +evaluation/blimp/determiner_noun_agreement: 89.11 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +evaluation/blimp/subject_verb_agreement: 66.72 +evaluation/blimp_average: 69.06500000000001 +Ending LM Evaluation Harness (duration: 0:07:40.776936) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.008 >= min_delta = 0.0. New best score: 2.961 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 02:03:10.651395] Computing e^(tensor([4711.7923], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 02:03:10.657358] Ending with PPL=10.547577032996044 (duration: 0:04:28.339202) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +evaluation/blimp/subject_verb_agreement: 67.24 +evaluation/blimp_average: 69.23416666666667 +Ending LM Evaluation Harness (duration: 0:07:40.028180) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.003 >= min_delta = 0.0. New best score: 2.958 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 02:32:19.575954] Computing e^(tensor([4714.1505], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 02:32:19.581937] Ending with PPL=10.560021050930258 (duration: 0:04:26.459877) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +evaluation/blimp/subject_verb_agreement: 65.93 +evaluation/blimp_average: 69.26333333333334 +Ending LM Evaluation Harness (duration: 0:07:40.655432) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.007 >= min_delta = 0.0. New best score: 2.951 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +[PPL Evaluation 2023-07-17 03:01:08.296266] Computing e^(tensor([4697.4575], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 03:01:08.302152] Ending with PPL=10.472248399116035 (duration: 0:04:27.961590) +Starting LM Evaluation Harness + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +evaluation/blimp/determiner_noun_agreement: 89.41 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +evaluation/blimp/npi_licensing: 54.77 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +evaluation/blimp/subject_verb_agreement: 67.71 +evaluation/blimp_average: 69.59916666666668 +Ending LM Evaluation Harness (duration: 0:07:39.264960) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.006 >= min_delta = 0.0. New best score: 2.945 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +[PPL Evaluation 2023-07-17 03:29:33.397167] Computing e^(tensor([4681.7004], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 03:29:33.403067] Ending with PPL=10.39006636954924 (duration: 0:04:32.491993) +Starting LM Evaluation Harness + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +evaluation/blimp/subject_verb_agreement: 68.53 +evaluation/blimp_average: 69.92416666666666 +Ending LM Evaluation Harness (duration: 0:07:38.860581) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.001 >= min_delta = 0.0. New best score: 2.945 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 03:57:57.201946] Computing e^(tensor([4686.7954], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 03:57:57.207915] Ending with PPL=10.41656877958814 (duration: 0:04:32.429077) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 + HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8751.ckpt' as top 11 6 +evaluation/blimp/subject_verb_agreement: 66.25 +evaluation/blimp_average: 69.405 +Ending LM Evaluation Harness (duration: 0:07:39.296108) +Epoch 0, global step 9565: 'evaluation/pseudo_perplexity' reached 9.86488 (best 9.86488), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Epoch 0, global step 9565: 'evaluation/pseudo_perplexity' reached 9.86488 (best 9.86488), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +[PPL Evaluation 2023-07-17 04:26:45.485899] Computing e^(tensor([4667.2989], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 04:26:45.491763] Ending with PPL=10.315518680169616 (duration: 0:04:25.981308) +Starting LM Evaluation Harness + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +evaluation/blimp/binding: 68.98 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +evaluation/blimp/determiner_noun_agreement: 90.23 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +evaluation/blimp/irregular_forms: 83.31 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +evaluation/blimp/quantifiers: 57.88 +evaluation/blimp/subject_verb_agreement: 66.96 +evaluation/blimp_average: 69.69500000000002 +Ending LM Evaluation Harness (duration: 0:07:41.312906) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.002 >= min_delta = 0.0. New best score: 2.937 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 04:55:28.703567] Computing e^(tensor([4666.7955], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 04:55:28.709364] Ending with PPL=10.312922676274159 (duration: 0:04:26.409130) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +evaluation/blimp/subject_verb_agreement: 66.76 +evaluation/blimp_average: 69.50999999999999 +Ending LM Evaluation Harness (duration: 0:07:41.271199) +[rank: 0] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.937. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 05:23:56.959879] Computing e^(tensor([4664.4546], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 05:23:56.965722] Ending with PPL=10.300858982141294 (duration: 0:04:26.052574) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +evaluation/blimp/subject_verb_agreement: 65.8 +evaluation/blimp_average: 69.21 +Ending LM Evaluation Harness (duration: 0:07:40.548538) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.018 >= min_delta = 0.0. New best score: 2.919 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +[PPL Evaluation 2023-07-17 05:52:33.423407] Computing e^(tensor([4675.2809], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 05:52:33.429290] Ending with PPL=10.356770434950128 (duration: 0:04:26.471126) +Starting LM Evaluation Harness + HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +evaluation/blimp/determiner_noun_agreement: 89.41 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +evaluation/blimp/filler_gap: 62.51 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +evaluation/blimp/quantifiers: 60.28 +evaluation/blimp/subject_verb_agreement: 67.06 +evaluation/blimp_average: 69.37916666666666 +Ending LM Evaluation Harness (duration: 0:07:39.676554) +[rank: 0] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.919. Wait count has been increased from 0. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +[PPL Evaluation 2023-07-17 06:21:02.014838] Computing e^(tensor([4628.2841], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 06:21:02.020836] Ending with PPL=10.116239999189522 (duration: 0:04:26.968100) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +evaluation/blimp/anaphor_agreement: 92.48 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +evaluation/blimp/subject_verb_agreement: 66.5 +evaluation/blimp_average: 69.51166666666667 +Ending LM Evaluation Harness (duration: 0:07:40.685364) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.003 >= min_delta = 0.0. New best score: 2.916 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +[PPL Evaluation 2023-07-17 06:49:19.646067] Computing e^(tensor([4619.3353], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 06:49:19.652006] Ending with PPL=10.071076770495424 (duration: 0:04:25.523764) +Starting LM Evaluation Harness + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +evaluation/blimp/subject_verb_agreement: 66.56 +evaluation/blimp_average: 69.80583333333334 +Ending LM Evaluation Harness (duration: 0:07:40.150879) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.004 >= min_delta = 0.0. New best score: 2.912 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 07:17:44.534566] Computing e^(tensor([4622.3392], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 07:17:44.540558] Ending with PPL=10.08621471016667 (duration: 0:04:29.802272) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +100%|██████████| 1/1 [00:00<00:00, 112.87it/s]mariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +100%|██████████| 1/1 [00:00<00:00, 112.87it/s]mariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)ext10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9565.ckpt' as top 1211 6 +evaluation/blimp/quantifiers: 58.5 +evaluation/blimp/subject_verb_agreement: 67.14 +evaluation/blimp_average: 69.75083333333333 +Ending LM Evaluation Harness (duration: 0:07:41.109540) +Epoch 0, global step 10376: 'evaluation/pseudo_perplexity' reached 9.59282 (best 9.59282), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Epoch 0, global step 10376: 'evaluation/pseudo_perplexity' reached 9.59282 (best 9.59282), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 07:46:31.324528] Computing e^(tensor([4606.5501], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 07:46:31.330486] Ending with PPL=10.006902018501924 (duration: 0:04:27.802890) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +evaluation/blimp/subject_verb_agreement: 65.22 +evaluation/blimp_average: 69.19416666666666 +Ending LM Evaluation Harness (duration: 0:07:39.939646) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.003 >= min_delta = 0.0. New best score: 2.904 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 08:14:32.929588] Computing e^(tensor([4591.4353], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 08:14:32.933857] Ending with PPL=9.93156088201508 (duration: 0:04:28.334626) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +evaluation/blimp/subject_verb_agreement: 67.48 +evaluation/blimp_average: 69.66 +Ending LM Evaluation Harness (duration: 0:07:40.628136) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.011 >= min_delta = 0.0. New best score: 2.893 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 08:42:53.328672] Computing e^(tensor([4587.6211], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 08:42:53.334586] Ending with PPL=9.912638554656146 (duration: 0:04:29.040264) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +evaluation/blimp/npi_licensing: 53.7 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +evaluation/blimp/subject_verb_agreement: 65.73 +evaluation/blimp_average: 69.78666666666668 +Ending LM Evaluation Harness (duration: 0:07:42.905542) +[rank: 0] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.893. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 09:11:41.414229] Computing e^(tensor([4589.1303], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 09:11:41.420244] Ending with PPL=9.920121220675531 (duration: 0:04:25.676038) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +evaluation/blimp/subject_verb_agreement: 66.81 +evaluation/blimp_average: 69.29 +Ending LM Evaluation Harness (duration: 0:07:38.383169) +[rank: 0] Monitored metric validation/losses/mlm_loss did not improve since the last 2 records. Best score: 2.893. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 09:40:41.969459] Computing e^(tensor([4572.1542], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 09:40:41.975379] Ending with PPL=9.836275268577287 (duration: 0:04:28.326412) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +evaluation/blimp/subject_verb_agreement: 67.46 +evaluation/blimp_average: 69.69916666666667 +Ending LM Evaluation Harness (duration: 0:07:40.037601) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.006 >= min_delta = 0.0. New best score: 2.887 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +[PPL Evaluation 2023-07-17 10:08:53.028035] Computing e^(tensor([4561.8768], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 10:08:53.033911] Ending with PPL=9.785859037888056 (duration: 0:04:25.752984) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +100%|██████████| 1/1 [00:00<00:00, 110.89it/s]mariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +evaluation/blimp/determiner_noun_agreement: 89.79 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10376.ckpt' as top 13 6 +evaluation/blimp/quantifiers: 56.96 +evaluation/blimp/subject_verb_agreement: 67.77 +evaluation/blimp_average: 70.2925 +Ending LM Evaluation Harness (duration: 0:07:39.658872) +Epoch 0, global step 11198: 'evaluation/pseudo_perplexity' reached 9.32132 (best 9.32132), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Epoch 0, global step 11198: 'evaluation/pseudo_perplexity' reached 9.32132 (best 9.32132), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 10:38:01.665532] Computing e^(tensor([4554.2365], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 10:38:01.671475] Ending with PPL=9.748546912882718 (duration: 0:04:27.335066) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +evaluation/blimp/subject_verb_agreement: 68.08 +evaluation/blimp_average: 69.34083333333334 +Ending LM Evaluation Harness (duration: 0:07:41.567215) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.004 >= min_delta = 0.0. New best score: 2.880 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 11:06:59.912196] Computing e^(tensor([4551.0314], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 11:06:59.918123] Ending with PPL=9.73293694817548 (duration: 0:04:27.728999) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +evaluation/blimp/subject_verb_agreement: 67.3 +evaluation/blimp_average: 69.23416666666667 +Ending LM Evaluation Harness (duration: 0:07:41.687282) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.004 >= min_delta = 0.0. New best score: 2.875 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 11:36:23.344084] Computing e^(tensor([4536.6500], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 11:36:23.350023] Ending with PPL=9.663201354654348 (duration: 0:04:30.287239) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +evaluation/blimp/determiner_noun_agreement: 88.84 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +evaluation/blimp/subject_verb_agreement: 68.83 +evaluation/blimp_average: 70.62166666666668 +Ending LM Evaluation Harness (duration: 0:07:40.112138) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.001 >= min_delta = 0.0. New best score: 2.875 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 12:05:03.317202] Computing e^(tensor([4535.0634], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 12:05:03.323169] Ending with PPL=9.655538376149105 (duration: 0:04:27.208052) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +evaluation/blimp/subject_verb_agreement: 67.24 +evaluation/blimp_average: 69.69416666666666 +Ending LM Evaluation Harness (duration: 0:07:39.701957) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.013 >= min_delta = 0.0. New best score: 2.862 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 12:33:46.720866] Computing e^(tensor([4527.2102], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 12:33:46.726751] Ending with PPL=9.617699743889112 (duration: 0:04:27.103630) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +evaluation/blimp/subject_verb_agreement: 66.61 +evaluation/blimp_average: 69.08750000000002 +Ending LM Evaluation Harness (duration: 0:07:40.964543) +[rank: 0] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.862. Wait count has been increased from 0. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +[PPL Evaluation 2023-07-17 13:02:44.602141] Computing e^(tensor([4511.7168], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 13:02:44.608042] Ending with PPL=9.543481895184163 (duration: 0:04:26.934732) +Starting LM Evaluation Harness + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +evaluation/blimp/npi_licensing: 56.53 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +evaluation/blimp/subject_verb_agreement: 66.47 +evaluation/blimp_average: 69.28666666666666 +Ending LM Evaluation Harness (duration: 0:07:41.049453) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.011 >= min_delta = 0.0. New best score: 2.850 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +[PPL Evaluation 2023-07-17 13:31:44.906087] Computing e^(tensor([4510.9435], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 13:31:44.912020] Ending with PPL=9.539792488319131 (duration: 0:04:26.848944) +Starting LM Evaluation Harness +Running on blimp... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +evaluation/blimp/argument_structure: 68.2 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11198.ckpt' as top 14 6 +evaluation/blimp/subject_verb_agreement: 67.3 +evaluation/blimp_average: 69.97583333333333 +Ending LM Evaluation Harness (duration: 0:07:44.129310) +Epoch 0, global step 12001: 'evaluation/pseudo_perplexity' reached 9.08805 (best 9.08805), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Epoch 0, global step 12001: 'evaluation/pseudo_perplexity' reached 9.08805 (best 9.08805), saving model to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 14:00:27.803237] Computing e^(tensor([4505.9503], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 14:00:27.809345] Ending with PPL=9.516005533721192 (duration: 0:04:25.288404) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 + eDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 + eDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +evaluation/blimp/subject_verb_agreement: 65.65 +evaluation/blimp_average: 69.24333333333335 +Ending LM Evaluation Harness (duration: 0:07:43.711513) +[rank: 0] Monitored metric validation/losses/mlm_loss did not improve since the last 2 records. Best score: 2.850. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 14:28:59.930029] Computing e^(tensor([4510.7124], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 14:28:59.935945] Ending with PPL=9.538690242344638 (duration: 0:04:26.475182) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 + eDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 + eDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 + HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +evaluation/blimp/determiner_noun_agreement: 90.68 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 + eDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)text10-vision0/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=12001.ckpt' as top 15 6 +evaluation/blimp/subject_verb_agreement: 67.53 +evaluation/blimp_average: 69.56083333333332 +Ending LM Evaluation Harness (duration: 0:07:40.754302) +[rank: 0] Monitored metric validation/losses/mlm_loss did not improve in the last 3 records. Best score: 2.850. Signaling model to ignore task. +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/plugins/environments/slurm.py:165: PossibleUserWarning: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/plugins/environments/slurm.py:165: PossibleUserWarning: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +2023-07-17 14:38:43.600178 Preparing data for datamodule: MLMDataModule +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Loading cached processed dataset at theodor1289wit-train[10%]gingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Loading cached processed dataset at theodor1289wit-testn[10%]gingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Loading cached processed dataset at theodor1289wit-testn[10%]gingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]1289wit-testn[10%]gingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]1289wit-testn[10%]gingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +2023-07-17 14:49:34.492158 Building val dataloader for datamodule: MLMDataModule +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-17 14:52:23.895845] Computing e^(tensor([4510.7124], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 14:52:23.902462] Ending with PPL=9.538690242344638 (duration: 0:02:48.452396) +Starting LM Evaluation Harness +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... + eDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-ae79e6548b05bf40/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-50ffc2dc33672f31/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... + eDatasets/json/default-d1126f4231c13bc2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-eee3686be74a7fc7/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-4ae88328549b04ea/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)2d4b021f398733dd84f3a6c569a7)lable on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... +evaluation/blimp/subject_verb_agreement: 67.53 +evaluation/blimp_average: 69.56083333333332 +Ending LM Evaluation Harness (duration: 0:07:32.660978) +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + Validate metric DataLoader 0 +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + evaluation/blimp/anaphor_agreement 94.98999786376953 + evaluation/blimp/argument_structure 67.27999877929688 + evaluation/blimp/binding 67.16000366210938 + evaluation/blimp/control_raising 66.41999816894531 +evaluation/blimp/determiner_noun_agreement 90.68000030517578 + evaluation/blimp/ellipsis 74.36000061035156 + evaluation/blimp/filler_gap 61.869998931884766 + evaluation/blimp/irregular_forms 80.91999816894531 + evaluation/blimp/island_effects 49.7400016784668 + evaluation/blimp/npi_licensing 55.91999816894531 + evaluation/blimp/quantifiers 57.86000061035156 + evaluation/blimp/subject_verb_agreement 67.52999877929688 + evaluation/blimp_average 69.56083679199219 + evaluation/pseudo_perplexity 9.067359924316406 + validation/losses/mlm_loss 2.8521814346313477 +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── \ No newline at end of file diff --git a/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/second_gpu.log b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/second_gpu.log new file mode 100644 index 0000000000000000000000000000000000000000..3e80b8256b8a4a6221c8f9761c6be2412738dde0 --- /dev/null +++ b/flava-wit/text10-vision0/half_bs4096_seed5501650_bf16-mixed/second_gpu.log @@ -0,0 +1,467 @@ +[update_ckeckpoint_dir] Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision0/bs4096_seed5501650_bf16-mixed/ +Text is the predominant modality (10 v.s. 0 vision), will sample proportionally for optimal BLiMP performance. +Assigning HuggingFace RAM +Found 747.32GBs of RAM available, assigning 5GBs to HuggingFace datasets (currently 5000000000 bytes). +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[rank: 1] Global seed set to 5501650 +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +MLMDataModule: batch_size is 32, num_workers is 4 +Only one modality, sampling weight will be [1.0] +2023-07-15 15:05:20.915000 Setting up datamodule: MLMDataModule +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor'] +[rank: 1] Global seed set to 5501650 +Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/2 +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-15 15:14:27.111917 Setting up datamodule: MLMDataModule +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Loading cached processed dataset at theodor1289wit-train[10%] +Counting words for split train[:10%] after collapsing... +Loading cached processed dataset at theodor1289wit-test +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-15 15:24:15.191792 Building train dataloader for datamodule: MLMDataModule +LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1] +2023-07-15 15:24:15.859659 Building val dataloader for datamodule: MLMDataModule +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 15:43:58.941471] Computing e^(tensor([18128.8627], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 15:43:58.951840] Ending with PPL=8642.363086006568 (duration: 0:03:38.169294) +[rank: 1] Metric validation/losses/mlm_loss improved. New best score: 9.020 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 16:14:45.772336] Computing e^(tensor([15762.9746], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 16:14:45.778191] Ending with PPL=2647.8077262700954 (duration: 0:04:27.604675) +[rank: 1] Metric validation/losses/mlm_loss improved by 1.253 >= min_delta = 0.0. New best score: 7.767 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 16:45:27.206333] Computing e^(tensor([15046.8851], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 16:45:27.212233] Ending with PPL=1850.9282330081776 (duration: 0:04:27.514420) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.488 >= min_delta = 0.0. New best score: 7.279 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 17:16:43.169922] Computing e^(tensor([14835.2436], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 17:16:43.175823] Ending with PPL=1665.0689493564334 (duration: 0:04:33.051131) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.176 >= min_delta = 0.0. New best score: 7.103 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 17:46:57.533155] Computing e^(tensor([14554.7926], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 17:46:57.539084] Ending with PPL=1447.2149834603997 (duration: 0:04:40.893253) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.174 >= min_delta = 0.0. New best score: 6.929 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 18:16:17.048021] Computing e^(tensor([14201.6934], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 18:16:17.053817] Ending with PPL=1212.9936660381966 (duration: 0:04:44.953646) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.162 >= min_delta = 0.0. New best score: 6.767 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 18:45:35.192575] Computing e^(tensor([13873.9739], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 18:45:35.196995] Ending with PPL=1029.6630898570393 (duration: 0:04:45.310888) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.136 >= min_delta = 0.0. New best score: 6.632 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 19:14:39.424503] Computing e^(tensor([13485.3226], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 19:14:39.430480] Ending with PPL=847.814032289094 (duration: 0:04:46.629442) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.147 >= min_delta = 0.0. New best score: 6.484 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 19:43:23.973721] Computing e^(tensor([12882.7859], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 19:43:23.979648] Ending with PPL=627.2799675496185 (duration: 0:04:46.522495) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.194 >= min_delta = 0.0. New best score: 6.291 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 20:12:26.015388] Computing e^(tensor([11995.2710], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 20:12:26.021241] Ending with PPL=402.47602219669955 (duration: 0:04:49.367632) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.271 >= min_delta = 0.0. New best score: 6.020 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 20:41:43.205756] Computing e^(tensor([10928.6739], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 20:41:43.211732] Ending with PPL=236.1192474968314 (duration: 0:04:46.352601) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.379 >= min_delta = 0.0. New best score: 5.640 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 21:10:46.451637] Computing e^(tensor([10160.9040], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 21:10:46.457584] Ending with PPL=160.84674397615396 (duration: 0:04:47.185416) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.296 >= min_delta = 0.0. New best score: 5.344 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 21:39:51.707781] Computing e^(tensor([9650.1904], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 21:39:51.713761] Ending with PPL=124.59832956870278 (duration: 0:04:45.016410) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.265 >= min_delta = 0.0. New best score: 5.080 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 22:08:48.430538] Computing e^(tensor([9198.0792], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 22:08:48.436377] Ending with PPL=99.38881576102021 (duration: 0:04:47.401169) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.182 >= min_delta = 0.0. New best score: 4.898 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 22:37:27.125266] Computing e^(tensor([8779.2553], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 22:37:27.131142] Ending with PPL=80.61039641208455 (duration: 0:04:46.410817) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.163 >= min_delta = 0.0. New best score: 4.734 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 23:06:44.470686] Computing e^(tensor([8387.0533], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 23:06:44.476706] Ending with PPL=66.25604014653881 (duration: 0:04:48.549990) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.137 >= min_delta = 0.0. New best score: 4.597 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-15 23:35:22.781895] Computing e^(tensor([8023.4376], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-15 23:35:22.787855] Ending with PPL=55.24173930062129 (duration: 0:04:47.180231) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.127 >= min_delta = 0.0. New best score: 4.470 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 00:04:47.228612] Computing e^(tensor([7664.5105], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 00:04:47.234915] Ending with PPL=46.16653705491442 (duration: 0:04:47.816792) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.133 >= min_delta = 0.0. New best score: 4.338 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 00:34:10.853858] Computing e^(tensor([7415.2122], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 00:34:10.859861] Ending with PPL=40.756123940417915 (duration: 0:04:49.638631) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.096 >= min_delta = 0.0. New best score: 4.241 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 01:03:28.567176] Computing e^(tensor([7152.9954], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 01:03:28.573144] Ending with PPL=35.74812077822303 (duration: 0:04:47.814016) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.109 >= min_delta = 0.0. New best score: 4.133 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 01:32:30.867274] Computing e^(tensor([6946.8900], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 01:32:30.873142] Ending with PPL=32.2476447240421 (duration: 0:04:46.349277) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.072 >= min_delta = 0.0. New best score: 4.061 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 02:01:27.970741] Computing e^(tensor([6752.8313], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 02:01:27.976787] Ending with PPL=29.265684106247914 (duration: 0:04:47.053383) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.089 >= min_delta = 0.0. New best score: 3.971 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 02:30:48.051779] Computing e^(tensor([6585.4508], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 02:30:48.057663] Ending with PPL=26.9161210335845 (duration: 0:04:46.726145) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.057 >= min_delta = 0.0. New best score: 3.914 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 02:59:25.076314] Computing e^(tensor([6429.5570], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 02:59:25.082276] Ending with PPL=24.89777632066803 (duration: 0:04:47.006310) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.063 >= min_delta = 0.0. New best score: 3.851 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 03:28:09.701670] Computing e^(tensor([6285.3447], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 03:28:09.707704] Ending with PPL=23.165691597254476 (duration: 0:04:47.331769) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.064 >= min_delta = 0.0. New best score: 3.787 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 03:57:22.245455] Computing e^(tensor([6159.9203], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 03:57:22.251353] Ending with PPL=21.757535509114046 (duration: 0:04:46.044102) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.058 >= min_delta = 0.0. New best score: 3.730 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 04:26:09.104004] Computing e^(tensor([6051.8718], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 04:26:09.110037] Ending with PPL=20.61328764142369 (duration: 0:04:46.786215) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.043 >= min_delta = 0.0. New best score: 3.686 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 04:54:56.608467] Computing e^(tensor([5948.6026], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 04:54:56.614500] Ending with PPL=19.575941148631916 (duration: 0:04:46.624935) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.040 >= min_delta = 0.0. New best score: 3.646 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 05:23:38.641067] Computing e^(tensor([5857.0348], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 05:23:38.645282] Ending with PPL=18.69988517569754 (duration: 0:04:46.736199) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.048 >= min_delta = 0.0. New best score: 3.598 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 05:52:22.735374] Computing e^(tensor([5759.8762], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 05:52:22.741280] Ending with PPL=17.813170564494044 (duration: 0:04:46.127134) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.050 >= min_delta = 0.0. New best score: 3.548 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 06:21:10.700512] Computing e^(tensor([5670.8995], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 06:21:10.706473] Ending with PPL=17.038061577157745 (duration: 0:04:45.445812) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.031 >= min_delta = 0.0. New best score: 3.517 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 06:50:12.976218] Computing e^(tensor([5580.0137], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 06:50:12.982174] Ending with PPL=16.28113097453333 (duration: 0:04:47.426949) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.041 >= min_delta = 0.0. New best score: 3.477 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 07:19:12.400615] Computing e^(tensor([5511.9229], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 07:19:12.404881] Ending with PPL=15.736162975944248 (duration: 0:04:47.471378) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.024 >= min_delta = 0.0. New best score: 3.452 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 07:47:44.186599] Computing e^(tensor([5441.9484], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 07:47:44.192618] Ending with PPL=15.195117960293896 (duration: 0:04:49.901273) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.032 >= min_delta = 0.0. New best score: 3.421 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 08:16:12.262776] Computing e^(tensor([5380.8338], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 08:16:12.268619] Ending with PPL=14.73781902289691 (duration: 0:04:46.089940) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.031 >= min_delta = 0.0. New best score: 3.389 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 08:44:31.584832] Computing e^(tensor([5328.4884], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 08:44:31.589229] Ending with PPL=14.357094144979621 (duration: 0:04:47.776748) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.024 >= min_delta = 0.0. New best score: 3.365 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 09:13:42.014246] Computing e^(tensor([5263.8137], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 09:13:42.020191] Ending with PPL=13.900250181235911 (duration: 0:04:46.533472) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.028 >= min_delta = 0.0. New best score: 3.337 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 09:42:14.199867] Computing e^(tensor([5211.7226], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 09:42:14.205825] Ending with PPL=13.54288507197612 (duration: 0:04:45.674707) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.022 >= min_delta = 0.0. New best score: 3.315 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 10:11:26.149818] Computing e^(tensor([5174.7102], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 10:11:26.155709] Ending with PPL=13.294562427057052 (duration: 0:04:46.186734) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.026 >= min_delta = 0.0. New best score: 3.289 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 10:40:31.886069] Computing e^(tensor([5130.9285], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 10:40:31.892052] Ending with PPL=13.006695110026035 (duration: 0:04:46.760813) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.017 >= min_delta = 0.0. New best score: 3.273 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 11:09:44.581921] Computing e^(tensor([5073.9001], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 11:09:44.587813] Ending with PPL=12.641057324600881 (duration: 0:04:46.888159) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.027 >= min_delta = 0.0. New best score: 3.245 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 11:38:27.557322] Computing e^(tensor([5050.5835], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 11:38:27.563117] Ending with PPL=12.494540194140688 (duration: 0:04:47.504389) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 3.240 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 12:07:52.545272] Computing e^(tensor([5003.6941], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 12:07:52.551190] Ending with PPL=12.205016609849677 (duration: 0:04:46.422213) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.032 >= min_delta = 0.0. New best score: 3.208 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 12:36:56.804013] Computing e^(tensor([4971.1588], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 12:36:56.809883] Ending with PPL=12.008075642849562 (duration: 0:04:48.240053) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.013 >= min_delta = 0.0. New best score: 3.195 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 13:06:03.299901] Computing e^(tensor([4930.3460], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 13:06:03.305808] Ending with PPL=11.765517594615874 (duration: 0:04:46.722271) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.017 >= min_delta = 0.0. New best score: 3.178 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 13:35:00.409835] Computing e^(tensor([4915.9020], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 13:35:00.415912] Ending with PPL=11.680853073031962 (duration: 0:04:45.737914) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.015 >= min_delta = 0.0. New best score: 3.163 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 14:04:11.430911] Computing e^(tensor([4877.5159], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 14:04:11.436821] Ending with PPL=11.458799503123318 (duration: 0:04:46.760586) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.011 >= min_delta = 0.0. New best score: 3.152 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 14:33:10.086462] Computing e^(tensor([4853.7953], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 14:33:10.092410] Ending with PPL=11.323697536555365 (duration: 0:04:46.031920) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.013 >= min_delta = 0.0. New best score: 3.138 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 15:02:18.130831] Computing e^(tensor([4820.3096], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 15:02:18.136778] Ending with PPL=11.135685082945127 (duration: 0:04:47.234756) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.019 >= min_delta = 0.0. New best score: 3.120 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 15:30:37.668291] Computing e^(tensor([4797.3102], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 15:30:37.674453] Ending with PPL=11.008361378471248 (duration: 0:04:46.631957) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.007 >= min_delta = 0.0. New best score: 3.113 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 15:59:31.467150] Computing e^(tensor([4781.3048], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 15:59:31.473051] Ending with PPL=10.920616116735738 (duration: 0:04:47.109517) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.007 >= min_delta = 0.0. New best score: 3.106 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 16:28:06.569099] Computing e^(tensor([4769.7222], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 16:28:06.574992] Ending with PPL=10.857554265386186 (duration: 0:04:45.628490) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.013 >= min_delta = 0.0. New best score: 3.094 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 16:56:53.195408] Computing e^(tensor([4729.4412], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 16:56:53.201238] Ending with PPL=10.641065400225221 (duration: 0:04:47.913909) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.009 >= min_delta = 0.0. New best score: 3.084 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 17:25:22.311551] Computing e^(tensor([4721.0314], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 17:25:22.315909] Ending with PPL=10.596414500522934 (duration: 0:04:46.562494) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.018 >= min_delta = 0.0. New best score: 3.067 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 17:54:28.529093] Computing e^(tensor([4707.5161], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 17:54:28.534998] Ending with PPL=10.525048960404906 (duration: 0:04:46.808524) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.007 >= min_delta = 0.0. New best score: 3.060 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 18:23:26.471677] Computing e^(tensor([4689.1851], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 18:23:26.477585] Ending with PPL=10.429022539097495 (duration: 0:04:47.428427) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.006 >= min_delta = 0.0. New best score: 3.054 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 18:52:08.047112] Computing e^(tensor([4684.4177], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 18:52:08.052982] Ending with PPL=10.404192736893805 (duration: 0:04:46.740314) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.004 >= min_delta = 0.0. New best score: 3.049 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 19:20:59.777639] Computing e^(tensor([4655.4140], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 19:20:59.783623] Ending with PPL=10.254401436682445 (duration: 0:04:45.356667) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.014 >= min_delta = 0.0. New best score: 3.036 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 19:50:07.001432] Computing e^(tensor([4640.3968], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 19:50:07.007376] Ending with PPL=10.177693130608489 (duration: 0:04:47.045830) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 3.030 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 20:19:06.889756] Computing e^(tensor([4632.4936], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 20:19:06.895717] Ending with PPL=10.137554376186241 (duration: 0:04:48.482978) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.008 >= min_delta = 0.0. New best score: 3.023 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 20:47:52.284357] Computing e^(tensor([4617.2953], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 20:47:52.290292] Ending with PPL=10.06080975073384 (duration: 0:04:50.862570) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.008 >= min_delta = 0.0. New best score: 3.014 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 21:16:24.923543] Computing e^(tensor([4595.8273], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 21:16:24.929417] Ending with PPL=9.953394280966032 (duration: 0:04:47.278002) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 3.010 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 21:44:47.138914] Computing e^(tensor([4595.7111], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 21:44:47.144847] Ending with PPL=9.952816466189493 (duration: 0:04:47.860061) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.010 >= min_delta = 0.0. New best score: 3.000 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 22:13:46.873652] Computing e^(tensor([4587.9910], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 22:13:46.879597] Ending with PPL=9.914471884207792 (duration: 0:04:46.746238) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.008 >= min_delta = 0.0. New best score: 2.992 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 22:42:41.061863] Computing e^(tensor([4571.0366], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 22:42:41.067794] Ending with PPL=9.830780043296569 (duration: 0:04:47.958045) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.992. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 23:11:21.972809] Computing e^(tensor([4563.4925], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 23:11:21.977072] Ending with PPL=9.793767621922237 (duration: 0:04:47.043184) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.008 >= min_delta = 0.0. New best score: 2.984 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-16 23:39:35.128172] Computing e^(tensor([4540.2056], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-16 23:39:35.134025] Ending with PPL=9.68039598283129 (duration: 0:04:46.664270) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 2.978 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 00:08:18.188354] Computing e^(tensor([4551.6806], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 00:08:18.194252] Ending with PPL=9.73609659240819 (duration: 0:04:47.023224) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.978. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 00:37:24.212146] Computing e^(tensor([4534.2147], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 00:37:24.218165] Ending with PPL=9.651441995804815 (duration: 0:04:48.422990) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 2.974 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 01:06:08.612493] Computing e^(tensor([4523.8994], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 01:06:08.618336] Ending with PPL=9.601791627772737 (duration: 0:04:47.993536) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 2.969 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 01:34:38.108523] Computing e^(tensor([4505.9601], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 01:34:38.114457] Ending with PPL=9.516051735346938 (duration: 0:04:46.247643) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.008 >= min_delta = 0.0. New best score: 2.961 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 02:03:29.293555] Computing e^(tensor([4493.2216], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 02:03:29.299465] Ending with PPL=9.455634593611125 (duration: 0:04:46.970981) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.003 >= min_delta = 0.0. New best score: 2.958 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 02:32:39.881714] Computing e^(tensor([4495.5272], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 02:32:39.887582] Ending with PPL=9.46654119517723 (duration: 0:04:46.790832) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.007 >= min_delta = 0.0. New best score: 2.951 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 03:01:27.317183] Computing e^(tensor([4479.7232], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 03:01:27.323209] Ending with PPL=9.39203142613244 (duration: 0:04:46.967158) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.006 >= min_delta = 0.0. New best score: 2.945 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 03:29:51.136837] Computing e^(tensor([4474.4659], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 03:29:51.142884] Ending with PPL=9.367375602347492 (duration: 0:04:50.218048) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.001 >= min_delta = 0.0. New best score: 2.945 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 03:58:14.633949] Computing e^(tensor([4462.8632], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 03:58:14.639903] Ending with PPL=9.313189259079865 (duration: 0:04:49.882672) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 2.939 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 04:27:05.936155] Computing e^(tensor([4467.5276], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 04:27:05.942156] Ending with PPL=9.334935002896673 (duration: 0:04:46.453959) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.002 >= min_delta = 0.0. New best score: 2.937 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 04:55:49.551876] Computing e^(tensor([4449.5206], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 04:55:49.557841] Ending with PPL=9.251265090146774 (duration: 0:04:47.252906) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.937. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 05:24:18.345109] Computing e^(tensor([4442.9353], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 05:24:18.351303] Ending with PPL=9.220853957440234 (duration: 0:04:47.400336) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.018 >= min_delta = 0.0. New best score: 2.919 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 05:52:54.481943] Computing e^(tensor([4437.3796], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 05:52:54.487824] Ending with PPL=9.195275460672004 (duration: 0:04:47.507264) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.919. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 06:21:21.292257] Computing e^(tensor([4427.6124], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 06:21:21.298186] Ending with PPL=9.150478889732927 (duration: 0:04:46.230869) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.003 >= min_delta = 0.0. New best score: 2.916 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 06:49:39.461046] Computing e^(tensor([4410.7150], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 06:49:39.467117] Ending with PPL=9.073494811355625 (duration: 0:04:45.388488) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.004 >= min_delta = 0.0. New best score: 2.912 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 07:18:02.418934] Computing e^(tensor([4416.4203], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 07:18:02.424846] Ending with PPL=9.099415395279102 (duration: 0:04:47.684884) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.004 >= min_delta = 0.0. New best score: 2.907 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 07:46:50.136141] Computing e^(tensor([4403.1098], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 07:46:50.141981] Ending with PPL=9.039057483848131 (duration: 0:04:46.618473) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.003 >= min_delta = 0.0. New best score: 2.904 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 08:14:52.515334] Computing e^(tensor([4388.6366], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 08:14:52.519635] Ending with PPL=8.97388160946113 (duration: 0:04:47.968447) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.011 >= min_delta = 0.0. New best score: 2.893 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 08:43:11.687443] Computing e^(tensor([4385.7451], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 08:43:11.693373] Ending with PPL=8.960916638488538 (duration: 0:04:47.424451) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.893. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 09:12:01.697441] Computing e^(tensor([4382.9727], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 09:12:01.703398] Ending with PPL=8.948504003269461 (duration: 0:04:45.961961) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 2 records. Best score: 2.893. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 09:40:59.659788] Computing e^(tensor([4367.8975], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 09:40:59.665686] Ending with PPL=8.881307214273752 (duration: 0:04:45.997049) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.006 >= min_delta = 0.0. New best score: 2.887 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 10:09:14.152844] Computing e^(tensor([4362.3652], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 10:09:14.158719] Ending with PPL=8.856774166791382 (duration: 0:04:46.884546) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.004 >= min_delta = 0.0. New best score: 2.883 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 10:38:19.775063] Computing e^(tensor([4356.9082], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 10:38:19.780948] Ending with PPL=8.832641327470064 (duration: 0:04:45.466491) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.004 >= min_delta = 0.0. New best score: 2.880 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 11:07:18.405356] Computing e^(tensor([4343.8260], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 11:07:18.411231] Ending with PPL=8.775054827400558 (duration: 0:04:46.199824) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.004 >= min_delta = 0.0. New best score: 2.875 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 11:36:41.360670] Computing e^(tensor([4334.8215], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 11:36:41.366667] Ending with PPL=8.735636203688317 (duration: 0:04:48.341376) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.001 >= min_delta = 0.0. New best score: 2.875 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 12:05:23.041003] Computing e^(tensor([4334.9122], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 12:05:23.046890] Ending with PPL=8.7360321244367 (duration: 0:04:46.943999) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.013 >= min_delta = 0.0. New best score: 2.862 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 12:34:05.479636] Computing e^(tensor([4344.7670], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 12:34:05.485515] Ending with PPL=8.779184129092808 (duration: 0:04:45.895145) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.862. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 13:03:05.017573] Computing e^(tensor([4324.7437], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 13:03:05.023580] Ending with PPL=8.691728541912672 (duration: 0:04:47.354836) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.011 >= min_delta = 0.0. New best score: 2.850 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 13:32:04.791948] Computing e^(tensor([4311.9507], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 13:32:04.797850] Ending with PPL=8.636309428542972 (duration: 0:04:46.720328) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.850. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 14:00:48.255978] Computing e^(tensor([4309.0015], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 14:00:48.261997] Ending with PPL=8.623583962263979 (duration: 0:04:45.747965) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 2 records. Best score: 2.850. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 14:29:19.467207] Computing e^(tensor([4302.6011], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 14:29:19.473094] Ending with PPL=8.596030541392796 (duration: 0:04:45.991208) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve in the last 3 records. Best score: 2.850. Signaling model to ignore task. +[rank: 1] Global seed set to 5501650 +Starting validation +2023-07-17 14:38:43.603026 Setting up datamodule: MLMDataModule +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1] +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-17 14:49:34.492129 Building val dataloader for datamodule: MLMDataModule +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-17 14:52:29.603121] Computing e^(tensor([4302.6011], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-17 14:52:29.609617] Ending with PPL=8.596030541392796 (duration: 0:02:54.172290) \ No newline at end of file diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=10155.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=10155.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..442456423aad05e88ab94937db60788936dcd48e --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=10155.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7692e1040453135f648289f6943ad0844e79913cb6a148d8cd830ce2a9496bf +size 3862235653 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=10376.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=10376.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..96c8b1203cfeebc57abd96ea8da348fa95eb61c1 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=10376.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c33740145367275995eb8722b1cdcf8fa1d8d950391f493235851c0f0ee4d6f +size 3862235972 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=10686.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=10686.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c5ccbf1a6cb757d053b84baa0398fd073c8b0fd0 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=10686.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4312360e9e8d14d7e7eab36ba8a55e7ca4c680fc9c3351e3d8403e61b3dcf86 +size 3862236355 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=11001.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=11001.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..cdca9c48f087d4588431df1f31e07ec0002ffdbe --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=11001.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a841d93b36011aae6ba286ddc1d3e8c6ab2bbe743fa57a5013432f687e9de42f +size 3862236866 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=11321.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=11321.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..0249cfab2cf8c086284e7d9de7221feda4b9b009 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=11321.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f1c3a1ddb5f13ff0a7f08631dd722fb5ad01746f87ad30e7664cf60403065a8 +size 3862237185 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=11626.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=11626.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..1cab912b58763a001838d44b66b6ef4ce6104376 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=11626.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2ed0b657d8fa208c8d5d968cc6455c50d6b2b8863091d02a4c4d65ee4a29b13 +size 3862237632 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=11950.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=11950.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..e5cd047e1e389453a7d54224eac208593114167a --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=11950.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed665a450e23f4e483eb85e92b272770503b282044c1aaf33685d610325d39f8 +size 3862238079 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=12251.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=12251.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..e07dabafab977055b676d6b5e06e75e48db3f943 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=12251.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:450fbbfc8b7ae0ef223933ed375c26b0b28a1aad191ef442dec359a9cdfa6cb1 +size 3862238462 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=12543.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=12543.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..88afbaadc194c440b5a2b956140d4ee915cd2a06 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=12543.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:888cc93a0e4c2112c5c702a776f15325f30643c0e78c466ec645851b484e9505 +size 3862238845 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1303.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1303.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..2856992e9c3439b4cc76703c802cfba651d141f3 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1303.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884208381081a5148f2dbcfb4dd3c8c9cdaca09af6e6444cea904c2c22d1543b +size 3848043495 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1703.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1703.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..585823b0ab26c381eda113a866873d25d0379921 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=1703.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1def266793601aa8393f9ebcfa14922525fc27786e82bbd495cb436ed96d29b +size 3848043942 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2126.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2126.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b505634b33e6fc4ae0a67ba239ee80772c7ee824 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2126.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2cc49b094da7e8fd4679d99c33b485817eb88396227cf87b161e6a4685489ab +size 3848044325 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2553.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2553.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..7dd0a8c1e8fe7e004f6e9f1d3aea0edcf4062c1f --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2553.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd16cd0f68517b3f0f0cacb4814d794dca64ebfb4fee4591256f85f9828187a4 +size 3848044772 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2960.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2960.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..9fa7cb39a0f5b1418b888403df21755bde4723f3 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=2960.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b47a010f9e07badd664f9bbcac7fa632452abb9f967ee82dafaa6a2a90fffe0e +size 3848045283 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3376.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3376.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..365c828b3c4721b0f4845cea6956c6f3a429a54e --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3376.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08a503b0907fbaabbd54bbf9f52a63bd3064662301f43c4fdc8303933d20bad8 +size 3848045602 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3771.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3771.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..830b32574cca4f10a10629b811eac98c4cd6d73a --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=3771.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9606f42f80dd4c2e0cdfcfffaa089fa8de8b3695d6f843287edc8286e37a04e +size 3848046049 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4163.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4163.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..da29f57e524afe759400d9cfbdb75c8cfb842ab9 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4163.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4096a67044c8d7b782493f12e1e1db4f9f3b8b4d4398e20ea638d76822bf6829 +size 3848046560 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=428.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=428.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..2fb12ffe4b8a9b38ef13e3648794faff97b97b17 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=428.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7e20456831b423c14203445c85fe7f4013c0f5e3fa213c4f8aa09eda5030e2a +size 3848042665 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4600.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4600.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b7547f84999a6afbeff82ae763141a03d2ed97cb --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4600.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2179bdf39b26e55c041346b14bbfbfeedb34b01882fada7d4954151095d19e71 +size 3848046879 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4953.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4953.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..ba482fa1ae41bc5878f648feeced142a3e35388d --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4953.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ed409f1dee34e108941401eb395425f2b48f867b67c7ab6c1e5244a9f9c7b7e +size 3848049001 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5330.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5330.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b422fd955ffa8d1c9ade2bba961a3de0c52ff0f6 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5330.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b979a1acfe7b81dbb5b3aa97b86f89f89365187099b1aa6e4954ae0c0428fdf2 +size 3848049384 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5687.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5687.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f418e0e254a4140772f254130b1f8b2fd0fafa3b --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=5687.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37282c112e47e8f510c20758937f43145d67823a19fe4ffa72e156e8a031f52b +size 3848049767 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=6107.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=6107.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c454cc72e8cce88c7c4d397abac82d0d794aa31f --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=6107.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:193bdc09ebbaedb74d53abd0ecc9cb8e7f2cbe4acfcca15085caba818e7ef1ad +size 3848050278 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=6425.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=6425.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..29b1b8bf7d607b41f583c14c3ab858c817f3b086 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=6425.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c365bef8a5905f73ea262ae72539aff8132a389f2844654d9f7ea85fb7d9961b +size 3848050597 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=6822.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=6822.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..61eec798c838f8f98d8e5e160f0402ae09bfc42e --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=6822.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ac7dafb034d0580ac866d7311d9b368984830d4415757d2bda39202d361960d +size 3848050980 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=7151.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=7151.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..fdae25d17aac10a636970c0cede4c2f60f592d16 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=7151.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e7cc7d3a5365edc77a2e56a7d6cd3a31f468171e21569da6afc4f41f33abfe6 +size 3848051491 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=7562.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=7562.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..38bc975acd431d23dac1df7930452ca47bd1ac1f --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=7562.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e21988bec3e682c91854a682d2041e544a9376daaeb178aa9f02a6b5f40ecf6 +size 3848051810 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=7882.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=7882.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..67fd92c4f0e6245b58f6df62755254f9cedccc1e --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=7882.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb0aab4e73895056e8ace45de371d3621f27ebad036e6e5c95652ab7259b8b88 +size 3848052257 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=8298.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=8298.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c9df01af74ede765c8435f4f7cb286019fb8502c --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=8298.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63d19bd27d673b84dc7a7be191b9728b24e059d6a0ec355e9e75662db63e40af +size 3848052640 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=8501.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=8501.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..366cbd7527a946cae7e443636a425518a0744937 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=8501.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b8d85caacb89c1adafe6cd7678fc8547821541e005633f0d6ef9d56ab4e7ef3 +size 3862232269 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=8502.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=8502.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..83d0bbe9dffaa6362b71e28885ee132393cc21b5 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=8502.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef65312d0531835a4f83e24bca8aab70b8f26d0cc321bf56c4bccae3f356a8dc +size 3848052959 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=8723.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=8723.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..9595f43cf0ad2a51dbd3d9028e259960e6cdbb7d --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=8723.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc25577ce0c6fba146d8d5060c6abea37298ebe71fa08ae968967b2de13eb82e +size 3862232652 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=876.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=876.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..991da6d3dfffd600d977865eb798f90aa81e2b69 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=876.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a7c238365ad2e7d33afc1a63760011606504622611ff7f116d430a9a3962e00 +size 3848043048 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=8909.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=8909.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..be323a55d7874544d7acdb2facf1193f225a32a1 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=8909.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d49e55e9e7e1fb881c220a79f244d7d5b89c569024ba2846509c317b4338e84 +size 3862233163 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=9126.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=9126.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..842abaac8fab867f7bec3fa17abdefe1e65831c7 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=9126.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:631ff7724d5e3426f2827b7eed621adb69cf139c52a3092397bfcc6bf6caf324 +size 3862233482 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=9350.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=9350.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..be9df1d08ec64d2ac8a23fa048d6a7ed56ccccc1 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=9350.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18c0c0d34fbffe90fa99a3d1cc7ee51fbd60778f559be46f3722d61b008815d7 +size 3862233929 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=9542.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=9542.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..88223b9ec22cd3c13b086527632a200840efa422 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=9542.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:164cd74454c3db333d7261002b1f0e80c354e1b1925655c776bfb91009e34fca +size 3862234376 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=9751.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=9751.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..ff150f39b3e6d68bbb7a95775c9783f888e1c1f9 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=9751.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05b79c887c5692412f078f871f70f3da495328693d12ce5619c3fb1c3433837b +size 3862234759 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=9966.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=9966.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..30bd9b2a6881804b1ecc22d34901bf1ceab51dad --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=9966.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9869dee8faa10c5b4005a4f9b57657d269e3773157d1b5aee8d04f673c163774 +size 3862235142 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/last-v1.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/last-v1.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..ef1cf7810555cf0192af1ce095b3b098c2eb33b3 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/last-v1.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4245f013457e6fe82b0bd3e0e8de3abab7d564c5cb19e0deca20599741db92bb +size 3862238845 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/last.ckpt b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/last.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..37bfa2ce1cfd3097d6e4f8fc1f39bebcfc2a4bd6 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/last.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8dc1af37fc01ba7460dc49a84966cd21d58b7a4723a260390e1db2aefa3ef598 +size 3848046879 diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/resume_singlenode_2023-07-09-19:15:11.log b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/resume_singlenode_2023-07-09-19:15:11.log new file mode 100644 index 0000000000000000000000000000000000000000..09f961f5460d47900152011848fdd6ee89476b97 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/resume_singlenode_2023-07-09-19:15:11.log @@ -0,0 +1,2243 @@ +wandb: Currently logged in as: tamariucai (rycolab). Use `wandb login --relogin` to force relogin +wandb: wandb version 0.15.5 is available! To upgrade, please run: +wandb: $ pip install wandb --upgrade +wandb: Tracking run with wandb version 0.15.4 +wandb: Run data is saved locally in ./wandb/run-20230709_191533-9hnl8w94 +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run dauntless-darkness-212 +wandb: ⭐️ View project at https://wandb.ai/rycolab/alkmi-wit +wandb: 🚀 View run at https://wandb.ai/rycolab/alkmi-wit/runs/9hnl8w94 +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +The current host is eu-a65-02 +The current cuDNN version is 8801 +[update_ckeckpoint_dir] Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/ +Text is the predominant modality (10 v.s. 1 vision), will sample proportionally for optimal BLiMP performance. +Assigning HuggingFace RAM +Found 747.21GBs of RAM available, assigning 5GBs to HuggingFace datasets (currently 5000000000 bytes). +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Found cached dataset imagenet-1k (/cluster/scratch/tamariucai/HuggingfaceDatasets/imagenet-1k/default/1.0.0/a1e9bfc56c3a7350165007d1176b15e9128fcaf9ab972147840529aed3ae52bc) +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/plugins/environments/slurm.py:165: PossibleUserWarning: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... + rank_zero_warn( +Using bfloat16 Automatic Mixed Precision (AMP) +GPU available: True (cuda), used: True +TPU available: False, using: 0 TPU cores +IPU available: False, using: 0 IPUs +HPU available: False, using: 0 HPUs +Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/2 +wandb: Currently logged in as: tamariucai (rycolab). Use `wandb login --relogin` to force relogin +wandb: wandb version 0.15.5 is available! To upgrade, please run: +wandb: $ pip install wandb --upgrade +wandb: Tracking run with wandb version 0.15.4 +wandb: Run data is saved locally in ./wandb/run-20230709_193040-9hnl8w94 +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run dauntless-darkness-212 +wandb: ⭐️ View project at https://wandb.ai/rycolab/alkmi-wit +wandb: 🚀 View run at https://wandb.ai/rycolab/alkmi-wit/runs/9hnl8w94 +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +The current host is eu-a65-02 +The current cuDNN version is 8801 +[update_ckeckpoint_dir] Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/ +Text is the predominant modality (10 v.s. 1 vision), will sample proportionally for optimal BLiMP performance. +Assigning HuggingFace RAM +Found 747.21GBs of RAM available, assigning 5GBs to HuggingFace datasets (currently 5000000000 bytes). +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Found cached dataset imagenet-1k (/cluster/scratch/tamariucai/HuggingfaceDatasets/imagenet-1k/default/1.0.0/a1e9bfc56c3a7350165007d1176b15e9128fcaf9ab972147840529aed3ae52bc) +Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/2 +---------------------------------------------------------------------------------------------------- +distributed_backend=nccl +All distributed processes registered. Starting with 2 processes +---------------------------------------------------------------------------------------------------- + +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at theodor1289wit-test +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed exists and is not empty. + rank_zero_warn(f"Checkpoint directory {dirpath} exists and is not empty.") +Restoring states from the checkpoint path at /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4600.ckpt +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:337: UserWarning: The dirpath has changed from "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/date(2023-07-03)_time(20:58:02)/magic({'enable': True})" to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed', therefore `best_model_score`, `kth_best_model_path`, `kth_value`, `last_model_path` and `best_k_models` won't be reloaded. Only `best_model_path` will be reloaded. + warnings.warn( +LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1] +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:337: UserWarning: The dirpath has changed from "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/date(2023-07-03)_time(20:58:02)/magic({'enable': True})" to '/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed', therefore `best_model_score`, `kth_best_model_path`, `kth_value`, `last_model_path` and `best_k_models` won't be reloaded. Only `best_model_path` will be reloaded. + warnings.warn( +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1] + + | Name | Type | Params +---------------------------------------------- +0 | model | FlavaForPreTraining | 357 M +---------------------------------------------- +303 M Trainable params +53.8 M Non-trainable params +357 M Total params +1,430.581 Total estimated model params size (MB) +Restored all states from the checkpoint at /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=4600.ckpt +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +VLDataModule: batch_size is 32, num_workers is 4 +ImageDataModule: batch_size is 32, num_workers is 4 +MLMDataModule: batch_size is 32, num_workers is 4 +Dataset sizes (un-normalized): [1, 1, 10] +Dataset sizes (normalized): [0.08333333333333333, 0.08333333333333333, 0.8333333333333334] +Sampling weights after temperature (1.0): [0.08333333333333333, 0.08333333333333333, 0.8333333333333334] +2023-07-09 19:16:18.722332 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-09 19:21:18.296225 Setting up datamodule: ImageDataModule +2023-07-09 19:21:22.087034 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Adding ImageNet zeroshot callback +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'ImageNetZeroshotCallback'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-09 19:30:22.933063 Preparing data for datamodule: VLDataModule +2023-07-09 19:30:22.933238 Preparing data for datamodule: ImageDataModule +2023-07-09 19:30:22.933350 Preparing data for datamodule: MLMDataModule +2023-07-09 19:44:21.073917 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-09 19:48:41.391375 Setting up datamodule: ImageDataModule +2023-07-09 19:48:45.291750 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-09 19:58:39.238393 Building train dataloader for datamodule: VLDataModule +2023-07-09 19:58:39.239141 Building train dataloader for datamodule: ImageDataModule +2023-07-09 19:58:39.239437 Building train dataloader for datamodule: MLMDataModule +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +VLDataModule: batch_size is 32, num_workers is 4 +ImageDataModule: batch_size is 32, num_workers is 4 +MLMDataModule: batch_size is 32, num_workers is 4 +Dataset sizes (un-normalized): [1, 1, 10] +Dataset sizes (normalized): [0.08333333333333333, 0.08333333333333333, 0.8333333333333334] +Sampling weights after temperature (1.0): [0.08333333333333333, 0.08333333333333333, 0.8333333333333334] +2023-07-09 19:31:08.350050 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-09 19:34:58.723413 Setting up datamodule: ImageDataModule +2023-07-09 19:35:03.179081 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Adding ImageNet zeroshot callback +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'ImageNetZeroshotCallback'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-09 19:44:21.073876 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-09 19:48:41.374155 Setting up datamodule: ImageDataModule +2023-07-09 19:48:45.090311 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-09 19:58:39.238511 Building train dataloader for datamodule: VLDataModule +2023-07-09 19:58:39.239359 Building train dataloader for datamodule: ImageDataModule +2023-07-09 19:58:39.239655 Building train dataloader for datamodule: MLMDataModule +2023-07-09 19:58:41.744906 Building val dataloader for datamodule: VLDataModule +2023-07-09 19:58:41.745946 Building val dataloader for datamodule: ImageDataModule +2023-07-09 19:58:41.746366 Building val dataloader for datamodule: MLMDataModule +2023-07-09 19:58:41.747242 Building val dataloader for datamodule: VLDataModule +2023-07-09 19:58:41.748205 Building val dataloader for datamodule: ImageDataModule +2023-07-09 19:58:41.748586 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 20:28:21.191786] Computing e^(tensor([5505.2625], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 20:28:21.203356] Ending with PPL=15.683845375877569 (duration: 0:19:59.949395) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.130 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 7 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 37 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 13 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 26 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.330. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 22:25:38.219116] Computing e^(tensor([5147.4765], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 22:25:38.226172] Ending with PPL=13.114759024971622 (duration: 0:12:39.363625) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.108 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 8 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 38 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 14 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 27 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.330. Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-09 23:22:24.183939] Computing e^(tensor([5104.3780], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 23:22:24.190886] Ending with PPL=12.835169040432374 (duration: 0:13:01.476184) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.093 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 9 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 39 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 15 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 28 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Metric validation/losses/mmm_text_loss improved by 0.041 >= min_delta = 0.0. New best score: 3.289 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 00:18:27.136628] Computing e^(tensor([5059.3839], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 00:18:27.141589] Ending with PPL=12.549639819529933 (duration: 0:12:07.331737) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.077 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 10 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 40 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 16 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 29 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Metric validation/losses/mmm_text_loss improved by 0.007 >= min_delta = 0.0. New best score: 3.282 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 01:23:30.479177] Computing e^(tensor([5038.2000], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 01:23:30.486220] Ending with PPL=12.417415823989177 (duration: 0:18:58.531335) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.063 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 11 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 41 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 17 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 30 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Metric validation/losses/mmm_text_loss improved by 0.031 >= min_delta = 0.0. New best score: 3.251 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 02:31:36.813862] Computing e^(tensor([5018.3764], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 02:31:36.820907] Ending with PPL=12.294945114728788 (duration: 0:22:29.939642) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.051 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 12 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 42 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 18 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 31 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Metric validation/losses/mmm_text_loss improved by 0.003 >= min_delta = 0.0. New best score: 3.248 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 03:33:03.839681] Computing e^(tensor([4989.4089], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 03:33:03.846786] Ending with PPL=12.118151258678708 (duration: 0:18:00.473902) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.042 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 13 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 43 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 19 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 32 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Metric validation/losses/mmm_text_loss improved by 0.017 >= min_delta = 0.0. New best score: 3.231 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 04:29:52.187048] Computing e^(tensor([4956.8648], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 04:29:52.192743] Ending with PPL=11.922559772837756 (duration: 0:12:44.310688) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.029 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 14 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 44 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 20 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 33 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Metric validation/losses/mmm_text_loss improved by 0.017 >= min_delta = 0.0. New best score: 3.214 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 05:25:18.210773] Computing e^(tensor([4938.9870], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 05:25:18.217750] Ending with PPL=11.816460069216118 (duration: 0:12:48.225313) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.012 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 15 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 45 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 21 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 34 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Metric validation/losses/mmm_text_loss improved by 0.015 >= min_delta = 0.0. New best score: 3.199 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 06:21:14.089945] Computing e^(tensor([4900.1674], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 06:21:14.097045] Ending with PPL=11.58931697258632 (duration: 0:12:34.522269) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.002 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 16 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 46 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 22 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 35 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Metric validation/losses/mmm_text_loss improved by 0.010 >= min_delta = 0.0. New best score: 3.188 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 07:17:44.442789] Computing e^(tensor([4879.1981], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 07:17:44.447813] Ending with PPL=11.468441518671739 (duration: 0:12:58.350217) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.999 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 17 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 47 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 23 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 36 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Metric validation/losses/mmm_text_loss improved by 0.006 >= min_delta = 0.0. New best score: 3.183 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 08:20:08.210058] Computing e^(tensor([4879.2598], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 08:20:08.215941] Ending with PPL=11.468795574081437 (duration: 0:17:21.896978) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.988 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 18 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 48 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 24 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 37 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Metric validation/losses/mmm_text_loss improved by 0.012 >= min_delta = 0.0. New best score: 3.170 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 09:29:22.615546] Computing e^(tensor([4855.6971], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 09:29:22.622505] Ending with PPL=11.334470484459404 (duration: 0:23:16.975015) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.977 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 19 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 49 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 25 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 38 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Metric validation/losses/mmm_text_loss improved by 0.003 >= min_delta = 0.0. New best score: 3.168 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 10:31:51.566635] Computing e^(tensor([4833.3703], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 10:31:51.573706] Ending with PPL=11.208642864942096 (duration: 0:17:27.599678) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.972 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 20 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 50 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 26 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 39 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Metric validation/losses/mmm_text_loss improved by 0.006 >= min_delta = 0.0. New best score: 3.162 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 11:29:33.439654] Computing e^(tensor([4805.9641], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 11:29:33.446836] Ending with PPL=11.05609686704331 (duration: 0:12:00.451163) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.955 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 21 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 51 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 27 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 40 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Metric validation/losses/mmm_text_loss improved by 0.013 >= min_delta = 0.0. New best score: 3.149 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 12:27:43.199967] Computing e^(tensor([4794.0089], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 12:27:43.207094] Ending with PPL=10.990205448609897 (duration: 0:12:51.460523) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.949 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 22 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 52 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 28 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 41 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.149. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 13:25:02.570993] Computing e^(tensor([4761.6697], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 13:25:02.578036] Ending with PPL=10.813927213204735 (duration: 0:12:40.393227) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.940 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 23 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 53 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 29 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 42 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Metric validation/losses/mmm_text_loss improved by 0.032 >= min_delta = 0.0. New best score: 3.117 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 14:31:04.773310] Computing e^(tensor([4755.6066], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 14:31:04.780309] Ending with PPL=10.781193615208359 (duration: 0:19:42.527889) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.932 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 24 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 54 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 30 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 43 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Metric validation/losses/mmm_text_loss improved by 0.010 >= min_delta = 0.0. New best score: 3.108 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 15:38:10.567322] Computing e^(tensor([4740.6892], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 15:38:10.574367] Ending with PPL=10.701079039540957 (duration: 0:20:03.049621) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.924 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 25 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 55 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 31 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 44 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Metric validation/losses/mmm_text_loss improved by 0.011 >= min_delta = 0.0. New best score: 3.097 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 16:37:44.760783] Computing e^(tensor([4720.6070], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 16:37:44.767907] Ending with PPL=10.59416618598225 (duration: 0:15:28.471586) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.915 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 26 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 56 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 32 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 45 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.097. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 17:35:03.443956] Computing e^(tensor([4709.4160], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 17:35:03.451051] Ending with PPL=10.535052176138757 (duration: 0:13:21.318916) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.911 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 27 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 57 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 33 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 46 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Metric validation/losses/mmm_text_loss improved by 0.010 >= min_delta = 0.0. New best score: 3.087 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 18:31:25.084431] Computing e^(tensor([4686.9820], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 18:31:25.091560] Ending with PPL=10.417540855111952 (duration: 0:12:00.132883) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.901 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 28 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 58 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 34 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 47 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Metric validation/losses/mmm_text_loss improved by 0.039 >= min_delta = 0.0. New best score: 3.048 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 19:28:11.066346] Computing e^(tensor([4700.7671], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 19:28:11.073309] Ending with PPL=10.48959244033098 (duration: 0:12:47.265039) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.047 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 20:29:59.135057] Computing e^(tensor([4657.3926], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 20:29:59.142195] Ending with PPL=10.264550786271771 (duration: 0:16:35.207364) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.883 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 30 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 60 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 36 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 49 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.047. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 21:40:22.247508] Computing e^(tensor([4647.7235], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 21:40:22.254585] Ending with PPL=10.215046082460166 (duration: 0:23:33.937280) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.880 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 31 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 61 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 37 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 50 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.047. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 22:42:36.261845] Computing e^(tensor([4629.6177], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 22:42:36.268827] Ending with PPL=10.122987818471294 (duration: 0:17:57.822221) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.875 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 32 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 62 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 38 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 51 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.047. Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-10 23:39:12.369426] Computing e^(tensor([4623.8358], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 23:39:12.375914] Ending with PPL=10.093765122848597 (duration: 0:12:33.594634) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00 + main() + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 169, in main + trainer.fit(model, datamodule=datamodule, ckpt_path=config.training.lightning_load_from_checkpoint) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 531, in fit + call._call_and_handle_interrupt( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 41, in _call_and_handle_interrupt + return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 91, in launch + return function(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 570, in _fit_impl + self._run(model, ckpt_path=ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 975, in _run + results = self._run_stage() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1018, in _run_stage + self.fit_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 201, in run + self.advance() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 354, in advance + self.epoch_loop.run(self._data_fetcher) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 133, in run + self.advance(data_fetcher) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 218, in advance + batch_output = self.automatic_optimization.run(trainer.optimizers[0], kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 178, in run + closure() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 140, in __call__ + self._result = self.closure(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 135, in closure + self._backward_fn(step_output.closure_loss) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 232, in backward_fn + call._call_strategy_hook(self.trainer, "backward", loss, optimizer) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 287, in _call_strategy_hook + output = fn(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 200, in backward + self.precision_plugin.backward(closure_loss, self.lightning_module, optimizer, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 67, in backward + model.backward(tensor, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/core/module.py", line 1046, in backward + loss.backward(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/_tensor.py", line 487, in backward + torch.autograd.backward( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/autograd/__init__.py", line 200, in backward + Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +RuntimeError: Expected to mark a variable ready only once. This error is caused by one of the following reasons: 1) Use of a module parameter outside the `forward` function. Please make sure model parameters are not shared across multiple concurrent forward-backward passes. or try to use _set_static_graph() as a workaround if this module graph does not change during training loop.2) Reused parameters in multiple reentrant backward passes. For example, if you use multiple `checkpoint` functions to wrap the same part of your model, it would result in the same set of parameters been used by different reentrant backward passes multiple times, and hence marking a variable ready multiple times. DDP does not support such use cases in default. You can try to use _set_static_graph() as a workaround if your module graph does not change over iterations. +Parameter at index 523 has been marked as ready twice. This means that multiple autograd engine hooks have fired for this particular parameter during this iteration. You can set the environment variable TORCH_DISTRIBUTED_DEBUG to either INFO or DETAIL to print parameter names for further debugging. +Traceback (most recent call last): + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 190, in + main() + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 169, in main + trainer.fit(model, datamodule=datamodule, ckpt_path=config.training.lightning_load_from_checkpoint) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 531, in fit + call._call_and_handle_interrupt( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 41, in _call_and_handle_interrupt + return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 91, in launch + return function(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 570, in _fit_impl + self._run(model, ckpt_path=ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 975, in _run + results = self._run_stage() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1018, in _run_stage + self.fit_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 201, in run + self.advance() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 354, in advance + self.epoch_loop.run(self._data_fetcher) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 133, in run + self.advance(data_fetcher) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 218, in advance + batch_output = self.automatic_optimization.run(trainer.optimizers[0], kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 178, in run + closure() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 140, in __call__ + self._result = self.closure(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 135, in closure + self._backward_fn(step_output.closure_loss) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 232, in backward_fn + call._call_strategy_hook(self.trainer, "backward", loss, optimizer) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 287, in _call_strategy_hook + output = fn(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 200, in backward + self.precision_plugin.backward(closure_loss, self.lightning_module, optimizer, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 67, in backward + model.backward(tensor, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/core/module.py", line 1046, in backward + loss.backward(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/_tensor.py", line 487, in backward + torch.autograd.backward( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/autograd/__init__.py", line 200, in backward + Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +RuntimeError: Expected to mark a variable ready only once. This error is caused by one of the following reasons: 1) Use of a module parameter outside the `forward` function. Please make sure model parameters are not shared across multiple concurrent forward-backward passes. or try to use _set_static_graph() as a workaround if this module graph does not change during training loop.2) Reused parameters in multiple reentrant backward passes. For example, if you use multiple `checkpoint` functions to wrap the same part of your model, it would result in the same set of parameters been used by different reentrant backward passes multiple times, and hence marking a variable ready multiple times. DDP does not support such use cases in default. You can try to use _set_static_graph() as a workaround if your module graph does not change over iterations. +Parameter at index 523 has been marked as ready twice. This means that multiple autograd engine hooks have fired for this particular parameter during this iteration. You can set the environment variable TORCH_DISTRIBUTED_DEBUG to either INFO or DETAIL to print parameter names for further debugging. +wandb: Waiting for W&B process to finish... (failed 1). Press Control-C to abort syncing. +Traceback (most recent call last): + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 190, in + main() + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 169, in main + trainer.fit(model, datamodule=datamodule, ckpt_path=config.training.lightning_load_from_checkpoint) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 531, in fit + call._call_and_handle_interrupt( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 41, in _call_and_handle_interrupt + return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 91, in launch + return function(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 570, in _fit_impl + self._run(model, ckpt_path=ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 975, in _run + results = self._run_stage() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1018, in _run_stage + self.fit_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 201, in run + self.advance() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 354, in advance + self.epoch_loop.run(self._data_fetcher) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 133, in run + self.advance(data_fetcher) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 218, in advance + batch_output = self.automatic_optimization.run(trainer.optimizers[0], kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 178, in run + closure() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 140, in __call__ + self._result = self.closure(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 135, in closure + self._backward_fn(step_output.closure_loss) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 232, in backward_fn + call._call_strategy_hook(self.trainer, "backward", loss, optimizer) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 287, in _call_strategy_hook + output = fn(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 200, in backward + self.precision_plugin.backward(closure_loss, self.lightning_module, optimizer, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 67, in backward + model.backward(tensor, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/core/module.py", line 1046, in backward + loss.backward(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/_tensor.py", line 487, in backward + torch.autograd.backward( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/autograd/__init__.py", line 200, in backward + Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +RuntimeError: Expected to mark a variable ready only once. This error is caused by one of the following reasons: 1) Use of a module parameter outside the `forward` function. Please make sure model parameters are not shared across multiple concurrent forward-backward passes. or try to use _set_static_graph() as a workaround if this module graph does not change during training loop.2) Reused parameters in multiple reentrant backward passes. For example, if you use multiple `checkpoint` functions to wrap the same part of your model, it would result in the same set of parameters been used by different reentrant backward passes multiple times, and hence marking a variable ready multiple times. DDP does not support such use cases in default. You can try to use _set_static_graph() as a workaround if your module graph does not change over iterations. +Parameter at index 523 has been marked as ready twice. This means that multiple autograd engine hooks have fired for this particular parameter during this iteration. You can set the environment variable TORCH_DISTRIBUTED_DEBUG to either INFO or DETAIL to print parameter names for further debugging. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 20:29:23.248708] Computing e^(tensor([5276.4919], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 20:29:23.259107] Ending with PPL=13.988645120717031 (duration: 0:21:01.951930) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 3.159. Wait count has been increased from 0. +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 36 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 12 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 25 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.330. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 21:30:24.220656] Computing e^(tensor([4908.4908], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 21:30:24.227652] Ending with PPL=11.63764853373668 (duration: 0:16:16.384508) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.029 >= min_delta = 0.0. New best score: 3.130 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 7 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 37 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 13 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 26 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.330. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 22:27:30.570506] Computing e^(tensor([4861.3647], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 22:27:30.577745] Ending with PPL=11.366635482228732 (duration: 0:14:31.705113) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.022 >= min_delta = 0.0. New best score: 3.108 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 8 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 38 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 14 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 27 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.330. Wait count has been increased from 2. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-09 23:23:57.101049] Computing e^(tensor([4817.4711], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-09 23:23:57.108005] Ending with PPL=11.11989176867155 (duration: 0:14:34.371367) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.015 >= min_delta = 0.0. New best score: 3.093 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 9 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 39 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 15 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 28 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Metric validation/losses/mmm_text_loss improved by 0.041 >= min_delta = 0.0. New best score: 3.289 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 00:23:04.380762] Computing e^(tensor([4803.5023], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 00:23:04.387693] Ending with PPL=11.04249641425781 (duration: 0:16:44.624080) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.016 >= min_delta = 0.0. New best score: 3.077 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 10 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 40 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 16 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 29 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Metric validation/losses/mmm_text_loss improved by 0.007 >= min_delta = 0.0. New best score: 3.282 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 01:28:03.125086] Computing e^(tensor([4769.2476], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 01:28:03.132142] Ending with PPL=10.85497813706371 (duration: 0:23:31.221543) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.014 >= min_delta = 0.0. New best score: 3.063 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 11 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 41 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 17 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 30 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Metric validation/losses/mmm_text_loss improved by 0.031 >= min_delta = 0.0. New best score: 3.251 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 02:34:22.879820] Computing e^(tensor([4743.7614], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 02:34:22.884944] Ending with PPL=10.717529711992045 (duration: 0:25:16.012909) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.012 >= min_delta = 0.0. New best score: 3.051 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 12 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 42 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 18 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 31 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Metric validation/losses/mmm_text_loss improved by 0.003 >= min_delta = 0.0. New best score: 3.248 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 03:34:57.722102] Computing e^(tensor([4710.4366], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 03:34:57.729055] Ending with PPL=10.540429818230402 (duration: 0:19:54.406844) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.010 >= min_delta = 0.0. New best score: 3.042 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 13 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 43 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 19 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 32 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Metric validation/losses/mmm_text_loss improved by 0.017 >= min_delta = 0.0. New best score: 3.231 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 04:32:07.255089] Computing e^(tensor([4694.6348], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 04:32:07.262175] Ending with PPL=10.457479035320329 (duration: 0:14:59.382095) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.013 >= min_delta = 0.0. New best score: 3.029 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 14 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 44 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 20 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 33 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Metric validation/losses/mmm_text_loss improved by 0.017 >= min_delta = 0.0. New best score: 3.214 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 05:26:58.465268] Computing e^(tensor([4673.2317], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 05:26:58.472256] Ending with PPL=10.346164193186937 (duration: 0:14:28.514350) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.017 >= min_delta = 0.0. New best score: 3.012 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 15 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 45 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 21 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 34 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Metric validation/losses/mmm_text_loss improved by 0.015 >= min_delta = 0.0. New best score: 3.199 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 06:23:00.180598] Computing e^(tensor([4648.4378], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 06:23:00.187782] Ending with PPL=10.218695061986782 (duration: 0:14:20.586477) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.010 >= min_delta = 0.0. New best score: 3.002 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 16 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 46 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 22 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 35 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Metric validation/losses/mmm_text_loss improved by 0.010 >= min_delta = 0.0. New best score: 3.188 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 07:21:05.980154] Computing e^(tensor([4636.9243], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 07:21:05.987096] Ending with PPL=10.160037619851263 (duration: 0:16:19.912589) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.003 >= min_delta = 0.0. New best score: 2.999 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 17 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 47 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 23 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 36 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Metric validation/losses/mmm_text_loss improved by 0.006 >= min_delta = 0.0. New best score: 3.183 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 08:25:53.521083] Computing e^(tensor([4624.3233], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 08:25:53.526145] Ending with PPL=10.096225812734119 (duration: 0:23:07.222594) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.011 >= min_delta = 0.0. New best score: 2.988 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 18 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 48 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 24 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 37 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Metric validation/losses/mmm_text_loss improved by 0.012 >= min_delta = 0.0. New best score: 3.170 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 09:32:07.132239] Computing e^(tensor([4585.6344], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 09:32:07.139344] Ending with PPL=9.90279666056585 (duration: 0:26:01.507453) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.011 >= min_delta = 0.0. New best score: 2.977 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 19 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 49 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 25 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 38 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Metric validation/losses/mmm_text_loss improved by 0.003 >= min_delta = 0.0. New best score: 3.168 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 10:33:26.402050] Computing e^(tensor([4565.0765], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 10:33:26.409172] Ending with PPL=9.801527635349158 (duration: 0:19:02.497652) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 2.972 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 20 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 50 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 26 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 39 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Metric validation/losses/mmm_text_loss improved by 0.006 >= min_delta = 0.0. New best score: 3.162 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 11:31:46.726646] Computing e^(tensor([4533.0200], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 11:31:46.733764] Ending with PPL=9.645678528308634 (duration: 0:14:13.798985) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.017 >= min_delta = 0.0. New best score: 2.955 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 21 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 51 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 27 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 40 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Metric validation/losses/mmm_text_loss improved by 0.013 >= min_delta = 0.0. New best score: 3.149 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 12:29:19.022871] Computing e^(tensor([4526.4192], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 12:29:19.030023] Ending with PPL=9.613896345144143 (duration: 0:14:27.223686) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.006 >= min_delta = 0.0. New best score: 2.949 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 22 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 52 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 28 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 41 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.149. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 13:26:44.023264] Computing e^(tensor([4512.6642], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 13:26:44.030357] Ending with PPL=9.548003871790435 (duration: 0:14:21.864628) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.009 >= min_delta = 0.0. New best score: 2.940 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 23 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 53 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 29 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 42 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Metric validation/losses/mmm_text_loss improved by 0.032 >= min_delta = 0.0. New best score: 3.117 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 14:35:11.131739] Computing e^(tensor([4510.1718], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 14:35:11.138948] Ending with PPL=9.536112322940088 (duration: 0:23:48.995369) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.007 >= min_delta = 0.0. New best score: 2.932 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 24 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 54 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 30 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 43 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Metric validation/losses/mmm_text_loss improved by 0.010 >= min_delta = 0.0. New best score: 3.108 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 15:41:05.340738] Computing e^(tensor([4483.9184], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 15:41:05.347876] Ending with PPL=9.411752654425648 (duration: 0:22:57.803082) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.008 >= min_delta = 0.0. New best score: 2.924 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 25 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 55 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 31 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 44 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Metric validation/losses/mmm_text_loss improved by 0.011 >= min_delta = 0.0. New best score: 3.097 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 16:39:47.191517] Computing e^(tensor([4461.5138], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 16:39:47.198531] Ending with PPL=9.306907939684628 (duration: 0:17:30.868989) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.009 >= min_delta = 0.0. New best score: 2.915 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 26 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 56 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 32 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 45 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.097. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 17:36:35.904102] Computing e^(tensor([4463.4062], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 17:36:35.911078] Ending with PPL=9.31571794732428 (duration: 0:14:53.722972) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 2.911 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 27 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 57 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 33 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 46 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Metric validation/losses/mmm_text_loss improved by 0.010 >= min_delta = 0.0. New best score: 3.087 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 18:33:38.819747] Computing e^(tensor([4448.1569], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 18:33:38.826802] Ending with PPL=9.244958959663304 (duration: 0:14:13.822591) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.009 >= min_delta = 0.0. New best score: 2.901 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 28 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 58 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 34 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 47 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Metric validation/losses/mmm_text_loss improved by 0.039 >= min_delta = 0.0. New best score: 3.048 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 19:29:54.998132] Computing e^(tensor([4443.4723], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 19:29:55.005136] Ending with PPL=9.223330009127379 (duration: 0:14:31.137589) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.901. Wait count has been increased from 0. +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 29 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 59 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 35 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 48 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Metric validation/losses/mmm_text_loss improved by 0.001 >= min_delta = 0.0. New best score: 3.047 +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 20:34:32.631064] Computing e^(tensor([4412.9285], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 20:34:32.638108] Ending with PPL=9.08354238311255 (duration: 0:21:08.651290) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.019 >= min_delta = 0.0. New best score: 2.883 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 30 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 60 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 36 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 49 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.047. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 21:43:15.576786] Computing e^(tensor([4393.3754], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 21:43:15.583821] Ending with PPL=8.995169492840777 (duration: 0:26:27.278826) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.003 >= min_delta = 0.0. New best score: 2.880 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 31 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 61 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 37 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 50 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.047. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 22:44:33.460799] Computing e^(tensor([4393.6197], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 22:44:33.467742] Ending with PPL=8.996268172095357 (duration: 0:19:55.041228) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 2.875 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 32 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 62 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 38 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 51 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.047. Wait count has been increased from 2. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-10 23:41:28.639985] Computing e^(tensor([4391.5483], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-10 23:41:28.646988] Ending with PPL=8.986955810829969 (duration: 0:14:49.921037) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.003 >= min_delta = 0.0. New best score: 2.872 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 33 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 63 records. Best score: 0.319. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 39 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 52 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 3.047. Signaling model to ignore task. +wandb: +wandb: Run history: +wandb: ImageDataModule ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: MLMDataModule ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: VLDataModule ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: epoch ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: evaluation/blimp/anaphor_agreement ▆█▆▅▆▅▁▄▂▅▆▁▂▇▆▇▅▇█▇▅█▇██▆▄▄ +wandb: evaluation/blimp/argument_structure ▁▄▅▅▆▆▃▇▂▆▂▅▄▃▆▂▅▅▇▂▂█▅▁▃▇▆▇ +wandb: evaluation/blimp/binding ▁▂▄▄▄▄▅▅▅▆▆▆▆▅▇▇▇▆▆▇▅▆▆▄▆▅█▆ +wandb: evaluation/blimp/control_raising ▄▁▃▄▃▆▆▇▆▄▆▇▆▅▇▅▇▆▇█▇▆▇▅█▇▇▇ +wandb: evaluation/blimp/determiner_noun_agreement ▁▃▂▄▄▄▅▄▃▅▄▅▅▃▄▆▅▇▆▇▇█▇▇██▇█ +wandb: evaluation/blimp/ellipsis ▂▂▂▂▁▃▂▄▅▄▄▅▃▃▄█▃▅▅▅▆▅▅▃▆▃▃▇ +wandb: evaluation/blimp/filler_gap ▆▅▆▆▅▅▄▃▄█▇▃▇▅▂▂▁█▆▆▆▆▅▅▅▄▂▇ +wandb: evaluation/blimp/irregular_forms █▃▄▄▄▂▁▁▃▅▅▄▄▃▂▂▅▄▅▅▄▅▅▂▃▂▁▁ +wandb: evaluation/blimp/island_effects ▁▃▃▂▅▅▁▃▅▆▆▆▇▆▄▇▄▃▇█▆▇▆▅▆███ +wandb: evaluation/blimp/npi_licensing ▆▂▄▄▁▃▂▂▂▁▄▇▇▄▅▅▅▄▅▅▃▅▃▆█▄▄▄ +wandb: evaluation/blimp/quantifiers ▃▂▅▅▃▆▄▂▄▅▆▃▁▄▂▅▇▆▃▄▆█▂▄▅▂▃▅ +wandb: evaluation/blimp/subject_verb_agreement ▁▃▂▃▄▄▂▄▂▄▅▃▃▃▅▅▅▇▇▆▅█▆▆▇▅▆▄ +wandb: evaluation/blimp_average ▂▁▂▂▂▃▁▂▃▄▅▅▄▄▄▆▅▆▆▇▅█▆▅▇▅▅▆ +wandb: evaluation/imagenet_zeroshot/top1 ▄▆▆▆▇▅▅█▇▅▇▄▃█▂▆▄▂▄▁▃▄▃▇▇▄▄▅ +wandb: evaluation/imagenet_zeroshot/top5 ▆▇▅▇▆█▅▇▇▆▇▃▄▅▃▄▃▁▃▂▂▆▃▆▁▁▂▅ +wandb: evaluation/pseudo_perplexity █▅▅▄▄▄▄▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁ +wandb: lr-AdamW/pg1 ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: lr-AdamW/pg2 ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: train/losses/global_contrastive_loss ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: train/losses/itm_loss ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: train/losses/mim_loss █▄▆▃▃▄▄▄▃▁▃▃▄▂▄▁▃▃▁▂▄▁▃▂▃▄▃▂▁▃▂▃▁▃▃▃▃▅▃▂ +wandb: train/losses/mlm_loss █▅▆▄▆▄▄▆▇▅▅▅▄▄▂▅▆▄▃▅▄▅▃▆▅▅▂▂▅▅▁▁▄▂▂▂▃▄▂▁ +wandb: train/losses/mmm_image_loss ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: train/losses/mmm_text_loss █▃▃▂▃▃▂▃▃▂▂▃▂▂▂▃▂▃▁▁▃▂▂▃▂▂▂▂▂▂▂▂▂▁▂▂▃▃▂▂ +wandb: trainer/global_step ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: validation/losses/global_contrastive_loss ▁ +wandb: validation/losses/itm_loss ▁ +wandb: validation/losses/mim_loss █▅▃▃▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/losses/mlm_loss █▅▅▅▄▄▄▄▄▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▁▁▁▁ +wandb: validation/losses/mmm_image_loss ▁ +wandb: validation/losses/mmm_text_loss █▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/monitor/global_contrastive ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/monitor/itm ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/monitor/mim ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/monitor/mlm ███████████████████████▁▁▁▁▁ +wandb: validation/monitor/mmm_image ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/monitor/mmm_text █▅▃▃▃▃▃▃▃▃▃▃▃▃▃▃▂▂▂▂▁▁▁▁▁▁▁▁ +wandb: +wandb: Run summary: +wandb: ImageDataModule 17968 +wandb: MLMDataModule 183460 +wandb: VLDataModule 17318 +wandb: epoch 0 +wandb: evaluation/blimp/anaphor_agreement 93.35 +wandb: evaluation/blimp/argument_structure 69.36 +wandb: evaluation/blimp/binding 67.01 +wandb: evaluation/blimp/control_raising 67.01 +wandb: evaluation/blimp/determiner_noun_agreement 90.61 +wandb: evaluation/blimp/ellipsis 80.83 +wandb: evaluation/blimp/filler_gap 62.9 +wandb: evaluation/blimp/irregular_forms 77.76 +wandb: evaluation/blimp/island_effects 53.59 +wandb: evaluation/blimp/npi_licensing 56.79 +wandb: evaluation/blimp/quantifiers 59.2 +wandb: evaluation/blimp/subject_verb_agreement 66.68 +wandb: evaluation/blimp_average 70.42416 +wandb: evaluation/imagenet_zeroshot/top1 0.00094 +wandb: evaluation/imagenet_zeroshot/top5 0.00474 +wandb: evaluation/pseudo_perplexity 9.54036 +wandb: lr-AdamW/pg1 0.0008 +wandb: lr-AdamW/pg2 0.0006 +wandb: train/losses/global_contrastive_loss 3.46825 +wandb: train/losses/itm_loss 0.35078 +wandb: train/losses/mim_loss 0.96148 +wandb: train/losses/mlm_loss 2.33651 +wandb: train/losses/mmm_image_loss 1.31719 +wandb: train/losses/mmm_text_loss 2.79106 +wandb: trainer/global_step 7999 +wandb: validation/losses/global_contrastive_loss 3.46555 +wandb: validation/losses/itm_loss 0.33735 +wandb: validation/losses/mim_loss 0.98974 +wandb: validation/losses/mlm_loss 2.87213 +wandb: validation/losses/mmm_image_loss 1.31868 +wandb: validation/losses/mmm_text_loss 3.08004 +wandb: validation/monitor/global_contrastive 0.0 +wandb: validation/monitor/itm 0.0 +wandb: validation/monitor/mim 0.0 +wandb: validation/monitor/mlm 0.20833 +wandb: validation/monitor/mmm_image 0.0 +wandb: validation/monitor/mmm_text 0.0 +wandb: +wandb: 🚀 View run dauntless-darkness-212 at: https://wandb.ai/rycolab/alkmi-wit/runs/9hnl8w94 +wandb: Synced 2 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20230709_193040-9hnl8w94/logs +[ImageNet Zero-Shot Evaluation 2023-07-10 23:49:46.786420] Ending with top5=0.00474 (duration: 0:02:13.200185) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.003 >= min_delta = 0.0. New best score: 2.872 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 33 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 63 records. Best score: 0.319. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 39 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 52 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 3.047. Signaling model to ignore task. diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/resume_singlenode_2023-07-11-21:17:35.log b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/resume_singlenode_2023-07-11-21:17:35.log new file mode 100644 index 0000000000000000000000000000000000000000..479d2ebb14108cd4e97f65714e256032301c7ab4 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/resume_singlenode_2023-07-11-21:17:35.log @@ -0,0 +1,805 @@ +wandb: Currently logged in as: tamariucai (rycolab). Use `wandb login --relogin` to force relogin +wandb: - Waiting for wandb.init()... wandb: \ Waiting for wandb.init()... wandb: wandb version 0.15.5 is available! To upgrade, please run: +wandb: $ pip install wandb --upgrade +wandb: Tracking run with wandb version 0.15.4 +wandb: Run data is saved locally in ./wandb/run-20230711_214056-9hnl8w94 +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run dauntless-darkness-212 +wandb: ⭐️ View project at https://wandb.ai/rycolab/alkmi-wit +wandb: 🚀 View run at https://wandb.ai/rycolab/alkmi-wit/runs/9hnl8w94 +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +The current host is eu-a65-04 +The current cuDNN version is 8801 +[update_ckeckpoint_dir] Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/ +Text is the predominant modality (10 v.s. 1 vision), will sample proportionally for optimal BLiMP performance. +Assigning HuggingFace RAM +Found 747.32GBs of RAM available, assigning 5GBs to HuggingFace datasets (currently 5000000000 bytes). +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Found cached dataset imagenet-1k (/cluster/scratch/tamariucai/HuggingfaceDatasets/imagenet-1k/default/1.0.0/a1e9bfc56c3a7350165007d1176b15e9128fcaf9ab972147840529aed3ae52bc) +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/plugins/environments/slurm.py:165: PossibleUserWarning: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... + rank_zero_warn( +Using bfloat16 Automatic Mixed Precision (AMP) +GPU available: True (cuda), used: True +TPU available: False, using: 0 TPU cores +IPU available: False, using: 0 IPUs +HPU available: False, using: 0 HPUs +Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/2 +wandb: Currently logged in as: tamariucai (rycolab). Use `wandb login --relogin` to force relogin +wandb: - Waiting for wandb.init()... wandb: \ Waiting for wandb.init()... wandb: wandb version 0.15.5 is available! To upgrade, please run: +wandb: $ pip install wandb --upgrade +wandb: Tracking run with wandb version 0.15.4 +wandb: Run data is saved locally in ./wandb/run-20230711_215650-9hnl8w94 +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run dauntless-darkness-212 +wandb: ⭐️ View project at https://wandb.ai/rycolab/alkmi-wit +wandb: 🚀 View run at https://wandb.ai/rycolab/alkmi-wit/runs/9hnl8w94 +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +The current host is eu-a65-04 +The current cuDNN version is 8801 +[update_ckeckpoint_dir] Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/ +Text is the predominant modality (10 v.s. 1 vision), will sample proportionally for optimal BLiMP performance. +Assigning HuggingFace RAM +Found 747.32GBs of RAM available, assigning 5GBs to HuggingFace datasets (currently 5000000000 bytes). +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Found cached dataset imagenet-1k (/cluster/scratch/tamariucai/HuggingfaceDatasets/imagenet-1k/default/1.0.0/a1e9bfc56c3a7350165007d1176b15e9128fcaf9ab972147840529aed3ae52bc) +Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/2 +---------------------------------------------------------------------------------------------------- +distributed_backend=nccl +All distributed processes registered. Starting with 2 processes +---------------------------------------------------------------------------------------------------- + +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at theodor1289wit-test +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed exists and is not empty. + rank_zero_warn(f"Checkpoint directory {dirpath} exists and is not empty.") +Restoring states from the checkpoint path at /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=7882.ckpt +LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1] +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1] + + | Name | Type | Params +---------------------------------------------- +0 | model | FlavaForPreTraining | 357 M +---------------------------------------------- +303 M Trainable params +53.8 M Non-trainable params +357 M Total params +1,430.581 Total estimated model params size (MB) +Restored all states from the checkpoint at /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=7882.ckpt +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +VLDataModule: batch_size is 32, num_workers is 4 +ImageDataModule: batch_size is 32, num_workers is 4 +MLMDataModule: batch_size is 32, num_workers is 4 +Dataset sizes (un-normalized): [1, 1, 10] +Dataset sizes (normalized): [0.08333333333333333, 0.08333333333333333, 0.8333333333333334] +Sampling weights after temperature (1.0): [0.08333333333333333, 0.08333333333333333, 0.8333333333333334] +2023-07-11 21:57:13.028772 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-11 22:00:57.699850 Setting up datamodule: ImageDataModule +2023-07-11 22:01:01.314810 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Adding ImageNet zeroshot callback +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'ImageNetZeroshotCallback'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-11 22:10:58.070398 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-11 22:15:49.003466 Setting up datamodule: ImageDataModule +2023-07-11 22:15:56.372445 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-11 22:27:41.511236 Building train dataloader for datamodule: VLDataModule +2023-07-11 22:27:41.512200 Building train dataloader for datamodule: ImageDataModule +2023-07-11 22:27:41.512535 Building train dataloader for datamodule: MLMDataModule +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +VLDataModule: batch_size is 32, num_workers is 4 +ImageDataModule: batch_size is 32, num_workers is 4 +MLMDataModule: batch_size is 32, num_workers is 4 +Dataset sizes (un-normalized): [1, 1, 10] +Dataset sizes (normalized): [0.08333333333333333, 0.08333333333333333, 0.8333333333333334] +Sampling weights after temperature (1.0): [0.08333333333333333, 0.08333333333333333, 0.8333333333333334] +2023-07-11 21:41:46.610265 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-11 21:46:45.619131 Setting up datamodule: ImageDataModule +2023-07-11 21:46:51.149812 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Adding ImageNet zeroshot callback +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'ImageNetZeroshotCallback'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-11 21:56:17.905693 Preparing data for datamodule: VLDataModule +2023-07-11 21:56:17.905942 Preparing data for datamodule: ImageDataModule +2023-07-11 21:56:17.906110 Preparing data for datamodule: MLMDataModule +2023-07-11 22:10:58.070350 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-11 22:15:49.026467 Setting up datamodule: ImageDataModule +2023-07-11 22:15:56.273372 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-11 22:27:41.511262 Building train dataloader for datamodule: VLDataModule +2023-07-11 22:27:41.512580 Building train dataloader for datamodule: ImageDataModule +2023-07-11 22:27:41.512941 Building train dataloader for datamodule: MLMDataModule +2023-07-11 22:27:44.089342 Building val dataloader for datamodule: VLDataModule +2023-07-11 22:27:44.092195 Building val dataloader for datamodule: ImageDataModule +2023-07-11 22:27:44.092665 Building val dataloader for datamodule: MLMDataModule +2023-07-11 22:27:44.082211 Building val dataloader for datamodule: VLDataModule +2023-07-11 22:27:44.083608 Building val dataloader for datamodule: ImageDataModule +2023-07-11 22:27:44.084173 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-11 23:09:54.652320] Computing e^(tensor([4833.7474], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 23:09:54.672052] Ending with PPL=11.210756228958989 (duration: 0:11:15.071494) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.315 +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 39 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 52 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 00:05:56.285036] Computing e^(tensor([4628.0173], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 00:05:56.293497] Ending with PPL=10.11489036723113 (duration: 0:12:16.282435) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.870 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 34 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.315. Wait count has been increased from 0. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 40 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 53 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 01:01:23.504076] Computing e^(tensor([4602.5966], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 01:01:23.511336] Ending with PPL=9.987140385024597 (duration: 0:12:24.071030) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.855 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 35 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.315. Wait count has been increased from 1. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 41 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 54 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 01:57:28.374787] Computing e^(tensor([4577.0626], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 01:57:28.388467] Ending with PPL=9.86044486538573 (duration: 0:11:55.360173) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.847 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 36 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.315. Wait count has been increased from 2. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 42 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 55 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_text_loss did not improve in the last 7 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-12 02:53:15.624921] Computing e^(tensor([4564.8592], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 02:53:15.633587] Ending with PPL=9.800462510069949 (duration: 0:12:24.397418) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00 + main() + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 169, in main + trainer.fit(model, datamodule=datamodule, ckpt_path=config.training.lightning_load_from_checkpoint) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 531, in fit + call._call_and_handle_interrupt( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 41, in _call_and_handle_interrupt + return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 91, in launch + return function(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 570, in _fit_impl + self._run(model, ckpt_path=ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 975, in _run + results = self._run_stage() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1018, in _run_stage + self.fit_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 201, in run + self.advance() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 354, in advance + self.epoch_loop.run(self._data_fetcher) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 133, in run + self.advance(data_fetcher) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 218, in advance + batch_output = self.automatic_optimization.run(trainer.optimizers[0], kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 178, in run + closure() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 140, in __call__ + self._result = self.closure(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 135, in closure + self._backward_fn(step_output.closure_loss) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 232, in backward_fn + call._call_strategy_hook(self.trainer, "backward", loss, optimizer) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 287, in _call_strategy_hook + output = fn(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 200, in backward + self.precision_plugin.backward(closure_loss, self.lightning_module, optimizer, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 67, in backward + model.backward(tensor, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/core/module.py", line 1046, in backward + loss.backward(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/_tensor.py", line 487, in backward + torch.autograd.backward( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/autograd/__init__.py", line 200, in backward + Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +RuntimeError: Expected to mark a variable ready only once. This error is caused by one of the following reasons: 1) Use of a module parameter outside the `forward` function. Please make sure model parameters are not shared across multiple concurrent forward-backward passes. or try to use _set_static_graph() as a workaround if this module graph does not change during training loop.2) Reused parameters in multiple reentrant backward passes. For example, if you use multiple `checkpoint` functions to wrap the same part of your model, it would result in the same set of parameters been used by different reentrant backward passes multiple times, and hence marking a variable ready multiple times. DDP does not support such use cases in default. You can try to use _set_static_graph() as a workaround if your module graph does not change over iterations. +Parameter at index 523 has been marked as ready twice. This means that multiple autograd engine hooks have fired for this particular parameter during this iteration. You can set the environment variable TORCH_DISTRIBUTED_DEBUG to either INFO or DETAIL to print parameter names for further debugging. +Traceback (most recent call last): + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 190, in + main() + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 169, in main + trainer.fit(model, datamodule=datamodule, ckpt_path=config.training.lightning_load_from_checkpoint) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 531, in fit + call._call_and_handle_interrupt( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 41, in _call_and_handle_interrupt + return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 91, in launch + return function(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 570, in _fit_impl + self._run(model, ckpt_path=ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 975, in _run + results = self._run_stage() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1018, in _run_stage + self.fit_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 201, in run + self.advance() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 354, in advance + self.epoch_loop.run(self._data_fetcher) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 133, in run + self.advance(data_fetcher) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 218, in advance + batch_output = self.automatic_optimization.run(trainer.optimizers[0], kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 178, in run + closure() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 140, in __call__ + self._result = self.closure(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 135, in closure + self._backward_fn(step_output.closure_loss) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 232, in backward_fn + call._call_strategy_hook(self.trainer, "backward", loss, optimizer) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 287, in _call_strategy_hook + output = fn(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 200, in backward + self.precision_plugin.backward(closure_loss, self.lightning_module, optimizer, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 67, in backward + model.backward(tensor, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/core/module.py", line 1046, in backward + loss.backward(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/_tensor.py", line 487, in backward + torch.autograd.backward( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/autograd/__init__.py", line 200, in backward + Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +RuntimeError: Expected to mark a variable ready only once. This error is caused by one of the following reasons: 1) Use of a module parameter outside the `forward` function. Please make sure model parameters are not shared across multiple concurrent forward-backward passes. or try to use _set_static_graph() as a workaround if this module graph does not change during training loop.2) Reused parameters in multiple reentrant backward passes. For example, if you use multiple `checkpoint` functions to wrap the same part of your model, it would result in the same set of parameters been used by different reentrant backward passes multiple times, and hence marking a variable ready multiple times. DDP does not support such use cases in default. You can try to use _set_static_graph() as a workaround if your module graph does not change over iterations. +Parameter at index 523 has been marked as ready twice. This means that multiple autograd engine hooks have fired for this particular parameter during this iteration. You can set the environment variable TORCH_DISTRIBUTED_DEBUG to either INFO or DETAIL to print parameter names for further debugging. +wandb: Waiting for W&B process to finish... (failed 1). Press Control-C to abort syncing. +wandb: - 0.000 MB of 0.000 MB uploaded (0.000 MB deduped) Traceback (most recent call last): + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 190, in + main() + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 169, in main + trainer.fit(model, datamodule=datamodule, ckpt_path=config.training.lightning_load_from_checkpoint) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 531, in fit + call._call_and_handle_interrupt( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 41, in _call_and_handle_interrupt + return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 91, in launch + return function(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 570, in _fit_impl + self._run(model, ckpt_path=ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 975, in _run + results = self._run_stage() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1018, in _run_stage + self.fit_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 201, in run + self.advance() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 354, in advance + self.epoch_loop.run(self._data_fetcher) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 133, in run + self.advance(data_fetcher) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 218, in advance + batch_output = self.automatic_optimization.run(trainer.optimizers[0], kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 178, in run + closure() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 140, in __call__ + self._result = self.closure(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 135, in closure + self._backward_fn(step_output.closure_loss) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 232, in backward_fn + call._call_strategy_hook(self.trainer, "backward", loss, optimizer) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 287, in _call_strategy_hook + output = fn(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 200, in backward + self.precision_plugin.backward(closure_loss, self.lightning_module, optimizer, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 67, in backward + model.backward(tensor, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/core/module.py", line 1046, in backward + loss.backward(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/_tensor.py", line 487, in backward + torch.autograd.backward( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/autograd/__init__.py", line 200, in backward + Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +RuntimeError: Expected to mark a variable ready only once. This error is caused by one of the following reasons: 1) Use of a module parameter outside the `forward` function. Please make sure model parameters are not shared across multiple concurrent forward-backward passes. or try to use _set_static_graph() as a workaround if this module graph does not change during training loop.2) Reused parameters in multiple reentrant backward passes. For example, if you use multiple `checkpoint` functions to wrap the same part of your model, it would result in the same set of parameters been used by different reentrant backward passes multiple times, and hence marking a variable ready multiple times. DDP does not support such use cases in default. You can try to use _set_static_graph() as a workaround if your module graph does not change over iterations. +Parameter at index 523 has been marked as ready twice. This means that multiple autograd engine hooks have fired for this particular parameter during this iteration. You can set the environment variable TORCH_DISTRIBUTED_DEBUG to either INFO or DETAIL to print parameter names for further debugging. +wandb: \ 0.000 MB of 0.060 MB uploaded (0.000 MB deduped) wandb: | 0.060 MB of 0.060 MB uploaded (0.000 MB deduped) wandb: +wandb: Run history: +wandb: ImageDataModule ▁▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▅▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇███ +wandb: MLMDataModule ▁▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇███ +wandb: VLDataModule ▁▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▅▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇███ +wandb: epoch ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: evaluation/blimp/anaphor_agreement ▆▁▆█▆ +wandb: evaluation/blimp/argument_structure ▁▄▆▄█ +wandb: evaluation/blimp/binding █▁▇▆█ +wandb: evaluation/blimp/control_raising ▁▃▅▄█ +wandb: evaluation/blimp/determiner_noun_agreement ▁▁▅▆█ +wandb: evaluation/blimp/ellipsis ▃▅▁▅█ +wandb: evaluation/blimp/filler_gap ▇▄▁▇█ +wandb: evaluation/blimp/irregular_forms ▁▄▆▇█ +wandb: evaluation/blimp/island_effects ▅▁▅▆█ +wandb: evaluation/blimp/npi_licensing ▁▅▅▄█ +wandb: evaluation/blimp/quantifiers ▄█▁▄▃ +wandb: evaluation/blimp/subject_verb_agreement ▁▁█▅▅ +wandb: evaluation/blimp_average ▁▂▃▅█ +wandb: evaluation/imagenet_zeroshot/top1 ▁▂▃█▃ +wandb: evaluation/imagenet_zeroshot/top5 ▁█▄▅▇ +wandb: evaluation/pseudo_perplexity █▃▂▁▁ +wandb: lr-AdamW/pg1 ▁▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▅▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇███ +wandb: lr-AdamW/pg2 ▁▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▅▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇███ +wandb: train/losses/global_contrastive_loss █▅▁▁▁▁▁▁▁▁▁▁▁ +wandb: train/losses/itm_loss ▂▂▁▅▄▂▇▂█▂▃▄▂ +wandb: train/losses/mim_loss █▃▃▃▄▄▁▂▄▂▂▁▂ +wandb: train/losses/mlm_loss █▄█▂▆▃▄▂▆▅▁▅▅ +wandb: train/losses/mmm_image_loss █▁▅▅▅▅▅▅▅▅▅▅▅ +wandb: train/losses/mmm_text_loss █▄▁▁▁▁▁▁▁▁▁▁▁ +wandb: trainer/global_step ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇███ +wandb: validation/losses/global_contrastive_loss ▁ +wandb: validation/losses/itm_loss ▁▃▅█▅ +wandb: validation/losses/mim_loss █▄▃▁▁ +wandb: validation/losses/mlm_loss █▃▂▁▁ +wandb: validation/losses/mmm_image_loss ▁ +wandb: validation/losses/mmm_text_loss ▁ +wandb: validation/monitor/global_contrastive ▁▁▁▁▁ +wandb: validation/monitor/itm █▅▃▂▁ +wandb: validation/monitor/mim ▁▁▁▁▁ +wandb: validation/monitor/mlm ▁▁▁▁▁ +wandb: validation/monitor/mmm_image ▁▁▁▁▁ +wandb: validation/monitor/mmm_text ▁▁▁▁▁ +wandb: +wandb: Run summary: +wandb: ImageDataModule 3332 +wandb: MLMDataModule 34072 +wandb: VLDataModule 3246 +wandb: epoch 0 +wandb: evaluation/blimp/anaphor_agreement 94.58 +wandb: evaluation/blimp/argument_structure 69.88 +wandb: evaluation/blimp/binding 67.85 +wandb: evaluation/blimp/control_raising 69.84 +wandb: evaluation/blimp/determiner_noun_agreement 91.01 +wandb: evaluation/blimp/ellipsis 77.42 +wandb: evaluation/blimp/filler_gap 63.87 +wandb: evaluation/blimp/irregular_forms 81.88 +wandb: evaluation/blimp/island_effects 53.55 +wandb: evaluation/blimp/npi_licensing 61.45 +wandb: evaluation/blimp/quantifiers 59.48 +wandb: evaluation/blimp/subject_verb_agreement 69.47 +wandb: evaluation/blimp_average 71.69 +wandb: evaluation/imagenet_zeroshot/top1 0.00098 +wandb: evaluation/imagenet_zeroshot/top5 0.00508 +wandb: evaluation/pseudo_perplexity 9.24253 +wandb: lr-AdamW/pg1 0.00085 +wandb: lr-AdamW/pg2 0.00064 +wandb: train/losses/global_contrastive_loss 3.43899 +wandb: train/losses/itm_loss 0.27725 +wandb: train/losses/mim_loss 1.01038 +wandb: train/losses/mlm_loss 2.58044 +wandb: train/losses/mmm_image_loss 1.15853 +wandb: train/losses/mmm_text_loss 4.99212 +wandb: trainer/global_step 8499 +wandb: validation/losses/global_contrastive_loss 3.43458 +wandb: validation/losses/itm_loss 0.32412 +wandb: validation/losses/mim_loss 1.00203 +wandb: validation/losses/mlm_loss 2.83872 +wandb: validation/losses/mmm_image_loss 1.0738 +wandb: validation/losses/mmm_text_loss 5.58073 +wandb: validation/monitor/global_contrastive 0.0 +wandb: validation/monitor/itm 0.0 +wandb: validation/monitor/mim 0.0 +wandb: validation/monitor/mlm 0.41667 +wandb: validation/monitor/mmm_image 0.0 +wandb: validation/monitor/mmm_text 0.0 +wandb: +wandb: 🚀 View run dauntless-darkness-212 at: https://wandb.ai/rycolab/alkmi-wit/runs/9hnl8w94 +wandb: Synced 3 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20230711_215650-9hnl8w94/logs +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-11 23:10:46.431537] Computing e^(tensor([4593.4365], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-11 23:10:46.447328] Ending with PPL=9.941503470954126 (duration: 0:12:06.827462) +[rank: 1] Monitored metric validation/losses/mlm_loss did not improve since the last 1 records. Best score: 2.875. Wait count has been increased from 0. +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 33 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Metric validation/losses/itm_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.315 +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 39 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 52 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-12 00:07:27.409542] Computing e^(tensor([4387.1071], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 00:07:27.420311] Ending with PPL=8.967021403005681 (duration: 0:13:47.424887) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.005 >= min_delta = 0.0. New best score: 2.870 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 34 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.315. Wait count has been increased from 0. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 40 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 53 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-12 01:02:47.141784] Computing e^(tensor([4368.3687], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 01:02:47.146828] Ending with PPL=8.88340000107075 (duration: 0:13:47.668061) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.014 >= min_delta = 0.0. New best score: 2.855 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 35 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.315. Wait count has been increased from 1. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 41 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 54 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-12 01:57:51.384712] Computing e^(tensor([4331.0260], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 01:57:51.391758] Ending with PPL=8.719073847268094 (duration: 0:12:18.362541) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.008 >= min_delta = 0.0. New best score: 2.847 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 36 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.315. Wait count has been increased from 2. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 42 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 55 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_text_loss did not improve in the last 7 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation 2023-07-12 02:54:49.020145] Computing e^(tensor([4323.1003], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-12 02:54:49.027408] Ending with PPL=8.6845894862785 (duration: 0:13:57.770802) +[rank: 1] Metric validation/losses/mlm_loss improved by 0.009 >= min_delta = 0.0. New best score: 2.839 +[rank: 1] Monitored metric validation/losses/mim_loss did not improve in the last 37 records. Best score: 0.985. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.315. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 43 records. Best score: 2.712. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_image_loss did not improve in the last 56 records. Best score: 0.988. Signaling model to ignore task. +[rank: 1] Monitored metric validation/losses/mmm_text_loss did not improve in the last 8 records. Best score: 3.047. Signaling model to ignore task. +[ImageNet Zero-Shot Evaluation 2023-07-12 03:04:20.817508] Ending with top5=0.00508 (duration: 0:02:15.416595) +[rank: 0] Metric validation/losses/mlm_loss improved by 0.009 >= min_delta = 0.0. New best score: 2.839 +[rank: 0] Monitored metric validation/losses/mim_loss did not improve in the last 37 records. Best score: 0.985. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.315. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/global_contrastive_loss did not improve in the last 43 records. Best score: 2.712. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_image_loss did not improve in the last 56 records. Best score: 0.988. Signaling model to ignore task. +[rank: 0] Monitored metric validation/losses/mmm_text_loss did not improve in the last 8 records. Best score: 3.047. Signaling model to ignore task. diff --git a/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/resume_singlenode_2023-07-22-11:44:46.log b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/resume_singlenode_2023-07-22-11:44:46.log new file mode 100644 index 0000000000000000000000000000000000000000..d967f09b8cb4ff6799c253bf210dc35d869ed7b9 --- /dev/null +++ b/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/resume_singlenode_2023-07-22-11:44:46.log @@ -0,0 +1,2991 @@ +wandb: Currently logged in as: tamariucai (rycolab). Use `wandb login --relogin` to force relogin +wandb: - Waiting for wandb.init()... wandb: \ Waiting for wandb.init()... wandb: wandb version 0.15.5 is available! To upgrade, please run: +wandb: $ pip install wandb --upgrade +wandb: Tracking run with wandb version 0.15.4 +wandb: Run data is saved locally in ./wandb/run-20230722_114508-9hnl8w94 +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run dauntless-darkness-212 +wandb: ⭐️ View project at https://wandb.ai/rycolab/alkmi-wit +wandb: 🚀 View run at https://wandb.ai/rycolab/alkmi-wit/runs/9hnl8w94 +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +The current host is eu-a65-05 +The current cuDNN version is 8801 +[update_ckeckpoint_dir] Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/ +Text is the predominant modality (10 v.s. 1 vision), will sample proportionally for optimal BLiMP performance. +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Found cached dataset imagenet-1k (/cluster/scratch/tamariucai/HuggingfaceDatasets/imagenet-1k/default/1.0.0/a1e9bfc56c3a7350165007d1176b15e9128fcaf9ab972147840529aed3ae52bc) +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/plugins/environments/slurm.py:165: PossibleUserWarning: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... + rank_zero_warn( +Using bfloat16 Automatic Mixed Precision (AMP) +GPU available: True (cuda), used: True +TPU available: False, using: 0 TPU cores +IPU available: False, using: 0 IPUs +HPU available: False, using: 0 HPUs +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:615: UserWarning: Checkpoint directory /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed exists and is not empty. + rank_zero_warn(f"Checkpoint directory {dirpath} exists and is not empty.") +Restoring states from the checkpoint path at /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=8298.ckpt +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0] + + | Name | Type | Params +---------------------------------------------- +0 | model | FlavaForPreTraining | 357 M +---------------------------------------------- +303 M Trainable params +53.8 M Non-trainable params +357 M Total params +1,430.581 Total estimated model params size (MB) +Restored all states from the checkpoint at /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision1/bs4096_seed-1_bf16-mixed/flava-epoch=00-step=8298.ckpt +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +VLDataModule: batch_size is 64, num_workers is 4 +ImageDataModule: batch_size is 64, num_workers is 4 +MLMDataModule: batch_size is 64, num_workers is 4 +Dataset sizes (un-normalized): [1, 1, 10] +Dataset sizes (normalized): [0.08333333333333333, 0.08333333333333333, 0.8333333333333334] +Sampling weights after temperature (1.0): [0.08333333333333333, 0.08333333333333333, 0.8333333333333334] +2023-07-22 11:46:15.109869 Setting up datamodule: VLDataModule +2023-07-22 11:46:44.571472 Setting up datamodule: ImageDataModule +2023-07-22 11:46:48.552439 Setting up datamodule: MLMDataModule +Registering multimodal callbacks (overfitting monitors) +Adding ImageNet zeroshot callback +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'ImageNetZeroshotCallback'] +Trainer successfully initialized (strategy=SingleDeviceStrategy) +Starting training +2023-07-22 11:48:08.042727 Preparing data for datamodule: VLDataModule +2023-07-22 11:48:08.042945 Preparing data for datamodule: ImageDataModule +2023-07-22 11:48:08.043092 Preparing data for datamodule: MLMDataModule +2023-07-22 11:48:08.144731 Setting up datamodule: VLDataModule +2023-07-22 11:48:29.187775 Setting up datamodule: ImageDataModule +2023-07-22 11:48:32.174414 Setting up datamodule: MLMDataModule +2023-07-22 11:49:40.014713 Building train dataloader for datamodule: VLDataModule +2023-07-22 11:49:40.015418 Building train dataloader for datamodule: ImageDataModule +2023-07-22 11:49:40.015687 Building train dataloader for datamodule: MLMDataModule +2023-07-22 11:49:41.250845 Building val dataloader for datamodule: VLDataModule +2023-07-22 11:49:41.251550 Building val dataloader for datamodule: ImageDataModule +2023-07-22 11:49:41.251832 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-22 12:57:01.164909] Computing e^(tensor([4532.6677], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-22 12:57:01.175823] Ending with PPL=9.643979668353179 (duration: 0:06:46.449883) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.771 +Monitored metric validation/losses/mim_loss did not improve in the last 36 records. Best score: 0.985. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.315. Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 42 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 55 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 7 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-22 14:39:46.641617] Computing e^(tensor([4414.8367], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-22 14:39:46.646547] Ending with PPL=9.092213372603235 (duration: 0:06:52.236696) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.719 +Monitored metric validation/losses/mim_loss did not improve in the last 37 records. Best score: 0.985. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 43 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 56 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 8 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-22 16:35:41.327773] Computing e^(tensor([4380.1595], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-22 16:35:41.333537] Ending with PPL=8.935925814026321 (duration: 0:07:11.275690) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.701 +Monitored metric validation/losses/mim_loss did not improve in the last 38 records. Best score: 0.985. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 44 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 57 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 9 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-22 18:20:33.552200] Computing e^(tensor([4377.9877], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-22 18:20:33.556804] Ending with PPL=8.926227600190147 (duration: 0:06:58.537776) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.695 +Monitored metric validation/losses/mim_loss did not improve in the last 40 records. Best score: 0.985. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 46 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 59 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 11 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-22 22:06:02.394945] Computing e^(tensor([4348.3398], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-22 22:06:02.405099] Ending with PPL=8.794881598655065 (duration: 0:06:46.392465) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.688 +Monitored metric validation/losses/mim_loss did not improve in the last 41 records. Best score: 0.985. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 8 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 47 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 60 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 12 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-22 23:59:00.340943] Computing e^(tensor([4338.0655], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-22 23:59:00.347555] Ending with PPL=8.749816540478566 (duration: 0:07:09.244085) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.683 +Monitored metric validation/losses/mim_loss did not improve in the last 42 records. Best score: 0.985. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 9 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 48 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 61 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 13 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-23 01:51:40.929606] Computing e^(tensor([4313.5446], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-23 01:51:40.934298] Ending with PPL=8.643195092252459 (duration: 0:07:00.387810) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.673 +Monitored metric validation/losses/mim_loss did not improve in the last 43 records. Best score: 0.985. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 10 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 49 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 62 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 14 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-23 03:39:10.665162] Computing e^(tensor([4316.8535], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-23 03:39:10.673277] Ending with PPL=8.657506303242833 (duration: 0:07:04.481900) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.671 +Metric validation/losses/mim_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.981 +Monitored metric validation/losses/itm_loss did not improve in the last 11 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 50 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 63 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 15 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-23 05:33:22.291660] Computing e^(tensor([4299.1877], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-23 05:33:22.296083] Ending with PPL=8.581372269919127 (duration: 0:06:58.438317) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.667 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.981. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 12 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 51 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 64 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 16 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-23 07:21:14.143586] Computing e^(tensor([4295.3913], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-23 07:21:14.147912] Ending with PPL=8.565098568451596 (duration: 0:07:00.151309) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.659 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.979 +Monitored metric validation/losses/itm_loss did not improve in the last 13 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 52 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 65 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 17 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-23 09:16:08.737079] Computing e^(tensor([4288.2341], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-23 09:16:08.742623] Ending with PPL=8.53450218799916 (duration: 0:07:02.909539) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.657 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.979. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 14 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 53 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 66 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 18 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-23 11:09:19.575527] Computing e^(tensor([4287.7549], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-23 11:09:19.581477] Ending with PPL=8.532457574514822 (duration: 0:06:59.354062) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.652 +Monitored metric validation/losses/mim_loss did not improve in the last 2 records. Best score: 0.979. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 15 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 54 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 67 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 19 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-23 12:55:57.591240] Computing e^(tensor([4278.5439], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-23 12:55:57.595416] Ending with PPL=8.493251753754448 (duration: 0:07:00.852874) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.649 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.974 +Monitored metric validation/losses/itm_loss did not improve in the last 16 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 55 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 68 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 20 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-23 14:50:53.071906] Computing e^(tensor([4267.1336], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-23 14:50:53.076595] Ending with PPL=8.444934627687438 (duration: 0:07:06.727088) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.642 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.974. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 17 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 56 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 69 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 21 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-23 16:38:50.795250] Computing e^(tensor([4248.3354], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-23 16:38:50.800017] Ending with PPL=8.365931434476826 (duration: 0:07:01.290303) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.636 +Monitored metric validation/losses/mim_loss did not improve in the last 2 records. Best score: 0.974. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 18 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 57 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 70 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 22 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-23 18:18:44.853598] Computing e^(tensor([4227.0907], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-23 18:18:44.858492] Ending with PPL=8.277536275555901 (duration: 0:06:55.252060) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.632 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 0.974. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 19 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 58 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 71 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 23 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-23 19:35:20.432337] Computing e^(tensor([4219.9806], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-23 19:35:20.436595] Ending with PPL=8.24816134801621 (duration: 0:06:55.817449) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.624 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 0.974. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 20 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 59 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 72 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 24 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-23 20:52:27.256394] Computing e^(tensor([4221.2854], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-23 20:52:27.260943] Ending with PPL=8.25354406648451 (duration: 0:06:57.925209) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.971 +Monitored metric validation/losses/itm_loss did not improve in the last 21 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 60 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 73 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 25 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-23 22:07:12.632244] Computing e^(tensor([4192.9070], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-23 22:07:12.636587] Ending with PPL=8.137259749958424 (duration: 0:06:54.288751) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.613 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.971. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 22 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 61 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 74 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 26 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-23 23:20:44.375254] Computing e^(tensor([4189.6755], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-23 23:20:44.379722] Ending with PPL=8.124122881683931 (duration: 0:06:55.613691) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.607 +Monitored metric validation/losses/mim_loss did not improve in the last 2 records. Best score: 0.971. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 23 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 62 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 75 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 27 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-24 00:35:03.421161] Computing e^(tensor([4167.0158], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-24 00:35:03.425750] Ending with PPL=8.032597196323424 (duration: 0:06:55.641483) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.600 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.970 +Monitored metric validation/losses/itm_loss did not improve in the last 24 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 63 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 76 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 28 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-24 01:50:03.894336] Computing e^(tensor([4183.6229], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-24 01:50:03.900310] Ending with PPL=8.099574027830835 (duration: 0:06:53.347922) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.600 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.970. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 25 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 64 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 77 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 29 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-24 03:04:47.175734] Computing e^(tensor([4156.2168], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-24 03:04:47.179864] Ending with PPL=7.989341835541894 (duration: 0:07:04.227822) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.593 +Monitored metric validation/losses/mim_loss did not improve in the last 2 records. Best score: 0.970. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 26 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 65 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 78 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 30 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-24 04:20:10.778931] Computing e^(tensor([4142.7029], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-24 04:20:10.783142] Ending with PPL=7.93554044718299 (duration: 0:07:03.232162) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.587 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 0.970. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 27 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 66 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 79 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 31 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-24 05:35:08.874188] Computing e^(tensor([4142.9835], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-24 05:35:08.880342] Ending with PPL=7.936653826332348 (duration: 0:06:59.201926) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.586 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.965 +Monitored metric validation/losses/itm_loss did not improve in the last 28 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 67 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 80 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 32 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-24 06:48:27.593680] Computing e^(tensor([4130.4027], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-24 06:48:27.597958] Ending with PPL=7.886885734615813 (duration: 0:07:04.018009) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.573 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.965. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 29 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 68 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 81 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 33 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-24 08:02:03.240670] Computing e^(tensor([4126.2344], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-24 08:02:03.245380] Ending with PPL=7.870465235059236 (duration: 0:06:59.981557) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.572 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 0.965. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 31 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 70 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 83 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 35 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-24 10:32:32.968955] Computing e^(tensor([4111.7767], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-24 10:32:32.973622] Ending with PPL=7.813776251205194 (duration: 0:07:06.069239) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.567 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 0.965. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 32 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 71 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 84 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 36 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-24 11:51:17.182434] Computing e^(tensor([4103.6423], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-24 11:51:17.188573] Ending with PPL=7.782060337218664 (duration: 0:06:56.407699) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.563 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 0.965. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 33 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 72 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 85 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 37 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-24 13:07:43.160902] Computing e^(tensor([4079.1245], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-24 13:07:43.167859] Ending with PPL=7.687243320861675 (duration: 0:07:30.094913) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.553 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.964 +Monitored metric validation/losses/itm_loss did not improve in the last 34 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 73 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 86 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 38 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-24 14:29:43.386473] Computing e^(tensor([4103.6973], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-24 14:29:43.390878] Ending with PPL=7.782274504268058 (duration: 0:08:26.468409) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.963 +Monitored metric validation/losses/itm_loss did not improve in the last 35 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 74 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 87 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 39 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-24 15:47:26.966407] Computing e^(tensor([4076.5113], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-24 15:47:26.972723] Ending with PPL=7.67720578491687 (duration: 0:07:04.429280) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.552 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.963. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 36 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 75 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 88 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 40 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-24 17:06:27.946852] Computing e^(tensor([4072.5205], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-24 17:06:27.951026] Ending with PPL=7.661902089043111 (duration: 0:07:06.513731) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.544 +Monitored metric validation/losses/mim_loss did not improve in the last 2 records. Best score: 0.963. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 37 records. Best score: 0.315. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 76 records. Best score: 2.712. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 89 records. Best score: 0.988. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 41 records. Best score: 3.047. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-24 18:24:13.045911] Computing e^(tensor([4075.9556], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-24 18:24:13.050174] Ending with PPL=7.675072777672177 (duration: 0:07:06.040568) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00 Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-03 21:17:24.412582 Setting up datamodule: ImageDataModule +2023-07-03 21:17:28.242315 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ImageNetZeroshotCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-03 21:25:41.904156 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-03 21:30:30.344372 Setting up datamodule: ImageDataModule +2023-07-03 21:30:33.768898 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-03 21:39:25.159475 Building train dataloader for datamodule: VLDataModule +2023-07-03 21:39:25.160216 Building train dataloader for datamodule: ImageDataModule +2023-07-03 21:39:25.160552 Building train dataloader for datamodule: MLMDataModule +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +VLDataModule: batch_size is 32, num_workers is 4 +ImageDataModule: batch_size is 32, num_workers is 4 +MLMDataModule: batch_size is 32, num_workers is 4 +Dataset sizes (un-normalized): [1, 1, 10] +Dataset sizes (normalized): [0.08333333333333333, 0.08333333333333333, 0.8333333333333334] +Sampling weights after temperature (1.0): [0.08333333333333333, 0.08333333333333333, 0.8333333333333334] +2023-07-03 20:58:50.872396 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-03 21:03:46.131401 Setting up datamodule: ImageDataModule +2023-07-03 21:03:50.151759 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ImageNetZeroshotCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-03 21:13:18.558581 Preparing data for datamodule: VLDataModule +2023-07-03 21:13:18.558811 Preparing data for datamodule: ImageDataModule +2023-07-03 21:13:18.558985 Preparing data for datamodule: MLMDataModule +2023-07-03 21:25:41.904198 Setting up datamodule: VLDataModule +Counting words for split train[:1%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:1%] ----> Total words: 426267, No. of unique: 65725 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:1%] after collapsing... +Split after collapsing: train[:1%] ----> Total words: 10283342, No. of unique: 257727 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-03 21:30:30.364585 Setting up datamodule: ImageDataModule +2023-07-03 21:30:33.919730 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-03 21:39:25.159488 Building train dataloader for datamodule: VLDataModule +2023-07-03 21:39:25.160327 Building train dataloader for datamodule: ImageDataModule +2023-07-03 21:39:25.160649 Building train dataloader for datamodule: MLMDataModule +2023-07-03 21:39:27.169338 Building val dataloader for datamodule: VLDataModule +2023-07-03 21:39:27.170381 Building val dataloader for datamodule: ImageDataModule +2023-07-03 21:39:27.170845 Building val dataloader for datamodule: MLMDataModule +2023-07-03 21:39:27.165988 Building val dataloader for datamodule: VLDataModule +2023-07-03 21:39:27.167161 Building val dataloader for datamodule: ImageDataModule +2023-07-03 21:39:27.167645 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-03 22:23:37.436511] Computing e^(tensor([16993.9415], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-03 22:23:37.447197] Ending with PPL=4899.903297318964 (duration: 0:11:20.751965) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00, 'Connection to proxy.ethz.ch timed out. (connect timeout=10)')))' thrown while requesting HEAD https://huggingface.co/facebook/flava-full/resolve/main/preprocessor_config.json +Traceback (most recent call last): + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connection.py", line 200, in _new_conn + sock = connection.create_connection( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/util/connection.py", line 85, in create_connection + raise err + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/util/connection.py", line 73, in create_connection + sock.connect(sa) +TimeoutError: timed out + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connectionpool.py", line 776, in urlopen + self._prepare_proxy(conn) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connectionpool.py", line 1041, in _prepare_proxy + conn.connect() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connection.py", line 604, in connect + self.sock = sock = self._new_conn() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connection.py", line 209, in _new_conn + raise ConnectTimeoutError( +urllib3.exceptions.ConnectTimeoutError: (, 'Connection to proxy.ethz.ch timed out. (connect timeout=10)') + +The above exception was the direct cause of the following exception: + +urllib3.exceptions.ProxyError: ('Unable to connect to proxy', ConnectTimeoutError(, 'Connection to proxy.ethz.ch timed out. (connect timeout=10)')) + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/adapters.py", line 486, in send + resp = conn.urlopen( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connectionpool.py", line 844, in urlopen + retries = retries.increment( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/util/retry.py", line 515, in increment + raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] +urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='huggingface.co', port=443): Max retries exceeded with url: /facebook/flava-full/resolve/main/preprocessor_config.json (Caused by ProxyError('Unable to connect to proxy', ConnectTimeoutError(, 'Connection to proxy.ethz.ch timed out. (connect timeout=10)'))) + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 41, in _call_and_handle_interrupt + return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 91, in launch + return function(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 570, in _fit_impl + self._run(model, ckpt_path=ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 975, in _run + results = self._run_stage() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1018, in _run_stage + self.fit_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 201, in run + self.advance() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 354, in advance + self.epoch_loop.run(self._data_fetcher) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 134, in run + self.on_advance_end() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 248, in on_advance_end + self.val_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/utilities.py", line 177, in _decorator + return loop_run(self, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 102, in run + self.on_run_start() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 234, in on_run_start + self._on_evaluation_start() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 283, in _on_evaluation_start + call._call_callback_hooks(trainer, hook_name, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 189, in _call_callback_hooks + fn(trainer, trainer.lightning_module, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context + return func(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_utilities/core/rank_zero.py", line 27, in wrapped_fn + return fn(*args, **kwargs) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/callbacks/blimp_eval.py", line 47, in on_validation_start + eval_model = FlavaLM(model=pl_module.model, + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/callbacks/flava_lm.py", line 37, in __init__ + self.tokenizer = FlavaProcessor.from_pretrained("facebook/flava-full").tokenizer + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/processing_utils.py", line 184, in from_pretrained + args = cls._get_arguments_from_pretrained(pretrained_model_name_or_path, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/processing_utils.py", line 228, in _get_arguments_from_pretrained + args.append(attribute_class.from_pretrained(pretrained_model_name_or_path, **kwargs)) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/image_processing_utils.py", line 165, in from_pretrained + image_processor_dict, kwargs = cls.get_image_processor_dict(pretrained_model_name_or_path, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/image_processing_utils.py", line 269, in get_image_processor_dict + resolved_image_processor_file = cached_file( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/utils/hub.py", line 417, in cached_file + resolved_file = hf_hub_download( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/utils/_validators.py", line 118, in _inner_fn + return fn(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/file_download.py", line 1195, in hf_hub_download + metadata = get_hf_file_metadata( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/utils/_validators.py", line 118, in _inner_fn + return fn(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/file_download.py", line 1532, in get_hf_file_metadata + r = _request_wrapper( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/file_download.py", line 407, in _request_wrapper + response = _request_wrapper( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/file_download.py", line 442, in _request_wrapper + return http_backoff( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/utils/_http.py", line 228, in http_backoff + raise err + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/utils/_http.py", line 212, in http_backoff + response = session.request(method=method, url=url, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/sessions.py", line 589, in request + resp = self.send(prep, **send_kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/sessions.py", line 703, in send + r = adapter.send(request, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/adapters.py", line 513, in send + raise ProxyError(e, request=request) +requests.exceptions.ProxyError: HTTPSConnectionPool(host='huggingface.co', port=443): Max retries exceeded with url: /facebook/flava-full/resolve/main/preprocessor_config.json (Caused by ProxyError('Unable to connect to proxy', ConnectTimeoutError(, 'Connection to proxy.ethz.ch timed out. (connect timeout=10)'))) + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 167, in + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 153, in main + wandb_logger.experiment.config.update(config) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 531, in fit + call._call_and_handle_interrupt( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 63, in _call_and_handle_interrupt + trainer.strategy.on_exception(exception) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/ddp.py", line 393, in on_exception + _augment_message( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/utilities/exceptions.py", line 40, in _augment_message + exception.args = tuple( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/utilities/exceptions.py", line 41, in + new_message if re.match(pattern, message, re.DOTALL) else message for message in exception.args + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/re.py", line 190, in match + return _compile(pattern, flags).match(string) +TypeError: expected string or bytes-like object +Traceback (most recent call last): + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connection.py", line 200, in _new_conn + sock = connection.create_connection( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/util/connection.py", line 85, in create_connection + raise err + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/util/connection.py", line 73, in create_connection + sock.connect(sa) +TimeoutError: timed out + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connectionpool.py", line 776, in urlopen + self._prepare_proxy(conn) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connectionpool.py", line 1041, in _prepare_proxy + conn.connect() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connection.py", line 604, in connect + self.sock = sock = self._new_conn() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connection.py", line 209, in _new_conn + raise ConnectTimeoutError( +urllib3.exceptions.ConnectTimeoutError: (, 'Connection to proxy.ethz.ch timed out. (connect timeout=10)') + +The above exception was the direct cause of the following exception: + +urllib3.exceptions.ProxyError: ('Unable to connect to proxy', ConnectTimeoutError(, 'Connection to proxy.ethz.ch timed out. (connect timeout=10)')) + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/adapters.py", line 486, in send + resp = conn.urlopen( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/connectionpool.py", line 844, in urlopen + retries = retries.increment( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/urllib3/util/retry.py", line 515, in increment + raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] +urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='huggingface.co', port=443): Max retries exceeded with url: /facebook/flava-full/resolve/main/preprocessor_config.json (Caused by ProxyError('Unable to connect to proxy', ConnectTimeoutError(, 'Connection to proxy.ethz.ch timed out. (connect timeout=10)'))) + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 41, in _call_and_handle_interrupt + return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 91, in launch + return function(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 570, in _fit_impl + self._run(model, ckpt_path=ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 975, in _run + results = self._run_stage() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1018, in _run_stage + self.fit_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 201, in run + self.advance() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 354, in advance + self.epoch_loop.run(self._data_fetcher) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 134, in run + self.on_advance_end() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 248, in on_advance_end + self.val_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/utilities.py", line 177, in _decorator + return loop_run(self, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 102, in run + self.on_run_start() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 234, in on_run_start + self._on_evaluation_start() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 283, in _on_evaluation_start + call._call_callback_hooks(trainer, hook_name, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 189, in _call_callback_hooks + fn(trainer, trainer.lightning_module, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context + return func(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_utilities/core/rank_zero.py", line 27, in wrapped_fn + return fn(*args, **kwargs) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/callbacks/blimp_eval.py", line 47, in on_validation_start + eval_model = FlavaLM(model=pl_module.model, + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/callbacks/flava_lm.py", line 37, in __init__ + self.tokenizer = FlavaProcessor.from_pretrained("facebook/flava-full").tokenizer + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/processing_utils.py", line 184, in from_pretrained + args = cls._get_arguments_from_pretrained(pretrained_model_name_or_path, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/processing_utils.py", line 228, in _get_arguments_from_pretrained + args.append(attribute_class.from_pretrained(pretrained_model_name_or_path, **kwargs)) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/image_processing_utils.py", line 165, in from_pretrained + image_processor_dict, kwargs = cls.get_image_processor_dict(pretrained_model_name_or_path, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/image_processing_utils.py", line 269, in get_image_processor_dict + resolved_image_processor_file = cached_file( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/utils/hub.py", line 417, in cached_file + resolved_file = hf_hub_download( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/utils/_validators.py", line 118, in _inner_fn + return fn(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/file_download.py", line 1195, in hf_hub_download + metadata = get_hf_file_metadata( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/utils/_validators.py", line 118, in _inner_fn + return fn(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/file_download.py", line 1532, in get_hf_file_metadata + r = _request_wrapper( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/file_download.py", line 407, in _request_wrapper + response = _request_wrapper( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/file_download.py", line 442, in _request_wrapper + return http_backoff( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/utils/_http.py", line 228, in http_backoff + raise err + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/utils/_http.py", line 212, in http_backoff + response = session.request(method=method, url=url, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/sessions.py", line 589, in request + resp = self.send(prep, **send_kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/sessions.py", line 703, in send + r = adapter.send(request, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/adapters.py", line 513, in send + raise ProxyError(e, request=request) +requests.exceptions.ProxyError: HTTPSConnectionPool(host='huggingface.co', port=443): Max retries exceeded with url: /facebook/flava-full/resolve/main/preprocessor_config.json (Caused by ProxyError('Unable to connect to proxy', ConnectTimeoutError(, 'Connection to proxy.ethz.ch timed out. (connect timeout=10)'))) + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 167, in + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 153, in main + wandb_logger.experiment.config.update(config) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 531, in fit + call._call_and_handle_interrupt( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 63, in _call_and_handle_interrupt + trainer.strategy.on_exception(exception) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/ddp.py", line 393, in on_exception + _augment_message( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/utilities/exceptions.py", line 40, in _augment_message + exception.args = tuple( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/utilities/exceptions.py", line 41, in + new_message if re.match(pattern, message, re.DOTALL) else message for message in exception.args + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/re.py", line 190, in match + return _compile(pattern, flags).match(string) +TypeError: expected string or bytes-like object +wandb: Waiting for W&B process to finish... (failed 1). Press Control-C to abort syncing. +wandb: Waiting for W&B process to finish... (failed 1). Press Control-C to abort syncing. +wandb: ERROR Error uploading "output.log": CommError, File ./wandb/run-20230703_211333-dqgqdbut/files/output.log size shrank from 18986 to 106 while it was being uploaded. +wandb: +wandb: Run history: +wandb: ImageDataModule ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: MLMDataModule ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: VLDataModule ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: epoch ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: evaluation/blimp/anaphor_agreement ▂▁▁▂▃▄▅▄▃▅▄▅▅▆▆▇▇▇▇▇███▇▇▇██████████ +wandb: evaluation/blimp/argument_structure ▁▃▃▃▃▃▄▄▄▅▅▅▅▅▅▆▅▆▅▆▆▆▅▆▇▆▇▇▇▆▆▇█▇██ +wandb: evaluation/blimp/binding ▅██▇▇▆▆▆▄▃▅▃▄▃▂▂▂▃▃▁▁▂▄▄▄▅▆▆█▇▆▇▆▇█▇ +wandb: evaluation/blimp/control_raising ▁▂▂▃▃▄▄▃▂▃▃▃▂▃▂▃▄▃▄▅▅▄▅▅▆▆▇▇▇▇▇▆▆█▇▇ +wandb: evaluation/blimp/determiner_noun_agreement ▁▁▁▁▁▂▂▂▂▃▃▄▅▆▆▆▆▇▇▇▇▇▇█████████████ +wandb: evaluation/blimp/ellipsis ▂▃▃▂▃▃▂▁▁▃▃▄▅▅▅▅▆▆▆▆▆▆▆▆▇▇▇▇▇▇█▇████ +wandb: evaluation/blimp/filler_gap ▁█▃▅▇▇▆▅▇▆▆▇▆▆▆▆▆▆▆▆▆▅▅▅▅▅▅▅▅▅▅▅▅▅▄▅ +wandb: evaluation/blimp/irregular_forms ▂▃▁▁▁▂▃▃▃▃▅▅▅▅▆▆▆▅▅▇▇█▇▆▆▆▆█▇▆▇▇█▇▇█ +wandb: evaluation/blimp/island_effects ▇▇▇▇▆█▇█▄▃▄▁▂▂▂▃▂▂▃▄▄▄▄▄▅▄▆▅▅▇▇▆▅▆▆▆ +wandb: evaluation/blimp/npi_licensing ▂▅▇▅▄▁▁▄▆▆▆▄▅▆▇▆██▇▆█▆▆▇▆▅▅▇▇▅▄▅▆▅▅▅ +wandb: evaluation/blimp/quantifiers ▄▂▁▁▂▂▂▄▆█▇████▇▆▆▇▇▇▇▆▆▇▆▆▅▆▆▅▅▆▆▅▅ +wandb: evaluation/blimp/subject_verb_agreement ▁▂▂▂▂▂▂▂▂▃▃▄▅▅▅▅▅▆▆▆▆▆▆▇▆▇▇▇█▇▇▇█▇██ +wandb: evaluation/blimp_average ▁▂▁▁▂▃▃▃▃▄▄▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇█████████ +wandb: evaluation/imagenet_zeroshot/top1 ▁▅▄▂▂▁▄▁▄▇█▆▇▆▅▇▄▄▃▄▃▇▇▅▂▆▅▅▇▅▃▅▆█▂▇ +wandb: evaluation/imagenet_zeroshot/top5 ▄▄▂▂▁▄▄▃▄▄▅▇▅█▄▅▆▄▅▅▅▆▆▄▄▅▄▇▇▄▄▅▆▆▄█ +wandb: evaluation/pseudo_perplexity █▄▃▃▃▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: lr-AdamW/pg1 ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: lr-AdamW/pg2 ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: train/losses/global_contrastive_loss ███▇▆▅▆▄▅▃▃▃▃▄▃▄▃▄▃▃▁▁▂▄▄▄▂▄▄▄▅▄▄▄▄▄▄▄▄▄ +wandb: train/losses/itm_loss █▂█▄▆▆▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: train/losses/mim_loss █▄▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▂▂▁▁▁▁▁▂▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: train/losses/mlm_loss █▆▆▅▅▅▅▅▄▄▄▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▁▁▂▂▁▁▁▁▁▁▁▁▁ +wandb: train/losses/mmm_image_loss █▃▂▂▂▂▁▁▁▁▁▁▁▁▁▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: train/losses/mmm_text_loss █▆▅▅▅▅▅▅▅▄▃▃▃▃▃▂▃▂▂▂▂▂▂▂▂▁▂▂▁▁▂▁▁▁▂▁▁▁▁▁ +wandb: trainer/global_step ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: validation/losses/global_contrastive_loss ██▆▅▄▄▄▃▃▃▃▃▂▂▂▂▂▂▁▁▁▁▁▁▁▁▂▂▂ +wandb: validation/losses/itm_loss ▁▆█▃▄ +wandb: validation/losses/mim_loss █▃▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/losses/mlm_loss █▇▆▆▆▅▅▅▄▄▃▃▃▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/losses/mmm_image_loss █▃▂▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/losses/mmm_text_loss █▆▆▆▆▅▅▅▄▄▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/monitor/global_contrastive █████▅▅▅▅▃▃▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/monitor/itm █▅▃▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/monitor/mim ███████▅▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/monitor/mlm ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/monitor/mmm_image █████▅▅▃▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/monitor/mmm_text █████████████████████████████▁▁▁▁▁▁▁ +wandb: +wandb: Run summary: +wandb: ImageDataModule 24317 +wandb: MLMDataModule 248231 +wandb: VLDataModule 23455 +wandb: epoch 0 +wandb: evaluation/blimp/anaphor_agreement 93.97 +wandb: evaluation/blimp/argument_structure 67.75 +wandb: evaluation/blimp/binding 63.46 +wandb: evaluation/blimp/control_raising 63.59 +wandb: evaluation/blimp/determiner_noun_agreement 87.47 +wandb: evaluation/blimp/ellipsis 76.39 +wandb: evaluation/blimp/filler_gap 61.78 +wandb: evaluation/blimp/irregular_forms 84.78 +wandb: evaluation/blimp/island_effects 48.84 +wandb: evaluation/blimp/npi_licensing 52.69 +wandb: evaluation/blimp/quantifiers 54.66 +wandb: evaluation/blimp/subject_verb_agreement 64.91 +wandb: evaluation/blimp_average 68.3575 +wandb: evaluation/imagenet_zeroshot/top1 0.00134 +wandb: evaluation/imagenet_zeroshot/top5 0.00664 +wandb: evaluation/pseudo_perplexity 12.87209 +wandb: lr-AdamW/pg1 0.00046 +wandb: lr-AdamW/pg2 0.00034 +wandb: train/losses/global_contrastive_loss 2.72632 +wandb: train/losses/itm_loss 0.17523 +wandb: train/losses/mim_loss 0.95932 +wandb: train/losses/mlm_loss 2.64568 +wandb: train/losses/mmm_image_loss 0.96786 +wandb: train/losses/mmm_text_loss 3.08949 +wandb: trainer/global_step 4599 +wandb: validation/losses/global_contrastive_loss 2.86443 +wandb: validation/losses/itm_loss 0.32719 +wandb: validation/losses/mim_loss 0.99233 +wandb: validation/losses/mlm_loss 3.15926 +wandb: validation/losses/mmm_image_loss 0.99464 +wandb: validation/losses/mmm_text_loss 3.32986 +wandb: validation/monitor/global_contrastive 0.0 +wandb: validation/monitor/itm 0.0 +wandb: validation/monitor/mim 0.0 +wandb: validation/monitor/mlm 0.83333 +wandb: validation/monitor/mmm_image 0.0 +wandb: validation/monitor/mmm_text 0.5 +wandb: +wandb: 🚀 View run dauntless-darkness-212 at: https://wandb.ai/rycolab/alkmi-wit/runs/9hnl8w94 +wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20230703_205758-9hnl8w94/logs +wandb: +wandb: Run history: +wandb: ImageDataModule ▁ +wandb: MLMDataModule ▁ +wandb: VLDataModule ▁ +wandb: +wandb: Run summary: +wandb: ImageDataModule 24317 +wandb: MLMDataModule 248231 +wandb: VLDataModule 23455 +wandb: +wandb: 🚀 View run glowing-planet-213 at: https://wandb.ai/rycolab/alkmi-wit/runs/dqgqdbut +wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20230703_211333-dqgqdbut/logs +[ImageNet Zero-Shot Evaluation 2023-07-05 07:44:25.612277] Ending with top5=0.00664 (duration: 0:02:07.569286) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-05 08:34:45.953674] Computing e^(tensor([5222.5291], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-05 08:34:45.960632] Ending with PPL=13.616258673532212 (duration: 0:19:34.514829) +Starting LM Evaluation Harness diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=10038.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=10038.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..929456d9312b72ec43a9a330f390dd27ca85a1c3 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=10038.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04e5f620b6ede7ad1d83a0c11add0ebcaeeba9eb96b9c896eda2fcd3e51ff57d +size 1849229565 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=10215.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=10215.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c6fe64495c078cdb6d7531fdf02c197ce56e881b --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=10215.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:223c34acb49fc1948171024bc35b4424cd3f675f6df572e6f736ca42ace74015 +size 1849230076 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=10383.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=10383.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..4a39a7989dc82890f063108ce1e7660437432641 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=10383.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d48a7c6a6feb3da4b9c52441bbb364ebbf85c73774bef1d5bd6119034836f40a +size 1849230587 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=10560.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=10560.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..91cf81e0f4c98593d620999f88e30e8e0e66dbba --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=10560.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d0c1eb68a5461fa3636f9c58057424091dc0e47d8b477375aebac4455f25c06 +size 1849231098 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=10740.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=10740.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..1be351259ddfee76449bdb31255b9098e8e8f3b1 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=10740.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71b329c0cc1920a20d4b857b3ca3a5c43513dcdc8e6d69a85f2b1a121854fdb8 +size 1849231545 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=10919.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=10919.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f3a07f671f9ed03985b195f077250db63adf347a --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=10919.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:889c21e415f8c15bd779f66060f6a6c05d0da7e8ee49e888a7a5b13db33c46d8 +size 1849231992 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=11106.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=11106.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..43a660f06a374ead451fefe6dcdf6a224d53b51b --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=11106.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9527bee70b65345b79e5f98bcb52f610dd865c1f8eff13b2dcc807122e3bbda6 +size 1849232503 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=11320.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=11320.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..ef2e1027de27306e486b68b1f0a081f146b5247c --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=11320.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1905aafdff814f2d0196303b0f1fb48564d19e5ce1ccd7d82bd1b15f78b84e8 +size 1849232950 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=11583.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=11583.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..1749f5f1c6f7b963620cb40db08f5eeb9dd13d90 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=11583.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d473f5299c660be83a33034c03813ff6dd87516e02f173d0bde85eddbe861a3b +size 1849233461 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=1168.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=1168.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..69c80248d3d9ee857699d6ec7fe29f9af86f31fb --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=1168.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8678b638f3da07462cd54d4bfcba80fdbf97324df8fd1ad1b312950c217011d +size 1849214931 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=11848.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=11848.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..85368262ad587e52461e2139b56a86ac02dc246c --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=11848.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25c5f61508bafaab12b37b029d1c91ce6650c7c20df5558f885fdc287172b187 +size 1849233908 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=12051.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=12051.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c6bd92748bc60959d5ec1e6c8bd709822709c23a --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=12051.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:162983fdd8accbe18c1975786aeead9fa277f537dbb55ba6a57481484609a9c0 +size 1849234483 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=12295.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=12295.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..a8e18c80ef1c70e50154360999edd51a56de5f80 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=12295.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ed7798258b21edf51bf67224c5e06fd9dec7615e3c2c2c976a6387b5344445e +size 1849234930 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=12518.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=12518.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..373372917f08e7d874aad61a25b849bd63c2c4c0 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=12518.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c81d3db87f7376e86704384f5020320ef250582ed3c7ff398c45263820180f87 +size 1849235377 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=12721.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=12721.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..977bc4184d3de2f430ab6f1a7cdeb39af229c018 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=12721.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:204aef5786e92d1d12d95bbcf41132a3eb4a0f8a1e4dddd93072ad3e04788cb0 +size 1849217303 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=12922.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=12922.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c4c7ab69d9b3c1c7900b4cb006ec1fe17a71720f --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=12922.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04af1a621edb50e2a48e98edb64c3e0d490e423c3a7244b0007f51866f08414a +size 1849217814 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=13119.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=13119.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..85e68426ce99ebaf27962bed78e30ee9da090edd --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=13119.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3d77cc399aa6120b2f395489b424b294a32c9cc793fffd8354f68b9b9d16abf +size 1849218197 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=13320.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=13320.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..35e257cd15876bd7660b2f10f92eb85c23ccf51a --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=13320.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9797fdee9f43a89162cfa848e490c5b63e4d1e18cab0389300292936a49a1ba0 +size 1849218708 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=13524.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=13524.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b946d7050656e65afc6b352ecb8cd7e9c5fb0cf1 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=13524.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40024352f3a062d68c0358cada118e37fcbaafc23448639eb95604d465704a43 +size 1849219219 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=13735.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=13735.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..a2a73ecaf0b7d5d01240ecd6a12cc2d4ef7f76ca --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=13735.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87cacb89ea39851faa211c67fb223a8f9e60d8144d4b337b626b3fb4b98d65c6 +size 1849219730 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=13926.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=13926.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..258e1537c11f46c09963c75a49d8eff49f4fa92c --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=13926.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f440d52b693ed130aa9f79370a03a49d8b1cf3d6861b822042b02ff24705a5a6 +size 1849220177 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=1407.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=1407.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..432d84ad4906cee5c399b0efecec98e1c6f1fdf3 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=1407.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94de9912246a050bb3d49388456f3e209fb5f454a9598aa7bb4c842865733f0b +size 1849215378 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=14116.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=14116.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..aeccc27c6d021eef885d12f1d33679628a7f0bea --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=14116.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00846e2e1a6678fec755053982a6cc5bb35e164fae578a542f4bd06b54ff7440 +size 1849220752 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=14301.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=14301.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..3886ead6139aa1375d29fec7c26b712b92995212 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=14301.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af4a5397b7527bdbd2ea7ee4def9dd583b709b7ab3a91a231c5c4cb330df35ad +size 1849221199 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=14481.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=14481.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..95b1c532a37505fc2537643f215bdf35b1837623 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=14481.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:677744aabab435d4b0d8fbfceddd9c166233ab002d558a269fa0e15feec39726 +size 1849221582 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=14658.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=14658.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..dcea516a6b2b025769967665764dbdfa820bb5ce --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=14658.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ce1ca9a251bdd2872a7fef8bd29e4b11223cea36baecba96b3f45728bc4103a +size 1849222093 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=14828.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=14828.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..7af37906b4ac4c8435a9abee14ebab85452546a0 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=14828.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fb42aba035e34399f6596af974a4699076ef322f64ed9b2c1412f82ba12fce3 +size 1849222604 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=15009.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=15009.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..604769cbc52be796ef82bebdcd81c00f93def397 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=15009.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79cdf81fd337106ccedcaadf9954efe4e597b7e746b476603087fef644b014d2 +size 1849223115 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=15195.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=15195.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..1460da022e955b61eff382d76d7c4e611afe878a --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=15195.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b468aac350e6a995b017c68f7d2c5d1e81e209024e788c11e227619911bd3126 +size 1849223562 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=15386.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=15386.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..606486a7c989202c9138a04b491eecf7ebb6988b --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=15386.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c81be4f39d96acde7ef83bbdc3429fddeecd17874b7e593ef60a684c02aa9ec0 +size 1849224073 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=15572.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=15572.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..d0dff1a3d8fa4318f034560b887e01c5e703b48d --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=15572.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:658ac108ee6f4f873098b36a316c84f7f10c72c0777280b180811c2f99e0f505 +size 1849224520 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=15790.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=15790.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c0dad29487e540a38c866f5347d9b8a8b6fbda82 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=15790.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:398793fbbe053b3e4df1a7b18107c0083000c423f47b3b6b1537c9f31792a6d2 +size 1849224967 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=16002.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=16002.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..e08017f1c4f3882f5664740ed7f0766bb2f60520 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=16002.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d87f5715fb820dd5a7c267f6ecbfcce7c2de40f9b544c68545b7b013bba59d8 +size 1849225414 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=1601.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=1601.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..a24f22c095577b154ddf0fd8b43ac2219cd9e849 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=1601.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab897d65deaab02850f0cd9702b5da1b81f992d0fa35ea28f921587386fa5da4 +size 1849215889 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=16198.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=16198.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..2025d1b88803623b4d378f2d5ddfe406b229adce --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=16198.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1dbe2175ffc9310a4643d165ff0675fb48126b8b5391eddcd31828d316d3ab9 +size 1849225925 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=16396.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=16396.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..9bd99d34fe56c9b93dec2f491232595d82dfdcd6 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=16396.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d4cd87cf93a670dfdce531ca63abbc2d26e75f69935adcbe08a0a5af0466160 +size 1849226308 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=16614.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=16614.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..0d9b6968746c822361849246a9258ae3b855aaf3 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=16614.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfd9acf1c25a1d715014d041a7604242d3177acb9e2d767659f58959e2088db3 +size 1849226691 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=16816.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=16816.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..a0bab1d544264faf05ef5e80db867a5eb41cec0d --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=16816.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26b4155efdcecab706bb8db36f1a786797d45fc820d5f5081f07fc6a510033b4 +size 1849227330 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=17031.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=17031.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..272add37003ce9e6e1ee25d9c7d1ba9df4e07135 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=17031.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c74927aa6e8bf0728bc624efd26b2f7793ab35740bddfa5a18a065084f59767 +size 1849227713 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=17215.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=17215.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..0c7d1366c1a236ded4898b293bcd727d6cdb4a23 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=17215.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19a0e9f0ece8beae6bccef6dc7c332c8c24746cbb4ac2447087cf60e32b62cf1 +size 1849228096 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=17409.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=17409.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..accd03ae35cc53975b72e1eb9327d8eec1a7abd6 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=17409.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4994578169dff66ecbffed8b3026be80639624c2ed45a81e8efaf35f56125656 +size 1849228671 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=17602.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=17602.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c02b5f569285ce8bb012d4dd19dab9a2870bf0c8 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=17602.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b691959fbd9f6b2dd3091ef83a84fe67f4a78a835befaaad2e87afaa1a1d38c4 +size 1849229182 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=17803.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=17803.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..35b0c740c7123cc2745c8a6a0340788daab183ce --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=17803.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40d866642433fc913357bbdbbcef69d2a25f36f29b6dfbe203d55abbf93ffbdf +size 1849229565 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=17992.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=17992.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..bcbe74a28e926215398d727e3d2beba4573292fc --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=17992.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aff233c5b8b7494de43ecb40c401485ff788bf0413991c869321c48a73e9439c +size 1849230012 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=18191.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=18191.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..57072cac1f6512e809c89807c8e04113e11b6608 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=18191.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24757d35a1a64c4c54041b7aa396f5ccf45e61919dca89c221b08d36d1b9daf6 +size 1849230587 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=1837.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=1837.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..dfe175b68bb9cd602a1fc6e03ed29de4144afc8f --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=1837.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52478de7a5b57ca34be7fee4752601143653f34a248ffd97e0cf93a19148b16b +size 1849216400 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=18394.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=18394.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..7a5d697a729b9bb0ffeb2766d57b510b70e68656 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=18394.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44925540ee352a005da2805f91d4450c5630af24a4e45ed7680f6bcffbd992e5 +size 1849231034 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=18598.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=18598.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..78827ac940443e65b85a268d0841f525fe403b76 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=18598.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32e5fd6ad220d463a844a88aee961c94109a6910eb595cb3c0ea3c04ecf1bc7b +size 1849231545 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=18807.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=18807.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..8c331a50e5adb87d9b6ccf523d3ce0b9b3c75654 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=18807.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:216fa54ef66069c006b818168fa9fc21bc819c080792554835c1d17adcb8ee12 +size 1849231992 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=19013.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=19013.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..6bf1ab50a0a4d2d19d62e7c4cfce2d290c0e56d1 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=19013.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:675a83b890088b32c6910edd16c033ddc37e9fb76174cd3fab64caf914e8e919 +size 1849232503 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=19220.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=19220.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..196702a6a2425c6b49cd481e029d3b24296e1ecf --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=19220.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6449705be2a17bcf29ca8a1003c3e3c0232e2c65e53531c1607ee2d96d76bf6 +size 1849232950 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=19420.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=19420.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..82554a958b7a246da25f66aa34c8d8d8a396bd2c --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=19420.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fc9c87461a8375618797fa041f8716bb2bef1c5b0c5ec3e06ac38f579306c4c +size 1849233397 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=19631.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=19631.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..853cf954a5241c8b17438a078ce7b8beb94e9ee2 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=19631.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cd929dfdcfdc20f14937e05e3b531e3c69cb5fc978acbc3e5f7e6fc97f297d2 +size 1849233908 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=19854.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=19854.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..42b0782ba3a743c8793cdce52ace02bee564b431 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=19854.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0185bda977e5111a0147eb3829cf8c377cec38a6ff4b15e337b3b8b9ff9ce271 +size 1849234419 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=20049.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=20049.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..4e31ff9e1d84dc24eaaeddbb96aeed3d1e7128fe --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=20049.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e21e5a839e24f26bbb47789025ea193468b8b2ac59fbc35d32c3846c21636690 +size 1849234930 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=20248.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=20248.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..36a874e70930e965dacb2ebd6836e38f779eb32e --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=20248.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d313a7cb94f2ea6035243e9dae92c185bec78e5c3eb96dfcb7635133c3b2f44 +size 1849235313 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=20583.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=20583.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..32689a555ee2c4836cce4d605ea0a2d016dd9880 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=20583.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d270314bcf4ea599df87c7f87941b4868e5b7afead9b2a8bf51e085bcca520e9 +size 1849217303 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=2071.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=2071.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..e01509ed13b3d3ed8e1895051cb75310e4deee90 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=2071.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d09b34bbc11ff2a10bc5f3921f7e78f176b3312fd38c394bef8f9b11e263958f +size 1849216911 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=20896.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=20896.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..81e025d8094d6cb56fb2dfba2621e577001ef63c --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=20896.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13d67fd8799cc338b5b2f6399376f6afaaf80ec398479f1d97ffccc0c3a3b63f +size 1849217814 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=21266.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=21266.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..4142cb26b9a0c9652df0c0bbb3f026f02858e7a2 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=21266.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39b15c3d9356044a597005e5cc7a31daa07f0a85f30b2fb824f51b26b40df0aa +size 1849218197 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=21554.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=21554.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b2a57ff65e07ba598f42c66ce8934afb19d60410 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=21554.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34ef8b663d5f039dfb57cbec0292d3f45855ff2525e99d8f0547d467d768a211 +size 1849218708 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=21855.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=21855.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f448ddec2a8482bded9172fb71cf52c4504410ec --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=21855.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbb271ffe46922b460b66a39a98a2cd6331e5d4f86912f116c7bc122f331c100 +size 1849219219 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=22119.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=22119.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..fd9d443c0ab161474a27f5d6b8ae93c110abff0a --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=22119.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b89a829f6dbb18111bc5c2146ff5e348d75186a5d3fe42a4221e870949dae24 +size 1849219730 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=22307.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=22307.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..289b6656827b2e68d117ed58af4c655c48c28885 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=22307.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:281ee48fe3d6098b8553361af834ed5666a3930404892d46867f8411e84b09b4 +size 1849220177 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=22496.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=22496.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..9d18282a7b1e25947df809bc82c8e052e48128a5 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=22496.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27a1b53ad4ccb58bda62bbe45aee324ef16927906b8f0c6f251ee9bfd4cb2edb +size 1849220752 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=22679.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=22679.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b6af774aa856d2a6d02fe8b3561f5b37cb24b71d --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=22679.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:033e2b9782d29cbe53bfe568c0a6988716c947c4cf1b2af7a8c3f0da7ce0dc89 +size 1849221199 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=22840.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=22840.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..45d0eccaa5e9d6abde531dc5b0984ca8ead6c6e7 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=22840.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4013013c462239b0c54bc1249b83cfc40f6fefce62b4240871d592b08cdd16e6 +size 1849221582 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=22996.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=22996.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..7a332f7b1de649b8f7c51f1b45520070acdbcfb2 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=22996.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03cf797ae03ebbc78af4923ea8506057c60fb5ac73df4d1fefab216820610cec +size 1849222093 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=2309.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=2309.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..ec6c4a2d449acafbd6d89cfcc2b223ca7974ddd6 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=2309.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:246f938d74222a2949fba456f51d59ef2a1cdbde217f58100b8ee4903a0617fc +size 1849217294 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=23166.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=23166.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..cb5d89692791bac54e28d076505f8839add02f02 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=23166.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef527bbb49bcfba2b5f70e24a627711d4adaf4163305bba5bf1ccbaafd6be201 +size 1849222604 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=23343.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=23343.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..aa68e459f3fb5094b6dc4a026f8e105dadcfd40d --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=23343.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0f9bfa0b4ca223e161be8f9f11cd0898bfc9fa7a5b6c7a3b1a657f86aac94dc +size 1849223115 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=23531.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=23531.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..5e15632df8cd5d9e6aafa300ee76ea2ec6a84a45 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=23531.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd451882ee1520df433b278c39bc73e488a24ba29f5013760c46455655e74680 +size 1849223562 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=236.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=236.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..558e972b2d5d8949723a6a98d26ab66c92bb4743 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=236.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2b02ecfa4ba9c23935dc29f042dcc9fb796e2a684eecb1dd1fe2f312098e239 +size 1849213015 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=23794.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=23794.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..bf498f54c9d55066eea199e24ca7c7bebd34c6b6 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=23794.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abc5e1dec08dc8cb6f8950c99495d5da1e8a00f1146672076583c069d29cc80e +size 1849224073 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=23982.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=23982.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..66cbaabcc3766a84af7676e0f486ef352cef34a2 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=23982.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb4200fb9f0edbf7320514989ddd73c1c1d7f99785f62dd3584571a85d79b074 +size 1849224520 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=24166.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=24166.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f59643d050188a8ec50497b59df49870bcc2699a --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=24166.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f6bb1ebf81d37530b3275ffccbcfdb920ab4682903424c63e8b9b55fb278d66 +size 1849224967 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=2532.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=2532.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..7b178175109a37bc71d0cd63855bacf35fcb9528 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=2532.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77ffe4e82e54d7584853c9b1771dacbda221450045f699ef3314271fce49c5ee +size 1849217741 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=2736.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=2736.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..a18e784317cecc0918ce749bd09b7085e267e1c8 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=2736.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6aac6ede1cf3cb217f73301d98d2f2c23cb43ea0a94d652dc66629706049239 +size 1849218316 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=2967.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=2967.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..ade4389ccc46a5f4a4dcc919b4c70d141c4e48ff --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=2967.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83f902919358cafce8c436d05ed4b3285c5ba894b01c2197690aeabcdfa2bf5f +size 1849218763 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=3192.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=3192.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c7701692bd38c4658b44d9751accd05bcc20da85 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=3192.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fee52bdf8559e7aa11599da4dc0b4f50cbedbd5f1a49dd2ce4545c7ca71cc18b +size 1849219274 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=3397.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=3397.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..454274a1d8721ec39a337d55dcad4471c0ca1fcf --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=3397.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a60ee9c74c7d187d48c6edb5bd0dfac9ce5491942d0e62a58271c76ed6c7809 +size 1849219721 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=3633.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=3633.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..4c8fafd1c4a3e1397df7da3c735c2b6633b135ed --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=3633.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a031e4cbc82e298edcc7db06cb2d53a292bb8938b451c708d6a0b78db1ed5569 +size 1849220232 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=3859.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=3859.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..64968d976b0dd07337983dd65dd66b524e9c85e0 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=3859.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48b092f7add74c5a4c4e9befe665e8ef693d2f52d1e1167482d0862a9d44a00c +size 1849220615 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=4055.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=4055.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..4104f24a0037e27ca61700137e925cbe7ecada07 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=4055.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4599eb62ec47488f6d4a8a904ccf0ee0021503b066c55b47bca6314c3a81752f +size 1849221126 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=4264.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=4264.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..6070f3c236455c45f20adff504f3809c823bb91f --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=4264.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdb6da1efb53fdc529a63d288237191fc7c1b197c1028a6db73ad898e8864d2f +size 1849217367 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=4484.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=4484.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..6e15e03e9e721f88a0d2a4d6af4b18f1a17f46ef --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=4484.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3920cdcb031dc9b20829c4922e75ecf9cecdff650bef481c407e1bfb88185b62 +size 1849217814 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=4678.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=4678.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..aff3b0403bac204fa90d4ea9cfaa1901f3b06ec2 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=4678.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8972a5e5900f1aa81926a2c9bba8117eb79da92f2515b9e4e5da96e0c24205c1 +size 1849218261 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=471.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=471.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..885f9d9040cd9399dfba8cded8bc69398564e750 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=471.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46302a9111d0b94f6edff6b435c2a5ce268a05fc6699c73865188f4f2953e07e +size 1849213462 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=4878.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=4878.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..d37ca467a05659f0673b36d61ba716e89616eac3 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=4878.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17ff2f842a538d3d654706860db7d493696cfbeecc025da2f47d2f5f0ae82334 +size 1849218708 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=5078.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=5078.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..3c03500c65ff82a84be6870899403e80cf0b302f --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=5078.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b11075f5d33d10d389f19a15a1010f66e395cbae38e2408831a83f6b907ac6de +size 1849219219 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=5267.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=5267.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..66af35d42158d09d0784b7e284f65efc62b97cb7 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=5267.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3a1d88508061265b1a6f5b87dde753c45adbdbb0077aa46d78687e2d1023ef9 +size 1849219730 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=5459.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=5459.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..020a9518d6bbdda67f3f1f3dfe55e2a6efd6ae53 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=5459.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:861cc8388bb9ebeccf6104c0759d7a5cd56482a9a6b5a340d83f578a9edf006f +size 1849220177 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=5673.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=5673.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..5c681dc35e4ab4e804be0420a8c36a8e25c35a63 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=5673.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:828deaae70abb7aa4649bdb486f6d5ac0959603e8eecefc371e9d091502a5bb2 +size 1849220752 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=5930.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=5930.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..2c0240e7aee5bd7ff1b82a7f1f00ee071002e511 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=5930.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ed35db2c0a19ba1ec381e8edb356f5dd7644e55ee0f7cd7dd2b41bf0b320136 +size 1849221199 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=6212.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=6212.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..e476aab2b45207ad22267a8a943c7b615469e631 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=6212.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:386cbe51b9343a0bebef40e7ce2dcf95ce5e0f36350f8354cf14c7304fa37845 +size 1849221646 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=6488.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=6488.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f7f4b806e305dcb12db338ae5ab6d9fd2724ae02 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=6488.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f03b1ef1ce79e77b808490b8a6bb44530833a40919a374668a2889a34ff8237c +size 1849222093 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=6722.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=6722.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..22965ba98154e21962d8c97b37a0bb8e7e05867f --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=6722.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7db5636267425629f6c1838ab50210dbfdd4a176d2f8d8332204d3c5bb906ea +size 1849222668 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=6981.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=6981.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..253770e1976a10d498bb495c3d2d6acb712d638f --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=6981.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72ddfeb3f542eb65b9467b3b8c0c25ada01a6254a03bfd80b0db8fee1575c284 +size 1849223115 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=708.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=708.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..d8f2aace65ac748074a77265e63e5a066c3cae92 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=708.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20a0362ee2d38e58b0be827259aad05468221a7db154169805c600eb8184d62c +size 1849213909 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=7248.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=7248.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..694cc09375aaf00bb84f162c7edd8c5a49d65328 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=7248.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd64e820fe8b1fcae45c9e1c2e76231f89ad4d171c9cd62ae65eb8bdf258ac84 +size 1849223626 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=7480.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=7480.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..6ef6d69d1ff82a51ec8adf63d9444d2d1b849588 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=7480.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2b975e463a4ff04e04b173066a541ebfe5dfcfe3d381a49f0e8e28896561e82 +size 1849224073 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=7713.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=7713.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..976f9596deb339ebc2ecfd0227562c316087d0a9 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=7713.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dee7ea375e4ee152afff9ac0bce8e1c41cd4468d8d425759a6b9e92f5a5c49a2 +size 1849224584 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=7923.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=7923.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..23d5327a4bbdd6805d53e857d54cda9d375b60b6 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=7923.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:566ed59366b2f51e726035d214a775870ccbe5291536506058ce150f045b09e2 +size 1849224967 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=8143.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=8143.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..ed05399c638689ffef84ae5e4c7bdd88406da04d --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=8143.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e97f6b82abd99118d2ecdd32cc2b1e6c32921c3af2cdb547356ee373e54f510 +size 1849225414 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=8352.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=8352.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..eda579300b526a21e4778018caa3fce5400fddc4 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=8352.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99d34f1fb82413c6e6cbd83e64c7fa8934a901e104336e481d69cfb5ece4c79e +size 1849225989 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=8565.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=8565.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..17b3487c1408bac14d2aeee6cb2e99b97f252a9b --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=8565.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcbcd21aeb37d393c87c1ae9c637823faeb6d87f4c05042d6766fb9d2a6af25c +size 1849226308 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=8791.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=8791.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..1fdf5f6fb7834cf61d8e2d9b773969d3348bcdd3 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=8791.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2ac4d8ba55eafd30a5419f930136387616719d2ba0cd7764112df966e8cbdd0 +size 1849226755 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=9012.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=9012.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..07204805a7e03a786314a3c2eef41b484e79e962 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=9012.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48468112b3e0aa0e211d03bb4709c1d96dac656fc2b29fbeb4df5fc96def13ef +size 1849227330 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=9224.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=9224.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..5f1715c1940e224694204478c167de870fd1b7d4 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=9224.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70f2138986d8a686cbac7b7351bba74ac59f6e8fed3777e27bfc4a67e6048f47 +size 1849227777 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=942.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=942.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..29a8ffc0da8a56cfb3c2a40151b4d9a0e44a24fb --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=942.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2046fe7e6471d698e66205b9a741a199c766d166fc495e9b98e0ddf983291913 +size 1849214420 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=9445.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=9445.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..3a12d97e2799d073938f09b6305b1eddc6edce77 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=9445.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec2aedafeb4a0022e64549193aedabb873ad5174d1a7722acf80438ba292ed7f +size 1849228096 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=9677.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=9677.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..8c8fee8d34c6ff626aa575e51ecd7af65f1bd36e --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=9677.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e56228e6bac36a83bdfb3667abfffa3c4f52dee1deac9da8966651fa64a2831 +size 1849228671 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=9854.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=9854.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..918900c8185e4b1d7e63ad45a5ec96242ce9fe82 --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=9854.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f901856726c67222a639acfab1ffe0ee3e17e71532b650682bf2005bfb7e23e2 +size 1849229246 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/last.ckpt b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/last.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..613b4a283322f30cb0a36b3d133ca5fc73ee403e --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/last.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:566dc4bf90e3af256240c131d23c8aa8e7be394eb0fb17d31b1329c926d66b0f +size 1849224967 diff --git a/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/resume_singlenode_2023-08-18-12:52:45.log b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/resume_singlenode_2023-08-18-12:52:45.log new file mode 100644 index 0000000000000000000000000000000000000000..2c19096cf4bbf3d97d5f2150d32f12fefc1f381d --- /dev/null +++ b/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/resume_singlenode_2023-08-18-12:52:45.log @@ -0,0 +1,7153 @@ +wandb: Currently logged in as: tamariucai (rycolab). Use `wandb login --relogin` to force relogin +wandb: - Waiting for wandb.init()... wandb: \ Waiting for wandb.init()... wandb: wandb version 0.15.8 is available! To upgrade, please run: +wandb: $ pip install wandb --upgrade +wandb: Tracking run with wandb version 0.15.4 +wandb: Run data is saved locally in ./wandb/run-20230818_125755-qva6a2jc +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run jolly-sweep-3 +wandb: ⭐️ View project at https://wandb.ai/rycolab/alkmi-wit +wandb: 🧹 View sweep at https://wandb.ai/rycolab/alkmi-wit/sweeps/p9z42he9 +wandb: 🚀 View run at https://wandb.ai/rycolab/alkmi-wit/runs/qva6a2jc +[rank: 0] Global seed set to 5501650 +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +The current host is eu-a65-03 +The current cuDNN version is 8801 +Parameter 'text_perc' already had the value 10. +Parameter 'vision_perc' already had the value 1. +Parameter 'seed' already had the value 5501650. +Total GPU memory: 42.296 GB +Detected half-sized run, (almost) doubling batch size to 48. +Detected half-sized run, (almost) halving gradient accumulation to 85 +[update_ckeckpoint_dir] Detected hyperparameter run! +[update_ckeckpoint_dir] Detected half-size run! +[update_ckeckpoint_dir] Found and updated model name flava in the checkpoint dirpath. +[update_ckeckpoint_dir] Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-18)_time(12:58:03)/seed(5501650)-magic({'enable': True})-text_perc(10)-vision_perc(1/ +Text is the predominant modality (10 v.s. 1 vision), will sample proportionally for optimal BLiMP performance. +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Found cached dataset imagenet-1k (/cluster/scratch/tamariucai/HuggingfaceDatasets/imagenet-1k/default/1.0.0/a1e9bfc56c3a7350165007d1176b15e9128fcaf9ab972147840529aed3ae52bc) +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/plugins/environments/slurm.py:165: PossibleUserWarning: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... + rank_zero_warn( +Using bfloat16 Automatic Mixed Precision (AMP) +GPU available: True (cuda), used: True +TPU available: False, using: 0 TPU cores +IPU available: False, using: 0 IPUs +HPU available: False, using: 0 HPUs +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[1%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-b1b82ac3a7df094e_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Restoring states from the checkpoint path at /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=4055.ckpt +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:337: UserWarning: The dirpath has changed from "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-15)_time(10:33:06)/seed(5501650)-text_perc(10)-vision_perc(1)-magic({'enable': True}" to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-18)_time(12:58:03)/seed(5501650)-magic({'enable': True})-text_perc(10)-vision_perc(1", therefore `best_model_score`, `kth_best_model_path`, `kth_value`, `last_model_path` and `best_k_models` won't be reloaded. Only `best_model_path` will be reloaded. + warnings.warn( +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0] + + | Name | Type | Params +---------------------------------------------- +0 | model | FlavaForPreTraining | 189 M +---------------------------------------------- +136 M Trainable params +53.8 M Non-trainable params +189 M Total params +759.666 Total estimated model params size (MB) +Restored all states from the checkpoint at /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision1/half_bs4080_seed5501650_bf16-mixed/flava-epoch=00-step=4055.ckpt +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +VLDataModule: batch_size is 48, num_workers is 4 +ImageDataModule: batch_size is 48, num_workers is 4 +MLMDataModule: batch_size is 48, num_workers is 4 +Dataset sizes (un-normalized): [1, 1, 10] +Dataset sizes (normalized): [0.08333333333333333, 0.08333333333333333, 0.8333333333333334] +Sampling weights after temperature (1.0): [0.08333333333333333, 0.08333333333333333, 0.8333333333333334] +2023-08-18 12:59:21.382707 Setting up datamodule: VLDataModule +2023-08-18 13:00:22.638702 Setting up datamodule: ImageDataModule +2023-08-18 13:00:31.800267 Setting up datamodule: MLMDataModule +Registering multimodal overfitting monitors +Adding ImageNet zeroshot callback +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'ImageNetZeroshotCallback'] +Trainer successfully initialized (strategy=SingleDeviceStrategy) +Starting training +2023-08-18 13:01:58.729177 Preparing data for datamodule: VLDataModule +2023-08-18 13:01:58.729366 Preparing data for datamodule: ImageDataModule +2023-08-18 13:01:58.729487 Preparing data for datamodule: MLMDataModule +2023-08-18 13:01:58.731507 Setting up datamodule: VLDataModule +2023-08-18 13:02:50.373307 Setting up datamodule: ImageDataModule +2023-08-18 13:02:57.952585 Setting up datamodule: MLMDataModule +Resetting wait count for `mlm` since training is resuming and they are special tasks. +Resetting wait count for `mmm_text` since training is resuming and they are special tasks. +2023-08-18 13:04:40.576646 Building train dataloader for datamodule: VLDataModule +2023-08-18 13:04:40.577386 Building train dataloader for datamodule: ImageDataModule +2023-08-18 13:04:40.577699 Building train dataloader for datamodule: MLMDataModule +2023-08-18 13:04:42.168194 Building val dataloader for datamodule: VLDataModule +2023-08-18 13:04:42.169404 Building val dataloader for datamodule: ImageDataModule +2023-08-18 13:04:42.169894 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 14:07:02.162841] Computing e^(tensor([5875.1059], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 14:07:02.173812] Ending with PPL=18.869614950663998 (duration: 0:04:05.867870) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.437 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 1.006. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.275. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.877. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 1.002. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.660. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 15:32:04.076367] Computing e^(tensor([5803.6728], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 15:32:04.082268] Ending with PPL=18.20755128822543 (duration: 0:04:12.722672) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.404 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 1.006. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.275. Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 2.877. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 4 records. Best score: 1.002. Wait count has been increased from 3. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.660. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 16:47:40.800520] Computing e^(tensor([5741.1930], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 16:47:40.806257] Ending with PPL=17.647541410272638 (duration: 0:04:09.164812) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.383 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 1.006. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.275. Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 8 records. Best score: 2.877. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 1.002. Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.036 >= min_delta = 0.0. New best score: 3.623 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 18:07:25.485421] Computing e^(tensor([5718.5310], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 18:07:25.491121] Ending with PPL=17.44870586515028 (duration: 0:04:09.027259) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.367 +Monitored metric validation/losses/mim_loss did not improve in the last 7 records. Best score: 1.006. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 4 records. Best score: 0.275. Wait count has been increased from 3. +Monitored metric validation/losses/global_contrastive_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 1.002. Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.004 >= min_delta = 0.0. New best score: 3.619 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 19:30:55.342735] Computing e^(tensor([5673.6551], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 19:30:55.348510] Ending with PPL=17.06155264770351 (duration: 0:04:11.463742) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.342 +Monitored metric validation/losses/mim_loss did not improve in the last 8 records. Best score: 1.006. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.877. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 7 records. Best score: 1.002. Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.013 >= min_delta = 0.0. New best score: 3.606 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 21:00:02.183373] Computing e^(tensor([5619.6496], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 21:00:02.188978] Ending with PPL=16.60700870870191 (duration: 0:06:14.380605) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.323 +Monitored metric validation/losses/mim_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.877. Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 8 records. Best score: 1.002. Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.113 >= min_delta = 0.0. New best score: 3.493 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 22:23:06.024072] Computing e^(tensor([5587.1706], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 22:23:06.029830] Ending with PPL=16.33949687553484 (duration: 0:04:07.996496) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.312 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.006. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.877. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_image_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Metric validation/losses/mmm_text_loss improved by 0.025 >= min_delta = 0.0. New best score: 3.468 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 23:52:30.601066] Computing e^(tensor([5558.4777], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 23:52:30.606972] Ending with PPL=16.106756312108345 (duration: 0:04:13.736391) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.300 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.006. Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve in the last 8 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 2.877. Wait count has been increased from 3. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.002. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.468. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 01:18:50.929769] Computing e^(tensor([5519.4277], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 01:18:50.935841] Ending with PPL=15.795322438267474 (duration: 0:04:10.835063) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.279 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 1.006. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.877. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.002. Wait count has been increased from 1. +Metric validation/losses/mmm_text_loss improved by 0.019 >= min_delta = 0.0. New best score: 3.449 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 02:50:21.629748] Computing e^(tensor([5486.1276], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 02:50:21.635617] Ending with PPL=15.534506911080472 (duration: 0:04:11.287121) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.267 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 1.006. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.275. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.877. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 1.002. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.449. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 04:12:49.543891] Computing e^(tensor([5463.0222], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 04:12:49.549761] Ending with PPL=15.35607391870846 (duration: 0:04:10.708156) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.255 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 1.006. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.275. Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 2.877. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 4 records. Best score: 1.002. Wait count has been increased from 3. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.449. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 05:45:37.297306] Computing e^(tensor([5431.5499], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 05:45:37.303172] Ending with PPL=15.116319972695637 (duration: 0:04:13.093405) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.247 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 1.006. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.275. Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 8 records. Best score: 2.877. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 1.002. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.449. Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 07:11:03.570497] Computing e^(tensor([5398.3426], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 07:11:03.576426] Ending with PPL=14.86740606427237 (duration: 0:04:10.206102) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.226 +Monitored metric validation/losses/mim_loss did not improve in the last 7 records. Best score: 1.006. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 4 records. Best score: 0.275. Wait count has been increased from 3. +Monitored metric validation/losses/global_contrastive_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 1.002. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 4 records. Best score: 3.449. Wait count has been increased from 3. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 08:34:42.297917] Computing e^(tensor([5384.3730], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 08:34:42.303871] Ending with PPL=14.76392207316689 (duration: 0:04:10.787524) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.218 +Monitored metric validation/losses/mim_loss did not improve in the last 8 records. Best score: 1.006. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.877. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 7 records. Best score: 1.002. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 3.449. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 09:59:05.296490] Computing e^(tensor([5357.0955], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 09:59:05.302447] Ending with PPL=14.56392759786929 (duration: 0:04:14.839865) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.216 +Monitored metric validation/losses/mim_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.877. Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 8 records. Best score: 1.002. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 3.449. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 11:19:27.826467] Computing e^(tensor([5334.8182], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 11:19:27.832312] Ending with PPL=14.402605316843426 (duration: 0:04:12.187151) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.201 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.006. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.877. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_image_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 7 records. Best score: 3.449. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 12:38:57.221882] Computing e^(tensor([5309.0174], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 12:38:57.227996] Ending with PPL=14.217999379537378 (duration: 0:04:16.835398) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.187 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.006. Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve in the last 8 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 2.877. Wait count has been increased from 3. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.002. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 8 records. Best score: 3.449. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 13:53:45.470362] Computing e^(tensor([5286.9571], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 13:53:45.476106] Ending with PPL=14.06203394726093 (duration: 0:04:13.832447) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.177 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 1.006. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.877. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.002. Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 15:08:17.560146] Computing e^(tensor([5276.6636], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 15:08:17.565841] Ending with PPL=13.989846330963976 (duration: 0:04:15.799261) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.174 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 1.006. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.275. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.877. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 1.002. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.449. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 16:07:00.168212] Computing e^(tensor([5244.3666], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 16:07:00.174109] Ending with PPL=13.765745627827743 (duration: 0:04:15.942572) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.160 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 1.006. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.275. Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 2.877. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 4 records. Best score: 1.002. Wait count has been increased from 3. +Metric validation/losses/mmm_text_loss improved by 0.020 >= min_delta = 0.0. New best score: 3.429 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 17:06:54.748160] Computing e^(tensor([5219.2638], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 17:06:54.754083] Ending with PPL=13.594046187564663 (duration: 0:04:20.473424) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.151 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 1.005 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.275. Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 8 records. Best score: 2.877. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 1.002. Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.004 >= min_delta = 0.0. New best score: 3.425 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 18:15:32.424992] Computing e^(tensor([5212.3352], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 18:15:32.430702] Ending with PPL=13.547033413225552 (duration: 0:04:15.190525) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.135 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.005. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 4 records. Best score: 0.275. Wait count has been increased from 3. +Monitored metric validation/losses/global_contrastive_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 1.002. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.425. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 19:20:07.159587] Computing e^(tensor([5174.8463], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 19:20:07.165455] Ending with PPL=13.295466691444581 (duration: 0:04:16.860196) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.131 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.005. Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.877. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 7 records. Best score: 1.002. Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.034 >= min_delta = 0.0. New best score: 3.391 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 20:22:16.042549] Computing e^(tensor([5170.6972], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 20:22:16.048750] Ending with PPL=13.267913609586039 (duration: 0:04:53.068495) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.120 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 1.005. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.877. Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 8 records. Best score: 1.002. Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.047 >= min_delta = 0.0. New best score: 3.345 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 21:42:57.497031] Computing e^(tensor([5153.4175], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 21:42:57.502697] Ending with PPL=13.153774728251067 (duration: 0:04:12.973223) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.117 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 1.004 +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.877. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_image_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.345. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 22:41:12.508205] Computing e^(tensor([5138.6807], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 22:41:12.514277] Ending with PPL=13.057208189402088 (duration: 0:04:17.982429) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.110 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.004. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 8 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 2.877. Wait count has been increased from 3. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.002. Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.010 >= min_delta = 0.0. New best score: 3.335 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 23:35:52.955947] Computing e^(tensor([5112.0164], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 23:35:52.961810] Ending with PPL=12.8842828722839 (duration: 0:04:52.455327) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.100 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.004. Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.877. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.001 >= min_delta = 0.0. New best score: 1.000 +Metric validation/losses/mmm_text_loss improved by 0.030 >= min_delta = 0.0. New best score: 3.305 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 01:02:49.198340] Computing e^(tensor([5109.1820], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 01:02:49.204272] Ending with PPL=12.866036545595602 (duration: 0:04:15.783678) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.999 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.305. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 01:56:06.368403] Computing e^(tensor([5088.4251], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 01:56:06.374311] Ending with PPL=12.733197574421702 (duration: 0:04:19.421252) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.085 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 1.004. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.275. Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 2.877. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.999. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.305. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 02:51:28.032888] Computing e^(tensor([5092.0554], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 02:51:28.038790] Ending with PPL=12.756331188943436 (duration: 0:04:42.257220) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.083 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 1.004. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.275. Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 8 records. Best score: 2.877. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.999. Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.305. Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 04:09:46.683995] Computing e^(tensor([5061.9162], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 04:09:46.689692] Ending with PPL=12.565539384190945 (duration: 0:04:15.179644) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.075 +Metric validation/losses/mim_loss improved by 0.003 >= min_delta = 0.0. New best score: 1.001 +Monitored metric validation/losses/itm_loss did not improve since the last 4 records. Best score: 0.275. Wait count has been increased from 3. +Monitored metric validation/losses/global_contrastive_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.999. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 4 records. Best score: 3.305. Wait count has been increased from 3. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 05:15:23.572378] Computing e^(tensor([5044.4325], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 05:15:23.578239] Ending with PPL=12.45617197518554 (duration: 0:04:14.285959) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.065 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.001. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.275. Signaling model to ignore task. +Metric validation/losses/global_contrastive_loss improved by 0.034 >= min_delta = 0.0. New best score: 2.843 +Metric validation/losses/mmm_image_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.998 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 3.305. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 06:11:15.438772] Computing e^(tensor([5044.0438], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 06:11:15.444515] Ending with PPL=12.453751428105118 (duration: 0:04:14.027335) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.062 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.001. Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.843. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.998. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 3.305. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 07:20:55.975180] Computing e^(tensor([5034.9295], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 07:20:55.981179] Ending with PPL=12.397127114906116 (duration: 0:04:15.893274) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 1.001 +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.843. Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.998. Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 7 records. Best score: 3.305. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 08:39:55.293513] Computing e^(tensor([5003.2494], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 08:39:55.299387] Ending with PPL=12.20230313697127 (duration: 0:04:12.306417) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.048 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.001. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 8 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.843. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.998. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 8 records. Best score: 3.305. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 09:58:37.195487] Computing e^(tensor([4994.6385], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 09:58:37.201313] Ending with PPL=12.149879520324944 (duration: 0:04:15.047432) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.043 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.999 +Monitored metric validation/losses/itm_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 2.843. Wait count has been increased from 3. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 4 records. Best score: 0.998. Wait count has been increased from 3. +Monitored metric validation/losses/mmm_text_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 11:08:40.172337] Computing e^(tensor([4968.8941], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 11:08:40.177942] Ending with PPL=11.994486323513737 (duration: 0:04:10.671792) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.041 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.999. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.275. Wait count has been increased from 0. +Metric validation/losses/global_contrastive_loss improved by 0.089 >= min_delta = 0.0. New best score: 2.754 +Metric validation/losses/mmm_image_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.995 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.305. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 12:29:12.522115] Computing e^(tensor([4975.4704], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 12:29:12.527853] Ending with PPL=12.033990375491205 (duration: 0:04:57.548329) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.040 +Metric validation/losses/mim_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.997 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.275. Wait count has been increased from 1. +Metric validation/losses/global_contrastive_loss improved by 0.024 >= min_delta = 0.0. New best score: 2.730 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.995. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.305. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 13:42:34.274557] Computing e^(tensor([4946.3017], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 13:42:34.280604] Ending with PPL=11.859756457415314 (duration: 0:04:19.693388) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.031 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.997. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.275. Wait count has been increased from 2. +Metric validation/losses/global_contrastive_loss improved by 0.001 >= min_delta = 0.0. New best score: 2.729 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.995. Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.305. Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 14:56:40.724579] Computing e^(tensor([4960.5546], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 14:56:40.730450] Ending with PPL=11.944575999795005 (duration: 0:04:20.967198) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.025 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 0.997. Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve since the last 4 records. Best score: 0.275. Wait count has been increased from 3. +Metric validation/losses/global_contrastive_loss improved by 0.000 >= min_delta = 0.0. New best score: 2.728 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.995. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 4 records. Best score: 3.305. Wait count has been increased from 3. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 16:10:57.868606] Computing e^(tensor([4955.6499], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 16:10:57.874445] Ending with PPL=11.915319532657172 (duration: 0:04:15.029378) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.025 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 0.997. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.728. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 4 records. Best score: 0.995. Wait count has been increased from 3. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 3.305. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 17:24:22.083065] Computing e^(tensor([4921.6702], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 17:24:22.088943] Ending with PPL=11.714590393536604 (duration: 0:04:18.864564) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.024 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 0.997. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.728. Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 0.995. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 3.305. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 18:46:27.502289] Computing e^(tensor([4914.0442], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 18:46:27.508148] Ending with PPL=11.67000738119761 (duration: 0:04:13.772512) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.011 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 0.997. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.728. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 0.995. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 7 records. Best score: 3.305. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 20:05:08.435032] Computing e^(tensor([4915.0514], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 20:05:08.440774] Ending with PPL=11.675886218498981 (duration: 0:04:12.066190) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.008 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 0.997. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 8 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 2.728. Wait count has been increased from 3. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 7 records. Best score: 0.995. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 8 records. Best score: 3.305. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 21:29:38.863686] Computing e^(tensor([4907.9187], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 21:29:38.869562] Ending with PPL=11.634319814590054 (duration: 0:04:12.377502) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.006 +Monitored metric validation/losses/mim_loss did not improve in the last 7 records. Best score: 0.997. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.728. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 8 records. Best score: 0.995. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 22:56:31.770360] Computing e^(tensor([4897.1809], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 22:56:31.776055] Ending with PPL=11.572023914733506 (duration: 0:04:13.625252) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.994 +Monitored metric validation/losses/mim_loss did not improve in the last 8 records. Best score: 0.997. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.275. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.728. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.305. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 00:07:35.744845] Computing e^(tensor([4864.9513], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 00:07:35.750850] Ending with PPL=11.387037660905788 (duration: 0:04:15.551497) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.990 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.305. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 01:33:48.615206] Computing e^(tensor([4878.7190], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 01:33:48.621166] Ending with PPL=11.465694433624364 (duration: 0:04:11.906001) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.992 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.997. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.275. Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 8 records. Best score: 2.728. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.990. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.305. Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 02:57:28.760922] Computing e^(tensor([4864.2133], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 02:57:28.766811] Ending with PPL=11.382836759635062 (duration: 0:04:14.947783) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.982 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 0.997. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.728. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.990. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 3.305. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 05:32:31.073680] Computing e^(tensor([4849.7384], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 05:32:31.079384] Ending with PPL=11.30075091720399 (duration: 0:04:13.490360) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.981 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.992 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.728. Wait count has been increased from 1. +Metric validation/losses/mmm_image_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.983 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 3.305. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 06:49:50.488532] Computing e^(tensor([4840.5638], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 06:49:50.494360] Ending with PPL=11.24902990023832 (duration: 0:04:16.128070) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.972 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.992. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.728. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.983. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 7 records. Best score: 3.305. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 08:05:03.827431] Computing e^(tensor([4827.2459], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 08:05:03.833324] Ending with PPL=11.174372359885485 (duration: 0:04:12.035469) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.969 +Metric validation/losses/mim_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.988 +Monitored metric validation/losses/itm_loss did not improve in the last 8 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 2.728. Wait count has been increased from 3. +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.979 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 8 records. Best score: 3.305. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 09:23:03.097269] Computing e^(tensor([4806.4567], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 09:23:03.103053] Ending with PPL=11.058820527691338 (duration: 0:04:19.507571) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.965 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.988. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.728. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.979. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 10:34:46.596993] Computing e^(tensor([4819.8667], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 10:34:46.602839] Ending with PPL=11.133218825594268 (duration: 0:04:16.352671) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.961 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.983 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.275. Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 2.728. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.979. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.305. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 13:12:52.221340] Computing e^(tensor([4793.7174], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 13:12:52.227221] Ending with PPL=10.988603773449878 (duration: 0:04:09.682775) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.953 +Metric validation/losses/mim_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.980 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.275. Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 8 records. Best score: 2.728. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.976 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.305. Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 14:26:11.785888] Computing e^(tensor([4792.5830], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 14:26:11.791556] Ending with PPL=10.982372828174597 (duration: 0:04:13.349257) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.976 +Metric validation/losses/mmm_text_loss improved by 0.012 >= min_delta = 0.0. New best score: 3.293 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 15:45:20.272536] Computing e^(tensor([4784.0192], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 15:45:20.278292] Ending with PPL=10.93544785208146 (duration: 0:04:12.909553) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.947 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 0.980. Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.275. Signaling model to ignore task. +Metric validation/losses/global_contrastive_loss improved by 0.012 >= min_delta = 0.0. New best score: 2.716 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.976. Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.040 >= min_delta = 0.0. New best score: 3.253 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 17:04:48.311793] Computing e^(tensor([4769.8947], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 17:04:48.317481] Ending with PPL=10.858490859165668 (duration: 0:04:09.600909) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.187 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 18:36:12.148497] Computing e^(tensor([4757.1793], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 18:36:12.159389] Ending with PPL=10.789674772155438 (duration: 0:04:32.737416) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.944 +Metric validation/losses/mim_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.973 +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.716. Wait count has been increased from 1. +Metric validation/losses/mmm_image_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.968 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.187. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 20:11:40.495834] Computing e^(tensor([4756.8598], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 20:11:40.501781] Ending with PPL=10.787951661442504 (duration: 0:04:09.264172) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.939 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.973. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 8 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.716. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.968. Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.020 >= min_delta = 0.0. New best score: 3.167 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 21:39:43.224742] Computing e^(tensor([4765.7366], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 21:39:43.230638] Ending with PPL=10.835939246667518 (duration: 0:04:12.937035) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.933 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 0.973. Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 2.716. Wait count has been increased from 3. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.968. Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.167. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 23:19:23.547242] Computing e^(tensor([4748.6978], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 23:19:23.553224] Ending with PPL=10.74401553442105 (duration: 0:04:15.105903) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.930 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 0.973. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.275. Wait count has been increased from 0. +Metric validation/losses/global_contrastive_loss improved by 0.100 >= min_delta = 0.0. New best score: 2.616 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.968. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.167. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 00:54:45.384346] Computing e^(tensor([4728.2062], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 00:54:45.390021] Ending with PPL=10.634496384376654 (duration: 0:04:14.936579) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.929 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 0.973. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.275. Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.616. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 4 records. Best score: 0.968. Wait count has been increased from 3. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.167. Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 02:31:57.625207] Computing e^(tensor([4746.6681], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 02:31:57.631232] Ending with PPL=10.73311743105477 (duration: 0:04:12.317614) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.924 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.972 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.275. Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.616. Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 0.968. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 4 records. Best score: 3.167. Wait count has been increased from 3. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 04:09:18.984346] Computing e^(tensor([4730.0870], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 04:09:18.990179] Ending with PPL=10.644501767368412 (duration: 0:04:11.787235) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.924 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.972. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 4 records. Best score: 0.275. Wait count has been increased from 3. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.616. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 0.968. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 3.167. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 05:41:27.830655] Computing e^(tensor([4716.7490], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 05:41:27.836405] Ending with PPL=10.573749614128143 (duration: 0:04:14.506200) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.918 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.966 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 2.616. Wait count has been increased from 3. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 7 records. Best score: 0.968. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 3.167. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 07:16:15.632594] Computing e^(tensor([4711.2346], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 07:16:15.638261] Ending with PPL=10.544636305918383 (duration: 0:04:12.088990) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.911 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.965 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.616. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 8 records. Best score: 0.968. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 7 records. Best score: 3.167. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 08:53:12.197914] Computing e^(tensor([4706.2999], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 08:53:12.203906] Ending with PPL=10.518650747653172 (duration: 0:04:10.696013) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.909 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.964 +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.616. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 8 records. Best score: 3.167. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 10:27:44.485488] Computing e^(tensor([4690.4653], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 10:27:44.491160] Ending with PPL=10.435700258787675 (duration: 0:04:17.077828) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.967 +Monitored metric validation/losses/mmm_text_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 12:01:03.381564] Computing e^(tensor([4688.5336], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 12:01:03.387270] Ending with PPL=10.425625639013864 (duration: 0:04:09.060655) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.899 +Metric validation/losses/mim_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.960 +Monitored metric validation/losses/itm_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 8 records. Best score: 2.616. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.959 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.167. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 13:34:11.312432] Computing e^(tensor([4677.1552], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 13:34:11.318091] Ending with PPL=10.366480941784094 (duration: 0:04:16.323636) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.896 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.958 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.275. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.959. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.167. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 15:05:29.084609] Computing e^(tensor([4684.9189], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 15:05:29.090306] Ending with PPL=10.406800026197628 (duration: 0:04:09.777452) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.956 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.275. Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.616. Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.953 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.167. Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 16:37:27.261412] Computing e^(tensor([4656.5458], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 16:37:27.267093] Ending with PPL=10.260206047662756 (duration: 0:04:04.180348) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.885 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.951 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.275. Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.616. Wait count has been increased from 1. +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.951 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 4 records. Best score: 3.167. Wait count has been increased from 3. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 17:50:57.191827] Computing e^(tensor([4659.6249], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 17:50:57.197599] Ending with PPL=10.276014329057077 (duration: 0:04:05.470713) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.949 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 3.167. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 19:14:08.887296] Computing e^(tensor([4666.1168], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 19:14:08.892985] Ending with PPL=10.309423542217845 (duration: 0:04:09.208636) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.941 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 3.167. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 20:35:08.714069] Computing e^(tensor([4641.0583], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 20:35:08.719747] Ending with PPL=10.181060358301986 (duration: 0:04:12.027656) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.881 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 0.951. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.616. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.941. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 7 records. Best score: 3.167. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 21:36:18.766030] Computing e^(tensor([4630.0200], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 21:36:18.771657] Ending with PPL=10.12502393717155 (duration: 0:04:10.124934) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.879 +Metric validation/losses/mim_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.951 +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.616. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.941. Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 8 records. Best score: 3.167. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 22:31:13.008494] Computing e^(tensor([4627.2964], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 22:31:13.014190] Ending with PPL=10.111245087232506 (duration: 0:04:18.568256) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.876 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.945 +Monitored metric validation/losses/itm_loss did not improve in the last 8 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 2.616. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.939 +Monitored metric validation/losses/mmm_text_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 23:54:37.953598] Computing e^(tensor([4632.9085], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 23:54:37.959235] Ending with PPL=10.139657628595119 (duration: 0:04:10.922602) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.870 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.945. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 8 records. Best score: 2.616. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.939. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.167. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 01:04:12.134439] Computing e^(tensor([4616.0351], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 01:04:12.140332] Ending with PPL=10.054472221558646 (duration: 0:04:20.349447) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.869 +Metric validation/losses/mim_loss improved by 0.017 >= min_delta = 0.0. New best score: 0.928 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.275. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Metric validation/losses/mmm_image_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.932 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.167. Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 02:00:35.528278] Computing e^(tensor([4622.6753], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 02:00:35.533972] Ending with PPL=10.087909588926182 (duration: 0:05:33.692342) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.868 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.928. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.275. Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.616. Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.932 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.167. Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 03:25:57.231739] Computing e^(tensor([4621.1746], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 03:25:57.237618] Ending with PPL=10.080343279245332 (duration: 0:04:16.147444) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.926 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 4 records. Best score: 3.167. Wait count has been increased from 3. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 04:35:24.767491] Computing e^(tensor([4625.6270], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 04:35:24.773316] Ending with PPL=10.102808720780496 (duration: 0:04:13.148105) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.868 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.927 +Monitored metric validation/losses/itm_loss did not improve since the last 4 records. Best score: 0.275. Wait count has been increased from 3. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.616. Wait count has been increased from 2. +Metric validation/losses/mmm_image_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.925 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 3.167. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 05:38:42.960685] Computing e^(tensor([4591.2346], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 05:38:42.966611] Ending with PPL=9.930564451903832 (duration: 0:07:27.132927) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.854 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.927. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 2.616. Wait count has been increased from 3. +Metric validation/losses/mmm_image_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.921 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 3.167. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 07:03:43.239025] Computing e^(tensor([4584.1378], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 07:03:43.244906] Ending with PPL=9.895389072791884 (duration: 0:04:17.555091) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.853 +Metric validation/losses/mim_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.921 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.616. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.918 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 7 records. Best score: 3.167. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 08:12:10.088753] Computing e^(tensor([4602.2435], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 08:12:10.094599] Ending with PPL=9.985377239246139 (duration: 0:04:14.520433) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.851 +Metric validation/losses/mim_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.915 +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.616. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.914 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 8 records. Best score: 3.167. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 09:14:31.378267] Computing e^(tensor([4581.4311], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 09:14:31.383935] Ending with PPL=9.882006355687096 (duration: 0:04:20.275146) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.845 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.915. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 8 records. Best score: 0.275. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 2.616. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.914. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 10:39:11.350003] Computing e^(tensor([4565.5970], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 10:39:11.355889] Ending with PPL=9.80407862400975 (duration: 0:04:51.752073) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.910 +Monitored metric validation/losses/itm_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 8 records. Best score: 2.616. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.908 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.167. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 11:54:18.909320] Computing e^(tensor([4559.8081], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 11:54:18.915242] Ending with PPL=9.775742400855941 (duration: 0:04:13.332460) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.894 +Metric validation/losses/mim_loss improved by 0.032 >= min_delta = 0.0. New best score: 0.944 +Metric validation/losses/mmm_image_loss improved by 0.034 >= min_delta = 0.0. New best score: 0.956 +Metric validation/losses/mmm_text_loss improved by 1.431 >= min_delta = 0.0. New best score: 3.985 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.324 (now 0.3287196755409241). Wait count has been increased from 0. +Metric validation/losses/global_contrastive_loss improved by 0.181 >= min_delta = 0.0. New best score: 2.646 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 20:27:33.147635] Computing e^(tensor([4671.3883], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 20:27:33.153447] Ending with PPL=10.336632753930909 (duration: 0:04:07.992492) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.888 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.944 (now 0.9457201957702637). Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.019 >= min_delta = 0.0. New best score: 0.937 +Metric validation/losses/mmm_text_loss improved by 0.490 >= min_delta = 0.0. New best score: 3.495 +Metric validation/losses/itm_loss improved by 0.019 >= min_delta = 0.0. New best score: 0.305 +Metric validation/losses/global_contrastive_loss improved by 0.060 >= min_delta = 0.0. New best score: 2.586 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 21:48:40.460593] Computing e^(tensor([4654.3621], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 21:48:40.466274] Ending with PPL=10.249009232750714 (duration: 0:04:10.444248) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.887 +Metric validation/losses/mim_loss improved by 0.019 >= min_delta = 0.0. New best score: 0.925 +Metric validation/losses/mmm_image_loss improved by 0.014 >= min_delta = 0.0. New best score: 0.923 +Metric validation/losses/mmm_text_loss improved by 0.082 >= min_delta = 0.0. New best score: 3.413 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.305 (now 0.3218728005886078). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.586 (now 2.632807731628418). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 23:17:22.358098] Computing e^(tensor([4646.1582], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 23:17:22.363773] Ending with PPL=10.20705461335306 (duration: 0:04:10.667531) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.873 +Metric validation/losses/mim_loss improved by 0.024 >= min_delta = 0.0. New best score: 0.901 +Metric validation/losses/mmm_image_loss improved by 0.022 >= min_delta = 0.0. New best score: 0.901 +Metric validation/losses/mmm_text_loss improved by 0.054 >= min_delta = 0.0. New best score: 3.359 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.305 (now 0.3351516127586365). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.586 (now 2.64322566986084). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 00:47:54.639396] Computing e^(tensor([4614.5727], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 00:47:54.645036] Ending with PPL=10.04712323929855 (duration: 0:04:11.639564) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.859 +Metric validation/losses/mim_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.893 +Metric validation/losses/mmm_image_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.893 +Metric validation/losses/mmm_text_loss improved by 0.061 >= min_delta = 0.0. New best score: 3.298 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.305 (now 0.3253622055053711). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.586 (now 2.6031992435455322). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 02:05:49.568271] Computing e^(tensor([4583.9605], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 02:05:49.572321] Ending with PPL=9.894511731583263 (duration: 0:04:14.667301) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.850 +Metric validation/losses/mim_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.889 +Metric validation/losses/mmm_image_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.885 +Metric validation/losses/mmm_text_loss improved by 0.055 >= min_delta = 0.0. New best score: 3.244 +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.305 (now 0.3182191550731659). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 2.586 (now 2.6385762691497803). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 03:33:31.247956] Computing e^(tensor([4561.5308], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 03:33:31.253761] Ending with PPL=9.784166192904998 (duration: 0:04:13.480170) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.843 +Metric validation/losses/mim_loss improved by 0.019 >= min_delta = 0.0. New best score: 0.870 +Metric validation/losses/mmm_image_loss improved by 0.020 >= min_delta = 0.0. New best score: 0.865 +Metric validation/losses/mmm_text_loss improved by 0.066 >= min_delta = 0.0. New best score: 3.177 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.305 (now 0.3182191550731659). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.586 (now 2.6385762691497803). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 05:02:47.513722] Computing e^(tensor([4555.2123], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 05:02:47.519501] Ending with PPL=9.753304488265066 (duration: 0:04:12.799732) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.831 +Metric validation/losses/mim_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.867 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.863 +Metric validation/losses/mmm_text_loss improved by 0.038 >= min_delta = 0.0. New best score: 3.139 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.305 (now 0.3182191550731659). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.586 (now 2.6385762691497803). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 06:19:00.982148] Computing e^(tensor([4539.9443], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 06:19:00.987940] Ending with PPL=9.679131428354951 (duration: 0:04:11.728458) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.862 +Metric validation/losses/mmm_text_loss improved by 0.047 >= min_delta = 0.0. New best score: 3.092 +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 07:50:38.539059] Computing e^(tensor([4635.5146], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 07:50:38.544647] Ending with PPL=10.152878845123066 (duration: 0:04:11.924414) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.820 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 0.867 (now 0.8732088804244995). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.862 (now 0.8713200688362122). Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.092 (now 3.135249137878418). Wait count has been increased from 2. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.305 (now 0.31590643525123596). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.586 (now 2.9117486476898193). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 11:58:45.011701] Computing e^(tensor([4522.7049], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 11:58:45.017264] Ending with PPL=9.59605874591979 (duration: 0:04:11.854055) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.812 +Metric validation/losses/mim_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.864 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.860 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 3.092 (now 3.1101431846618652). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.305 (now 0.33944612741470337). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 2.586 (now 2.9085161685943604). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 13:31:45.060981] Computing e^(tensor([4503.8972], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 13:31:45.066632] Ending with PPL=9.506241696031424 (duration: 0:04:09.336431) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.860 +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.855 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 3.092 (now 3.1101431846618652). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.305 (now 0.33944612741470337). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.586 (now 2.9085161685943604). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 15:01:00.702446] Computing e^(tensor([4504.7753], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 15:01:00.708207] Ending with PPL=9.510416308108953 (duration: 0:04:48.631397) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.809 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.859 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.855 (now 0.8566834926605225). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 18:04:32.533019] Computing e^(tensor([4596.3286], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 18:04:32.538667] Ending with PPL=9.955889569598243 (duration: 0:04:10.787452) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.806 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 0.859 (now 0.8616237044334412). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 0.855 (now 0.8719851970672607). Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.005 >= min_delta = 0.0. New best score: 3.087 +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.305 (now 0.33633795380592346). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 2.586 (now 3.076284646987915). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 00:11:25.867388] Computing e^(tensor([4508.1099], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 00:11:25.873138] Ending with PPL=9.5262860419246 (duration: 0:04:14.390963) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.793 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 0.859 (now 0.8670850992202759). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Metric validation/losses/mmm_text_loss improved by 0.055 >= min_delta = 0.0. New best score: 3.031 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.305 (now 0.33633795380592346). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.586 (now 3.076284646987915). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 01:47:25.338583] Computing e^(tensor([4484.5584], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 01:47:25.344228] Ending with PPL=9.414765112412917 (duration: 0:04:14.135816) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.985 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.305 (now 0.33633795380592346). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.586 (now 3.076284646987915). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 03:25:52.712054] Computing e^(tensor([4470.5446], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 03:25:52.717642] Ending with PPL=9.349027071148925 (duration: 0:04:13.620653) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.789 +Monitored metric validation/losses/mim_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.855 (now 0.8561944961547852). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.985 (now 3.0023298263549805). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 04:56:33.874063] Computing e^(tensor([4584.7751], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 04:56:33.879737] Ending with PPL=9.898542923203834 (duration: 0:04:20.000715) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.783 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 0.859 (now 0.8631399869918823). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.855 (now 0.8610655665397644). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.305 (now 0.3162252604961395). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.586 (now 3.325246810913086). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 14:10:09.793487] Computing e^(tensor([4465.1669], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 14:10:09.799178] Ending with PPL=9.323922725618456 (duration: 0:04:30.026087) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.779 +Metric validation/losses/mim_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.854 +Monitored metric validation/losses/mmm_image_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 2.985 (now 3.0729825496673584). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.305 (now 0.33114272356033325). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.586 (now 3.487175703048706). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 22:16:10.931250] Computing e^(tensor([4451.3016], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 22:16:10.936860] Ending with PPL=9.25950692707805 (duration: 0:04:07.833800) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.779 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.854 (now 0.8591603636741638). Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.852 +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.305 (now 0.33114272356033325). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.586 (now 3.487175703048706). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 23:37:40.937604] Computing e^(tensor([4451.5949], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 23:37:40.943277] Ending with PPL=9.260864751802163 (duration: 0:04:08.152390) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.849 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 2.985 (now 3.0597832202911377). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.305 (now 0.31688910722732544). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.586 (now 3.778892993927002). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 05:23:58.467840] Computing e^(tensor([4444.3279], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 05:23:58.473538] Ending with PPL=9.227276798934719 (duration: 0:04:10.255801) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.770 +Monitored metric validation/losses/mim_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.849 (now 0.8513005971908569). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 2.985 (now 3.0597832202911377). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.305 (now 0.32866379618644714). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.586 (now 3.7842299938201904). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 08:07:13.461597] Computing e^(tensor([4427.4854], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 08:07:13.467228] Ending with PPL=9.14989748285146 (duration: 0:05:29.191344) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.767 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.854 (now 0.862072765827179). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.849 (now 0.8565381169319153). Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.305 (now 0.32866379618644714). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.586 (now 3.7842299938201904). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 09:36:26.705071] Computing e^(tensor([4442.7626], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 09:36:26.710717] Ending with PPL=9.220057751726412 (duration: 0:04:13.174137) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.297 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.586 (now 3.712536573410034). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 13:39:08.982035] Computing e^(tensor([4458.4673], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 13:39:08.987671] Ending with PPL=9.292741667732805 (duration: 0:04:13.678684) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.282 +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 2.586 (now 3.7550456523895264). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 16:41:44.091716] Computing e^(tensor([4434.5487], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 16:41:44.097439] Ending with PPL=9.182269274412288 (duration: 0:04:29.135151) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.764 +Monitored metric validation/losses/mim_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.849 (now 0.8520420789718628). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 2.985 (now 3.069577932357788). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.282 (now 0.2921837270259857). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.586 (now 3.7550456523895264). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 18:04:55.751619] Computing e^(tensor([4418.4693], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 18:04:55.757298] Ending with PPL=9.108742153957675 (duration: 0:05:28.697207) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.759 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.854 (now 0.8609479665756226). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.849 (now 0.8534638285636902). Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Metric validation/losses/itm_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.273 +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.586 (now 3.7550456523895264). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 19:37:02.583770] Computing e^(tensor([4418.9072], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 19:37:02.589475] Ending with PPL=9.110736762790097 (duration: 0:04:15.622912) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.255 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.586 (now 3.5025525093078613). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 22:23:14.445007] Computing e^(tensor([4459.1438], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 22:23:14.449079] Ending with PPL=9.295885730866036 (duration: 0:04:20.598896) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.758 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 0.854 (now 0.8589944839477539). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.849 (now 0.858772337436676). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 2.985 (now 3.053874969482422). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.255 (now 0.2572108209133148). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 2.586 (now 3.6157829761505127). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 02:40:32.754173] Computing e^(tensor([4403.3815], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 02:40:32.759785] Ending with PPL=9.040285376005015 (duration: 0:04:33.111478) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.750 +Monitored metric validation/losses/mim_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.849 (now 0.8609395027160645). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 2.985 (now 3.053874969482422). Signaling model to ignore task. +Metric validation/losses/itm_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.252 +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.586 (now 3.6157829761505127). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 04:13:08.184208] Computing e^(tensor([4409.4774], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 04:13:08.189872] Ending with PPL=9.067881909187276 (duration: 0:04:44.610552) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.246 +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 07:07:26.467776] Computing e^(tensor([4433.1030], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 07:07:26.473457] Ending with PPL=9.175634161359422 (duration: 0:04:15.607100) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.741 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 0.854 (now 0.863511860370636). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.849 (now 0.8605883717536926). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 2.985 (now 3.0149080753326416). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.246 (now 0.281973659992218). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 2.586 (now 3.852733612060547). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 12:43:09.043149] Computing e^(tensor([4397.4048], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 12:43:09.048803] Ending with PPL=9.01331035411931 (duration: 0:04:13.632010) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.739 +Monitored metric validation/losses/mim_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.849 (now 0.8641515374183655). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 2.985 (now 3.0149080753326416). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.246 (now 0.281973659992218). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.586 (now 3.852733612060547). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 14:03:17.845480] Computing e^(tensor([4389.8998], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 14:03:17.851279] Ending with PPL=8.979551081150554 (duration: 0:04:39.323225) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.737 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.854 (now 0.8676209449768066). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.849 (now 0.8624650239944458). Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.246 (now 0.3224860429763794). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.586 (now 3.7841827869415283). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 01:17:02.883419] Computing e^(tensor([4380.6089], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 01:17:02.889117] Ending with PPL=8.93793388564623 (duration: 0:04:17.066549) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.733 +Monitored metric validation/losses/mim_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.849 (now 0.858325183391571). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 2.985 (now 3.010132312774658). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.246 (now 0.29857203364372253). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.586 (now 3.7149178981781006). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 09:30:52.936835] Computing e^(tensor([4356.7486], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 09:30:52.942405] Ending with PPL=8.831936574482683 (duration: 0:04:23.152862) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.733 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.854 (now 0.8696660399436951). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.849 (now 0.8665866851806641). Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.246 (now 0.29857203364372253). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.586 (now 3.7149178981781006). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 10:56:11.897871] Computing e^(tensor([4360.6490], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 10:56:11.903509] Ending with PPL=8.849177244140318 (duration: 0:04:44.475437) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.798 +Metric validation/losses/mim_loss improved by 0.015 >= min_delta = 0.0. New best score: 0.885 +Metric validation/losses/mmm_image_loss improved. New best score: 0.893 +Metric validation/losses/mmm_text_loss improved by 0.042 >= min_delta = 0.0. New best score: 3.196 +Metric validation/losses/itm_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.247 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 3.477 (now 3.519721269607544). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 20:37:12.806854] Computing e^(tensor([4438.3298], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 20:37:12.812393] Ending with PPL=9.199644807498778 (duration: 0:07:55.546474) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.770 +Metric validation/losses/mim_loss improved by 0.010 >= min_delta = 0.0. New best score: 0.876 +Metric validation/losses/mmm_image_loss improved by 0.021 >= min_delta = 0.0. New best score: 0.872 +Metric validation/losses/mmm_text_loss improved by 0.074 >= min_delta = 0.0. New best score: 3.122 +Metric validation/losses/itm_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.244 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 3.477 (now 3.557244062423706). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 21:38:22.508742] Computing e^(tensor([4422.0194], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 21:38:22.514510] Ending with PPL=9.124925143471845 (duration: 0:06:17.487430) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.759 +Metric validation/losses/mim_loss improved by 0.015 >= min_delta = 0.0. New best score: 0.860 +Metric validation/losses/mmm_image_loss improved by 0.017 >= min_delta = 0.0. New best score: 0.854 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.122 (now 3.126352310180664). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.244 (now 0.2648835778236389). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 3.477 (now 3.6175718307495117). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 22:24:48.105017] Computing e^(tensor([4382.4343], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 22:24:48.110564] Ending with PPL=8.946095087775507 (duration: 0:06:04.517830) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.735 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.860 (now 0.8647965788841248). Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.853 +Metric validation/losses/mmm_text_loss improved by 0.052 >= min_delta = 0.0. New best score: 3.069 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.244 (now 0.2677534520626068). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.477 (now 3.7223167419433594). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 23:26:33.189269] Computing e^(tensor([4372.1653], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 23:26:33.195047] Ending with PPL=8.900279289053312 (duration: 0:04:22.281729) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.729 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 0.860 (now 0.8638901114463806). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.853 (now 0.861332356929779). Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.010 >= min_delta = 0.0. New best score: 3.059 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.244 (now 0.2852664887905121). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.477 (now 3.7223167419433594). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 00:18:06.974672] Computing e^(tensor([4354.0451], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 00:18:06.980335] Ending with PPL=8.82000607521696 (duration: 0:04:18.067010) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.045 +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.244 (now 0.32437899708747864). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 3.477 (now 3.7223167419433594). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 01:05:51.720951] Computing e^(tensor([4377.2306], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 01:05:51.726743] Ending with PPL=8.922848994773778 (duration: 0:05:26.515110) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.021 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.244 (now 0.32437899708747864). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 02:03:22.808366] Computing e^(tensor([4395.0906], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 02:03:22.813961] Ending with PPL=9.002886909685424 (duration: 0:04:36.765260) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.005 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.244 (now 0.32437899708747864). Signaling model to ignore task. +Metric validation/losses/global_contrastive_loss improved by 0.015 >= min_delta = 0.0. New best score: 3.463 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 02:51:28.984516] Computing e^(tensor([4380.4536], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 02:51:28.990186] Ending with PPL=8.937239970833247 (duration: 0:05:05.988522) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.985 +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Metric validation/losses/global_contrastive_loss improved by 0.020 >= min_delta = 0.0. New best score: 3.443 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 03:40:16.244529] Computing e^(tensor([4401.6170], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 03:40:16.248579] Ending with PPL=9.032312962432448 (duration: 0:06:21.681057) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.715 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 0.860 (now 0.8715371489524841). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.853 (now 0.8683581948280334). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 2.985 (now 3.0064916610717773). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.244 (now 0.41365697979927063). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.443 (now 3.6239418983459473). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 07:21:04.967914] Computing e^(tensor([4335.5248], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 07:21:04.973532] Ending with PPL=8.738708285363758 (duration: 0:04:37.343244) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.714 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 0.860 (now 0.8813304305076599). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.853 (now 0.8833534121513367). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 2.985 (now 3.003140449523926). Wait count has been increased from 2. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.244 (now 0.32056015729904175). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 3.443 (now 3.811096429824829). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 14:11:51.084910] Computing e^(tensor([4320.0962], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 14:11:51.090441] Ending with PPL=8.671554621877961 (duration: 0:04:12.543940) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.713 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 0.860 (now 0.880665123462677). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.853 (now 0.8833609223365784). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 2.985 (now 2.987083911895752). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.244 (now 0.3593738377094269). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.443 (now 3.7622323036193848). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 15:48:09.168482] Computing e^(tensor([4337.3075], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 15:48:09.172603] Ending with PPL=8.746500995358424 (duration: 0:04:17.764735) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.708 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 0.860 (now 0.8917713165283203). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.853 (now 0.883549153804779). Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 2.985 (now 2.987083911895752). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.244 (now 0.3593738377094269). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.443 (now 3.7622323036193848). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 17:23:28.852242] Computing e^(tensor([4333.9284], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 17:23:28.857946] Ending with PPL=8.731736135952705 (duration: 0:04:13.565548) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.971 +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.244 (now 0.34734970331192017). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.443 (now 3.9313855171203613). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 03:30:34.679044] Computing e^(tensor([4318.5405], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 03:30:34.684804] Ending with PPL=8.664812173654033 (duration: 0:04:10.487244) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.937 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.244 (now 0.34734970331192017). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.443 (now 3.9313855171203613). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 05:08:45.304459] Computing e^(tensor([4319.8110], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 05:08:45.310098] Ending with PPL=8.67031815097133 (duration: 0:04:11.820655) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.705 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 0.860 (now 0.9174327254295349). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 4 records. Best score: 0.853 (now 0.9086042046546936). Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.013 >= min_delta = 0.0. New best score: 2.924 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.244 (now 0.34734970331192017). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 3.443 (now 3.9313855171203613). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 06:47:02.623519] Computing e^(tensor([4311.7057], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 06:47:02.629104] Ending with PPL=8.63525177465509 (duration: 0:04:17.387474) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.702 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 0.860 (now 0.9164105653762817). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 0.853 (now 0.9086042046546936). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.924 (now 2.9249844551086426). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 08:30:50.502531] Computing e^(tensor([4430.7735], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 08:30:50.508089] Ending with PPL=9.16495287370654 (duration: 0:05:00.213053) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.699 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 0.860 (now 0.8963720202445984). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.853 (now 0.8957487344741821). Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 2.924 (now 2.9731948375701904). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.244 (now 0.3419637680053711). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.443 (now 3.9362518787384033). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 15:20:21.122586] Computing e^(tensor([4314.6953], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 15:20:21.128171] Ending with PPL=8.648169161508264 (duration: 0:06:22.022242) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00 Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 + Collapsing WiT text for split train[:10%] of None: 0%| | 0/392187 [00:00 Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 + Creating a copy of the dataset (to be ITM-transformed)... (num_proc=16): 0%| | 0/1166454 [00:00 Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Callbacks registered: ['LearningRateMonitor', 'LMEvalHarnessCallback', 'PseudoPerplexityCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-06-27 21:20:30.985274 Preparing data for datamodule: VLDataModule +2023-06-27 21:20:30.985464 Preparing data for datamodule: ImageDataModule +2023-06-27 21:20:30.985572 Preparing data for datamodule: MLMDataModule +2023-06-27 21:32:22.450487 Setting up datamodule: VLDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-06-27 21:38:51.778394 Setting up datamodule: ImageDataModule +2023-06-27 21:39:04.928566 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-06-27 21:46:41.733965 Building train dataloader for datamodule: VLDataModule +2023-06-27 21:46:41.734864 Building train dataloader for datamodule: ImageDataModule +2023-06-27 21:46:41.735203 Building train dataloader for datamodule: MLMDataModule +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[PPL Evaluation] Length of the dataset is 1295591 +Initializing datamodule +VLDataModule: batch_size is 32, num_workers is 4 +ImageDataModule: batch_size is 32, num_workers is 4 +MLMDataModule: batch_size is 32, num_workers is 4 +Dataset sizes (un-normalized): [10, 10, 10] +Dataset sizes (normalized): [0.3333333333333333, 0.3333333333333333, 0.3333333333333333] +Sampling weights after temperature (1.0): [0.3333333333333333, 0.3333333333333333, 0.3333333333333333] +2023-06-27 21:21:13.420052 Setting up datamodule: VLDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-06-27 21:25:56.133038 Setting up datamodule: ImageDataModule +2023-06-27 21:26:05.377929 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Callbacks registered: ['LearningRateMonitor', 'LMEvalHarnessCallback', 'PseudoPerplexityCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-06-27 21:32:22.450357 Setting up datamodule: VLDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-06-27 21:38:51.776672 Setting up datamodule: ImageDataModule +2023-06-27 21:39:04.990161 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-06-27 21:46:41.734336 Building train dataloader for datamodule: VLDataModule +2023-06-27 21:46:41.735203 Building train dataloader for datamodule: ImageDataModule +2023-06-27 21:46:41.735648 Building train dataloader for datamodule: MLMDataModule +2023-06-27 21:46:44.140922 Building val dataloader for datamodule: VLDataModule +2023-06-27 21:46:44.142733 Building val dataloader for datamodule: ImageDataModule +2023-06-27 21:46:44.143659 Building val dataloader for datamodule: MLMDataModule +2023-06-27 21:46:44.156772 Building val dataloader for datamodule: VLDataModule +2023-06-27 21:46:44.158370 Building val dataloader for datamodule: ImageDataModule +2023-06-27 21:46:44.159058 Building val dataloader for datamodule: MLMDataModule +WARNING:datasets.builder:Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00 + wandb.finish() # [optional] finish the wandb run, necessary in notebooks + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 145, in main + wandb_logger.experiment.config.update(config) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 531, in fit + call._call_and_handle_interrupt( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 41, in _call_and_handle_interrupt + return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 91, in launch + return function(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 570, in _fit_impl + self._run(model, ckpt_path=ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 975, in _run + results = self._run_stage() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1018, in _run_stage + self.fit_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 201, in run + self.advance() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 354, in advance + self.epoch_loop.run(self._data_fetcher) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 134, in run + self.on_advance_end() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 248, in on_advance_end + self.val_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/utilities.py", line 177, in _decorator + return loop_run(self, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 102, in run + self.on_run_start() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 234, in on_run_start + self._on_evaluation_start() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 283, in _on_evaluation_start + call._call_callback_hooks(trainer, hook_name, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 189, in _call_callback_hooks + fn(trainer, trainer.lightning_module, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context + return func(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_utilities/core/rank_zero.py", line 27, in wrapped_fn + return fn(*args, **kwargs) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/callbacks/blimp_eval.py", line 46, in on_validation_start + optimized_text_model = replace_flava_submodel_with_orig_for_eval(pl_module.model) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/callbacks/flava_lm.py", line 38, in __init__ + self.tokenizer.model_max_length = self.max_length + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/processing_utils.py", line 184, in from_pretrained + args = cls._get_arguments_from_pretrained(pretrained_model_name_or_path, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/processing_utils.py", line 228, in _get_arguments_from_pretrained + args.append(attribute_class.from_pretrained(pretrained_model_name_or_path, **kwargs)) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/image_processing_utils.py", line 165, in from_pretrained + image_processor_dict, kwargs = cls.get_image_processor_dict(pretrained_model_name_or_path, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/image_processing_utils.py", line 269, in get_image_processor_dict + resolved_image_processor_file = cached_file( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/utils/hub.py", line 433, in cached_file + raise EnvironmentError( +OSError: facebook/flava-full is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models' +If this is a private repository, make sure to pass a token having permission to this repo with `use_auth_token` or log in with `huggingface-cli login` and pass `use_auth_token=True`. +evaluation/blimp/subject_verb_agreement: 68.62 +evaluation/blimp_average: 69.20333333333333 +Ending LM Evaluation Harness (duration: 0:08:29.763944) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation] Based on the available VRAM (39.39GBs), we will use a max phrase length of 200. +[PPL Evaluation] Computing e^(5241.188715837896 / 2000) +[PPL Evaluation] Ending with PPL=13.743889947793965 (duration: 0:01:38.453485) +Starting LM Evaluation Harness +2023-07-01 21:23:34,869 - wandb.wandb_agent - INFO - Cleaning up finished run: asgf2rm1 +wandb: Waiting for W&B process to finish... (failed 1). Press Control-C to abort syncing. +wandb: +wandb: Run history: +wandb: ImageDataModule ▁▁▁▁▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: MLMDataModule ▁▁▁▁▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: VLDataModule ▁▁▁▁▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: epoch ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: evaluation/blimp/anaphor_agreement ▂▁▂▃▄▃▃▃▄▄▅▄▆▅▆▆▇▆▇▇▇▇▇▇▇▇▇▇▇▇▇█████████ +wandb: evaluation/blimp/argument_structure ▁▂▃▃▄▃▄▄▅▅▅▄▄▅▅▅▆▆▆▆▆▆▇▆▆▆▇▆▆▆▇▇▆▇▇▇▇▇▇█ +wandb: evaluation/blimp/binding ▂▅▃▃▄▄▃▃▁▁▃▃▄▄▄▄▃▅▄▅▅▅▅▄▆▆▇▆▅██▇▆▇▅▇█▇█▅ +wandb: evaluation/blimp/control_raising ▁▂▃▃▄▄▂▁▂▂▃▃▃▃▄▃▄▄▃▅▅▅▅▆▆▆▇▅▇▆▅▇▆▇▆▇▆▇▇█ +wandb: evaluation/blimp/determiner_noun_agreement ▁▁▁▁▁▂▂▂▃▃▄▅▅▆▆▇▆▆▆▇▇▇▇▇▇▇▇█▇▇██▇███████ +wandb: evaluation/blimp/ellipsis ▂▂▃▃▃▂▁▁▃▃▄▅▅▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇█▇█████ +wandb: evaluation/blimp/filler_gap ▁█▆▅▇▇▆▆▆▇▆▆▆▆▅▆▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅ +wandb: evaluation/blimp/irregular_forms ▁▂▃▃▃▄▅▅▅▇▇▇▇█▇█▇█▇▇██▇▇█▇▆█▇▇▇▇▇▆▆▇▇█▇█ +wandb: evaluation/blimp/island_effects ▅▄▅▅▇█▅▄▁▁▁▁▁▂▂▂▂▁▂▂▂▄▂▃▂▄▂▃▃▃▃▃▃▃▄▄▄▄▃▃ +wandb: evaluation/blimp/npi_licensing ▁▅▆▄▄▃▅▃▄▄▃▄▅▅▇▆▇▇▆▆▆▇▇▇▇▆▆▅▇█▇▇▆█▆▆█▇▇▆ +wandb: evaluation/blimp/quantifiers ▃▄▂▁▃▅▆▅▇▇███▇█▇▇▇█▇▇▇▇▇▆▇▇▇▆▇▆▆▆▆▆▆▆▆▇▆ +wandb: evaluation/blimp/subject_verb_agreement ▁▁▁▁▁▁▁▂▂▂▃▄▄▅▄▅▅▅▅▅▆▅▆▆▆▇▇▇▆▇▆▇▇▇▇▇▇███ +wandb: evaluation/blimp_average ▁▂▂▂▃▃▃▃▄▄▅▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇▇█▇█▇█████ +wandb: evaluation/pseudo_perplexity █▃▃▃▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: lr-AdamW/pg1 ▁▁▁▁▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: lr-AdamW/pg2 ▁▁▁▁▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: train/losses/global_contrastive_loss ██▆▄▄▃▁▃▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂ +wandb: train/losses/itm_loss ██▁▁▃▅▆▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂ +wandb: train/losses/mim_loss █▅▃▄▃▃▃▃▃▃▃▂▃▂▂▁▂▂▂▁▃▃▃▁▂▃▂▃▃▂▃▂▂▁▁▂▂▁▂▂ +wandb: train/losses/mlm_loss █▆▆▆▆▅▅▅▄▃▃▃▃▃▂▂▂▂▂▂▁▂▁▁▂▁▁▂▂▁▁▂▁▁▁▁▂▁▁▁ +wandb: train/losses/mmm_image_loss █▇▁▂▁▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂ +wandb: train/losses/mmm_text_loss █▆▆▆▆▅▄▄▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▁▂▁▂▁▂▁▂▁▁▂▁▁▁▁▁▁ +wandb: trainer/global_step ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███ +wandb: validation/losses/global_contrastive_loss ██▆▅▄▃▂▂▁▁▂▂ +wandb: validation/losses/itm_loss ███▇▆▄▃▁▂▂ +wandb: validation/losses/mim_loss █▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/losses/mlm_loss █▆▆▆▆▅▅▄▄▃▃▃▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/losses/mmm_image_loss █▄▁▁▁▁▁▁▁▁ +wandb: validation/losses/mmm_text_loss █▇▆▆▆▅▅▄▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/monitor/global_contrastive ███████▅▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/monitor/itm ██▅▅▅▅▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/monitor/mim ██████▅▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/monitor/mlm ██████████████████████████████████▅▅▃▃▁▁ +wandb: validation/monitor/mmm_image ██████▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: validation/monitor/mmm_text ████████████████████████████████▄▄▂▂▂▂▁▁ +wandb: +wandb: Run summary: +wandb: ImageDataModule 163737 +wandb: MLMDataModule 167189 +wandb: VLDataModule 165077 +wandb: epoch 0 +wandb: evaluation/blimp/anaphor_agreement 90.13 +wandb: evaluation/blimp/argument_structure 68.4 +wandb: evaluation/blimp/binding 64.8 +wandb: evaluation/blimp/control_raising 66.22 +wandb: evaluation/blimp/determiner_noun_agreement 90.15 +wandb: evaluation/blimp/ellipsis 78.23 +wandb: evaluation/blimp/filler_gap 61.33 +wandb: evaluation/blimp/irregular_forms 85.14 +wandb: evaluation/blimp/island_effects 43.2 +wandb: evaluation/blimp/npi_licensing 56.83 +wandb: evaluation/blimp/quantifiers 57.39 +wandb: evaluation/blimp/subject_verb_agreement 68.62 +wandb: evaluation/blimp_average 69.20333 +wandb: evaluation/pseudo_perplexity 12.8362 +wandb: lr-AdamW/pg1 0.00077 +wandb: lr-AdamW/pg2 0.00058 +wandb: train/losses/global_contrastive_loss 2.32867 +wandb: train/losses/itm_loss 0.19559 +wandb: train/losses/mim_loss 0.56371 +wandb: train/losses/mlm_loss 3.00859 +wandb: train/losses/mmm_image_loss 0.68354 +wandb: train/losses/mmm_text_loss 2.40413 +wandb: trainer/global_step 7749 +wandb: validation/losses/global_contrastive_loss 2.62358 +wandb: validation/losses/itm_loss 0.28444 +wandb: validation/losses/mim_loss 0.65336 +wandb: validation/losses/mlm_loss 3.13987 +wandb: validation/losses/mmm_image_loss 0.67712 +wandb: validation/losses/mmm_text_loss 2.89136 +wandb: validation/monitor/global_contrastive 0.0 +wandb: validation/monitor/itm 0.0 +wandb: validation/monitor/mim 0.0 +wandb: validation/monitor/mlm 0.0 +wandb: validation/monitor/mmm_image 0.0 +wandb: validation/monitor/mmm_text 0.0625 +wandb: +wandb: 🚀 View run balmy-sweep-3 at: https://wandb.ai/rycolab/alkmi-wit/runs/asgf2rm1 +wandb: Synced 3 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20230627_212047-asgf2rm1/logs +wandb: Terminating and syncing runs. Press ctrl-c to kill. diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1001.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1001.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..51171f45a2563d3a2de7b6d1c90db0429c284c09 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1001.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32439de02640fafa1366f80985a2f7c4d711eb580b242ce56ba534a850ddb3b7 +size 3848043751 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10073.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10073.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..a7958c5fd9d3829da823746b50e32c3a916f9b38 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10073.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7b5906dff9b99c08ffc36758f000335b11f06e63fdb9cc6fd266df5f7378d83 +size 3862234570 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10193.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10193.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..edd41bff10fa9ef876c4b3861171364471a2a11a --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10193.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10f4026d469cff524f0786d5ef27d07ab7f81186c7d5409a510914a0d5d0b7e4 +size 3862235017 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10316.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10316.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..0690449f75853457ad28212077ffea5c152dabf6 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10316.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fff549ad58beedde7e12e5750c0cffa1354429f3394322f44ebba74aaf107c10 +size 3862235528 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10434.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10434.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..58e965a1557589050971676d658fff1ed6f440b6 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10434.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa0da9e36a77425744d7db7ff138a30d1df1495d140d7a9254cfa141aefb6f11 +size 3862235975 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10547.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10547.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..cd8bcae46839c63cc2a17fc64a3ea0d82d0cc1a6 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10547.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5d251766dc60d6c7583c93fde7f77b5258335ec2f307b75ee8a7c6904529374 +size 3862236422 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10626.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10626.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..4f957378f266690643caa0ad09cba1ddec405081 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10626.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6eedf6e1b6879b04adbf9be4ce36efb2247e4cbec3c0342f91f20f25912a3043 +size 3862236933 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10723.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10723.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..1f59bc5b6c4a6595f86125e1efaf5d7ad46319df --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10723.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bbbe816c114bedc006f01cf8cf3358fbe3a04c8e9c45490b95d01e5a3960c07 +size 3862237380 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10804.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10804.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c7ca830180ee1400c25a53c9354e031d4baf59a6 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10804.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ded52f1261a37d8d7057cf1cf013f7d4dca1cd67c3670ad5025e05bc11d1349 +size 3862237827 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10887.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10887.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..64bbe2c9d29235174752e64110daf652c1b81450 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10887.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd9f642ffc37d8be303f6055a95f516ba9771dc79d0f40da708a0bd22d033dc7 +size 3862238402 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10980.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10980.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..60408e92475b630bb209d13216e1617737593a6a --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10980.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d81b2b6b8db77745908e47b7d98b4219c5b9cf890ed757ab5e218f1a09c50e89 +size 3862238785 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11101.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11101.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..281f73068ac115498f78744b8043dc0a900ca0f2 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11101.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99d1e8135897cc1b004690e6c83b1e9607244df72860b04806683270aafcbbfe +size 3862228439 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11245.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11245.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..3489902ab88852b46ebc5116c85755260bbdeb93 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11245.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f802022cba398d199730bdab80b4541adf102579c5fc0e6ec34735b7ba4b91f7 +size 3862228822 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11376.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11376.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..10b96d8a89471a35b76747f52dd608dbea9d6b5e --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11376.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e01698bb421b9f3f4b51208ad5064fcef10338a78eeca5df324cc3adc1fdd294 +size 3862229333 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11558.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11558.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..85f8bd135df13f7af03cf4cc361befcabcb19d44 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11558.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd7bbf3fbbe1fd947cec4905ba80c07ea0c85be48b4c6012370722b23adcbc6d +size 3862229908 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11728.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11728.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..316b5bc6c05892d8416c938df4cf91941688237a --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11728.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a1998eb0531e61f827c24004a0045895e12b00a15abbcc0b857fb508ee7d60b +size 3862230291 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11876.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11876.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..8d25322433a18b679dbd7037c59c7600f90c4615 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11876.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6655da4e5bd2e18026f3d84daf5b647d4125760de420f28fd29bad42b044b936 +size 3862230738 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1371.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1371.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c7b363175522098ed49c9db8c52ec17f0be24256 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1371.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecf80b2afcc1aad8c477abdbcd0027b4dfb6e7d8f649304c1000c840dd088fc5 +size 3848044262 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1705.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1705.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..5fb7e14e554abb251dfca3d31b45a3d7f3983f66 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1705.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:963b41d27815804dddcf689dfdf2bba8aa4398b0063131e0111b67f09b849166 +size 3848044709 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1923.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1923.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..a55ec8b23e8eb1deb7b36d16c9311abdc22d7616 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1923.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e4dbab84f5401c0771d6ad980a6f9b16975546a49f78e8ecdd21efa31b1ded1 +size 3848045156 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2140.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2140.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..7857499682c6d0897ac9bbce8ec34ec8d4986a2e --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2140.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c0caa59d470912d28cbac1fde3606f6b5d5723c283555f6e7a072b491c85bc2 +size 3848045667 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2376.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2376.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..be603e9e9679211497aaa348837564aaf40eae26 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2376.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7c773730fe0a61b922fc3abb7c27b7d8e200156c72bc924f3e87f6a960678a6 +size 3848046114 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2619.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2619.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..6bb52797f66ece1db580f6888f338dcf5edb5d62 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2619.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8f0d6c11785750d5a4ba4d82d8ee18351a7dea5da7fd474446bfc6ba96163b6 +size 3848046561 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2840.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2840.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..72cc25ae092e21c3a85502ab6d2452bd3a7acdb7 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2840.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a7792af95a9756cb7b77cb642834e710634693a63e604862b5076e12b6cb3b6 +size 3848047136 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3056.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3056.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..60f982c269622b6b4352e8ea514b61aac40d6aca --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3056.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8178297bbd1e5c2140fbb935227e56264d29c7e4f34f5ffe6fb3a3a57f2cf5ca +size 3848047519 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=308.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=308.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..6f982521e09cca45b7ef14c5b66971d63d937b62 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=308.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c47b70674b4f9b8498ec18b2166140c9bf1fbd24d77b683687edf27e8c41ffb +size 3848042857 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3275.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3275.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..08d6aa18b03da5be01c9c2bfd6e63f97fae6ecc4 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3275.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95721f114bedb4b3ad9e29bf5567b16c52c8c72e0ea9c6176fb07475a96ddb57 +size 3848047966 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3501.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3501.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..e6e01d5b58b8697c35b91351cac14e28b3eb9db0 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3501.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc737a905c838d6ec6be2e7f20535a5bd85f2f43238909fceb1932d1c6ea4493 +size 3848048541 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3730.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3730.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..e7fbd2e933e9302219aac367fc0b7385634337e0 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3730.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8029be959d3487636254747c91c99df06e7367d621ddc45687428a15718c549 +size 3848048924 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3940.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3940.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b08be7bb0aeb9ec42d8922905630756ed01006f1 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3940.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44fe8c6ebab91a0bb775a3ef02cf7a3af1debdac7d8d9f91c8a955c450a6bb0e +size 3848049371 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4154.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4154.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c97cdd7960d5a07e09fbb85fa93d8a5c2971acfa --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4154.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee72c8d1ac79c7fe2ba2b508664b12fd44f50a0fb8d7b676894b836b9887ec41 +size 3848049946 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4376.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4376.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..5e29ac9c3d9ad2bac0d375928f52082e7fee1d43 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4376.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2c5a40c77c92b7f5cadca86be98f31c55cb826427d54ab0362d7f6f6b9e299c +size 3848050329 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4608.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4608.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..944987a12c200c2d327c9b87e7fe40c097c25790 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4608.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5409ea77a3e87ed64f6ea79c809d4365a2d1d722b90a65d394a826f54325905 +size 3848050840 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4822.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4822.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..299da243cc4220d2cbd7d7565bbc6ab771b55fef --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4822.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15fa8ac404d49f44179acc0f15abe9e29d2b7762c0d42610d81839ea6196fa62 +size 3848051351 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5041.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5041.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..2cee1fa40fcd91d7dfced4e31069497285103fa6 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5041.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3503c66f11886d61ac89bee2dcb6f94d1cdd780c36a1dc6e2a7eee98731cbbbe +size 3848051734 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5256.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5256.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..df4b46bcf322415985c5dcc21505479d9cb7fbb0 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5256.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6ebbbe841a64f1f69be138335ea6fa9c0906bcaac67845c87316672ef204414 +size 3848052245 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5499.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5499.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..fedfdcf71e2b731283ab6ffb3aa80627587c6704 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5499.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fccd2ac3208b9a709d56bfec4d57832a623f739d1830ddd69dfa736eec900ee +size 3848052756 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5716.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5716.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..90f510e7a72e80aec390d31d0cf10a8fc2a8bd50 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5716.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6598960bf56e222de013b0f5122678077fb9ab97dcc38364698cb3c95d8f14a9 +size 3848053203 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5942.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5942.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..70e6b7050d36092c5a49907fffae479de0820d3a --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5942.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9bc107770708063c45e9cb20df892ea12c182edbfb589770d24c20b1d968c85 +size 3848053650 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6167.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6167.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c47cd4eaf33fa4b03f39b8ab936c8c1bba84937b --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6167.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c49f004c5714bc78f3fc9fae602368b2cd679ab5578a619d9733014b8190355c +size 3848054161 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6397.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6397.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..21b11a2f4757c586c5637d445b43f63b71424a68 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6397.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe55e388d33878cd64789cd427df4bd49b55eee89ac00ba04d32a795c9bd361a +size 3848054608 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=651.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=651.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..44b8670ba52d7d2bf445802be38dcc7556d4d0b0 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=651.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9959bc741e90347633925f1bba947b2f35be99cad8c213f1a41e762a45ac9510 +size 3848043240 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6627.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6627.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..8066b5dd9a9fff8e7f9d8bf60f2464d7fa0f6fb8 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6627.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed9d9c960b330d56d93c5b3eee275869916ee0e25a3c5e875d95ee1a33949606 +size 3848055055 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6876.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6876.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..db20b639e19d455ddb100bbe7deca6d79724c90d --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6876.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67f82d5026d51d642f9a677ef20ef6c5b9cda3a02968ff7545504a0eaeeb7f27 +size 3848055566 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7123.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7123.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..0bffc42da0e39d5c2fa5310c1ef95d2292c760a7 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7123.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95d265191570f15431334d61483ed6cbbc2f6093db6bc80b3e54a9122732bfa1 +size 3848056013 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7355.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7355.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..dce2f98247b2357eee15dd6b0284cae7fd223cf3 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7355.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c825c152155051675bf4ba1983ce40d49a27d337e11a99c85b9706571575da0 +size 3848056460 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7587.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7587.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..d3b7ea9d062e117bf34a99dd32c004c230662b70 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7587.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a686e2d1ef5b3778844c265e5e02ab9e2ee6f73b8154d047df83a8368d93b1d +size 3848057035 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7686.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7686.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..dbc39dea69881433bfdcc8ec70c2f3bcdb56d556 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7686.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:821117133b63e2bb6678a27ad16dc7fea8e0280703bddca440ff68934856e8ed +size 3862228567 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7782.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7782.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..314a3751c61da6f9a281c57689f189affa850ea4 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7782.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbdc2bc32dc01503c74f8ca31c7694020323e179071184556708ee1c58261a0c +size 3862228950 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7880.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7880.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..8bfbca85dba339d181413c7878d29bc1111760fd --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7880.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94aeffd701d8e96981480b9b1e9b8d82b193845d66c7eefabf67bd8764cef879 +size 3862229397 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7992.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7992.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..3904528cc609d030634c996fa6a289544e741512 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7992.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ea0fc8ca9a6206ec900ae9b405e3efd1aca3ad58e0335a7f96dcb7353c6bd5c +size 3862229972 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8090.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8090.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c1ea10e8818dec505064637d9e4f6b5ca94f8387 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8090.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6485dd3fac18b76b11083b974a08cc679b8f71337b295868d8ed74f29db7e42 +size 3862230355 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8190.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8190.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..9801c300623c2abfe85f06e041e29fd1263c05f6 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8190.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3d9cc1b46c2b4e33788a922100942654c8b7ba5dd1c2d9fb09ec0b0a445ea33 +size 3862230866 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8287.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8287.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..372a6be37052f53b480ea8e8602beca917582be9 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8287.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4532a34967999e4a538f2b92c934ee9024fe58bc9597d18105f8b87ca232830 +size 3862231377 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8383.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8383.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..1ec8d66202708c8661770bb3f49687996b38ff95 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8383.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f50e06341c4dd155c98bcfb68fdf43f13255888a8eac2ce454287b2bc45f1e4d +size 3862231760 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8501.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8501.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f5a428ee4eb61fc7945e18b6d13df3077c10621c --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8501.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32bc9bc92afe2b40d833cc19adc1494981d73c651ced5f76f22fe3cc434b6d45 +size 3862228439 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8631.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8631.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..15823b54449ac9a6eba20e3c9f8ce7732da0f1de --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8631.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afd28937970333696adefd4ba7b01272d84cfbd6925b568f3ef9743c12b3d209 +size 3862228886 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8755.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8755.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..d9495d67c8a8bc3c5179297c3a65bb2a9ef3dbc8 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8755.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14a4fe279ca706eeacf6ac6adc40525d848c879cb5cc66f7c3bfbe956b8336b2 +size 3862229333 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8876.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8876.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b57fc8d51183fa7acedd851540f24197bad0484c --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8876.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fa3d5dbce31b2241a1e75e6d65c50f52abd1307435c20dfa99a961db0a0a888 +size 3862229844 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9001.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9001.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..5dbac9b16840dd2a054924185692f3578803c505 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9001.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3037b608cc5b2cdd7b2caab2ab40086584aa5a5880cf0bcdc0d6c49ecaa879f8 +size 3862230291 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9126.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9126.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..a21f66d20a64c2b09f3ef4622c2689a64098e446 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9126.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e4d8c468f8a1e034a3bff74285fa8076a066c7bb15dd0102ad711554d84d25c +size 3862230738 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9251.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9251.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..fce793fd3f3cd6bfb8d98c297992b00e7b08928a --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9251.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4a2ed8abe3f9a72b48cae1de9980759a71d70844dd5650e5f02523381b6f48a +size 3862231313 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9348.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9348.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..515008697b24c92646df0ef4cc4f0d84faf85f39 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9348.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c28d5de42b7900eee9402760e181c7f8708fbc6d4d1cc9b449c82fd0d7ec6e9d +size 3862231696 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9461.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9461.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..45dc472d173c76f5aeccb0ffd0d633a1de7a46c2 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9461.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cef43a869296f7ba0818263b220231d307786fed31fa6aaa5a1e07cd76a41890 +size 3862232143 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9593.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9593.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..006b0daf9bad73575a9a65225a4f4b8c3496b597 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9593.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52847095561b2569bb0abf094e0bf936abc8e2058e54361c0dd6b9bb30ac1f92 +size 3862232718 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9722.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9722.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..7a454a15383b0cfd8e776969526909ee846de116 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9722.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cb09b9df5c5fe354b9c1dae07271a0dd661d164b2708cb8cf11c465ed95e05a +size 3862233165 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9837.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9837.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..50a7cc7d3d34004800add8939cac5d2779da3bf9 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9837.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93f07e3b476583881ac85d6057031242738257f2c273ec52e8137ac65a35073f +size 3862233612 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9952.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9952.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..172642b6e12b1133e41e2430519c42cbd4842fcc --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9952.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:199bd48379735aed14ffaed9145d62396505e63a9ca2096bb9d1d479aff7833b +size 3862234123 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/last.ckpt b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/last.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..7154ce698aff212034843703ddfc3625fd552ba4 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/last.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c233e6f1cdb3929fd873a6e9cd8fa610f2e8d982fa427a742b692df56a1acb0c +size 3862230802 diff --git a/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/resume_bugTimeLimitReachedElapsedTimeIs5days_singlenode_2023-07-26-08:50:28.log b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/resume_bugTimeLimitReachedElapsedTimeIs5days_singlenode_2023-07-26-08:50:28.log new file mode 100644 index 0000000000000000000000000000000000000000..39c0a1d02af1348c8eb45d06686a5482dfc415d8 --- /dev/null +++ b/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/resume_bugTimeLimitReachedElapsedTimeIs5days_singlenode_2023-07-26-08:50:28.log @@ -0,0 +1,381 @@ +wandb: Currently logged in as: tamariucai (rycolab). Use `wandb login --relogin` to force relogin +wandb: - Waiting for wandb.init()... wandb: \ Waiting for wandb.init()... wandb: wandb version 0.15.7 is available! To upgrade, please run: +wandb: $ pip install wandb --upgrade +wandb: Tracking run with wandb version 0.15.4 +wandb: Run data is saved locally in ./wandb/run-20230726_085054-asgf2rm1 +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run text10-vision10 +wandb: ⭐️ View project at https://wandb.ai/rycolab/alkmi-wit +wandb: 🚀 View run at https://wandb.ai/rycolab/alkmi-wit/runs/asgf2rm1 +[rank: 0] Global seed set to 5501650 +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +The current host is eu-a65-05 +The current cuDNN version is 8801 +Parameter 'text_perc' already had the value 10. +Parameter 'vision_perc' already had the value 10. +Parameter 'seed' already had the value 5501650. +Total GPU memory: 84.991 GB +Detected 80 GB of RAM, doubling batch size to 64. +Detected 80 GB of RAM, halving gradient accumulation to 64 +[update_ckeckpoint_dir] Detected hyperparameter run! +[update_ckeckpoint_dir] Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/date(2023-07-26)_time(08:50:56)/seed(5501650)-magic({'enable': True})-text_perc(10)-vision_perc(10/ +Text is the predominant modality (10 v.s. 10 vision), will sample proportionally for optimal BLiMP performance. +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-aa925072b83a8ef4_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/plugins/environments/slurm.py:165: PossibleUserWarning: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... + rank_zero_warn( +Using bfloat16 Automatic Mixed Precision (AMP) +GPU available: True (cuda), used: True +TPU available: False, using: 0 TPU cores +IPU available: False, using: 0 IPUs +HPU available: False, using: 0 HPUs +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-aa925072b83a8ef4_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Restoring states from the checkpoint path at /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8383.ckpt +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:337: UserWarning: The dirpath has changed from "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/date(2023-07-24)_time(18:44:53)/seed(5501650)-magic({'enable': True})-text_perc(10)-vision_perc(10" to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/date(2023-07-26)_time(08:50:56)/seed(5501650)-magic({'enable': True})-text_perc(10)-vision_perc(10", therefore `best_model_score`, `kth_best_model_path`, `kth_value`, `last_model_path` and `best_k_models` won't be reloaded. Only `best_model_path` will be reloaded. + warnings.warn( +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0] + + | Name | Type | Params +---------------------------------------------- +0 | model | FlavaForPreTraining | 357 M +---------------------------------------------- +303 M Trainable params +53.8 M Non-trainable params +357 M Total params +1,430.581 Total estimated model params size (MB) +Restored all states from the checkpoint at /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision10/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8383.ckpt +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +VLDataModule: batch_size is 64, num_workers is 4 +ImageDataModule: batch_size is 64, num_workers is 4 +MLMDataModule: batch_size is 64, num_workers is 4 +Dataset sizes (un-normalized): [10, 10, 10] +Dataset sizes (normalized): [0.3333333333333333, 0.3333333333333333, 0.3333333333333333] +Sampling weights after temperature (1.0): [0.3333333333333333, 0.3333333333333333, 0.3333333333333333] +2023-07-26 08:52:00.559170 Setting up datamodule: VLDataModule +2023-07-26 08:54:01.262186 Setting up datamodule: ImageDataModule +2023-07-26 08:54:07.518836 Setting up datamodule: MLMDataModule +Registering multimodal overfitting monitors +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor'] +Trainer successfully initialized (strategy=SingleDeviceStrategy) +Starting training +2023-07-26 08:55:05.574293 Preparing data for datamodule: VLDataModule +2023-07-26 08:55:05.574433 Preparing data for datamodule: ImageDataModule +2023-07-26 08:55:05.574521 Preparing data for datamodule: MLMDataModule +2023-07-26 08:55:05.575486 Setting up datamodule: VLDataModule +2023-07-26 08:55:44.191848 Setting up datamodule: ImageDataModule +2023-07-26 08:55:49.501239 Setting up datamodule: MLMDataModule +Resetting wait count for `mlm` since training is resuming and they are special tasks. +Resetting wait count for `mmm_text` since training is resuming and they are special tasks. +2023-07-26 08:56:31.825446 Building train dataloader for datamodule: VLDataModule +2023-07-26 08:56:31.826284 Building train dataloader for datamodule: ImageDataModule +2023-07-26 08:56:31.826568 Building train dataloader for datamodule: MLMDataModule +2023-07-26 08:56:33.052983 Building val dataloader for datamodule: VLDataModule +2023-07-26 08:56:33.053905 Building val dataloader for datamodule: ImageDataModule +2023-07-26 08:56:33.054201 Building val dataloader for datamodule: MLMDataModule +Time limit reached. Elapsed time is 5 days, 0:00:00. Signaling Trainer to stop. +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-26 11:26:42.679102] Computing e^(tensor([5289.3738], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-26 11:26:42.691313] Ending with PPL=14.079035950808176 (duration: 0:06:56.691220) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.101 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 0.654. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in 52 records. This exceeds the restart patience 10, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve in 50 records. This exceeds the restart patience 10, so we will restart the task with renewed patience. +Monitored metric validation/losses/mmm_image_loss did not improve in 52 records. This exceeds the restart patience 10, so we will restart the task with renewed patience. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 2 records. Best score: 2.900. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-24 23:41:11.792906] Computing e^(tensor([5044.1541], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-24 23:41:11.797172] Ending with PPL=12.454438535598515 (duration: 0:06:54.247858) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.999 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 0.654. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.277. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.540. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.676. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 3 records. Best score: 2.900. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-25 03:25:04.377425] Computing e^(tensor([4982.6393], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-25 03:25:04.381583] Ending with PPL=12.077203415632733 (duration: 0:06:57.739940) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.981 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 0.654. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 2 records. Best score: 0.277. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 2 records. Best score: 2.540. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.672 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 2.900. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-25 07:08:48.142623] Computing e^(tensor([4948.3597], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-25 07:08:48.146792] Ending with PPL=11.871966221358347 (duration: 0:06:57.215453) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.965 +Monitored metric validation/losses/mim_loss did not improve in the last 7 records. Best score: 0.654. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 3 records. Best score: 0.277. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 3 records. Best score: 2.540. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.666 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 2.900. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-25 10:54:13.565916] Computing e^(tensor([4932.8614], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-25 10:54:13.570101] Ending with PPL=11.780324107918508 (duration: 0:06:51.006140) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.964 +Monitored metric validation/losses/mim_loss did not improve in the last 8 records. Best score: 0.654. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.277. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 2.540. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.665 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 2.900. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-25 14:34:19.230212] Computing e^(tensor([4909.9151], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-25 14:34:19.234361] Ending with PPL=11.645939406486743 (duration: 0:06:55.796029) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.955 +Monitored metric validation/losses/mim_loss did not improve in the last 9 records. Best score: 0.654. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.277. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.540. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.662 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 7 records. Best score: 2.900. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-25 18:19:48.022972] Computing e^(tensor([4895.6531], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-25 18:19:48.027179] Ending with PPL=11.563187568248745 (duration: 0:06:53.783990) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.940 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.652 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.277. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.540. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.655 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 8 records. Best score: 2.900. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-25 21:21:26.695211] Computing e^(tensor([4904.8309], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-25 21:21:26.699402] Ending with PPL=11.616371873408044 (duration: 0:06:56.034473) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.651 +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.277. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 2.540. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.652 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 9 records. Best score: 2.900. Signaling model to ignore task. +Starting validation +2023-07-25 21:38:27.142675 Preparing data for datamodule: VLDataModule +2023-07-25 21:38:27.142811 Preparing data for datamodule: ImageDataModule +2023-07-25 21:38:27.142899 Preparing data for datamodule: MLMDataModule +2023-07-25 21:38:27.143220 Setting up datamodule: VLDataModule +2023-07-25 21:39:49.625459 Setting up datamodule: ImageDataModule +2023-07-25 21:39:58.481676 Setting up datamodule: MLMDataModule +2023-07-25 21:41:17.283293 Building val dataloader for datamodule: VLDataModule +2023-07-25 21:41:17.283845 Building val dataloader for datamodule: ImageDataModule +2023-07-25 21:41:17.284118 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-25 21:44:41.632733] Computing e^(tensor([4904.8309], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-25 21:44:41.640056] Ending with PPL=11.616371873408044 (duration: 0:03:10.676100) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.892 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-26 22:52:39.543279] Computing e^(tensor([4946.0842], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-26 22:52:39.551019] Ending with PPL=11.858466819839876 (duration: 0:07:31.781116) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.867 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-27 01:58:34.102317] Computing e^(tensor([4934.2827], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-27 01:58:34.106736] Ending with PPL=11.788698842278366 (duration: 0:07:02.035161) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.866 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-27 08:30:26.952144] Computing e^(tensor([4863.7884], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-27 08:30:26.956715] Ending with PPL=11.380418674626023 (duration: 0:07:03.779783) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.853 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-27 11:54:58.134563] Computing e^(tensor([4834.0565], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-27 11:54:58.141027] Ending with PPL=11.212488888528263 (duration: 0:07:03.947954) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.914 +Monitored metric validation/losses/mim_loss did not improve in the last 7 records. Best score: 0.652. Signaling model to ignore task. +Metric validation/losses/itm_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.276 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 2.540. Wait count has been increased from 3. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 7 records. Best score: 0.655. Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.008 >= min_delta = 0.0. New best score: 2.845 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-27 16:18:21.543466] Computing e^(tensor([4805.4272], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-27 16:18:21.548325] Ending with PPL=11.05312936709418 (duration: 0:07:10.891880) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.903 +Monitored metric validation/losses/mim_loss did not improve in the last 8 records. Best score: 0.652. Signaling model to ignore task. +Metric validation/losses/itm_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.270 +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.540. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 8 records. Best score: 0.655. Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.010 >= min_delta = 0.0. New best score: 2.835 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-27 19:20:45.217626] Computing e^(tensor([4793.8127], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-27 19:20:45.222881] Ending with PPL=10.989127490153415 (duration: 0:07:14.722409) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.893 +Monitored metric validation/losses/mim_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Metric validation/losses/itm_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.266 +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.540. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.835. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-27 22:12:40.678391] Computing e^(tensor([4759.7074], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-27 22:12:40.684668] Ending with PPL=10.803322416884757 (duration: 0:07:17.740877) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.871 +Metric validation/losses/mim_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.649 +Metric validation/losses/itm_loss improved by 0.011 >= min_delta = 0.0. New best score: 0.255 +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 2.540. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.649 +Metric validation/losses/mmm_text_loss improved by 0.047 >= min_delta = 0.0. New best score: 2.788 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-28 01:12:13.714201] Computing e^(tensor([4739.2922], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-28 01:12:13.719153] Ending with PPL=10.693607346647639 (duration: 0:07:09.261914) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.776 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-28 04:23:13.707188] Computing e^(tensor([4708.3171], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-28 04:23:13.713761] Ending with PPL=10.529265400851628 (duration: 0:08:27.850991) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.858 +Metric validation/losses/mim_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.640 +Metric validation/losses/itm_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.255 +Monitored metric validation/losses/global_contrastive_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Metric validation/losses/mmm_image_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.641 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.776. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-28 07:35:51.723771] Computing e^(tensor([4715.1017], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-28 07:35:51.728761] Ending with PPL=10.56504442116853 (duration: 0:07:20.341360) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.853 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.639 +Metric validation/losses/itm_loss improved by 0.012 >= min_delta = 0.0. New best score: 0.243 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.540. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.641. Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.025 >= min_delta = 0.0. New best score: 2.751 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-28 10:44:29.459157] Computing e^(tensor([4706.9636], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-28 10:44:29.465036] Ending with PPL=10.522142260589515 (duration: 0:07:13.663576) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.851 +Metric validation/losses/mim_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.633 +Metric validation/losses/itm_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.238 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.540. Wait count has been increased from 1. +Metric validation/losses/mmm_image_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.636 +Metric validation/losses/mmm_text_loss improved by 0.010 >= min_delta = 0.0. New best score: 2.741 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-28 13:52:10.598432] Computing e^(tensor([4695.1371], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-28 13:52:10.603968] Ending with PPL=10.460105700062348 (duration: 0:07:01.537705) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.844 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.633. Wait count has been increased from 0. +Metric validation/losses/itm_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.235 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.540. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.636. Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.001 >= min_delta = 0.0. New best score: 2.741 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-28 17:02:52.471250] Computing e^(tensor([4680.3498], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-28 17:02:52.481187] Ending with PPL=10.383052438317149 (duration: 0:06:59.398187) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.833 +Metric validation/losses/mim_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.626 +Metric validation/losses/itm_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.226 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 2.540. Wait count has been increased from 3. +Metric validation/losses/mmm_image_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.628 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.741. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-28 20:19:45.071555] Computing e^(tensor([4677.5738], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-28 20:19:45.076725] Ending with PPL=10.36865048179292 (duration: 0:07:09.528361) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.831 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.626. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.226. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.540. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.628. Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.015 >= min_delta = 0.0. New best score: 2.726 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-29 00:13:23.251392] Computing e^(tensor([4643.5855], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-29 00:13:23.256696] Ending with PPL=10.193933185935586 (duration: 0:07:06.357139) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.826 +Metric validation/losses/mim_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.622 +Metric validation/losses/itm_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.223 +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.540. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.624 +Metric validation/losses/mmm_text_loss improved by 0.008 >= min_delta = 0.0. New best score: 2.718 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-29 04:34:58.817185] Computing e^(tensor([4648.3176], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-29 04:34:58.824774] Ending with PPL=10.218080943959029 (duration: 0:07:05.938325) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.814 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.617 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.223. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 2.540. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.619 +Metric validation/losses/mmm_text_loss improved by 0.009 >= min_delta = 0.0. New best score: 2.709 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-29 08:55:26.292117] Computing e^(tensor([4637.4901], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-29 08:55:26.296857] Ending with PPL=10.16291257502816 (duration: 0:08:03.754629) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.812 +Metric validation/losses/mim_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.614 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.223. Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.617 +Metric validation/losses/mmm_text_loss improved by 0.003 >= min_delta = 0.0. New best score: 2.707 +Starting validation +2023-07-29 13:42:14.267887 Preparing data for datamodule: VLDataModule +2023-07-29 13:42:14.268174 Preparing data for datamodule: ImageDataModule +2023-07-29 13:42:14.268349 Preparing data for datamodule: MLMDataModule +2023-07-29 13:42:14.268878 Setting up datamodule: VLDataModule +2023-07-29 13:43:54.021055 Setting up datamodule: ImageDataModule +2023-07-29 13:44:02.267176 Setting up datamodule: MLMDataModule +2023-07-29 13:44:50.077401 Building val dataloader for datamodule: VLDataModule +2023-07-29 13:44:50.078097 Building val dataloader for datamodule: ImageDataModule +2023-07-29 13:44:50.078381 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-29 13:48:18.070043] Computing e^(tensor([4619.4497], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-29 13:48:18.078553] Ending with PPL=10.071653185824571 (duration: 0:03:11.249332) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.805 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 0.617. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.223. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.540. Wait count has been increased from 2. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 0.619. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 4 records. Best score: 2.709. Wait count has been increased from 3. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-01 02:43:55.158963] Computing e^(tensor([4595.9773], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-01 02:43:55.163309] Ending with PPL=9.95414089711192 (duration: 0:07:11.610017) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.703 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-01 05:01:30.344543] Computing e^(tensor([4594.7861], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-01 05:01:30.350411] Ending with PPL=9.948214315687311 (duration: 0:07:14.701964) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.697 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-01 07:18:35.694565] Computing e^(tensor([4593.2825], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-01 07:18:35.698941] Ending with PPL=9.940737986330936 (duration: 0:07:03.230360) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.801 +Metric validation/losses/mim_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.616 +Monitored metric validation/losses/itm_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.540. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 8 records. Best score: 0.619. Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.007 >= min_delta = 0.0. New best score: 2.690 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-01 09:37:00.275429] Computing e^(tensor([4585.0641], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-01 09:37:00.279713] Ending with PPL=9.899973111607094 (duration: 0:07:18.500516) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.798 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.616. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.223. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 2.540. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.690. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-01 12:00:25.229377] Computing e^(tensor([4579.6733], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-01 12:00:25.233745] Ending with PPL=9.87332466421661 (duration: 0:07:19.999917) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.792 +Metric validation/losses/mim_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.610 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.223. Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 8 records. Best score: 2.540. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.611 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 2.690. Wait count has been increased from 1. +Starting validation +2023-08-01 12:17:51.055281 Preparing data for datamodule: VLDataModule +2023-08-01 12:17:51.055555 Preparing data for datamodule: ImageDataModule +2023-08-01 12:17:51.055753 Preparing data for datamodule: MLMDataModule +2023-08-01 12:17:51.056155 Setting up datamodule: VLDataModule +2023-08-01 12:19:04.393885 Setting up datamodule: ImageDataModule +2023-08-01 12:19:13.933039 Setting up datamodule: MLMDataModule +2023-08-01 12:20:16.168773 Building val dataloader for datamodule: VLDataModule +2023-08-01 12:20:16.169310 Building val dataloader for datamodule: ImageDataModule +2023-08-01 12:20:16.169590 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-01 12:23:40.718548] Computing e^(tensor([4579.6733], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-01 12:23:40.724605] Ending with PPL=9.87332466421661 (duration: 0:03:11.114062) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.158 +Metric validation/losses/mim_loss improved by 0.014 >= min_delta = 0.0. New best score: 0.977 +Metric validation/losses/itm_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.215 +Metric validation/losses/global_contrastive_loss improved by 0.845 >= min_delta = 0.0. New best score: 2.237 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.996. Wait count has been increased from 2. +Metric validation/losses/mmm_text_loss improved by 0.018 >= min_delta = 0.0. New best score: 3.711 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 23:18:39.107644] Computing e^(tensor([7505.6457], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 23:18:39.113244] Ending with PPL=42.641283144994304 (duration: 0:04:16.377142) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.123 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.975 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.215. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.237. Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.010 >= min_delta = 0.0. New best score: 0.985 +Metric validation/losses/mmm_text_loss improved by 0.055 >= min_delta = 0.0. New best score: 3.656 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 03:30:13.327972] Computing e^(tensor([7385.1237], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 03:30:13.333635] Ending with PPL=40.14756756042479 (duration: 0:04:10.656408) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.076 +Metric validation/losses/mim_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.966 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.215. Wait count has been increased from 1. +Metric validation/losses/global_contrastive_loss improved by 0.033 >= min_delta = 0.0. New best score: 2.204 +Metric validation/losses/mmm_image_loss improved by 0.013 >= min_delta = 0.0. New best score: 0.972 +Metric validation/losses/mmm_text_loss improved by 0.004 >= min_delta = 0.0. New best score: 3.652 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 07:39:36.670328] Computing e^(tensor([7263.1206], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 07:39:36.675921] Ending with PPL=37.77170538715712 (duration: 0:05:57.649722) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.032 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.962 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.215. Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.204. Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.967 +Metric validation/losses/mmm_text_loss improved by 0.037 >= min_delta = 0.0. New best score: 3.615 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 11:30:50.260322] Computing e^(tensor([7133.1702], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 11:30:50.266010] Ending with PPL=35.39551385153069 (duration: 0:04:08.935276) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.984 +Metric validation/losses/mim_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.955 +Metric validation/losses/itm_loss improved by 0.012 >= min_delta = 0.0. New best score: 0.203 +Metric validation/losses/global_contrastive_loss improved by 0.073 >= min_delta = 0.0. New best score: 2.131 +Metric validation/losses/mmm_image_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.959 +Metric validation/losses/mmm_text_loss improved by 0.047 >= min_delta = 0.0. New best score: 3.568 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 15:29:06.431803] Computing e^(tensor([7024.8679], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 15:29:06.437463] Ending with PPL=33.52977898267011 (duration: 0:04:13.262644) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.940 +Metric validation/losses/mim_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.946 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.203. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.131. Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.950 +Metric validation/losses/mmm_text_loss improved by 0.014 >= min_delta = 0.0. New best score: 3.554 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 19:37:07.186849] Computing e^(tensor([6903.5232], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 19:37:07.190944] Ending with PPL=31.5559327504869 (duration: 0:04:08.024637) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.900 +Metric validation/losses/mim_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.944 +Metric validation/losses/itm_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.201 +Metric validation/losses/global_contrastive_loss improved by 0.040 >= min_delta = 0.0. New best score: 2.091 +Metric validation/losses/mmm_image_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.947 +Metric validation/losses/mmm_text_loss improved by 0.028 >= min_delta = 0.0. New best score: 3.525 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 23:48:04.796022] Computing e^(tensor([6830.2491], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 23:48:04.801801] Ending with PPL=30.42073777496611 (duration: 0:04:09.465096) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.867 +Metric validation/losses/mim_loss improved by 0.014 >= min_delta = 0.0. New best score: 0.930 +Metric validation/losses/itm_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.194 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.091. Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.013 >= min_delta = 0.0. New best score: 0.934 +Metric validation/losses/mmm_text_loss improved by 0.042 >= min_delta = 0.0. New best score: 3.483 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 03:58:43.558286] Computing e^(tensor([6748.3988], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 03:58:43.564031] Ending with PPL=29.200896270195518 (duration: 0:04:09.498367) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.830 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.925 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.194. Wait count has been increased from 0. +Metric validation/losses/global_contrastive_loss improved by 0.025 >= min_delta = 0.0. New best score: 2.067 +Metric validation/losses/mmm_image_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.927 +Metric validation/losses/mmm_text_loss improved by 0.023 >= min_delta = 0.0. New best score: 3.460 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 07:59:55.871175] Computing e^(tensor([6667.1622], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 07:59:55.876756] Ending with PPL=28.03857131782274 (duration: 0:04:10.602945) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.809 +Metric validation/losses/mim_loss improved by 0.015 >= min_delta = 0.0. New best score: 0.909 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.194. Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.067. Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.015 >= min_delta = 0.0. New best score: 0.912 +Metric validation/losses/mmm_text_loss improved by 0.045 >= min_delta = 0.0. New best score: 3.416 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 11:56:57.512787] Computing e^(tensor([6600.6580], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 11:56:57.518321] Ending with PPL=27.121560954277264 (duration: 0:04:30.823325) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.783 +Metric validation/losses/mim_loss improved by 0.017 >= min_delta = 0.0. New best score: 0.892 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.194. Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.067. Wait count has been increased from 1. +Metric validation/losses/mmm_image_loss improved by 0.018 >= min_delta = 0.0. New best score: 0.893 +Metric validation/losses/mmm_text_loss improved by 0.004 >= min_delta = 0.0. New best score: 3.412 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 15:56:00.786157] Computing e^(tensor([6519.1680], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 15:56:00.791840] Ending with PPL=26.038702775935164 (duration: 0:04:12.673932) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.753 +Metric validation/losses/mim_loss improved by 0.014 >= min_delta = 0.0. New best score: 0.878 +Monitored metric validation/losses/itm_loss did not improve since the last 4 records. Best score: 0.194. Wait count has been increased from 3. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.067. Wait count has been increased from 2. +Metric validation/losses/mmm_image_loss improved by 0.012 >= min_delta = 0.0. New best score: 0.881 +Metric validation/losses/mmm_text_loss improved by 0.021 >= min_delta = 0.0. New best score: 3.391 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 20:27:06.746473] Computing e^(tensor([6460.7791], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 20:27:06.752105] Ending with PPL=25.289506047954276 (duration: 0:04:42.002390) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.732 +Metric validation/losses/mim_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.869 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.194. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 2.067. Wait count has been increased from 3. +Metric validation/losses/mmm_image_loss improved by 0.014 >= min_delta = 0.0. New best score: 0.867 +Metric validation/losses/mmm_text_loss improved by 0.029 >= min_delta = 0.0. New best score: 3.362 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 01:16:31.105967] Computing e^(tensor([6398.9740], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 01:16:31.111549] Ending with PPL=24.519948491601337 (duration: 0:04:11.211993) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.699 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.864 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.194. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.067. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.863 +Metric validation/losses/mmm_text_loss improved by 0.073 >= min_delta = 0.0. New best score: 3.289 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 06:04:35.096584] Computing e^(tensor([6363.5392], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 06:04:35.102149] Ending with PPL=24.089343896074936 (duration: 0:04:14.596912) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.678 +Metric validation/losses/mim_loss improved by 0.026 >= min_delta = 0.0. New best score: 0.838 +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.194. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.067. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.023 >= min_delta = 0.0. New best score: 0.839 +Metric validation/losses/mmm_text_loss improved by 0.014 >= min_delta = 0.0. New best score: 3.276 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 10:51:44.407844] Computing e^(tensor([6304.4579], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 10:51:44.413488] Ending with PPL=23.388137883965335 (duration: 0:04:42.504786) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.655 +Metric validation/losses/mim_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.833 +Monitored metric validation/losses/itm_loss did not improve in the last 8 records. Best score: 0.194. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 2.067. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.835 +Metric validation/losses/mmm_text_loss improved by 0.029 >= min_delta = 0.0. New best score: 3.247 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 15:53:07.175279] Computing e^(tensor([6260.9281], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 15:53:07.181159] Ending with PPL=22.88459681934565 (duration: 0:04:20.389331) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.629 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.829 +Monitored metric validation/losses/itm_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 8 records. Best score: 2.067. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.829 +Metric validation/losses/mmm_text_loss improved by 0.010 >= min_delta = 0.0. New best score: 3.236 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 20:22:07.721120] Computing e^(tensor([6225.3589], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 20:22:07.727047] Ending with PPL=22.481200721032945 (duration: 0:04:29.946668) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.615 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.829. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.194. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Metric validation/losses/mmm_image_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.829 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.236. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 01:22:45.337047] Computing e^(tensor([6140.4611], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 01:22:45.342676] Ending with PPL=21.546869496477857 (duration: 0:04:10.586890) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.562 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.827 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.194. Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.067. Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.827 +Metric validation/losses/mmm_text_loss improved by 0.031 >= min_delta = 0.0. New best score: 3.205 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 06:14:33.737471] Computing e^(tensor([6106.5069], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 06:14:33.743029] Ending with PPL=21.184154468212338 (duration: 0:04:11.229687) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.554 +Metric validation/losses/mim_loss improved by 0.011 >= min_delta = 0.0. New best score: 0.817 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.194. Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.067. Wait count has been increased from 1. +Metric validation/losses/mmm_image_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.820 +Metric validation/losses/mmm_text_loss improved by 0.005 >= min_delta = 0.0. New best score: 3.200 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 11:14:54.894261] Computing e^(tensor([6078.8356], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 11:14:54.899841] Ending with PPL=20.89307547520629 (duration: 0:04:10.494929) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.532 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.817. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 4 records. Best score: 0.194. Wait count has been increased from 3. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.067. Wait count has been increased from 2. +Metric validation/losses/mmm_image_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.820 +Metric validation/losses/mmm_text_loss improved by 0.001 >= min_delta = 0.0. New best score: 3.200 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 15:50:01.976253] Computing e^(tensor([6034.4736], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 15:50:01.981920] Ending with PPL=20.434748418045594 (duration: 0:04:09.506172) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.530 +Metric validation/losses/mim_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.810 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.194. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 2.067. Wait count has been increased from 3. +Metric validation/losses/mmm_image_loss improved by 0.012 >= min_delta = 0.0. New best score: 0.808 +Metric validation/losses/mmm_text_loss improved by 0.004 >= min_delta = 0.0. New best score: 3.196 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 20:09:16.708075] Computing e^(tensor([6013.3178], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 20:09:16.713618] Ending with PPL=20.219730496798533 (duration: 0:05:01.425030) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.511 +Metric validation/losses/mim_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.801 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.194. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.067. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.805 +Metric validation/losses/mmm_text_loss improved by 0.053 >= min_delta = 0.0. New best score: 3.144 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 00:18:13.426815] Computing e^(tensor([5970.9347], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 00:18:13.432634] Ending with PPL=19.79575201034063 (duration: 0:04:08.889654) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.496 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.800 +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.194. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.067. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.799 +Metric validation/losses/mmm_text_loss improved by 0.001 >= min_delta = 0.0. New best score: 3.142 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 04:21:46.027962] Computing e^(tensor([5968.4540], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 04:21:46.033594] Ending with PPL=19.771212879210875 (duration: 0:04:10.282456) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.799 +Monitored metric validation/losses/itm_loss did not improve in the last 8 records. Best score: 0.194. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 2.067. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.799. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.142. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 08:25:47.901389] Computing e^(tensor([5882.3980], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 08:25:47.907089] Ending with PPL=18.938540143971938 (duration: 0:04:07.643219) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.456 +Metric validation/losses/mim_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.791 +Monitored metric validation/losses/itm_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 8 records. Best score: 2.067. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.792 +Metric validation/losses/mmm_text_loss improved by 0.069 >= min_delta = 0.0. New best score: 3.073 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 12:24:30.813538] Computing e^(tensor([5858.5858], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 12:24:30.819150] Ending with PPL=18.714392837928706 (duration: 0:04:05.871198) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.449 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.791. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.194. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.792. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.073. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 16:06:27.995077] Computing e^(tensor([5839.2037], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 16:06:28.000682] Ending with PPL=18.533906848184426 (duration: 0:04:07.728404) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.260 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 10:19:26.442437] Computing e^(tensor([6051.6216], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 10:19:26.448083] Ending with PPL=20.610708617792447 (duration: 0:04:19.761928) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.226 +Metric validation/losses/global_contrastive_loss improved by 0.021 >= min_delta = 0.0. New best score: 2.239 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 14:56:42.101080] Computing e^(tensor([6068.6478], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 14:56:42.106767] Ending with PPL=20.786919092826377 (duration: 0:04:24.899386) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.846 +Metric validation/losses/mmm_text_loss improved by 0.005 >= min_delta = 0.0. New best score: 3.195 +Metric validation/losses/itm_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.223 +Metric validation/losses/global_contrastive_loss improved by 0.070 >= min_delta = 0.0. New best score: 2.169 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 19:47:30.387265] Computing e^(tensor([6079.3953], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 19:47:30.392983] Ending with PPL=20.898923854293574 (duration: 0:04:31.188533) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.844 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.195 (now 3.197561502456665). Wait count has been increased from 0. +Metric validation/losses/itm_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.221 +Metric validation/losses/global_contrastive_loss improved by 0.057 >= min_delta = 0.0. New best score: 2.112 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 00:51:29.892962] Computing e^(tensor([6025.4861], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 00:51:29.898490] Ending with PPL=20.343125553500336 (duration: 0:04:11.345443) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.179 +Metric validation/losses/itm_loss improved by 0.010 >= min_delta = 0.0. New best score: 0.211 +Metric validation/losses/global_contrastive_loss improved by 0.044 >= min_delta = 0.0. New best score: 2.068 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 05:42:00.739037] Computing e^(tensor([6009.4302], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 05:42:00.744675] Ending with PPL=20.180465537593516 (duration: 0:04:09.450901) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.506 +Metric validation/losses/mim_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.839 +Metric validation/losses/mmm_image_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.837 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.179 (now 3.1888558864593506). Wait count has been increased from 0. +Metric validation/losses/itm_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.207 +Metric validation/losses/global_contrastive_loss improved by 0.052 >= min_delta = 0.0. New best score: 2.016 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 10:33:25.977806] Computing e^(tensor([6013.7035], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 10:33:25.983442] Ending with PPL=20.223630906632927 (duration: 0:04:12.059991) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.497 +Metric validation/losses/mim_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.835 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.835 +Metric validation/losses/mmm_text_loss improved by 0.012 >= min_delta = 0.0. New best score: 3.167 +Metric validation/losses/itm_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.205 +Metric validation/losses/global_contrastive_loss improved by 0.010 >= min_delta = 0.0. New best score: 2.006 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 15:24:30.394837] Computing e^(tensor([5994.1620], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 15:24:30.400600] Ending with PPL=20.026992464300978 (duration: 0:04:17.620427) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.488 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.835 (now 0.8388342261314392). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.835 (now 0.8373665809631348). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.167 (now 3.1675193309783936). Wait count has been increased from 0. +Metric validation/losses/itm_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.204 +Metric validation/losses/global_contrastive_loss improved by 0.031 >= min_delta = 0.0. New best score: 1.976 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 19:46:59.707997] Computing e^(tensor([5989.6326], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 19:46:59.713637] Ending with PPL=19.98168880137396 (duration: 0:04:45.311910) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.151 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.204 (now 0.2101156860589981). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 1.976 (now 2.092069387435913). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 23:51:24.430296] Computing e^(tensor([5909.6264], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 23:51:24.435872] Ending with PPL=19.198136118109616 (duration: 0:06:04.639481) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.452 +Metric validation/losses/mim_loss improved by 0.017 >= min_delta = 0.0. New best score: 0.818 +Metric validation/losses/mmm_image_loss improved by 0.018 >= min_delta = 0.0. New best score: 0.818 +Metric validation/losses/mmm_text_loss improved by 0.029 >= min_delta = 0.0. New best score: 3.122 +Metric validation/losses/itm_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.203 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 1.976 (now 2.028102397918701). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 04:29:21.600638] Computing e^(tensor([5853.5952], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 04:29:21.606324] Ending with PPL=18.66775320311226 (duration: 0:04:14.301269) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.421 +Metric validation/losses/mim_loss improved by 0.010 >= min_delta = 0.0. New best score: 0.809 +Metric validation/losses/mmm_image_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.811 +Metric validation/losses/mmm_text_loss improved by 0.011 >= min_delta = 0.0. New best score: 3.111 +Metric validation/losses/itm_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.202 +Metric validation/losses/global_contrastive_loss improved by 0.018 >= min_delta = 0.0. New best score: 1.958 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 08:42:10.735875] Computing e^(tensor([5821.3992], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 08:42:10.741559] Ending with PPL=18.36964506780487 (duration: 0:04:15.102208) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.415 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.809 (now 0.8092057108879089). Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.806 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.111 (now 3.111330986022949). Wait count has been increased from 0. +Metric validation/losses/itm_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.193 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 1.958 (now 1.9678369760513306). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 13:04:08.406813] Computing e^(tensor([5779.8365], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 13:04:08.412463] Ending with PPL=17.991839111267453 (duration: 0:05:09.716764) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.396 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.808 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.806 (now 0.8098016381263733). Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.014 >= min_delta = 0.0. New best score: 3.097 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.193 (now 0.1967204362154007). Wait count has been increased from 0. +Metric validation/losses/global_contrastive_loss improved by 0.016 >= min_delta = 0.0. New best score: 1.942 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 17:25:04.944803] Computing e^(tensor([5757.5198], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 17:25:04.950462] Ending with PPL=17.792195371784974 (duration: 0:04:20.330139) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.388 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.808 (now 0.8114915490150452). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.806 (now 0.811784565448761). Wait count has been increased from 1. +Metric validation/losses/mmm_text_loss improved by 0.019 >= min_delta = 0.0. New best score: 3.077 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.193 (now 0.19440199434757233). Wait count has been increased from 1. +Metric validation/losses/global_contrastive_loss improved by 0.006 >= min_delta = 0.0. New best score: 1.936 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 21:55:36.559941] Computing e^(tensor([5731.0938], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 21:55:36.565533] Ending with PPL=17.55865377276801 (duration: 0:04:18.520702) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.378 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.805 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.803 +Metric validation/losses/mmm_text_loss improved by 0.009 >= min_delta = 0.0. New best score: 3.069 +Metric validation/losses/itm_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.190 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 1.936 (now 1.9454690217971802). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 02:18:41.173252] Computing e^(tensor([5698.4079], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 02:18:41.178963] Ending with PPL=17.27402504397933 (duration: 0:04:14.080572) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.362 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.801 +Metric validation/losses/mmm_image_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.800 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.069 (now 3.079806327819824). Wait count has been increased from 0. +Metric validation/losses/itm_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.186 +Metric validation/losses/global_contrastive_loss improved by 0.016 >= min_delta = 0.0. New best score: 1.920 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 06:41:37.531263] Computing e^(tensor([5690.4277], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 06:41:37.536866] Ending with PPL=17.205237234929857 (duration: 0:04:13.285499) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.358 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.801 (now 0.8020022511482239). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.800 (now 0.8005017638206482). Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.011 >= min_delta = 0.0. New best score: 3.058 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.186 (now 0.1889059692621231). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 1.920 (now 1.92125403881073). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 10:59:20.546992] Computing e^(tensor([5643.1813], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 10:59:20.552606] Ending with PPL=16.803558284848126 (duration: 0:04:18.177599) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.335 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.799 +Metric validation/losses/mmm_image_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.796 +Metric validation/losses/mmm_text_loss improved by 0.020 >= min_delta = 0.0. New best score: 3.038 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.186 (now 0.20485661923885345). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 1.920 (now 2.0774805545806885). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 15:10:11.986137] Computing e^(tensor([5625.4407], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 15:10:11.991817] Ending with PPL=16.65516441593167 (duration: 0:04:17.246499) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.327 +Metric validation/losses/mim_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.790 +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.791 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.038 (now 3.0390939712524414). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.186 (now 0.18992705643177032). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 1.920 (now 2.0536768436431885). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 19:23:25.027102] Computing e^(tensor([5586.8936], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 19:23:25.032836] Ending with PPL=16.33723416375161 (duration: 0:04:15.407144) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.312 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.787 +Metric validation/losses/mmm_image_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.788 +Metric validation/losses/mmm_text_loss improved by 0.023 >= min_delta = 0.0. New best score: 3.015 +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.186 (now 0.18979933857917786). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 1.920 (now 2.0488123893737793). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 23:34:50.113449] Computing e^(tensor([5564.5698], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 23:34:50.119035] Ending with PPL=16.155893087198873 (duration: 0:04:16.750313) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.311 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.786 +Metric validation/losses/mmm_image_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.786 +Metric validation/losses/mmm_text_loss improved by 0.060 >= min_delta = 0.0. New best score: 2.956 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.186 (now 0.18979933857917786). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 1.920 (now 2.0488123893737793). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 03:44:18.996293] Computing e^(tensor([5522.3623], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 03:44:19.001978] Ending with PPL=15.818515883629017 (duration: 0:04:20.526600) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.300 +Metric validation/losses/mim_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.783 +Metric validation/losses/mmm_image_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.785 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.956 (now 2.9577338695526123). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.186 (now 0.18979933857917786). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 1.920 (now 2.0488123893737793). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 07:59:14.616097] Computing e^(tensor([5526.5946], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 07:59:14.621691] Ending with PPL=15.852026063665642 (duration: 0:05:04.847286) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.289 +Metric validation/losses/mim_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.783 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.783 +Metric validation/losses/mmm_text_loss improved by 0.024 >= min_delta = 0.0. New best score: 2.932 +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 12:11:25.961987] Computing e^(tensor([5809.5321], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 12:11:25.966039] Ending with PPL=18.260970821802122 (duration: 0:04:13.934889) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.374 +Metric validation/losses/mim_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.820 +Metric validation/losses/mmm_image_loss improved by 0.010 >= min_delta = 0.0. New best score: 0.823 +Metric validation/losses/mmm_text_loss improved by 0.076 >= min_delta = 0.0. New best score: 3.239 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.205 (now 0.2066977322101593). Wait count has been increased from 0. +Metric validation/losses/global_contrastive_loss improved by 0.085 >= min_delta = 0.0. New best score: 2.583 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 08:59:47.307407] Computing e^(tensor([5793.8511], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 08:59:47.311436] Ending with PPL=18.11835565563289 (duration: 0:03:22.566172) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.359 +Metric validation/losses/mim_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.813 +Metric validation/losses/mmm_image_loss improved by 0.010 >= min_delta = 0.0. New best score: 0.814 +Metric validation/losses/mmm_text_loss improved by 0.068 >= min_delta = 0.0. New best score: 3.171 +Metric validation/losses/itm_loss improved by 0.011 >= min_delta = 0.0. New best score: 0.194 +Metric validation/losses/global_contrastive_loss improved by 0.053 >= min_delta = 0.0. New best score: 2.529 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 18:29:25.504493] Computing e^(tensor([5772.5083], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 18:29:25.508046] Ending with PPL=17.926035308265213 (duration: 0:03:22.850450) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.334 +Metric validation/losses/mim_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.813 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.814 (now 0.8143738508224487). Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.037 >= min_delta = 0.0. New best score: 3.134 +Metric validation/losses/itm_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.189 +Metric validation/losses/global_contrastive_loss improved by 0.049 >= min_delta = 0.0. New best score: 2.480 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-01 02:49:22.320053] Computing e^(tensor([5740.2169], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-01 02:49:22.323930] Ending with PPL=17.638931463248422 (duration: 0:03:20.884661) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.332 +Metric validation/losses/mim_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.809 +Metric validation/losses/mmm_image_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.810 +Metric validation/losses/mmm_text_loss improved by 0.020 >= min_delta = 0.0. New best score: 3.114 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.189 (now 0.20271900296211243). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.480 (now 2.703260898590088). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-01 09:25:18.858051] Computing e^(tensor([5653.0954], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-01 09:25:18.862155] Ending with PPL=16.887060640230008 (duration: 0:03:16.710004) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.285 +Metric validation/losses/mim_loss improved by 0.013 >= min_delta = 0.0. New best score: 0.796 +Metric validation/losses/mmm_image_loss improved by 0.011 >= min_delta = 0.0. New best score: 0.799 +Metric validation/losses/mmm_text_loss improved by 0.057 >= min_delta = 0.0. New best score: 3.057 +Metric validation/losses/itm_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.183 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.480 (now 2.6094307899475098). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-01 16:06:26.453818] Computing e^(tensor([5581.2580], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-01 16:06:26.457587] Ending with PPL=16.291263823612077 (duration: 0:03:22.732875) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.252 +Metric validation/losses/mim_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.796 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.796 +Metric validation/losses/mmm_text_loss improved by 0.023 >= min_delta = 0.0. New best score: 3.035 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.183 (now 0.18375247716903687). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.480 (now 2.544569730758667). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-01 22:48:13.157492] Computing e^(tensor([5541.3251], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-01 22:48:13.161111] Ending with PPL=15.969210863614695 (duration: 0:03:26.687524) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.225 +Metric validation/losses/mim_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.787 +Metric validation/losses/mmm_image_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.788 +Metric validation/losses/mmm_text_loss improved by 0.042 >= min_delta = 0.0. New best score: 2.992 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.183 (now 0.18496039509773254). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 2.480 (now 2.6187260150909424). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-02 06:19:51.604939] Computing e^(tensor([5429.5915], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-02 06:19:51.608867] Ending with PPL=15.101524961727124 (duration: 0:03:18.005383) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.205 +Metric validation/losses/mim_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.780 +Metric validation/losses/mmm_image_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.779 +Metric validation/losses/mmm_text_loss improved by 0.038 >= min_delta = 0.0. New best score: 2.954 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.183 (now 0.18455882370471954). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.480 (now 2.6187260150909424). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-02 14:12:07.761602] Computing e^(tensor([5393.1320], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-02 14:12:07.765135] Ending with PPL=14.828722337014169 (duration: 0:03:22.984567) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.190 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.778 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.777 +Metric validation/losses/mmm_text_loss improved by 0.009 >= min_delta = 0.0. New best score: 2.945 +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.183 (now 0.18643388152122498). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.480 (now 2.6187260150909424). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-02 22:38:03.829782] Computing e^(tensor([5340.0378], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-02 22:38:03.833304] Ending with PPL=14.440242248990353 (duration: 0:03:21.455456) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.174 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.773 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.775 +Metric validation/losses/mmm_text_loss improved by 0.082 >= min_delta = 0.0. New best score: 2.863 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.183 (now 0.18643388152122498). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-03 07:10:19.606598] Computing e^(tensor([5480.0472], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-03 07:10:19.611546] Ending with PPL=15.487350926068087 (duration: 0:03:23.432923) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.165 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 0.773 (now 0.7843025326728821). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.775 (now 0.7841938734054565). Wait count has been increased from 1. +Metric validation/losses/mmm_text_loss improved by 0.025 >= min_delta = 0.0. New best score: 2.839 +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.480 (now 2.914065361022949). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-03 23:14:09.608072] Computing e^(tensor([5397.0514], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-03 23:14:09.611585] Ending with PPL=14.857811003911642 (duration: 0:03:23.562699) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00 Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +WARNING:datasets.arrow_dataset:Loading cached processed dataset at train[10%] +Counting words for split train[:10%] after collapsing... +WARNING:datasets.arrow_dataset:Loading cached processed dataset at test +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +WARNING:datasets.arrow_dataset:Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-209d0ec87f0e85fd_*_of_00016.arrow +WARNING:datasets.arrow_dataset:Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-d473e62d5d49e2de_*_of_00016.arrow +2023-06-27 21:25:54.856054 Setting up datamodule: ImageDataModule +WARNING:datasets.builder:Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +WARNING:datasets.builder:Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +2023-06-27 21:28:16.244589 Setting up datamodule: MLMDataModule +WARNING:datasets.builder:Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Counting words for split train[:10%] before (i.e. captions only) collapsing... +WARNING:datasets.builder:Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +WARNING:datasets.arrow_dataset:Loading cached processed dataset at train[10%] +Counting words for split train[:10%] after collapsing... +WARNING:datasets.arrow_dataset:Loading cached processed dataset at test +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Callbacks registered: ['LearningRateMonitor', 'LMEvalHarnessCallback', 'PseudoPerplexityCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +[rank: 1] Global seed set to 5501650 +Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/2 +---------------------------------------------------------------------------------------------------- +distributed_backend=nccl +All distributed processes registered. Starting with 2 processes +---------------------------------------------------------------------------------------------------- +2023-06-27 21:32:21.247353 Setting up datamodule: VLDataModule2023-06-27 21:32:21.247294 Setting up datamodule: VLDataModule +WARNING:datasets.builder:Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +WARNING:datasets.builder:Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +WARNING:datasets.builder:Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +WARNING:datasets.builder:Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +WARNING:datasets.arrow_dataset:Loading cached processed dataset at train[10%] +WARNING:datasets.arrow_dataset:Loading cached processed dataset at train[10%] +Counting words for split train[:10%] after collapsing... +Counting words for split train[:10%] after collapsing... +WARNING:datasets.arrow_dataset:Loading cached processed dataset at test +WARNING:datasets.arrow_dataset:Loading cached processed dataset at test +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +WARNING:datasets.arrow_dataset:Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-209d0ec87f0e85fd_*_of_00016.arrow +WARNING:datasets.arrow_dataset:Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-209d0ec87f0e85fd_*_of_00016.arrow +2023-06-27 21:38:51.900215 Setting up datamodule: ImageDataModule +2023-06-27 21:38:51.940119 Setting up datamodule: ImageDataModule +WARNING:datasets.arrow_dataset:Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-d473e62d5d49e2de_*_of_00016.arrow +WARNING:datasets.arrow_dataset:Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-d473e62d5d49e2de_*_of_00016.arrow +WARNING:datasets.builder:Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +WARNING:datasets.builder:Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +WARNING:datasets.builder:Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +WARNING:datasets.builder:Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +2023-06-27 21:41:14.784332 Setting up datamodule: MLMDataModule +2023-06-27 21:41:14.841176 Setting up datamodule: MLMDataModule +WARNING:datasets.builder:Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +WARNING:datasets.builder:Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Counting words for split train[:10%] before (i.e. captions only) collapsing... +WARNING:datasets.builder:Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +WARNING:datasets.builder:Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +WARNING:datasets.arrow_dataset:Loading cached processed dataset at train[10%] +WARNING:datasets.arrow_dataset:Loading cached processed dataset at train[10%] +WARNING:datasets.arrow_dataset:Loading cached processed dataset at test +WARNING:datasets.arrow_dataset:Loading cached processed dataset at test +Counting words for split train[:10%] after collapsing... +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1] +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1] +2023-06-27 21:46:38.977236 Building train dataloader for datamodule: VLDataModule +2023-06-27 21:46:38.977388 Building train dataloader for datamodule: VLDataModule +2023-06-27 21:46:38.978104 Building train dataloader for datamodule: ImageDataModule +2023-06-27 21:46:38.978424 Building train dataloader for datamodule: MLMDataModule +2023-06-27 21:46:38.978357 Building train dataloader for datamodule: ImageDataModule +2023-06-27 21:46:38.978686 Building train dataloader for datamodule: MLMDataModule + | Name | Type | Params +---------------------------------------------- +0 | model | FlavaForPreTraining | 357 M +---------------------------------------------- +303 M Trainable params +53.8 M Non-trainable params +357 M Total params +1,430.581 Total estimated model params size (MB) +2023-06-27 21:46:41.184768 Building val dataloader for datamodule: VLDataModule2023-06-27 21:46:41.184970 Building val dataloader for datamodule: VLDataModule +2023-06-27 21:46:41.186282 Building val dataloader for datamodule: ImageDataModule +2023-06-27 21:46:41.186461 Building val dataloader for datamodule: ImageDataModule +2023-06-27 21:46:41.187018 Building val dataloader for datamodule: MLMDataModule +2023-06-27 21:46:41.187137 Building val dataloader for datamodule: MLMDataModule +Starting LM Evaluation Harness +[PPL Evaluation] Starting from index 2000 to index 4000. +[PPL Evaluation] Based on the available VRAM (39.39GBs), we will use a max phrase length of 200. +Running on blimp... +WARNING:datasets.builder:Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 148.61it/s] +[2023-06-27 22:43:51,511] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'forward' (/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/models/flava/modeling_flava.py:1014) + reasons: self.training == True +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +evaluation/blimp/anaphor_agreement: 33.28 +WARNING:datasets.builder:Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-bcbe0de2681c3396/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 110.01it/s] + +Map: 81%|████████▏ | 6708/8248 [00:01<00:00, 6555.29 examples/s] +[2023-06-27 22:47:35,455] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'forward' (/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/models/flava/modeling_flava.py:1014) + reasons: self.training == True +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[2023-06-27 22:55:34,514] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'get_extended_attention_mask' (/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/modeling_utils.py:845) + reasons: tensor 'attention_mask' size mismatch at index 0. expected 195, actual 120 +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[2023-06-27 22:55:35,023] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'forward' (/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/models/flava/modeling_flava.py:390) + reasons: tensor 'input_ids' size mismatch at index 0. expected 195, actual 120 +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[2023-06-27 22:55:35,026] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'forward' (/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/models/flava/modeling_flava.py:640) + reasons: tensor 'hidden_states' size mismatch at index 0. expected 195, actual 120 +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[2023-06-27 22:55:44,337] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'forward' (/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/models/flava/modeling_flava.py:696) + reasons: tensor 'hidden_states' size mismatch at index 0. expected 195, actual 120 +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[2023-06-27 22:56:17,283] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'forward' (/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/models/flava/modeling_flava.py:602) + reasons: ___check_obj_id(self, 47128437866160) +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[2023-06-27 22:57:03,851] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'forward' (/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/models/flava/modeling_flava.py:537) + reasons: ___check_obj_id(self, 47128437861696) +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[2023-06-27 22:57:04,506] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'forward' (/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/models/flava/modeling_flava.py:564) + reasons: ___check_obj_id(self, 47128437860256) +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[2023-06-27 22:57:04,834] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'forward' (/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/models/flava/modeling_flava.py:578) + reasons: ___check_obj_id(self, 47128437860064) +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[2023-06-27 22:57:38,867] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'forward' (/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/models/flava/modeling_flava.py:449) + reasons: ___check_obj_id(self, 47128437861504) +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[2023-06-27 22:57:38,890] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'forward' (/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/models/flava/modeling_flava.py:505) + reasons: ___check_obj_id(self, 47128437865104) +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[2023-06-27 22:57:38,892] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'forward' (/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/activations.py:77) + reasons: ___check_obj_id(self, 47128437864096) +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[2023-06-27 22:57:40,165] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'transpose_for_scores' (/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/models/flava/modeling_flava.py:444) + reasons: ___check_obj_id(self, 47128437865968) +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[PPL Evaluation] Computing e^(17278.34717464447 / 2000) +[PPL Evaluation] Ending with PPL=5648.659770959 (duration: 0:23:30.453922) +[2023-06-27 22:59:56,409] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'get_extended_attention_mask' (/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/modeling_utils.py:845) + reasons: tensor 'attention_mask' size mismatch at index 0. expected 243, actual 242 +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[2023-06-27 22:59:56,884] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'forward' (/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/models/flava/modeling_flava.py:390) + reasons: tensor 'input_ids' size mismatch at index 0. expected 243, actual 242 +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[2023-06-27 22:59:56,887] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'forward' (/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/models/flava/modeling_flava.py:640) + reasons: tensor 'hidden_states' size mismatch at index 0. expected 243, actual 242 +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[2023-06-27 23:00:07,120] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'forward' (/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/models/flava/modeling_flava.py:696) + reasons: tensor 'hidden_states' size mismatch at index 0. expected 243, actual 242 +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[2023-06-27 23:00:52,585] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'forward' (/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/models/flava/modeling_flava.py:602) + reasons: ___check_obj_id(self, 47224780770896) +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[2023-06-27 23:01:38,066] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'forward' (/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/models/flava/modeling_flava.py:537) + reasons: ___check_obj_id(self, 47224780586000) +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[2023-06-27 23:01:38,729] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'forward' (/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/models/flava/modeling_flava.py:564) + reasons: ___check_obj_id(self, 47224780585376) +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[2023-06-27 23:01:39,096] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'forward' (/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/models/flava/modeling_flava.py:578) + reasons: ___check_obj_id(self, 47224780585136) +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[2023-06-27 23:02:20,772] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'forward' (/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/models/flava/modeling_flava.py:449) + reasons: ___check_obj_id(self, 47224780572848) +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[2023-06-27 23:02:20,795] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'forward' (/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/models/flava/modeling_flava.py:505) + reasons: ___check_obj_id(self, 47224780575008) +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[2023-06-27 23:02:20,796] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'forward' (/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/activations.py:77) + reasons: ___check_obj_id(self, 47224780582640) +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +[2023-06-27 23:02:22,120] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64) + function: 'transpose_for_scores' (/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/models/flava/modeling_flava.py:444) + reasons: ___check_obj_id(self, 47224780770752) +to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html. +evaluation/blimp/argument_structure: 58.66 +WARNING:datasets.builder:Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-46d7356e699f4da2/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 121.26it/s] +evaluation/blimp/binding: 61.06 +WARNING:datasets.builder:Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-98827a9e77ac98da/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 89.22it/s] +evaluation/blimp/control_raising: 55.24 +WARNING:datasets.builder:Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-f80f492c055aba32/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 134.57it/s] + +Map: 65%|██████▌ | 4914/7542 [00:00<00:00, 7491.21 examples/s] +evaluation/blimp/determiner_noun_agreement: 51.58 +WARNING:datasets.builder:Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-939eb7fe84c2f940/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 167.42it/s] +evaluation/blimp/ellipsis: 42.26 +WARNING:datasets.builder:Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-89e27f5e6a682b1a/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +100%|██████████| 1/1 [00:00<00:00, 133.81it/s] + +Map: 0%| | 0/6426 [00:00 + wandb.finish() # [optional] finish the wandb run, necessary in notebooks + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 145, in main + wandb_logger.experiment.config.update(config) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 531, in fit + call._call_and_handle_interrupt( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 41, in _call_and_handle_interrupt + return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 91, in launch + return function(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 570, in _fit_impl + self._run(model, ckpt_path=ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 975, in _run + results = self._run_stage() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1018, in _run_stage + self.fit_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 201, in run + self.advance() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 354, in advance + self.epoch_loop.run(self._data_fetcher) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 134, in run + self.on_advance_end() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 248, in on_advance_end + self.val_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/utilities.py", line 177, in _decorator + return loop_run(self, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 102, in run + self.on_run_start() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 234, in on_run_start + self._on_evaluation_start() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 283, in _on_evaluation_start + call._call_callback_hooks(trainer, hook_name, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 189, in _call_callback_hooks + fn(trainer, trainer.lightning_module, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context + return func(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_utilities/core/rank_zero.py", line 27, in wrapped_fn + return fn(*args, **kwargs) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/callbacks/blimp_eval.py", line 46, in on_validation_start + optimized_text_model = replace_flava_submodel_with_orig_for_eval(pl_module.model) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/callbacks/flava_lm.py", line 38, in __init__ + self.tokenizer.model_max_length = self.max_length + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/processing_utils.py", line 184, in from_pretrained + args = cls._get_arguments_from_pretrained(pretrained_model_name_or_path, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/processing_utils.py", line 228, in _get_arguments_from_pretrained + args.append(attribute_class.from_pretrained(pretrained_model_name_or_path, **kwargs)) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/image_processing_utils.py", line 165, in from_pretrained + image_processor_dict, kwargs = cls.get_image_processor_dict(pretrained_model_name_or_path, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/image_processing_utils.py", line 269, in get_image_processor_dict + resolved_image_processor_file = cached_file( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/transformers/utils/hub.py", line 433, in cached_file + raise EnvironmentError( +OSError: facebook/flava-full is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models' +If this is a private repository, make sure to pass a token having permission to this repo with `use_auth_token` or log in with `huggingface-cli login` and pass `use_auth_token=True`. \ No newline at end of file diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10002.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10002.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..93b296ddb6ab54abfaf64ba81fcb8f59d2aa1a89 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10002.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9187be1bd87278210296ac9bf01d0afc98b8d00c767aeded27868007d6cec38d +size 3862234570 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1001.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1001.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..11868d4e6c03f976de0273625feb25f1c5afbb15 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1001.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d528a2da55fd74f7484c8a1ee6335993914352858f4c0bb97aab9ab223235860 +size 3848043751 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10127.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10127.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c2a2e0bc02290421aa555b00090900fd7695b405 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10127.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfcd40b1a82a1c124ef81b7eac8211fcc5ea38cbb2ba9fb80270a876e8efabe2 +size 3862235017 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10252.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10252.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..d292532f0a412e97783b3144541340b03996501a --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10252.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5088704be68dacfd9727c7ba7ea91e1aedc4e0007b4e5299f10eaf3e1058b63e +size 3862235592 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10377.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10377.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b9a814846881a8c763fed51fe3f44f49a08804c1 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10377.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1541a128adee2cbef61d1124da225dbff2a9aa859a584db2762916320bca629 +size 3862235975 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10503.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10503.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..15bd492136d86e66a0b70c8427e4fe207a9682f1 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10503.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5e08c4c94569d0cf1c4dd174b2eb26f6572858308234748a4d43b4387352cd2 +size 3862228439 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10660.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10660.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..28707c3eb77ed9aa831f0fd8c57fc4323c5a49a4 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10660.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c28af83b7014820103513b6c98258c1db3f4366899bedbf727712085aa28ddc +size 3862228886 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10816.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10816.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..d37535e804e1af78938f8287f93d33754b1278e4 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10816.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b9f16bb9b58c55c5af72e88f53fe36972def48d4e22e39c898684779433841a +size 3862229333 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10985.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10985.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c3075437d2f4e58b214f018496546b8a9a37452c --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=10985.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26c959cded4280f03c66ea4a0fd6ab2d886a3b1ac34aa08c6eaa49597caa404f +size 3862229908 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11134.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11134.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..da3d81c1a7a4f7dd57a76252a78c384291fec3b6 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11134.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:992e2e43e52978628dd65d55fd20585a5f60ad1b4de338f3b41e00d152aa2a56 +size 3862230291 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11297.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11297.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b88b97147df394b5b2b6d05c6977dfd3ac6f8b40 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11297.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:398ce02f5c6d6f6f55f149dca3acfb9889fa2ff36af048ce1de3c8c9d085eaa4 +size 3862230802 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11449.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11449.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c37bdab12b883a0f31eb8fc83ec72cab885d69c1 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11449.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec4479a01e4c926ae85f03c319efae38ffe49f9d525c7aab93ff2d8709da5499 +size 3862231313 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11562.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11562.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..cec8cfa191855bc479a32e03a7a7814daea1503d --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=11562.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3f3b7cf5d33011126b00463d9d1b39ae4ab5d0ff623408a17b29ede81a80616 +size 3862231760 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1371.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1371.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f34b2607696adddfc0bcc1d920131204ba627f45 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1371.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b04b030e3a92415b10bf6adf0da7c44efbca8ee404597c8c64a0a4bdf46f5f1 +size 3848044262 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1706.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1706.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..840e45aedd6e7880fab08fc301bd7d4cc77433ac --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1706.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d20c2c32a2259824c65943b71c92eaab042d58279a5a831cc7076806723bcc3 +size 3848044709 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1918.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1918.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..e168df007f7d4edaa6eeb57bc05e629711305ebc --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=1918.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74ce5a3ec5d5273889a2b88e8973f0d0aa75376693d47e2e612a05b2f8683843 +size 3848045156 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2135.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2135.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..3fb41ecb5603d0ba2094f59675ae30fe40ba3863 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2135.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcee1b3ac5615c484a2c6351542721873d288c068e489b153ece9948b19cf18a +size 3848045731 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2376.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2376.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c235319e4fbb3c1e44b5cb3be0b06f3ba6d899d0 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2376.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebc378fabf49d5b953664debc29b070bf92d13849dde56174b5d9e884ce510c9 +size 3848046114 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2619.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2619.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..4f954261da4e0c1c9f40964d14bf45a92ae933bb --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2619.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36e71d661daafd7d9ef814f1da2fbcc46dfa7a77fd2c78730d86c6548311da4c +size 3848046561 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2837.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2837.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..24884ae289db3ec90168bc0290f82de95b893014 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=2837.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77a00a3c9f467ec08d838a1f68076113f951e1494abde72f3396dafb538cdb1b +size 3848047136 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3051.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3051.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..3ebb0333dd623c6860df63affe9bd7053758dd64 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3051.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87ef6f815c9277358eafed875e069ca477dc5a6ad66a82d5c1b690112923ba2d +size 3848047519 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=308.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=308.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..bc2f267f3be74aa6a843504151e7198a5ebc6a8c --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=308.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecc80d62edbc9da4d2bd528ea7ceb7f23bb70535ec46a6ebf5fd7889613882c0 +size 3848042857 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3266.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3266.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..a70b6aaaf412145a87f2e0148df906cbc611985e --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3266.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aac60a1993ff14aad3203667d24a43c9214d1e2a8abe77f272ab4c4ab88e9856 +size 3848047966 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3496.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3496.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f22d3e0f6dca05e802dee23ec9de769e84c4096a --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3496.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e060973f9052ee9eadaf3fbf408a9b32fc1a502769d30cf285b207b0048620c0 +size 3848048541 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3713.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3713.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..b347b8833d6440dddf1c5a590ac6ae5a7e8407cf --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3713.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fe04dee9b1e7826efc0355b33d6232966bbb48df05abb8221ec9d7a6c7a152f +size 3848048924 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3925.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3925.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..6c30268cb740b97af48a39ebffb6765671d6d3a7 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=3925.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d97bfafc3c04262ea29e6c3c0267c82376da1d16a577c668fa5f35f5a096f8d +size 3848049435 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4137.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4137.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..67e36e748a5481a19685dfc6c40eef7066bb5b24 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4137.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2a909e447d74f4ec30776a55fea5f8a3f15d5ae9c370f06893c4340861cfe70 +size 3848049946 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4369.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4369.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c2ebf063d663c0dc6b6b83906e8bd03a077e56ec --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4369.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31f24f020dde445eb535ab2d38d6b2b9a7cc1ee791cb2635128d17a735e67816 +size 3848050393 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4580.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4580.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..8d635234b86ca97f0704aff7f61689f661604b0c --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4580.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5133d68b3c399def2f333464aa9941f75a6c412aac796ae64758c6b49f073b28 +size 3848050840 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4800.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4800.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..274beab411394a333e625ad787b8ceac348a9992 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=4800.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d41295cc9713a741cdefea63543105e773f02aa477cc169c2d1c5cfad11f9b2 +size 3848051351 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5013.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5013.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..4d8bd9582fcc7fe30e7aad42cd5fabc4957f839a --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5013.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de00d1a2dce90b48b8d570c302d5737386a1babd41c481fbada0977323f830f5 +size 3848051798 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5251.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5251.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..ee666ad1f5eba62151d83f4caa9fd6ac8ce115da --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5251.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:929e1ef4958ba05497972ec130b55ad76861c74fc3fa6a7d5f73aeb2eef657f3 +size 3848052245 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5491.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5491.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..36f08d48e28ab4dbec8ea482140afe6d354b347f --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5491.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0826a98123b5295a54410b420dbeb4ac0f65fea6dd3ff181edcc2b7f005c4c9f +size 3848052820 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5711.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5711.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..4a940de094d0f17116819e578f7f7b4f8000dc40 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5711.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a643908cce18436ebfa0a625db5266e0b36ab93f6880c78a04ae72f6734cdf5 +size 3848053203 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5934.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5934.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..2a5e932fc1b1b9f8e1218287364a358363835cca --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=5934.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2343d3e70ad6d518ac42e731c3cbc476cee6e4c14f7ebc5c46817d407e5d7a7 +size 3848053650 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6151.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6151.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..432f687497d5a879d6f8bfd3849d897ad6b07278 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6151.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad5e82692e1521dcc0a003ac33c56eba2902d811bf707d2782d975964aa0a117 +size 3848054225 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6376.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6376.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..3ec4edbd1c67fc65ce092d59b95cb46be6c33ad6 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6376.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9405bb46ef191963cf079e1b3f84d0e7bd207eb04ee623633867cc31537477b +size 3848054608 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=651.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=651.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..89cb6977fe7dac1bd85404e97ae5835a1e1dccec --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=651.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:687ed9d1506a99d4671297bf7c63e3f830cf2fc2283c4eaeef973c908ea08430 +size 3848043240 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6626.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6626.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..8576fb48a4e9c3d3b93fb58e5106c8bdd4c26fb9 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6626.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5739f690883fc649eed973b61b2d7531a1f7dc79f0fd0e6eb839773b2311d9c9 +size 3848055055 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6876.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6876.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..666f64171927b5b7578f2875973d9d634c8504ac --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=6876.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac1869446328cbd921f60442ca3ce1a19200d46edfeed54052437f34f5906203 +size 3848055630 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..bb3c03a927f6623a7d45a373d5143535c265400e --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7126.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f327de7c665f9d2c5f74c93167cb76fc87588e82ac5952adb273188c19f5d82 +size 3848056013 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7376.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7376.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..fffde56ac706929d468c76a05a6d9c49743f82c4 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7376.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef0aabf719fa6a25c3ef667123b209955a937c78131bc88da539876b02694075 +size 3848056524 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7594.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7594.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..d5a6968d4c65a313fc5876821e7854ba020df567 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7594.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:381d4456c84ef695f70ff72d070136ef4a171230dd448fd0ddc2031d6ac170ec +size 3862228567 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7626.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7626.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..40a71688e62b9ea85efa5cc242969d6f45c0ac27 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7626.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6bddb8baca25c7f84ea6594c75a887d44f1e7b79243dd7dd90891ce68abb496 +size 3848057035 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7694.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7694.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..7eb2f88766da8013c8930163873173c8228c75bc --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7694.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eed854f83bcab94e71c5da3891512cc02a561aa644872315f8223849c72e366a +size 3862228950 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7794.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7794.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..4c617a78a08c6dfc10e7ffa1fa88067528695792 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7794.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f85995e6626abec1071204dbf7d25257897e8c9c3008ca41e90ec9755d5d100 +size 3862229397 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7894.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7894.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..c896fa5da20ae60cf5252369369bebf0ac4cefe8 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7894.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efce6691ffbf77b25790cd2fb8a09b6ded785f1c860cd1cf0b676c1708e41546 +size 3862229972 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8002.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8002.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..f8832e59b2b1cb2f310a9ed708cc2577221499ff --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8002.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fc823969bcded5503bd9289cd2f6da703bd6dd4fc007c37a7893f9ade479bc1 +size 3862230355 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8124.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8124.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..63dc8f23ecae8c6fb030aae68e4c36921c733d96 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8124.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8ead01b91271a41a6d0bc77aed63cf980436cf730abdccdbacf6a58fa60c630 +size 3862230866 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8232.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8232.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..9bfe9ba5bbbde8d9b02a8cfa83b2fe39329d1483 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8232.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3adfe3b8ae661b0e00dedeeac26fdd8d1b391cabbaf682e1d44fac4a4a0ae8e +size 3862231377 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8335.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8335.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..ffea8c8d94ef618bf6429a584925e6ff1abadb05 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8335.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1727ef244d6dc18b650c0882b3cc03c7c7a21763cea7592ec1572796aa33583d +size 3862231760 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8469.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8469.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..48ffedb7efb210a46d958da95d497504228f391d --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8469.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a48631be0fa55f96680df09aa4ee6b082e2042e056c8ed46ebf82a82c229e875 +size 3862228439 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8595.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8595.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..e5013c86450f6592c411f63ad42e455a638aaa80 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8595.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d3491bd9571f55d3ddfec79c4e579e06fbcb1ab39c85874d905c985943eba8f +size 3862228886 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8719.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8719.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..2af76ea41014cfae0f709e937c8865ad5890532f --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8719.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4b73a6986ec1b4341bdcd87e6932ae6dfebbc4b8d8f48f136cd7122b690de36 +size 3862229333 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8842.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8842.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..6f1f8ce51f2927aca78a0197f9c217794c2f732b --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8842.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5a1665e083d5431ad9373bf2ec7738e4840139531e46f203ecd49f8442f623d +size 3862229908 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8958.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8958.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..5ab2d7f5f2f4941177de1c83d546ddfc524db321 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8958.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d11ecb0a652998fe60b84fd2af39e900439b0d5df8760e1f68d0b41c204191e +size 3862230291 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9075.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9075.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..008ae41f507c5013552da1f63df3ee24e1db0994 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9075.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d8ecf11f58da5b40dfc85a5b67a85aff9566c3acdf2abf45f2e3d1e5f8aa031 +size 3862230802 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9184.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9184.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..511cb69a111c946047b5c762c3946c264ffd3293 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9184.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c634bc7983097e43b1522698457cf90955a8ee8c68f40daebf2bc1fa7591bc32 +size 3862231313 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9268.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9268.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..12220084e6fdd87e2b5d6957a04aae39396b061b --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9268.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c278b25e733dea72719cb87e78808bf3db77a1aa547153ad282eba779551ea90 +size 3862231696 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9368.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9368.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..39d6b5ef6c581253f1384536b1ce9ac84ead1756 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9368.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1a63b2c88f43e1047aea31c152036a535bc49fabfb6189ace8b2b4f3954b9fd +size 3862232207 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9496.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9496.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..880a5f8314bc908ea85671c7f481d436e617ebeb --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9496.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:463ae1ded654808ecb1a53cd6ddfcb75fde1ad798baae1b3c5fbebebd4c50e84 +size 3862232718 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9627.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9627.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..2c575ba0aceb407ac4ffe9746ca3374bced63c32 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9627.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffeb4178b45b179ecb2bc57d482b5301729a809d62cae3f177823305f770a162 +size 3862233165 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9752.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9752.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..8a54a0c27063777907f1b3339c40d4a7f53ee01a --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9752.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f3b8152f5be60a0c58a5902b72a057841ab7e6a37b221fdffbecc42685dabf8 +size 3862233612 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9877.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9877.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..3df45488c628f6dcda455a8984955b52e394f841 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=9877.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6384ef6c78f4b21a6261d58bd0af321fb639818c9ea60d0ea96949bccd7f796e +size 3862234123 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/last.ckpt b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/last.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..17e460bf458c392486083633a5e419b95790e7bd --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/last.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84f39914954bbc77d458f07489ffd2a3fc2e3b42ef8b90c45acd09b0c00a799b +size 3862231760 diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/resume_bugTimeLimitReachedElapsedTimeIs5days_singlenode_2023-07-24-23:04:27.log b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/resume_bugTimeLimitReachedElapsedTimeIs5days_singlenode_2023-07-24-23:04:27.log new file mode 100644 index 0000000000000000000000000000000000000000..e5da55e3bd56bd918647e056f57d2da77778d1c5 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/resume_bugTimeLimitReachedElapsedTimeIs5days_singlenode_2023-07-24-23:04:27.log @@ -0,0 +1,975 @@ +wandb: Currently logged in as: tamariucai (rycolab). Use `wandb login --relogin` to force relogin +wandb: - Waiting for wandb.init()... wandb: \ Waiting for wandb.init()... wandb: wandb version 0.15.6 is available! To upgrade, please run: +wandb: $ pip install wandb --upgrade +wandb: Tracking run with wandb version 0.15.4 +wandb: Run data is saved locally in ./wandb/run-20230725_055332-189lqx4a +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run text10-vision100 +wandb: ⭐️ View project at https://wandb.ai/rycolab/alkmi-wit +wandb: 🚀 View run at https://wandb.ai/rycolab/alkmi-wit/runs/189lqx4a +[rank: 0] Global seed set to 5501650 +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +The current host is eu-a65-06 +The current cuDNN version is 8801 +Parameter 'text_perc' already had the value 10. +Parameter 'vision_perc' already had the value 100. +Parameter 'seed' already had the value 5501650. +Total GPU memory: 84.991 GB +Detected 80 GB of RAM, doubling batch size to 64. +Detected 80 GB of RAM, halving gradient accumulation to 64 +[update_ckeckpoint_dir] Detected hyperparameter run! +[update_ckeckpoint_dir] Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/date(2023-07-25)_time(05:53:35)/seed(5501650)-magic({'enable': True})-text_perc(10)-vision_perc(100/ +Text isn't the predominant modality (10 v.s. 100 vision), will over-sample text (uniform rates) for better BLiMP performance. +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-aa925072b83a8ef4_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/plugins/environments/slurm.py:165: PossibleUserWarning: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... + rank_zero_warn( +Using bfloat16 Automatic Mixed Precision (AMP) +GPU available: True (cuda), used: True +TPU available: False, using: 0 TPU cores +IPU available: False, using: 0 IPUs +HPU available: False, using: 0 HPUs +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-aa925072b83a8ef4_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Restoring states from the checkpoint path at /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7376.ckpt +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:337: UserWarning: The dirpath has changed from "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/date(2023-06-27)_time(20:42:57)/seed(5501650)-text_perc(10)-vision_perc(100)-magic({'enable': True})" to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/date(2023-07-25)_time(05:53:35)/seed(5501650)-magic({'enable': True})-text_perc(10)-vision_perc(100", therefore `best_model_score`, `kth_best_model_path`, `kth_value`, `last_model_path` and `best_k_models` won't be reloaded. Only `best_model_path` will be reloaded. + warnings.warn( +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0] + + | Name | Type | Params +---------------------------------------------- +0 | model | FlavaForPreTraining | 357 M +---------------------------------------------- +303 M Trainable params +53.8 M Non-trainable params +357 M Total params +1,430.581 Total estimated model params size (MB) +Restored all states from the checkpoint at /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=7376.ckpt +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +VLDataModule: batch_size is 64, num_workers is 4 +ImageDataModule: batch_size is 64, num_workers is 4 +MLMDataModule: batch_size is 64, num_workers is 4 +Dataset sizes (un-normalized): [10, 100, 10] +Dataset sizes (normalized): [0.08333333333333333, 0.8333333333333334, 0.08333333333333333] +Sampling weights after temperature (0.0): [1.0, 1.0, 1.0] +2023-07-25 05:54:29.960078 Setting up datamodule: VLDataModule +2023-07-25 05:55:17.328189 Setting up datamodule: ImageDataModule +2023-07-25 05:56:08.603803 Setting up datamodule: MLMDataModule +Registering multimodal overfitting monitors +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor'] +Trainer successfully initialized (strategy=SingleDeviceStrategy) +Starting training +2023-07-25 05:56:58.182971 Preparing data for datamodule: VLDataModule +2023-07-25 05:56:58.183346 Preparing data for datamodule: ImageDataModule +2023-07-25 05:56:58.183548 Preparing data for datamodule: MLMDataModule +2023-07-25 05:56:58.188191 Setting up datamodule: VLDataModule +2023-07-25 05:57:30.151015 Setting up datamodule: ImageDataModule +2023-07-25 05:58:24.008058 Setting up datamodule: MLMDataModule +2023-07-25 05:59:39.069765 Building train dataloader for datamodule: VLDataModule +2023-07-25 05:59:39.070487 Building train dataloader for datamodule: ImageDataModule +2023-07-25 05:59:39.070756 Building train dataloader for datamodule: MLMDataModule +2023-07-25 05:59:40.722748 Building val dataloader for datamodule: VLDataModule +2023-07-25 05:59:40.723680 Building val dataloader for datamodule: ImageDataModule +2023-07-25 05:59:40.724030 Building val dataloader for datamodule: MLMDataModule +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:359: UserWarning: `ModelCheckpoint(monitor='evaluation/pseudo_perplexity')` could not find the monitored key in the returned metrics: ['train/losses/mlm_loss', 'train/losses/itm_loss', 'train/losses/global_contrastive_loss', 'train/losses/mmm_image_loss', 'train/losses/mmm_text_loss', 'train/losses/mim_loss', 'epoch', 'step']. HINT: Did you call `log('evaluation/pseudo_perplexity', value)` in the `LightningModule`? + warning_cache.warn(m) +Epoch 0, global step 7493: 'evaluation/pseudo_perplexity' was not in top -1 +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-25 09:19:09.933568] Computing e^(tensor([5286.8242], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-25 09:19:09.942499] Ending with PPL=14.061099665107196 (duration: 0:06:52.887612) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.128 +Monitored metric validation/losses/mim_loss did not improve in the last 2 records. Best score: 0.635. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in 57 records. This exceeds the restart patience 10, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve in 17 records. This exceeds the restart patience 10, so we will restart the task with renewed patience. +Monitored metric validation/losses/mmm_image_loss did not improve in 51 records. This exceeds the restart patience 10, so we will restart the task with renewed patience. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.917. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-25 12:56:05.755672] Computing e^(tensor([5068.9552], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-25 12:56:05.760242] Ending with PPL=12.609841556402566 (duration: 0:07:03.846211) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.021 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 0.635. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.322. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.126. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.675. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 2 records. Best score: 2.917. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-25 16:36:10.440272] Computing e^(tensor([5005.3712], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-25 16:36:10.444862] Ending with PPL=12.215255417615529 (duration: 0:06:59.642291) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.999 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 0.635. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 2 records. Best score: 0.322. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 2 records. Best score: 2.126. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 2 records. Best score: 0.675. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 3 records. Best score: 2.917. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-25 20:15:25.621156] Computing e^(tensor([4973.3183], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-25 20:15:25.626063] Ending with PPL=12.021048344156881 (duration: 0:06:52.460447) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.982 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 0.635. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 3 records. Best score: 0.322. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 3 records. Best score: 2.126. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 3 records. Best score: 0.675. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 2.917. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-25 23:43:45.238756] Computing e^(tensor([4947.4466], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-25 23:43:45.246219] Ending with PPL=11.866547683617048 (duration: 0:06:48.043355) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.977 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 0.635. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.322. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 2.126. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 4 records. Best score: 0.675. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 2.917. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-26 03:12:20.630519] Computing e^(tensor([4926.0973], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-26 03:12:20.635671] Ending with PPL=11.740550115251777 (duration: 0:06:57.253570) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.970 +Monitored metric validation/losses/mim_loss did not improve in the last 7 records. Best score: 0.635. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.322. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.126. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 0.675. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 2.917. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-26 06:34:54.487976] Computing e^(tensor([4912.8365], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-26 06:34:54.493438] Ending with PPL=11.662962634637777 (duration: 0:06:53.333137) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.957 +Monitored metric validation/losses/mim_loss did not improve in the last 8 records. Best score: 0.635. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.322. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.126. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 0.675. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 7 records. Best score: 2.917. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-26 10:18:51.749369] Computing e^(tensor([4903.1664], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-26 10:18:51.755059] Ending with PPL=11.606707876967558 (duration: 0:06:52.106822) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.957 +Monitored metric validation/losses/mim_loss did not improve in the last 9 records. Best score: 0.635. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.322. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 2.126. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 7 records. Best score: 0.675. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 8 records. Best score: 2.917. Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-26 11:01:44.776785] Computing e^(tensor([4916.8436], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-26 11:01:44.782528] Ending with PPL=11.686353604605724 (duration: 0:07:01.053631) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.955 +Monitored metric validation/losses/mim_loss did not improve in the last 10 records. Best score: 0.635. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 8 records. Best score: 0.322. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 8 records. Best score: 2.126. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 8 records. Best score: 0.675. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 9 records. Best score: 2.917. Signaling model to ignore task. +Starting validation +2023-07-26 11:19:26.067887 Preparing data for datamodule: VLDataModule +2023-07-26 11:19:26.068034 Preparing data for datamodule: ImageDataModule +2023-07-26 11:19:26.068130 Preparing data for datamodule: MLMDataModule +2023-07-26 11:19:26.068400 Setting up datamodule: VLDataModule +2023-07-26 11:21:22.910503 Setting up datamodule: ImageDataModule +2023-07-26 11:22:22.364105 Setting up datamodule: MLMDataModule +2023-07-26 11:24:11.016855 Building val dataloader for datamodule: VLDataModule +2023-07-26 11:24:11.017509 Building val dataloader for datamodule: ImageDataModule +2023-07-26 11:24:11.017792 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-26 11:27:28.174010] Computing e^(tensor([4916.8436], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-26 11:27:28.184809] Ending with PPL=11.686353604605724 (duration: 0:03:06.970902) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00 Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-08 11:36:30.349984 Setting up datamodule: ImageDataModule +2023-07-08 11:37:58.281930 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Adding ImageNet zeroshot callback +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'ImageNetZeroshotCallback'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-08 11:48:16.232338 Setting up datamodule: VLDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-08 11:59:27.470107 Setting up datamodule: ImageDataModule +2023-07-08 12:00:54.433468 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-08 12:12:28.229775 Building train dataloader for datamodule: VLDataModule +2023-07-08 12:12:28.230664 Building train dataloader for datamodule: ImageDataModule +2023-07-08 12:12:28.231235 Building train dataloader for datamodule: MLMDataModule +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +VLDataModule: batch_size is 32, num_workers is 4 +ImageDataModule: batch_size is 32, num_workers is 4 +MLMDataModule: batch_size is 32, num_workers is 4 +Dataset sizes (un-normalized): [10, 100, 10] +Dataset sizes (normalized): [0.08333333333333333, 0.8333333333333334, 0.08333333333333333] +Sampling weights after temperature (0.0): [1.0, 1.0, 1.0] +2023-07-08 11:06:04.334544 Setting up datamodule: VLDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-08 11:15:33.815115 Setting up datamodule: ImageDataModule +2023-07-08 11:17:04.528184 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +Registering multimodal callbacks (overfitting monitors) +Adding ImageNet zeroshot callback +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'ImageNetZeroshotCallback'] +Trainer successfully initialized (strategy=DDPStrategy) +Starting training +2023-07-08 11:25:32.596744 Preparing data for datamodule: VLDataModule +2023-07-08 11:25:32.596915 Preparing data for datamodule: ImageDataModule +2023-07-08 11:25:32.597022 Preparing data for datamodule: MLMDataModule +2023-07-08 11:48:16.232814 Setting up datamodule: VLDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-08 11:59:27.442780 Setting up datamodule: ImageDataModule +2023-07-08 12:00:54.667169 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-08 12:12:28.230451 Building train dataloader for datamodule: VLDataModule +2023-07-08 12:12:28.232904 Building train dataloader for datamodule: ImageDataModule +2023-07-08 12:12:28.234179 Building train dataloader for datamodule: MLMDataModule +2023-07-08 12:12:31.436162 Building val dataloader for datamodule: VLDataModule +2023-07-08 12:12:31.437124 Building val dataloader for datamodule: ImageDataModule +2023-07-08 12:12:31.437519 Building val dataloader for datamodule: MLMDataModule +2023-07-08 12:12:30.967356 Building val dataloader for datamodule: VLDataModule +2023-07-08 12:12:30.969331 Building val dataloader for datamodule: ImageDataModule +2023-07-08 12:12:30.969848 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7871cfcbbb4c64f4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-08 13:35:55.903619] Computing e^(tensor([5533.5028], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-08 13:35:55.917517] Ending with PPL=15.906874598687425 (duration: 0:11:49.966885) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00 Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-08 15:43:43.978458 Setting up datamodule: ImageDataModule +2023-07-08 15:45:27.731017 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-08 15:56:53.886596 Building val dataloader for datamodule: VLDataModule +2023-07-08 15:56:53.887303 Building val dataloader for datamodule: ImageDataModule +2023-07-08 15:56:53.887602 Building val dataloader for datamodule: MLMDataModule +[ImageNet Zero-Shot Evaluation 2023-07-08 15:25:42.900750] Ending with top5=0.00564 (duration: 0:02:17.754587) +Starting validation +2023-07-08 15:31:44.248527 Preparing data for datamodule: VLDataModule +2023-07-08 15:31:44.248728 Preparing data for datamodule: ImageDataModule +2023-07-08 15:31:44.248841 Preparing data for datamodule: MLMDataModule +2023-07-08 15:31:44.255051 Setting up datamodule: VLDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-08 15:43:44.683365 Setting up datamodule: ImageDataModule +2023-07-08 15:45:27.668092 Setting up datamodule: MLMDataModule +Counting words for split train[:10%] before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: train[:10%] ----> Total words: 4327460, No. of unique: 290428 +Counting words for split test before (i.e. captions only) collapsing... +Split before (i.e. captions only) collapsing: test ----> Total words: 4814827, No. of unique: 309990 +Counting words for split train[:10%] after collapsing... +Split after collapsing: train[:10%] ----> Total words: 103097255, No. of unique: 1018549 +Counting words for split test after collapsing... +Split after collapsing: test ----> Total words: 114495389, No. of unique: 1079620 +2023-07-08 15:56:53.886691 Building val dataloader for datamodule: VLDataModule +2023-07-08 15:56:53.887522 Building val dataloader for datamodule: ImageDataModule +2023-07-08 15:56:53.887856 Building val dataloader for datamodule: MLMDataModule +Traceback (most recent call last): + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 180, in + main() + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 172, in main + trainer.validate(model, datamodule=datamodule) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 630, in validate + return call._call_and_handle_interrupt( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 41, in _call_and_handle_interrupt + return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 91, in launch + return function(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 673, in _validate_impl + results = self._run(model, ckpt_path=ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 975, in _run + results = self._run_stage() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1011, in _run_stage + return self._evaluation_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/utilities.py", line 177, in _decorator + return loop_run(self, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 98, in run + self.setup_data() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 168, in setup_data + _check_dataloader_iterable(dl, source, trainer_fn) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/connectors/data_connector.py", line 385, in _check_dataloader_iterable + iter(dataloader) # type: ignore[call-overload] + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/multidata.py", line 73, in __iter__ + self.change_dataloader() + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/multidata.py", line 126, in change_dataloader + choice = [self.sampling_func()] + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/multidata.py", line 173, in + self.sampling_func = lambda: random.choices(population=range(len(sampling_weights)), + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/random.py", line 535, in choices + raise ValueError('Total of weights must be greater than zero') +ValueError: Total of weights must be greater than zero +Traceback (most recent call last): + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 180, in + main() + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 172, in main + trainer.validate(model, datamodule=datamodule) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 630, in validate + return call._call_and_handle_interrupt( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 41, in _call_and_handle_interrupt + return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 91, in launch + return function(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 673, in _validate_impl + results = self._run(model, ckpt_path=ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 975, in _run + results = self._run_stage() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1011, in _run_stage + return self._evaluation_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/utilities.py", line 177, in _decorator + return loop_run(self, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 98, in run + self.setup_data() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 168, in setup_data + _check_dataloader_iterable(dl, source, trainer_fn) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/connectors/data_connector.py", line 385, in _check_dataloader_iterable + iter(dataloader) # type: ignore[call-overload] + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/multidata.py", line 73, in __iter__ + self.change_dataloader() + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/multidata.py", line 126, in change_dataloader + choice = [self.sampling_func()] + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/multidata.py", line 173, in + self.sampling_func = lambda: random.choices(population=range(len(sampling_weights)), + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/random.py", line 535, in choices + raise ValueError('Total of weights must be greater than zero') +ValueError: Total of weights must be greater than zero +Traceback (most recent call last): + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 180, in + main() + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 172, in main + trainer.validate(model, datamodule=datamodule) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 630, in validate + return call._call_and_handle_interrupt( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 41, in _call_and_handle_interrupt + return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 91, in launch + return function(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 673, in _validate_impl + results = self._run(model, ckpt_path=ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 975, in _run + results = self._run_stage() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1011, in _run_stage + return self._evaluation_loop.run() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/utilities.py", line 177, in _decorator + return loop_run(self, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 98, in run + self.setup_data() + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/loops/evaluation_loop.py", line 168, in setup_data + _check_dataloader_iterable(dl, source, trainer_fn) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/connectors/data_connector.py", line 385, in _check_dataloader_iterable + iter(dataloader) # type: ignore[call-overload] + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/multidata.py", line 73, in __iter__ + self.change_dataloader() + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/multidata.py", line 126, in change_dataloader + choice = [self.sampling_func()] + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/multidata.py", line 173, in + self.sampling_func = lambda: random.choices(population=range(len(sampling_weights)), + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/random.py", line 535, in choices + raise ValueError('Total of weights must be greater than zero') +ValueError: Total of weights must be greater than zero +wandb: Waiting for W&B process to finish... (failed 1). Press Control-C to abort syncing. +wandb: - 0.000 MB of 0.000 MB uploaded (0.000 MB deduped) wandb: \ 0.000 MB of 0.045 MB uploaded (0.000 MB deduped) wandb: | 0.045 MB of 0.045 MB uploaded (0.000 MB deduped) wandb: +wandb: Run history: +wandb: ImageDataModule ▁▁▁▁▁▁▁▁▃▃▃▃▃▃▃▃▄▄▄▄▄▄▄▄▆▆▆▆▆▆▆▆████████ +wandb: MLMDataModule ▁▁▁▁▁▁▁▁▃▃▃▃▃▃▃▃▄▄▄▄▄▅▅▅▆▆▆▆▆▆▆▆████████ +wandb: VLDataModule ▁▁▁▁▁▁▁▁▃▃▃▃▃▃▃▃▅▅▅▅▅▅▅▅▆▆▆▆▆▆▆▆████████ +wandb: epoch ▁▁▁▁▁▁▁ +wandb: evaluation/blimp/anaphor_agreement █▁ +wandb: evaluation/blimp/argument_structure ▁█ +wandb: evaluation/blimp/binding ▁█ +wandb: evaluation/blimp/control_raising ▁█ +wandb: evaluation/blimp/determiner_noun_agreement ▁█ +wandb: evaluation/blimp/ellipsis ▁█ +wandb: evaluation/blimp/filler_gap █▁ +wandb: evaluation/blimp/irregular_forms ▁█ +wandb: evaluation/blimp/island_effects ▁█ +wandb: evaluation/blimp/npi_licensing █▁ +wandb: evaluation/blimp/quantifiers ▁█ +wandb: evaluation/blimp/subject_verb_agreement ▁█ +wandb: evaluation/blimp_average ▁█ +wandb: evaluation/imagenet_zeroshot/top1 █▁ +wandb: evaluation/imagenet_zeroshot/top5 ▁█ +wandb: evaluation/pseudo_perplexity █▁ +wandb: lr-AdamW/pg1 ▁▁▁▁▁▁▁▁▃▃▃▃▃▃▃▃▅▅▅▅▅▅▅▅▆▆▆▆▆▆▆▆████████ +wandb: lr-AdamW/pg2 ▁▁▁▁▁▁▁▁▃▃▃▃▃▃▃▃▅▅▅▅▅▅▅▅▆▆▆▆▆▆▆▆████████ +wandb: train/losses/global_contrastive_loss █▁▃▃▃ +wandb: train/losses/itm_loss █▁▃▃▃ +wandb: train/losses/mim_loss ▂█▁▆▇ +wandb: train/losses/mlm_loss ▅█▄▇▁ +wandb: train/losses/mmm_image_loss █▁▃▃▃ +wandb: train/losses/mmm_text_loss ▇▁▅▃█ +wandb: trainer/global_step ▁▁▁▁▁▁▁▁▃▃▃▃▃▃▃▃▅▅▅▅▅▅▅▅▆▆▆▆▆▆▆▆████████ +wandb: validation/losses/global_contrastive_loss ▁ +wandb: validation/losses/itm_loss ▁ +wandb: validation/losses/mim_loss █▁ +wandb: validation/losses/mlm_loss █▁ +wandb: validation/losses/mmm_image_loss ▁ +wandb: validation/losses/mmm_text_loss █▁ +wandb: validation/monitor/global_contrastive ▁▁ +wandb: validation/monitor/itm ▁▁ +wandb: validation/monitor/mim █▁ +wandb: validation/monitor/mlm █▁ +wandb: validation/monitor/mmm_image ▁▁ +wandb: validation/monitor/mmm_text █▁ +wandb: +wandb: Run summary: +wandb: ImageDataModule 5258 +wandb: MLMDataModule 5405 +wandb: VLDataModule 5277 +wandb: epoch 0 +wandb: evaluation/blimp/anaphor_agreement 91.41 +wandb: evaluation/blimp/argument_structure 68.0 +wandb: evaluation/blimp/binding 67.75 +wandb: evaluation/blimp/control_raising 64.49 +wandb: evaluation/blimp/determiner_noun_agreement 87.06 +wandb: evaluation/blimp/ellipsis 78.46 +wandb: evaluation/blimp/filler_gap 60.63 +wandb: evaluation/blimp/irregular_forms 85.29 +wandb: evaluation/blimp/island_effects 49.74 +wandb: evaluation/blimp/npi_licensing 56.33 +wandb: evaluation/blimp/quantifiers 57.44 +wandb: evaluation/blimp/subject_verb_agreement 66.43 +wandb: evaluation/blimp_average 69.41917 +wandb: evaluation/imagenet_zeroshot/top1 0.0011 +wandb: evaluation/imagenet_zeroshot/top5 0.00564 +wandb: evaluation/pseudo_perplexity 13.1394 +wandb: lr-AdamW/pg1 0.00078 +wandb: lr-AdamW/pg2 0.00059 +wandb: train/losses/global_contrastive_loss 3.09187 +wandb: train/losses/itm_loss 0.50165 +wandb: train/losses/mim_loss 0.78864 +wandb: train/losses/mlm_loss 2.61549 +wandb: train/losses/mmm_image_loss 0.7283 +wandb: train/losses/mmm_text_loss 3.21014 +wandb: trainer/global_step 7874 +wandb: validation/losses/global_contrastive_loss 3.02728 +wandb: validation/losses/itm_loss 0.3324 +wandb: validation/losses/mim_loss 0.70603 +wandb: validation/losses/mlm_loss 3.17218 +wandb: validation/losses/mmm_image_loss 0.73865 +wandb: validation/losses/mmm_text_loss 3.01337 +wandb: validation/monitor/global_contrastive 0.0 +wandb: validation/monitor/itm 0.0 +wandb: validation/monitor/mim 0.0 +wandb: validation/monitor/mlm 0.0 +wandb: validation/monitor/mmm_image 0.0 +wandb: validation/monitor/mmm_text 0.0 +wandb: +wandb: 🚀 View run text10-vision100 at: https://wandb.ai/rycolab/alkmi-wit/runs/189lqx4a +wandb: Synced 3 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20230708_112549-189lqx4a/logs diff --git a/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/resume_singlenode_2023-07-26-14:14:40.log b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/resume_singlenode_2023-07-26-14:14:40.log new file mode 100644 index 0000000000000000000000000000000000000000..ac1bbf58fda63a66791568112505dbcc37afc0d9 --- /dev/null +++ b/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/resume_singlenode_2023-07-26-14:14:40.log @@ -0,0 +1,1642 @@ +wandb: Currently logged in as: tamariucai (rycolab). Use `wandb login --relogin` to force relogin +wandb: - Waiting for wandb.init()... wandb: \ Waiting for wandb.init()... wandb: wandb version 0.15.7 is available! To upgrade, please run: +wandb: $ pip install wandb --upgrade +wandb: Tracking run with wandb version 0.15.4 +wandb: Run data is saved locally in ./wandb/run-20230726_141453-189lqx4a +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run text10-vision100 +wandb: ⭐️ View project at https://wandb.ai/rycolab/alkmi-wit +wandb: 🚀 View run at https://wandb.ai/rycolab/alkmi-wit/runs/189lqx4a +[rank: 0] Global seed set to 5501650 +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +The current host is eu-a65-05 +The current cuDNN version is 8801 +Parameter 'text_perc' already had the value 10. +Parameter 'vision_perc' already had the value 100. +Parameter 'seed' already had the value 5501650. +Total GPU memory: 84.991 GB +Detected 80 GB of RAM, doubling batch size to 64. +Detected 80 GB of RAM, halving gradient accumulation to 64 +[update_ckeckpoint_dir] Detected hyperparameter run! +[update_ckeckpoint_dir] Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/date(2023-07-26)_time(14:14:56)/seed(5501650)-magic({'enable': True})-text_perc(10)-vision_perc(100/ +Text isn't the predominant modality (10 v.s. 100 vision), will over-sample text (uniform rates) for better BLiMP performance. +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-aa925072b83a8ef4_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/plugins/environments/slurm.py:165: PossibleUserWarning: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... + rank_zero_warn( +Using bfloat16 Automatic Mixed Precision (AMP) +GPU available: True (cuda), used: True +TPU available: False, using: 0 TPU cores +IPU available: False, using: 0 IPUs +HPU available: False, using: 0 HPUs +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-aa925072b83a8ef4_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Restoring states from the checkpoint path at /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8335.ckpt +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py:247: UserWarning: Be aware that when using `ckpt_path`, callbacks used to create the checkpoint need to be provided during `Trainer` instantiation. Please add the following callbacks: ['Timer']. + rank_zero_warn( +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:337: UserWarning: The dirpath has changed from "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/date(2023-07-25)_time(05:53:35)/seed(5501650)-magic({'enable': True})-text_perc(10)-vision_perc(100" to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/date(2023-07-26)_time(14:14:56)/seed(5501650)-magic({'enable': True})-text_perc(10)-vision_perc(100", therefore `best_model_score`, `kth_best_model_path`, `kth_value`, `last_model_path` and `best_k_models` won't be reloaded. Only `best_model_path` will be reloaded. + warnings.warn( +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0] + + | Name | Type | Params +---------------------------------------------- +0 | model | FlavaForPreTraining | 357 M +---------------------------------------------- +303 M Trainable params +53.8 M Non-trainable params +357 M Total params +1,430.581 Total estimated model params size (MB) +Restored all states from the checkpoint at /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text10-vision100/bs4096_seed5501650_bf16-mixed/flava-epoch=00-step=8335.ckpt +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +VLDataModule: batch_size is 64, num_workers is 4 +ImageDataModule: batch_size is 64, num_workers is 4 +MLMDataModule: batch_size is 64, num_workers is 4 +Dataset sizes (un-normalized): [10, 100, 10] +Dataset sizes (normalized): [0.08333333333333333, 0.8333333333333334, 0.08333333333333333] +Sampling weights after temperature (0.0): [1.0, 1.0, 1.0] +2023-07-26 14:15:12.860800 Setting up datamodule: VLDataModule +2023-07-26 14:15:58.088581 Setting up datamodule: ImageDataModule +2023-07-26 14:16:47.945672 Setting up datamodule: MLMDataModule +Registering multimodal overfitting monitors +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor'] +Trainer successfully initialized (strategy=SingleDeviceStrategy) +Starting training +2023-07-26 14:16:59.559087 Preparing data for datamodule: VLDataModule +2023-07-26 14:16:59.559339 Preparing data for datamodule: ImageDataModule +2023-07-26 14:16:59.559501 Preparing data for datamodule: MLMDataModule +2023-07-26 14:16:59.561023 Setting up datamodule: VLDataModule +2023-07-26 14:17:20.143515 Setting up datamodule: ImageDataModule +2023-07-26 14:18:08.072821 Setting up datamodule: MLMDataModule +Resetting wait count for `mlm` since training is resuming and they are special tasks. +Resetting wait count for `mmm_text` since training is resuming and they are special tasks. +2023-07-26 14:19:05.596042 Building train dataloader for datamodule: VLDataModule +2023-07-26 14:19:05.596619 Building train dataloader for datamodule: ImageDataModule +2023-07-26 14:19:05.596878 Building train dataloader for datamodule: MLMDataModule +2023-07-26 14:19:07.096791 Building val dataloader for datamodule: VLDataModule +2023-07-26 14:19:07.097625 Building val dataloader for datamodule: ImageDataModule +2023-07-26 14:19:07.098036 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-26 15:11:45.712030] Computing e^(tensor([5160.2461], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-26 15:11:45.719458] Ending with PPL=13.19876243496051 (duration: 0:06:59.058532) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.872 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-26 21:09:46.681284] Computing e^(tensor([4926.6794], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-26 21:09:46.685377] Ending with PPL=11.74396747072004 (duration: 0:07:10.267892) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.855 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-27 00:09:21.993275] Computing e^(tensor([4931.5019], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-27 00:09:21.997391] Ending with PPL=11.772319529327088 (duration: 0:07:02.240484) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.302 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.126. Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.668 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.855. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-27 03:22:06.150794] Computing e^(tensor([4864.7871], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-27 03:22:06.154922] Ending with PPL=11.38610248451502 (duration: 0:07:13.592882) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.952 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 0.635. Signaling model to ignore task. +Metric validation/losses/itm_loss improved by 0.033 >= min_delta = 0.0. New best score: 0.269 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.126. Wait count has been increased from 1. +Metric validation/losses/mmm_image_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.660 +Metric validation/losses/mmm_text_loss improved by 0.005 >= min_delta = 0.0. New best score: 2.850 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-27 06:33:25.934135] Computing e^(tensor([4838.4254], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-27 06:33:25.938194] Ending with PPL=11.237009122806047 (duration: 0:07:10.743427) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.951 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 0.635. Signaling model to ignore task. +Metric validation/losses/itm_loss improved by 0.013 >= min_delta = 0.0. New best score: 0.256 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.126. Wait count has been increased from 2. +Metric validation/losses/mmm_image_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.654 +Metric validation/losses/mmm_text_loss improved by 0.010 >= min_delta = 0.0. New best score: 2.840 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-27 09:51:09.215058] Computing e^(tensor([4842.3175], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-27 09:51:09.219132] Ending with PPL=11.258898234561714 (duration: 0:07:09.343815) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.940 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 0.635. Signaling model to ignore task. +Metric validation/losses/itm_loss improved by 0.014 >= min_delta = 0.0. New best score: 0.242 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 2.126. Wait count has been increased from 3. +Metric validation/losses/mmm_image_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.648 +Metric validation/losses/mmm_text_loss improved by 0.006 >= min_delta = 0.0. New best score: 2.834 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-27 13:21:21.921230] Computing e^(tensor([4854.2832], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-27 13:21:21.925322] Ending with PPL=11.326460274031172 (duration: 0:07:13.264631) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.238 +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.126. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.640 +Metric validation/losses/mmm_text_loss improved by 0.001 >= min_delta = 0.0. New best score: 2.833 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-27 17:36:58.887443] Computing e^(tensor([4834.5938], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-27 17:36:58.891486] Ending with PPL=11.215501807795812 (duration: 0:07:11.949070) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.925 +Monitored metric validation/losses/mim_loss did not improve in the last 8 records. Best score: 0.635. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.238. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.126. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.637 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.833. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-27 20:32:44.707044] Computing e^(tensor([4786.6346], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-27 20:32:44.711176] Ending with PPL=10.949757297291526 (duration: 0:07:20.276753) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.895 +Monitored metric validation/losses/mim_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Metric validation/losses/itm_loss improved by 0.013 >= min_delta = 0.0. New best score: 0.225 +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 2.126. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.629 +Metric validation/losses/mmm_text_loss improved by 0.052 >= min_delta = 0.0. New best score: 2.781 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-27 23:23:11.080166] Computing e^(tensor([4756.5636], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-27 23:23:11.084252] Ending with PPL=10.786353633476898 (duration: 0:07:19.084100) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.893 +Metric validation/losses/mim_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.626 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.225. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 8 records. Best score: 2.126. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.626 +Metric validation/losses/mmm_text_loss improved by 0.017 >= min_delta = 0.0. New best score: 2.764 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-28 02:28:46.189224] Computing e^(tensor([4735.5113], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-28 02:28:46.193290] Ending with PPL=10.67341063411208 (duration: 0:07:20.026837) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.877 +Metric validation/losses/mim_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.620 +Metric validation/losses/itm_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.223 +Monitored metric validation/losses/global_contrastive_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Metric validation/losses/mmm_image_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.622 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.764. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-28 05:43:04.829342] Computing e^(tensor([4728.9523], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-28 05:43:04.833421] Ending with PPL=10.638464160531454 (duration: 0:07:16.040835) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.872 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.618 +Metric validation/losses/itm_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.221 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.126. Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.620 +Metric validation/losses/mmm_text_loss improved by 0.018 >= min_delta = 0.0. New best score: 2.746 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-28 08:52:38.798475] Computing e^(tensor([4723.0405], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-28 08:52:38.802613] Ending with PPL=10.607064533224717 (duration: 0:07:20.862025) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.611 +Metric validation/losses/itm_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.215 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.126. Wait count has been increased from 1. +Metric validation/losses/mmm_image_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.613 +Metric validation/losses/mmm_text_loss improved by 0.002 >= min_delta = 0.0. New best score: 2.744 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-28 12:02:05.823470] Computing e^(tensor([4714.2215], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-28 12:02:05.827569] Ending with PPL=10.560395932315116 (duration: 0:07:17.332485) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.867 +Metric validation/losses/mim_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.608 +Metric validation/losses/itm_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.213 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.126. Wait count has been increased from 2. +Metric validation/losses/mmm_image_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.610 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.744. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-28 15:07:01.471009] Computing e^(tensor([4686.5601], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-28 15:07:01.475113] Ending with PPL=10.415343403217967 (duration: 0:07:19.953182) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.859 +Metric validation/losses/mim_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.604 +Metric validation/losses/itm_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.207 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 2.126. Wait count has been increased from 3. +Metric validation/losses/mmm_image_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.606 +Metric validation/losses/mmm_text_loss improved by 0.014 >= min_delta = 0.0. New best score: 2.730 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-28 18:22:23.385686] Computing e^(tensor([4669.2468], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-28 18:22:23.389779] Ending with PPL=10.325570721005214 (duration: 0:07:22.537711) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.856 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.604. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.207. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.126. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.604 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.730. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-28 21:41:27.652929] Computing e^(tensor([4664.4123], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-28 21:41:27.657017] Ending with PPL=10.300641322846392 (duration: 0:07:09.105531) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.848 +Metric validation/losses/mim_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.595 +Metric validation/losses/itm_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.205 +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.126. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.597 +Metric validation/losses/mmm_text_loss improved by 0.019 >= min_delta = 0.0. New best score: 2.711 +Starting validation +2023-07-28 22:07:45.184271 Preparing data for datamodule: VLDataModule +2023-07-28 22:07:45.184539 Preparing data for datamodule: ImageDataModule +2023-07-28 22:07:45.184719 Preparing data for datamodule: MLMDataModule +2023-07-28 22:07:45.185103 Setting up datamodule: VLDataModule +2023-07-28 22:09:16.217707 Setting up datamodule: ImageDataModule +2023-07-28 22:10:07.726739 Setting up datamodule: MLMDataModule +2023-07-28 22:11:30.067843 Building val dataloader for datamodule: VLDataModule +2023-07-28 22:11:30.068326 Building val dataloader for datamodule: ImageDataModule +2023-07-28 22:11:30.068587 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-07-28 22:15:06.489432] Computing e^(tensor([4664.4123], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-07-28 22:15:06.495229] Ending with PPL=10.300641322846392 (duration: 0:03:14.512281) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.848 +Metric validation/losses/mim_loss improved by 0.021 >= min_delta = 0.0. New best score: 0.583 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.207. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Metric validation/losses/mmm_image_loss improved by 0.022 >= min_delta = 0.0. New best score: 0.582 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 4 records. Best score: 2.730. Wait count has been increased from 3. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-01 04:34:25.885144] Computing e^(tensor([4656.3009], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-01 04:34:25.891579] Ending with PPL=10.258949459928038 (duration: 0:07:05.311425) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.574 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.207. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.126. Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.574 +Metric validation/losses/mmm_text_loss improved by 0.022 >= min_delta = 0.0. New best score: 2.707 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-01 06:51:32.260108] Computing e^(tensor([4633.4914], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-01 06:51:32.264325] Ending with PPL=10.142613685995705 (duration: 0:06:58.945077) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.848 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.569 +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.207. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.126. Wait count has been increased from 1. +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.570 +Metric validation/losses/mmm_text_loss improved by 0.006 >= min_delta = 0.0. New best score: 2.701 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-01 09:11:43.613735] Computing e^(tensor([4651.2274], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-01 09:11:43.618084] Ending with PPL=10.232958267517988 (duration: 0:07:18.934763) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.844 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.568 +Monitored metric validation/losses/itm_loss did not improve in the last 8 records. Best score: 0.207. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.126. Wait count has been increased from 2. +Metric validation/losses/mmm_image_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.569 +Metric validation/losses/mmm_text_loss improved by 0.004 >= min_delta = 0.0. New best score: 2.697 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-01 11:33:41.352380] Computing e^(tensor([4625.3454], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-01 11:33:41.356635] Ending with PPL=10.10138673574363 (duration: 0:06:53.736820) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.837 +Metric validation/losses/mim_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.561 +Monitored metric validation/losses/itm_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 2.126. Wait count has been increased from 3. +Metric validation/losses/mmm_image_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.561 +Metric validation/losses/mmm_text_loss improved by 0.017 >= min_delta = 0.0. New best score: 2.680 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-01 14:30:35.274933] Computing e^(tensor([4626.1020], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-01 14:30:35.279185] Ending with PPL=10.105208536612883 (duration: 0:06:59.009881) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.833 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.561. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.207. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.126. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.561 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.680. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-01 17:52:13.702235] Computing e^(tensor([4612.7940], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-01 17:52:13.706678] Ending with PPL=10.038191678995233 (duration: 0:07:02.194676) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.823 +Metric validation/losses/mim_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.557 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.207. Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.126. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.554 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 2.680. Wait count has been increased from 1. +Starting validation +2023-08-01 18:12:08.494647 Preparing data for datamodule: VLDataModule +2023-08-01 18:12:08.494795 Preparing data for datamodule: ImageDataModule +2023-08-01 18:12:08.494890 Preparing data for datamodule: MLMDataModule +2023-08-01 18:12:08.495133 Setting up datamodule: VLDataModule +2023-08-01 18:14:03.447020 Setting up datamodule: ImageDataModule +2023-08-01 18:14:57.178336 Setting up datamodule: MLMDataModule +2023-08-01 18:16:44.896689 Building val dataloader for datamodule: VLDataModule +2023-08-01 18:16:44.897226 Building val dataloader for datamodule: ImageDataModule +2023-08-01 18:16:44.897506 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-01 18:20:03.068621] Computing e^(tensor([4612.7940], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-01 18:20:03.074441] Ending with PPL=10.038191678995233 (duration: 0:03:05.983953) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.257 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.973 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.664. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-17 22:04:32.267066] Computing e^(tensor([7628.6671], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-17 22:04:32.272773] Ending with PPL=45.34652623514945 (duration: 0:03:59.790259) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.158 +Metric validation/losses/mim_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.960 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.211. Wait count has been increased from 1. +Metric validation/losses/global_contrastive_loss improved by 0.001 >= min_delta = 0.0. New best score: 2.256 +Metric validation/losses/mmm_image_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.967 +Metric validation/losses/mmm_text_loss improved by 0.033 >= min_delta = 0.0. New best score: 3.631 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 01:42:02.951410] Computing e^(tensor([7591.8522], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 01:42:02.957130] Ending with PPL=44.5194473178257 (duration: 0:04:08.537269) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.138 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.955 +Metric validation/losses/itm_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.209 +Metric validation/losses/global_contrastive_loss improved by 0.005 >= min_delta = 0.0. New best score: 2.251 +Metric validation/losses/mmm_image_loss improved by 0.010 >= min_delta = 0.0. New best score: 0.957 +Metric validation/losses/mmm_text_loss improved by 0.005 >= min_delta = 0.0. New best score: 3.626 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 05:17:21.153636] Computing e^(tensor([7523.9393], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 05:17:21.159294] Ending with PPL=43.033101876135845 (duration: 0:04:00.727333) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.111 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.953 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.209. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.251. Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.956 +Metric validation/losses/mmm_text_loss improved by 0.009 >= min_delta = 0.0. New best score: 3.617 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 08:40:04.897148] Computing e^(tensor([7403.1468], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 08:40:04.903264] Ending with PPL=40.510995169702156 (duration: 0:04:08.700944) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.072 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.951 +Metric validation/losses/itm_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.206 +Metric validation/losses/global_contrastive_loss improved by 0.092 >= min_delta = 0.0. New best score: 2.158 +Metric validation/losses/mmm_image_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.953 +Metric validation/losses/mmm_text_loss improved by 0.013 >= min_delta = 0.0. New best score: 3.604 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 12:26:05.335209] Computing e^(tensor([7299.8173], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 12:26:05.341233] Ending with PPL=38.47115126317305 (duration: 0:04:07.853692) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.026 +Metric validation/losses/mim_loss improved by 0.019 >= min_delta = 0.0. New best score: 0.932 +Metric validation/losses/itm_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.204 +Metric validation/losses/global_contrastive_loss improved by 0.033 >= min_delta = 0.0. New best score: 2.126 +Metric validation/losses/mmm_image_loss improved by 0.021 >= min_delta = 0.0. New best score: 0.931 +Metric validation/losses/mmm_text_loss improved by 0.013 >= min_delta = 0.0. New best score: 3.591 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 16:33:35.674426] Computing e^(tensor([7241.1243], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 16:33:35.680152] Ending with PPL=37.358562979200165 (duration: 0:04:15.046414) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.005 +Metric validation/losses/mim_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.926 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.204. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.126. Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.931 +Metric validation/losses/mmm_text_loss improved by 0.013 >= min_delta = 0.0. New best score: 3.578 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 21:00:11.235712] Computing e^(tensor([7091.3673], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 21:00:11.239831] Ending with PPL=34.663375128033316 (duration: 0:06:02.839343) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.959 +Metric validation/losses/mim_loss improved by 0.019 >= min_delta = 0.0. New best score: 0.907 +Metric validation/losses/itm_loss improved by 0.011 >= min_delta = 0.0. New best score: 0.193 +Metric validation/losses/global_contrastive_loss improved by 0.063 >= min_delta = 0.0. New best score: 2.063 +Metric validation/losses/mmm_image_loss improved by 0.021 >= min_delta = 0.0. New best score: 0.910 +Metric validation/losses/mmm_text_loss improved by 0.039 >= min_delta = 0.0. New best score: 3.538 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 01:24:31.642991] Computing e^(tensor([7015.4010], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 01:24:31.648843] Ending with PPL=33.371441890337564 (duration: 0:04:34.462928) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.921 +Metric validation/losses/mim_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.898 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.193. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.063. Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.011 >= min_delta = 0.0. New best score: 0.899 +Metric validation/losses/mmm_text_loss improved by 0.008 >= min_delta = 0.0. New best score: 3.530 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 06:00:09.861352] Computing e^(tensor([6927.2369], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 06:00:09.867085] Ending with PPL=31.932313677880646 (duration: 0:04:21.102230) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.899 +Metric validation/losses/mim_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.892 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.193. Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.063. Wait count has been increased from 1. +Metric validation/losses/mmm_image_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.891 +Metric validation/losses/mmm_text_loss improved by 0.032 >= min_delta = 0.0. New best score: 3.498 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 10:23:32.286095] Computing e^(tensor([6842.3127], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 10:23:32.291784] Ending with PPL=30.604783719917975 (duration: 0:04:13.537231) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.865 +Metric validation/losses/mim_loss improved by 0.017 >= min_delta = 0.0. New best score: 0.875 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.193. Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.063. Wait count has been increased from 2. +Metric validation/losses/mmm_image_loss improved by 0.016 >= min_delta = 0.0. New best score: 0.875 +Metric validation/losses/mmm_text_loss improved by 0.025 >= min_delta = 0.0. New best score: 3.473 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 14:37:41.084455] Computing e^(tensor([6731.0568], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 14:37:41.090205] Ending with PPL=28.948790021646044 (duration: 0:04:07.361812) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.829 +Metric validation/losses/mim_loss improved by 0.010 >= min_delta = 0.0. New best score: 0.865 +Monitored metric validation/losses/itm_loss did not improve since the last 4 records. Best score: 0.193. Wait count has been increased from 3. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 2.063. Wait count has been increased from 3. +Metric validation/losses/mmm_image_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.868 +Metric validation/losses/mmm_text_loss improved by 0.026 >= min_delta = 0.0. New best score: 3.447 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 17:41:36.554814] Computing e^(tensor([6655.1262], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 17:41:36.560562] Ending with PPL=27.87034135727125 (duration: 0:04:13.319526) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.802 +Metric validation/losses/mim_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.857 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.193. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.063. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.014 >= min_delta = 0.0. New best score: 0.854 +Metric validation/losses/mmm_text_loss improved by 0.030 >= min_delta = 0.0. New best score: 3.417 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 22:07:49.750220] Computing e^(tensor([6568.2552], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 22:07:49.755913] Ending with PPL=26.685694154329152 (duration: 0:04:18.267667) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.771 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.855 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.193. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.063. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.854 +Metric validation/losses/mmm_text_loss improved by 0.074 >= min_delta = 0.0. New best score: 3.344 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 02:23:55.858924] Computing e^(tensor([6486.7681], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 02:23:55.864625] Ending with PPL=25.620276080102872 (duration: 0:04:15.804665) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.737 +Metric validation/losses/mim_loss improved by 0.010 >= min_delta = 0.0. New best score: 0.845 +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.193. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 2.063. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.847 +Metric validation/losses/mmm_text_loss improved by 0.029 >= min_delta = 0.0. New best score: 3.315 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 06:30:31.601902] Computing e^(tensor([6448.7684], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 06:30:31.607622] Ending with PPL=25.138088792151404 (duration: 0:04:16.278286) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.718 +Metric validation/losses/mim_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.837 +Monitored metric validation/losses/itm_loss did not improve in the last 8 records. Best score: 0.193. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 8 records. Best score: 2.063. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.010 >= min_delta = 0.0. New best score: 0.837 +Metric validation/losses/mmm_text_loss improved by 0.026 >= min_delta = 0.0. New best score: 3.289 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 10:24:53.217959] Computing e^(tensor([6395.8985], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 10:24:53.223716] Ending with PPL=24.48227122115072 (duration: 0:04:09.704658) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.695 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.835 +Monitored metric validation/losses/itm_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.837. Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.012 >= min_delta = 0.0. New best score: 3.277 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 14:20:14.284109] Computing e^(tensor([6367.4875], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 14:20:14.289834] Ending with PPL=24.13694740417408 (duration: 0:04:16.602798) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.680 +Metric validation/losses/mim_loss improved by 0.011 >= min_delta = 0.0. New best score: 0.824 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.193. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.063. Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.014 >= min_delta = 0.0. New best score: 0.823 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.277. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 18:11:32.059232] Computing e^(tensor([6267.4796], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 18:11:32.064888] Ending with PPL=22.959683731106107 (duration: 0:04:15.548873) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.617 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.824. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.193. Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.063. Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.823. Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.043 >= min_delta = 0.0. New best score: 3.235 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 22:17:36.547931] Computing e^(tensor([6217.5622], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 22:17:36.553733] Ending with PPL=22.39373241047046 (duration: 0:04:06.619996) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.607 +Metric validation/losses/mim_loss improved by 0.010 >= min_delta = 0.0. New best score: 0.814 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.193. Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.063. Wait count has been increased from 2. +Metric validation/losses/mmm_image_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.817 +Metric validation/losses/mmm_text_loss improved by 0.002 >= min_delta = 0.0. New best score: 3.233 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 02:22:18.566295] Computing e^(tensor([6167.7779], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 02:22:18.572020] Ending with PPL=21.84318440560706 (duration: 0:04:11.426727) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.573 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.813 +Monitored metric validation/losses/itm_loss did not improve since the last 4 records. Best score: 0.193. Wait count has been increased from 3. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 2.063. Wait count has been increased from 3. +Metric validation/losses/mmm_image_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.813 +Metric validation/losses/mmm_text_loss improved by 0.003 >= min_delta = 0.0. New best score: 3.230 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 06:27:20.332288] Computing e^(tensor([6127.8217], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 06:27:20.338036] Ending with PPL=21.411129107583044 (duration: 0:04:12.115712) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.192 +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.063. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.812 +Metric validation/losses/mmm_text_loss improved by 0.004 >= min_delta = 0.0. New best score: 3.226 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 10:18:00.909271] Computing e^(tensor([6091.8021], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 10:18:00.914904] Ending with PPL=21.028970554782074 (duration: 0:04:12.438996) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.551 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 0.813. Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.192. Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.063. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.812. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.226. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 14:13:53.980905] Computing e^(tensor([6032.1214], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 14:13:53.986589] Ending with PPL=20.410729133342457 (duration: 0:04:12.541901) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.507 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.807 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.192. Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 2.063. Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.806 +Metric validation/losses/mmm_text_loss improved by 0.070 >= min_delta = 0.0. New best score: 3.157 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 18:27:57.703607] Computing e^(tensor([5996.7520], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 18:27:57.709359] Ending with PPL=20.052944963904583 (duration: 0:04:08.666929) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.505 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.807. Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.192. Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 8 records. Best score: 2.063. Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.806. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.157. Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 23:29:35.588428] Computing e^(tensor([5938.6619], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 23:29:35.594141] Ending with PPL=19.478883210627863 (duration: 0:04:13.174200) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.481 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 0.807. Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve since the last 4 records. Best score: 0.192. Wait count has been increased from 3. +Monitored metric validation/losses/global_contrastive_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.806. Wait count has been increased from 1. +Metric validation/losses/mmm_text_loss improved by 0.049 >= min_delta = 0.0. New best score: 3.108 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 04:26:41.749519] Computing e^(tensor([5949.9376], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 04:26:41.755258] Ending with PPL=19.58901173940465 (duration: 0:04:12.476492) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.473 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 0.807. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.192. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.063. Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.806. Wait count has been increased from 2. +Metric validation/losses/mmm_text_loss improved by 0.009 >= min_delta = 0.0. New best score: 3.099 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 09:21:23.574399] Computing e^(tensor([5900.7044], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 09:21:23.580229] Ending with PPL=19.112684084767825 (duration: 0:04:14.437471) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.466 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 0.807. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.192. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.063. Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 4 records. Best score: 0.806. Wait count has been increased from 3. +Metric validation/losses/mmm_text_loss improved by 0.053 >= min_delta = 0.0. New best score: 3.046 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 14:16:35.600703] Computing e^(tensor([5864.9451], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 14:16:35.609254] Ending with PPL=18.773992919482 (duration: 0:04:10.492683) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.858 +Metric validation/losses/mmm_image_loss improved by 0.022 >= min_delta = 0.0. New best score: 0.864 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.204 (now 3.275930404663086). Wait count has been increased from 0. +Metric validation/losses/itm_loss improved by 0.026 >= min_delta = 0.0. New best score: 0.237 +Metric validation/losses/global_contrastive_loss improved by 0.151 >= min_delta = 0.0. New best score: 2.315 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 05:38:59.892759] Computing e^(tensor([6095.4598], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 05:38:59.898355] Ending with PPL=21.067465003166106 (duration: 0:07:18.473729) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.264 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 10:32:42.652557] Computing e^(tensor([6167.8271], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 10:32:42.658214] Ending with PPL=21.843722163440486 (duration: 0:04:15.662066) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.860 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.204 (now 3.2077882289886475). Wait count has been increased from 2. +Metric validation/losses/itm_loss improved by 0.011 >= min_delta = 0.0. New best score: 0.227 +Metric validation/losses/global_contrastive_loss improved by 0.044 >= min_delta = 0.0. New best score: 2.220 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 15:08:42.396968] Computing e^(tensor([6185.9993], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 15:08:42.402648] Ending with PPL=22.04310076900048 (duration: 0:04:12.434547) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.852 +Metric validation/losses/mmm_image_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.853 +Metric validation/losses/mmm_text_loss improved by 0.013 >= min_delta = 0.0. New best score: 3.191 +Metric validation/losses/itm_loss improved by 0.011 >= min_delta = 0.0. New best score: 0.216 +Metric validation/losses/global_contrastive_loss improved by 0.062 >= min_delta = 0.0. New best score: 2.158 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 19:27:55.549419] Computing e^(tensor([6179.6191], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 19:27:55.555222] Ending with PPL=21.972892692432296 (duration: 0:04:11.788261) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.851 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.853 (now 0.853738009929657). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.191 (now 3.2038707733154297). Wait count has been increased from 0. +Metric validation/losses/itm_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.210 +Metric validation/losses/global_contrastive_loss improved by 0.024 >= min_delta = 0.0. New best score: 2.134 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 23:37:26.674549] Computing e^(tensor([6162.0997], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 23:37:26.680211] Ending with PPL=21.781257878025038 (duration: 0:04:08.766250) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.086 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 03:50:37.771778] Computing e^(tensor([6128.9863], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 03:50:37.777469] Ending with PPL=21.423601026450218 (duration: 0:04:12.169422) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.851 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.191 (now 3.1971793174743652). Wait count has been increased from 2. +Metric validation/losses/itm_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.205 +Metric validation/losses/global_contrastive_loss improved by 0.033 >= min_delta = 0.0. New best score: 2.053 +Starting validation +2023-08-24 04:19:36.656823 Preparing data for datamodule: VLDataModule +2023-08-24 04:19:36.657080 Preparing data for datamodule: ImageDataModule +2023-08-24 04:19:36.657254 Preparing data for datamodule: MLMDataModule +2023-08-24 04:19:36.657701 Setting up datamodule: VLDataModule +2023-08-24 04:21:40.437839 Setting up datamodule: ImageDataModule +2023-08-24 04:23:40.410809 Setting up datamodule: MLMDataModule +2023-08-24 04:25:36.698620 Building val dataloader for datamodule: VLDataModule +2023-08-24 04:25:36.699171 Building val dataloader for datamodule: ImageDataModule +2023-08-24 04:25:36.699475 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 04:29:22.083613] Computing e^(tensor([6128.9863], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 04:29:22.092122] Ending with PPL=21.423601026450218 (duration: 0:03:30.649167) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.838 +Metric validation/losses/mmm_text_loss improved by 0.026 >= min_delta = 0.0. New best score: 3.369 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.214 (now 0.22135213017463684). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.167 (now 2.2502083778381348). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 06:08:02.227594] Computing e^(tensor([6041.8921], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 06:08:02.233328] Ending with PPL=20.510686993712557 (duration: 0:04:10.898138) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.522 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.829 +Metric validation/losses/mmm_image_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.832 +Metric validation/losses/mmm_text_loss improved by 0.031 >= min_delta = 0.0. New best score: 3.339 +Metric validation/losses/itm_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.214 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.167 (now 2.2701919078826904). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 10:52:24.654614] Computing e^(tensor([5978.9241], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 10:52:24.660203] Ending with PPL=19.874987989349492 (duration: 0:04:07.102960) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.490 +Metric validation/losses/mim_loss improved by 0.015 >= min_delta = 0.0. New best score: 0.814 +Metric validation/losses/mmm_image_loss improved by 0.019 >= min_delta = 0.0. New best score: 0.813 +Metric validation/losses/mmm_text_loss improved by 0.046 >= min_delta = 0.0. New best score: 3.293 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.214 (now 0.21730507910251617). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.167 (now 2.1960630416870117). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 15:43:55.487617] Computing e^(tensor([5918.0771], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 15:43:55.493349] Ending with PPL=19.27942667674679 (duration: 0:04:08.251682) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.466 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.809 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.811 +Metric validation/losses/mmm_text_loss improved by 0.049 >= min_delta = 0.0. New best score: 3.244 +Metric validation/losses/itm_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.212 +Metric validation/losses/global_contrastive_loss improved by 0.007 >= min_delta = 0.0. New best score: 2.160 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 20:52:48.161069] Computing e^(tensor([5910.1822], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 20:52:48.166764] Ending with PPL=19.203471957268793 (duration: 0:04:09.327607) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.456 +Metric validation/losses/mim_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.809 +Metric validation/losses/mmm_image_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.811 +Metric validation/losses/mmm_text_loss improved by 0.009 >= min_delta = 0.0. New best score: 3.235 +Metric validation/losses/itm_loss improved by 0.010 >= min_delta = 0.0. New best score: 0.202 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.160 (now 2.2237460613250732). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 02:01:42.728712] Computing e^(tensor([5860.8479], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 02:01:42.734257] Ending with PPL=18.735571744015587 (duration: 0:04:45.957962) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.440 +Metric validation/losses/mim_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.805 +Metric validation/losses/mmm_image_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.806 +Metric validation/losses/mmm_text_loss improved by 0.020 >= min_delta = 0.0. New best score: 3.215 +Metric validation/losses/itm_loss improved by 0.010 >= min_delta = 0.0. New best score: 0.192 +Metric validation/losses/global_contrastive_loss improved by 0.199 >= min_delta = 0.0. New best score: 1.961 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 07:06:50.261815] Computing e^(tensor([5827.3701], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 07:06:50.267576] Ending with PPL=18.424568634905313 (duration: 0:04:08.790352) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.421 +Metric validation/losses/mim_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.801 +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.801 +Metric validation/losses/mmm_text_loss improved by 0.039 >= min_delta = 0.0. New best score: 3.175 +Metric validation/losses/itm_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.192 +Metric validation/losses/global_contrastive_loss improved by 0.028 >= min_delta = 0.0. New best score: 1.933 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 12:09:04.715788] Computing e^(tensor([5812.6450], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 12:09:04.721533] Ending with PPL=18.289415573255468 (duration: 0:04:06.691101) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.412 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.801 (now 0.8048014640808105). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.801 (now 0.8053436875343323). Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.004 >= min_delta = 0.0. New best score: 3.171 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.192 (now 0.19565017521381378). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 1.933 (now 1.952521562576294). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 17:12:41.693427] Computing e^(tensor([5768.6419], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 17:12:41.699170] Ending with PPL=17.8914141946102 (duration: 0:04:10.150030) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.400 +Metric validation/losses/mim_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.796 +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.796 +Metric validation/losses/mmm_text_loss improved by 0.036 >= min_delta = 0.0. New best score: 3.135 +Metric validation/losses/itm_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.191 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 1.933 (now 2.0757205486297607). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 21:36:06.219924] Computing e^(tensor([5760.5935], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 21:36:06.225594] Ending with PPL=17.819560153622877 (duration: 0:05:28.679021) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.389 +Metric validation/losses/mim_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.794 +Metric validation/losses/mmm_image_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.793 +Metric validation/losses/mmm_text_loss improved by 0.001 >= min_delta = 0.0. New best score: 3.134 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.191 (now 0.20545783638954163). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 1.933 (now 2.0911741256713867). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 02:07:15.519209] Computing e^(tensor([5711.7941], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 02:07:15.524781] Ending with PPL=17.390030253965726 (duration: 0:04:16.784835) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.379 +Metric validation/losses/mim_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.791 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.793 (now 0.7945088148117065). Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.007 >= min_delta = 0.0. New best score: 3.127 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.191 (now 0.19637709856033325). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 1.933 (now 2.061845302581787). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 06:33:25.576600] Computing e^(tensor([5675.1643], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 06:33:25.582217] Ending with PPL=17.074432055433487 (duration: 0:04:16.435855) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.370 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.788 +Metric validation/losses/mmm_image_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.786 +Metric validation/losses/mmm_text_loss improved by 0.020 >= min_delta = 0.0. New best score: 3.106 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.191 (now 0.19603154063224792). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 1.933 (now 2.061845302581787). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 10:49:28.682041] Computing e^(tensor([5634.5422], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 10:49:28.687691] Ending with PPL=16.731131080260816 (duration: 0:04:31.227213) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.359 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.787 +Metric validation/losses/mmm_image_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.786 +Metric validation/losses/mmm_text_loss improved by 0.016 >= min_delta = 0.0. New best score: 3.091 +Metric validation/losses/itm_loss improved by 0.011 >= min_delta = 0.0. New best score: 0.181 +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 1.933 (now 2.061845302581787). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 15:12:47.944619] Computing e^(tensor([5625.4137], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 15:12:47.950217] Ending with PPL=16.654939293756406 (duration: 0:07:49.439908) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.346 +Metric validation/losses/mim_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.783 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.784 +Metric validation/losses/mmm_text_loss improved by 0.011 >= min_delta = 0.0. New best score: 3.079 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.181 (now 0.1853049099445343). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 19:43:38.964619] Computing e^(tensor([5736.3065], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 19:43:38.970223] Ending with PPL=17.604477362869073 (duration: 0:04:23.585132) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.043 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.181 (now 0.19220036268234253). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 1.933 (now 2.0850794315338135). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 04:28:55.386498] Computing e^(tensor([5647.8846], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 04:28:55.392154] Ending with PPL=16.843120910840085 (duration: 0:04:22.640886) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.333 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 0.783 (now 0.7971897125244141). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.784 (now 0.7957203388214111). Wait count has been increased from 2. +Metric validation/losses/mmm_text_loss improved by 0.004 >= min_delta = 0.0. New best score: 3.039 +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.181 (now 0.18371324241161346). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 1.933 (now 2.051375150680542). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 08:55:40.361763] Computing e^(tensor([5604.7016], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 08:55:40.367406] Ending with PPL=16.483350629250534 (duration: 0:05:50.539556) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.311 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 0.783 (now 0.7905482053756714). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 4 records. Best score: 0.784 (now 0.7878834009170532). Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.069 >= min_delta = 0.0. New best score: 2.970 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.181 (now 0.18371324241161346). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 1.933 (now 2.4245784282684326). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 13:07:26.257420] Computing e^(tensor([5561.1383], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 13:07:26.263110] Ending with PPL=16.12819798524996 (duration: 0:04:22.186166) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.308 +Metric validation/losses/mim_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.783 +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 0.784 (now 0.7878834009170532). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.970 (now 2.984549045562744). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.181 (now 0.18371324241161346). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 1.933 (now 2.4245784282684326). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 17:27:02.682959] Computing e^(tensor([5510.4488], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 17:27:02.688701] Ending with PPL=15.724568883764146 (duration: 0:04:51.519011) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.275 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.782 +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 0.784 (now 0.7878834009170532). Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.040 >= min_delta = 0.0. New best score: 2.930 +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 1.933 (now 2.4245784282684326). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 21:42:42.893933] Computing e^(tensor([5523.3051], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 21:42:42.899744] Ending with PPL=15.825974534681208 (duration: 0:04:28.915686) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.825 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.831 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.176 (now 3.1900746822357178). Wait count has been increased from 0. +Metric validation/losses/itm_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.218 +Metric validation/losses/global_contrastive_loss improved by 0.119 >= min_delta = 0.0. New best score: 2.230 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 09:11:18.479915] Computing e^(tensor([5803.4189], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 09:11:18.486020] Ending with PPL=18.20524014053406 (duration: 0:04:42.304130) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.397 +Metric validation/losses/mim_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.818 +Metric validation/losses/mmm_image_loss improved by 0.011 >= min_delta = 0.0. New best score: 0.820 +Metric validation/losses/mmm_text_loss improved by 0.036 >= min_delta = 0.0. New best score: 3.140 +Metric validation/losses/itm_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.209 +Metric validation/losses/global_contrastive_loss improved by 0.005 >= min_delta = 0.0. New best score: 2.224 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 13:37:06.619416] Computing e^(tensor([5769.4124], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 13:37:06.625144] Ending with PPL=17.89830798407429 (duration: 0:04:12.764070) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.387 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.818 (now 0.8201166987419128). Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.819 +Metric validation/losses/mmm_text_loss improved by 0.011 >= min_delta = 0.0. New best score: 3.129 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.209 (now 0.21092838048934937). Wait count has been increased from 0. +Metric validation/losses/global_contrastive_loss improved by 0.068 >= min_delta = 0.0. New best score: 2.156 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 18:56:20.945213] Computing e^(tensor([5763.3988], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 18:56:20.950912] Ending with PPL=17.844572909701164 (duration: 0:04:10.439305) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.380 +Metric validation/losses/mim_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.809 +Metric validation/losses/mmm_image_loss improved by 0.010 >= min_delta = 0.0. New best score: 0.810 +Metric validation/losses/mmm_text_loss improved by 0.016 >= min_delta = 0.0. New best score: 3.113 +Metric validation/losses/itm_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.201 +Metric validation/losses/global_contrastive_loss improved by 0.043 >= min_delta = 0.0. New best score: 2.113 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 00:30:30.666975] Computing e^(tensor([5755.4475], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 00:30:30.672739] Ending with PPL=17.773769691811683 (duration: 0:04:24.465551) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.369 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.809 (now 0.8157520294189453). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.810 (now 0.8173198103904724). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.113 (now 3.125105619430542). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.201 (now 0.2015400528907776). Wait count has been increased from 0. +Metric validation/losses/global_contrastive_loss improved by 0.025 >= min_delta = 0.0. New best score: 2.088 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 06:02:17.683160] Computing e^(tensor([5710.4369], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 06:02:17.689025] Ending with PPL=17.378233070618073 (duration: 0:04:15.785271) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.356 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 0.809 (now 0.8106369972229004). Wait count has been increased from 1. +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.808 +Metric validation/losses/mmm_text_loss improved by 0.031 >= min_delta = 0.0. New best score: 3.082 +Metric validation/losses/itm_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.192 +Metric validation/losses/global_contrastive_loss improved by 0.086 >= min_delta = 0.0. New best score: 2.002 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 11:27:09.528946] Computing e^(tensor([5695.9101], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 11:27:09.534932] Ending with PPL=17.252465401957235 (duration: 0:04:11.000418) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.342 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 0.809 (now 0.8090618252754211). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.808 (now 0.8095826506614685). Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.013 >= min_delta = 0.0. New best score: 3.070 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.192 (now 0.19432321190834045). Wait count has been increased from 0. +Metric validation/losses/global_contrastive_loss improved by 0.000 >= min_delta = 0.0. New best score: 2.002 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 16:35:36.392499] Computing e^(tensor([5652.1914], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 16:35:36.398423] Ending with PPL=16.879429596409217 (duration: 0:05:06.945070) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.339 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.807 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.806 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.070 (now 3.070329427719116). Wait count has been increased from 0. +Metric validation/losses/itm_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.190 +Metric validation/losses/global_contrastive_loss improved by 0.028 >= min_delta = 0.0. New best score: 1.974 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 21:30:08.874527] Computing e^(tensor([5632.9313], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 21:30:08.880438] Ending with PPL=16.7176604752886 (duration: 0:04:12.419135) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.329 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.802 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.804 +Metric validation/losses/mmm_text_loss improved by 0.040 >= min_delta = 0.0. New best score: 3.030 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.190 (now 0.20896200835704803). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 1.974 (now 2.134364604949951). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-01 02:03:21.334369] Computing e^(tensor([5576.7667], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-01 02:03:21.340109] Ending with PPL=16.254720183559918 (duration: 0:04:09.328979) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.302 +Metric validation/losses/mim_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.794 +Metric validation/losses/mmm_image_loss improved by 0.011 >= min_delta = 0.0. New best score: 0.793 +Metric validation/losses/mmm_text_loss improved by 0.013 >= min_delta = 0.0. New best score: 3.017 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.190 (now 0.19527816772460938). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 1.974 (now 2.0809268951416016). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-01 05:51:21.479169] Computing e^(tensor([5527.8346], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-01 05:51:21.485058] Ending with PPL=15.86185687193826 (duration: 0:04:10.390099) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.276 +Metric validation/losses/mim_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.786 +Metric validation/losses/mmm_image_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.789 +Metric validation/losses/mmm_text_loss improved by 0.003 >= min_delta = 0.0. New best score: 3.014 +Metric validation/losses/itm_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.186 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 1.974 (now 2.0339603424072266). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-01 09:33:08.695479] Computing e^(tensor([5500.1055], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-01 09:33:08.701199] Ending with PPL=15.643456692053606 (duration: 0:04:38.258932) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.257 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.784 +Metric validation/losses/mmm_image_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.781 +Metric validation/losses/mmm_text_loss improved by 0.004 >= min_delta = 0.0. New best score: 3.010 +Metric validation/losses/itm_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.186 +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 1.974 (now 2.053138494491577). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-01 13:25:44.337819] Computing e^(tensor([5430.2406], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-01 13:25:44.343703] Ending with PPL=15.106427126514458 (duration: 0:04:17.127578) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.255 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.783 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.781 (now 0.7845971584320068). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.010 (now 3.016385078430176). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.186 (now 0.20268356800079346). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 1.974 (now 2.053138494491577). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-01 17:13:10.118638] Computing e^(tensor([5388.7190], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-01 17:13:10.124511] Ending with PPL=14.79603871792976 (duration: 0:04:09.978545) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.231 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.778 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.779 +Metric validation/losses/mmm_text_loss improved by 0.055 >= min_delta = 0.0. New best score: 2.955 +Metric validation/losses/itm_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.185 +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 1.974 (now 2.053138494491577). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-01 21:08:06.448198] Computing e^(tensor([5371.3375], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-01 21:08:06.452459] Ending with PPL=14.668007443979633 (duration: 0:04:11.414561) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.218 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.777 +Metric validation/losses/mmm_image_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.778 +Metric validation/losses/mmm_text_loss improved by 0.007 >= min_delta = 0.0. New best score: 2.948 +Metric validation/losses/itm_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.181 +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-02 00:55:58.528213] Computing e^(tensor([5483.2590], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-02 00:55:58.534401] Ending with PPL=15.512241423036079 (duration: 0:04:12.686099) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.948 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.181 (now 0.19328585267066956). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 1.974 (now 2.0815629959106445). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-02 10:09:18.302392] Computing e^(tensor([5392.2284], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-02 10:09:18.306640] Ending with PPL=14.822024042513254 (duration: 0:04:14.476619) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.215 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 0.777 (now 0.782050371170044). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.778 (now 0.7795664668083191). Wait count has been increased from 2. +Metric validation/losses/mmm_text_loss improved by 0.013 >= min_delta = 0.0. New best score: 2.935 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.181 (now 0.2027672976255417). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 1.974 (now 2.243521213531494). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-02 14:20:39.100440] Computing e^(tensor([5373.7796], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-02 14:20:39.106312] Ending with PPL=14.685928439725117 (duration: 0:04:16.350468) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.208 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 0.777 (now 0.7778112888336182). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 4 records. Best score: 0.778 (now 0.7786700129508972). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.935 (now 2.941282272338867). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.181 (now 0.19442035257816315). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 1.974 (now 2.2434091567993164). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-02 18:52:28.779393] Computing e^(tensor([5309.8174], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-02 18:52:28.785465] Ending with PPL=14.223687114287314 (duration: 0:04:15.811394) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.187 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.775 +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 0.778 (now 0.7786700129508972). Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.090 >= min_delta = 0.0. New best score: 2.845 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.181 (now 0.19442035257816315). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 1.974 (now 2.2434091567993164). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-02 23:36:33.556509] Computing e^(tensor([5286.5222], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-02 23:36:33.562310] Ending with PPL=14.05897654801943 (duration: 0:04:12.148028) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.187 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.775 (now 0.7764624953269958). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 0.778 (now 0.7786700129508972). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.845 (now 2.846285820007324). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.181 (now 0.19442035257816315). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 1.974 (now 2.2434091567993164). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-03 04:21:23.644746] Computing e^(tensor([5268.8432], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-03 04:21:23.650488] Ending with PPL=13.935250094260628 (duration: 0:04:10.937429) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.181 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.774 +Monitored metric validation/losses/mmm_image_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Metric validation/losses/mmm_text_loss improved by 0.025 >= min_delta = 0.0. New best score: 2.820 +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-09-03 08:57:40.635279] Computing e^(tensor([5481.1138], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-09-03 08:57:40.641047] Ending with PPL=15.495612280951201 (duration: 0:04:13.234682) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.448 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 13:33:13.705835] Computing e^(tensor([5797.6025], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 13:33:13.711104] Ending with PPL=18.152372336883413 (duration: 0:04:06.625458) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.419 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 14:40:23.834766] Computing e^(tensor([5748.7880], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 14:40:23.840083] Ending with PPL=17.714685935014472 (duration: 0:04:07.686097) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.393 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 15:50:00.725552] Computing e^(tensor([5708.4172], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 15:50:00.730856] Ending with PPL=17.360692279081878 (duration: 0:04:11.000440) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.376 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 17:06:01.577975] Computing e^(tensor([5671.3757], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 17:06:01.583349] Ending with PPL=17.04211891949142 (duration: 0:04:06.996822) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.351 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 18:24:19.144117] Computing e^(tensor([5622.1666], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 18:24:19.149432] Ending with PPL=16.62792145416098 (duration: 0:04:10.250881) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.337 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 19:47:12.311438] Computing e^(tensor([5586.3956], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 19:47:12.316683] Ending with PPL=16.33316618694188 (duration: 0:04:06.529068) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.317 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 21:10:58.025457] Computing e^(tensor([5512.2741], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 21:10:58.030875] Ending with PPL=15.738926485914865 (duration: 0:04:05.244375) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.299 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-18 22:32:23.775011] Computing e^(tensor([5480.0492], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-18 22:32:23.780279] Ending with PPL=15.487366238584793 (duration: 0:04:04.702080) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.279 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 00:00:19.336860] Computing e^(tensor([5460.4286], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 00:00:19.342238] Ending with PPL=15.336172955392422 (duration: 0:04:05.950853) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.264 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 01:22:14.909715] Computing e^(tensor([5424.6010], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 01:22:14.915004] Ending with PPL=15.063890371147805 (duration: 0:04:04.558032) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.248 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 02:48:19.012416] Computing e^(tensor([5373.3947], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 02:48:19.020031] Ending with PPL=14.683102694863766 (duration: 0:04:11.139082) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.232 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 04:11:27.735574] Computing e^(tensor([5347.0620], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 04:11:27.740821] Ending with PPL=14.491046686704946 (duration: 0:04:06.528941) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.219 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 05:41:12.705006] Computing e^(tensor([5312.9766], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 05:41:12.710192] Ending with PPL=14.246172678332353 (duration: 0:04:08.745970) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.203 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 07:01:39.639585] Computing e^(tensor([5289.0615], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 07:01:39.645001] Ending with PPL=14.076838258265582 (duration: 0:04:04.903897) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.180 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 08:20:11.424634] Computing e^(tensor([5256.1836], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 08:20:11.429842] Ending with PPL=13.84732142111972 (duration: 0:04:08.907348) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.174 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 09:35:34.982295] Computing e^(tensor([5217.1983], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 09:35:34.987644] Ending with PPL=13.580014028820898 (duration: 0:04:07.126162) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.155 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 10:50:50.206175] Computing e^(tensor([5204.1837], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 10:50:50.211469] Ending with PPL=13.491931685934745 (duration: 0:04:03.309598) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.152 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 12:12:58.707116] Computing e^(tensor([5186.8826], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 12:12:58.712506] Ending with PPL=13.375722621504986 (duration: 0:04:08.043749) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.136 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 13:34:56.547710] Computing e^(tensor([5163.0370], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 13:34:56.553181] Ending with PPL=13.217193373211165 (duration: 0:04:06.506829) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.123 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 14:50:58.651919] Computing e^(tensor([5142.3186], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 14:50:58.657452] Ending with PPL=13.080980704764924 (duration: 0:04:12.355856) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.119 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 15:53:58.350763] Computing e^(tensor([5110.0834], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 15:53:58.356329] Ending with PPL=12.871836164453207 (duration: 0:04:06.401250) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.111 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 16:56:56.054557] Computing e^(tensor([5097.5775], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 16:56:56.060169] Ending with PPL=12.791600742074502 (duration: 0:04:08.100344) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.098 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 18:08:24.364438] Computing e^(tensor([5078.5512], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 18:08:24.369695] Ending with PPL=12.670488977911505 (duration: 0:04:10.742021) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.087 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 19:23:03.243850] Computing e^(tensor([5057.3129], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 19:23:03.249091] Ending with PPL=12.536650948872797 (duration: 0:04:14.751701) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.082 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 20:37:54.294073] Computing e^(tensor([5041.1628], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 20:37:54.299353] Ending with PPL=12.435824802374572 (duration: 0:04:06.727012) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.074 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 21:55:30.644461] Computing e^(tensor([5025.7633], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 21:55:30.649661] Ending with PPL=12.340439492237719 (duration: 0:04:07.563573) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.060 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-19 23:06:47.273268] Computing e^(tensor([5002.3437], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-19 23:06:47.278646] Ending with PPL=12.19677837671533 (duration: 0:04:06.342370) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.055 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 00:15:51.125971] Computing e^(tensor([4992.7054], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 00:15:51.131185] Ending with PPL=12.138141607919353 (duration: 0:04:11.049304) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.043 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 01:34:08.844610] Computing e^(tensor([4974.9525], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 01:34:08.849927] Ending with PPL=12.030874818077981 (duration: 0:04:06.050276) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.035 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 02:45:19.491691] Computing e^(tensor([4960.1071], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 02:45:19.496933] Ending with PPL=11.941903627599615 (duration: 0:04:04.453264) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.033 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 03:57:01.738518] Computing e^(tensor([4944.9001], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 03:57:01.743869] Ending with PPL=11.851448126676102 (duration: 0:04:08.807739) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.019 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 05:16:52.630941] Computing e^(tensor([4930.8073], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 05:16:52.636143] Ending with PPL=11.768231385438824 (duration: 0:04:04.784745) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.018 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 06:28:27.656713] Computing e^(tensor([4914.3553], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 06:28:27.661985] Ending with PPL=11.67182326125226 (duration: 0:04:12.646386) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.010 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 07:39:01.427485] Computing e^(tensor([4885.4845], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 07:39:01.432809] Ending with PPL=11.504545629500143 (duration: 0:06:01.725028) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.997 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 08:48:22.195112] Computing e^(tensor([4888.3414], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 08:48:22.200560] Ending with PPL=11.520991394215201 (duration: 0:04:11.536302) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.987 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 11:04:06.710782] Computing e^(tensor([4864.2305], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 11:04:06.716043] Ending with PPL=11.382934502703131 (duration: 0:04:06.550594) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.984 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 13:24:25.027175] Computing e^(tensor([4832.4234], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 13:24:25.032575] Ending with PPL=11.20333733326268 (duration: 0:04:13.794903) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.976 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 14:28:29.424776] Computing e^(tensor([4820.2656], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 14:28:29.430329] Ending with PPL=11.135439599945297 (duration: 0:04:14.160233) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.970 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 15:35:23.684402] Computing e^(tensor([4818.4150], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 15:35:23.689681] Ending with PPL=11.125140740177013 (duration: 0:04:41.968019) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.963 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 16:42:30.939002] Computing e^(tensor([4797.1808], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 16:42:30.944305] Ending with PPL=11.007649230493227 (duration: 0:04:08.643158) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.954 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 17:55:09.660446] Computing e^(tensor([4786.3808], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 17:55:09.665740] Ending with PPL=10.948368142816838 (duration: 0:04:06.614415) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.945 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 20:15:08.123801] Computing e^(tensor([4775.0292], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 20:15:08.129006] Ending with PPL=10.886403411120844 (duration: 0:04:08.468338) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.938 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 21:31:55.055490] Computing e^(tensor([4766.9564], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 21:31:55.060825] Ending with PPL=10.842550071278994 (duration: 0:04:11.926519) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.937 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 22:49:03.014312] Computing e^(tensor([4752.3625], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 22:49:03.019517] Ending with PPL=10.76372055420455 (duration: 0:04:09.794846) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.936 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-20 23:56:46.369425] Computing e^(tensor([4750.1119], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-20 23:56:46.374798] Ending with PPL=10.751614641610134 (duration: 0:04:32.147155) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.925 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 01:11:27.366170] Computing e^(tensor([4731.4294], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 01:11:27.371391] Ending with PPL=10.651648659543163 (duration: 0:04:05.715039) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.921 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 02:21:05.265962] Computing e^(tensor([4743.1062], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 02:21:05.271251] Ending with PPL=10.714019460153027 (duration: 0:04:05.622948) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.920 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 03:32:57.568633] Computing e^(tensor([4716.2548], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 03:32:57.573935] Ending with PPL=10.571137212189424 (duration: 0:04:11.295268) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.917 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 04:39:41.900103] Computing e^(tensor([4705.4605], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 04:39:41.905448] Ending with PPL=10.514236864280958 (duration: 0:04:04.606835) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.907 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 05:48:37.315836] Computing e^(tensor([4705.8025], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 05:48:37.321194] Ending with PPL=10.516035176027536 (duration: 0:04:08.896878) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.904 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 06:58:21.653654] Computing e^(tensor([4710.9679], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 06:58:21.659071] Ending with PPL=10.54323001880966 (duration: 0:04:05.823238) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.902 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 08:08:18.344001] Computing e^(tensor([4684.3312], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 08:08:18.349488] Ending with PPL=10.403742281571613 (duration: 0:04:31.605361) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.900 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 09:19:51.954576] Computing e^(tensor([4672.3352], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 09:19:51.960002] Ending with PPL=10.341527471432507 (duration: 0:04:12.107120) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.893 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 11:48:13.080653] Computing e^(tensor([4660.0735], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 11:48:13.086040] Ending with PPL=10.278319034532913 (duration: 0:04:08.246148) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.885 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 13:00:25.285459] Computing e^(tensor([4656.7201], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 13:00:25.290933] Ending with PPL=10.261099953637759 (duration: 0:04:09.198197) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.882 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 14:09:16.679404] Computing e^(tensor([4656.4192], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 14:09:16.684718] Ending with PPL=10.25955661991089 (duration: 0:04:11.082532) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.876 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 16:30:01.795293] Computing e^(tensor([4643.3288], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 16:30:01.800568] Ending with PPL=10.192624682703888 (duration: 0:04:08.363543) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.870 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 18:59:54.824218] Computing e^(tensor([4617.1822], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 18:59:54.829558] Ending with PPL=10.060240969209659 (duration: 0:04:09.962868) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.868 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 20:18:51.106037] Computing e^(tensor([4611.1823], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 20:18:51.111211] Ending with PPL=10.030105580834839 (duration: 0:04:13.794927) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.861 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 21:34:30.339751] Computing e^(tensor([4604.9462], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 21:34:30.345001] Ending with PPL=9.998880114262002 (duration: 0:04:06.505589) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.857 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 22:56:39.652254] Computing e^(tensor([4591.3892], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 22:56:39.657521] Ending with PPL=9.931331872230903 (duration: 0:04:12.747888) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.856 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 00:14:44.141074] Computing e^(tensor([4589.2305], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 00:14:44.146278] Ending with PPL=9.920618320419045 (duration: 0:04:12.053340) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.846 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 01:33:28.193503] Computing e^(tensor([4573.6026], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 01:33:28.198808] Ending with PPL=9.843401286615727 (duration: 0:04:13.369971) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.845 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 02:54:14.554362] Computing e^(tensor([4576.4268], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 02:54:14.559604] Ending with PPL=9.857310974731506 (duration: 0:04:11.316089) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.839 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 04:12:25.243481] Computing e^(tensor([4571.0249], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 04:12:25.248709] Ending with PPL=9.830722500788653 (duration: 0:04:12.353502) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.837 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 05:28:03.331057] Computing e^(tensor([4561.8707], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 05:28:03.336266] Ending with PPL=9.785829157329488 (duration: 0:04:11.410911) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.833 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 06:43:51.831242] Computing e^(tensor([4567.3987], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 06:43:51.836562] Ending with PPL=9.812914813826726 (duration: 0:04:10.072543) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.830 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 08:04:36.157418] Computing e^(tensor([4539.6419], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 08:04:36.162904] Ending with PPL=9.67766771575397 (duration: 0:04:08.214351) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.829 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 09:23:10.290078] Computing e^(tensor([4530.7885], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 09:23:10.293903] Ending with PPL=9.634922642540829 (duration: 0:04:07.883473) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.820 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 10:44:04.664827] Computing e^(tensor([4511.3006], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 10:44:04.670138] Ending with PPL=9.541496260692693 (duration: 0:04:11.605005) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.817 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 12:12:13.993848] Computing e^(tensor([4511.9349], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 12:12:13.999077] Ending with PPL=9.544522706595622 (duration: 0:04:09.858757) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.812 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 14:01:36.359471] Computing e^(tensor([4502.8925], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 14:01:36.365235] Ending with PPL=9.501467483312117 (duration: 0:04:11.727102) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.810 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 15:38:42.915752] Computing e^(tensor([4496.3426], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 15:38:42.921300] Ending with PPL=9.470401469945948 (duration: 0:04:17.245874) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.805 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 17:15:29.572676] Computing e^(tensor([4501.0965], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 17:15:29.576775] Ending with PPL=9.49293911158625 (duration: 0:05:19.592848) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.804 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 18:40:40.777777] Computing e^(tensor([4495.4599], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 18:40:40.783293] Ending with PPL=9.466222557696021 (duration: 0:04:15.319902) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.802 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 20:13:07.431546] Computing e^(tensor([4491.4211], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 20:13:07.437191] Ending with PPL=9.447125791761222 (duration: 0:04:11.657171) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.796 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 21:48:32.701316] Computing e^(tensor([4469.2196], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 21:48:32.706953] Ending with PPL=9.342835754923493 (duration: 0:04:55.971797) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.791 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 23:25:59.510689] Computing e^(tensor([4478.0459], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 23:25:59.516123] Ending with PPL=9.384158145590694 (duration: 0:04:09.522414) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.791 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 02:38:34.354474] Computing e^(tensor([4455.7269], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 02:38:34.359741] Ending with PPL=9.280017616339524 (duration: 0:04:13.334302) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.777 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 04:20:14.197094] Computing e^(tensor([4459.1773], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 04:20:14.202428] Ending with PPL=9.29604137134959 (duration: 0:04:07.422752) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.776 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 05:49:35.237109] Computing e^(tensor([4457.5102], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 05:49:35.242515] Ending with PPL=9.288295800581354 (duration: 0:04:11.142845) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.773 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 10:50:17.925921] Computing e^(tensor([4443.7614], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 10:50:17.931201] Ending with PPL=9.224663297913382 (duration: 0:04:08.446181) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.769 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 09:04:01.151777] Computing e^(tensor([4435.6536], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 09:04:01.157087] Ending with PPL=9.187343147441634 (duration: 0:04:06.072894) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.768 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 10:17:33.831776] Computing e^(tensor([4411.8024], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 10:17:33.837032] Ending with PPL=9.078429531008014 (duration: 0:04:09.685901) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.766 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 11:27:20.845183] Computing e^(tensor([4413.9332], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 11:27:20.850530] Ending with PPL=9.088106614563467 (duration: 0:04:09.169997) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.756 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 12:35:16.012693] Computing e^(tensor([4407.2476], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 12:35:16.018103] Ending with PPL=9.057777501494582 (duration: 0:04:08.762557) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.753 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 14:29:48.023318] Computing e^(tensor([4384.5908], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 14:29:48.028629] Ending with PPL=8.955746497936126 (duration: 0:04:07.205418) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.750 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 17:59:59.303787] Computing e^(tensor([4377.9996], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 17:59:59.309172] Ending with PPL=8.9262804114866 (duration: 0:04:12.652538) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.748 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 19:29:44.601646] Computing e^(tensor([4365.5540], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 19:29:44.607023] Ending with PPL=8.870906681837365 (duration: 0:04:11.592946) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.744 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 20:48:37.423923] Computing e^(tensor([4374.5143], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 20:48:37.429281] Ending with PPL=8.910738690201045 (duration: 0:04:10.036115) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.742 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 22:13:36.034476] Computing e^(tensor([4361.7591], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 22:13:36.039827] Ending with PPL=8.854090425096933 (duration: 0:04:13.189269) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.732 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 01:11:57.930029] Computing e^(tensor([4356.4631], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 01:11:57.935569] Ending with PPL=8.830675685047595 (duration: 0:04:16.228542) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.730 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 04:01:25.861141] Computing e^(tensor([4358.7021], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 04:01:25.866504] Ending with PPL=8.840567335374116 (duration: 0:04:08.818851) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.727 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 06:48:36.150331] Computing e^(tensor([4343.8268], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 06:48:36.155783] Ending with PPL=8.775058369240591 (duration: 0:04:13.829226) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.724 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 08:17:12.673572] Computing e^(tensor([4324.9447], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 08:17:12.678946] Ending with PPL=8.692602161033825 (duration: 0:04:16.771953) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.721 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 12:41:24.321911] Computing e^(tensor([4330.0550], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 12:41:24.327356] Ending with PPL=8.714841436398697 (duration: 0:04:15.603147) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.720 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 14:13:01.734613] Computing e^(tensor([4327.3482], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 14:13:01.739942] Ending with PPL=8.703054828409858 (duration: 0:04:13.338853) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.717 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 15:46:58.768597] Computing e^(tensor([4316.7414], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 15:46:58.774050] Ending with PPL=8.657021400393226 (duration: 0:04:16.603764) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.715 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 18:53:54.053859] Computing e^(tensor([4312.5284], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 18:53:54.059271] Ending with PPL=8.638804636015111 (duration: 0:04:08.311607) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.712 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 20:29:26.242217] Computing e^(tensor([4303.1128], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 20:29:26.247622] Ending with PPL=8.598230260084058 (duration: 0:04:13.428419) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.707 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 22:07:19.674858] Computing e^(tensor([4307.5211], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 22:07:19.680275] Ending with PPL=8.617202754779155 (duration: 0:04:13.454478) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.700 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 02:51:58.418197] Computing e^(tensor([4293.1118], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 02:51:58.423730] Ending with PPL=8.555342245197496 (duration: 0:04:13.371352) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.698 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 07:33:26.764842] Computing e^(tensor([4288.7298], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 07:33:26.770135] Ending with PPL=8.536617882180927 (duration: 0:04:15.071664) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.691 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 10:44:41.790045] Computing e^(tensor([4288.6244], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 10:44:41.794006] Ending with PPL=8.536168053752988 (duration: 0:04:15.254232) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 8.440 +Metric validation/losses/mim_loss improved by 0.721 >= min_delta = 0.0. New best score: 2.438 +Metric validation/losses/mmm_image_loss improved by 0.952 >= min_delta = 0.0. New best score: 2.297 +Metric validation/losses/mmm_text_loss improved by 0.757 >= min_delta = 0.0. New best score: 9.056 +Metric validation/losses/itm_loss improved by 0.012 >= min_delta = 0.0. New best score: 0.309 +Metric validation/losses/global_contrastive_loss improved by 0.009 >= min_delta = 0.0. New best score: 3.872 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 03:26:37.697882] Computing e^(tensor([15424.1922], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 03:26:37.703478] Ending with PPL=2235.22259407767 (duration: 0:04:12.437987) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 7.481 +Metric validation/losses/mim_loss improved by 0.761 >= min_delta = 0.0. New best score: 1.677 +Metric validation/losses/mmm_image_loss improved by 0.724 >= min_delta = 0.0. New best score: 1.573 +Metric validation/losses/mmm_text_loss improved by 0.807 >= min_delta = 0.0. New best score: 8.249 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.309 (now 0.3412454128265381). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 3.872 (now 3.8771886825561523). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 04:40:22.884149] Computing e^(tensor([15140.7219], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 04:40:22.889734] Ending with PPL=1939.8403664652772 (duration: 0:06:49.837514) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 7.214 +Metric validation/losses/mim_loss improved by 0.434 >= min_delta = 0.0. New best score: 1.244 +Metric validation/losses/mmm_image_loss improved by 0.375 >= min_delta = 0.0. New best score: 1.198 +Metric validation/losses/mmm_text_loss improved by 0.644 >= min_delta = 0.0. New best score: 7.605 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.309 (now 0.3209057152271271). Wait count has been increased from 1. +Metric validation/losses/global_contrastive_loss improved by 0.001 >= min_delta = 0.0. New best score: 3.871 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 05:54:45.306417] Computing e^(tensor([14951.0630], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 05:54:45.311958] Ending with PPL=1764.3392174495486 (duration: 0:04:16.454795) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 7.066 +Metric validation/losses/mim_loss improved by 0.099 >= min_delta = 0.0. New best score: 1.145 +Metric validation/losses/mmm_image_loss improved by 0.028 >= min_delta = 0.0. New best score: 1.170 +Metric validation/losses/mmm_text_loss improved by 0.178 >= min_delta = 0.0. New best score: 7.427 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.309 (now 0.33438432216644287). Wait count has been increased from 2. +Metric validation/losses/global_contrastive_loss improved by 0.000 >= min_delta = 0.0. New best score: 3.871 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 07:01:50.060933] Computing e^(tensor([14715.0797], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 07:01:50.066751] Ending with PPL=1567.9743896405184 (duration: 0:04:26.277530) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.917 +Metric validation/losses/mim_loss improved by 0.031 >= min_delta = 0.0. New best score: 1.113 +Metric validation/losses/mmm_image_loss improved by 0.092 >= min_delta = 0.0. New best score: 1.078 +Metric validation/losses/mmm_text_loss improved by 0.058 >= min_delta = 0.0. New best score: 7.368 +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.309 (now 0.32441791892051697). Signaling model to ignore task. +Metric validation/losses/global_contrastive_loss improved by 0.034 >= min_delta = 0.0. New best score: 3.837 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 08:11:39.015826] Computing e^(tensor([14466.4106], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 08:11:39.021584] Ending with PPL=1384.6536235442197 (duration: 0:04:37.181719) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.787 +Metric validation/losses/mim_loss improved by 0.024 >= min_delta = 0.0. New best score: 1.090 +Metric validation/losses/mmm_image_loss improved by 0.032 >= min_delta = 0.0. New best score: 1.046 +Metric validation/losses/mmm_text_loss improved by 0.130 >= min_delta = 0.0. New best score: 7.238 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.309 (now 0.32441791892051697). Signaling model to ignore task. +Metric validation/losses/global_contrastive_loss improved by 0.032 >= min_delta = 0.0. New best score: 3.805 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 09:24:03.787497] Computing e^(tensor([14178.7386], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 09:24:03.793267] Ending with PPL=1199.151231783321 (duration: 0:04:18.035672) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.653 +Metric validation/losses/mim_loss improved by 0.021 >= min_delta = 0.0. New best score: 1.069 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.046 (now 1.0471864938735962). Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.046 >= min_delta = 0.0. New best score: 7.192 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.309 (now 0.32441791892051697). Signaling model to ignore task. +Metric validation/losses/global_contrastive_loss improved by 0.061 >= min_delta = 0.0. New best score: 3.743 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 10:34:37.552532] Computing e^(tensor([13912.5843], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 10:34:37.558357] Ending with PPL=1049.7340916596731 (duration: 0:06:09.775786) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.542 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.069 (now 1.0720385313034058). Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.010 >= min_delta = 0.0. New best score: 1.037 +Metric validation/losses/mmm_text_loss improved by 0.035 >= min_delta = 0.0. New best score: 7.157 +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Metric validation/losses/global_contrastive_loss improved by 0.060 >= min_delta = 0.0. New best score: 3.683 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 11:48:19.900402] Computing e^(tensor([13656.6316], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 11:48:19.906089] Ending with PPL=923.633930819971 (duration: 0:04:17.188837) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.444 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.069 (now 1.082824468612671). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.037 (now 1.0431228876113892). Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.079 >= min_delta = 0.0. New best score: 7.078 +Metric validation/losses/itm_loss improved by 0.012 >= min_delta = 0.0. New best score: 0.296 +Metric validation/losses/global_contrastive_loss improved by 0.049 >= min_delta = 0.0. New best score: 3.633 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 12:59:55.748929] Computing e^(tensor([13323.6104], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 12:59:55.754650] Ending with PPL=781.961247840238 (duration: 0:04:15.056710) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.331 +Metric validation/losses/mim_loss improved by 0.007 >= min_delta = 0.0. New best score: 1.062 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.037 (now 1.0392088890075684). Wait count has been increased from 1. +Metric validation/losses/mmm_text_loss improved by 0.105 >= min_delta = 0.0. New best score: 6.973 +Metric validation/losses/itm_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.290 +Metric validation/losses/global_contrastive_loss improved by 0.039 >= min_delta = 0.0. New best score: 3.594 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 14:12:00.685661] Computing e^(tensor([12898.0080], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 14:12:00.691238] Ending with PPL=632.0724277084527 (duration: 0:04:13.688017) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.207 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.062 (now 1.0804604291915894). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 1.037 (now 1.0457748174667358). Wait count has been increased from 2. +Metric validation/losses/mmm_text_loss improved by 0.055 >= min_delta = 0.0. New best score: 6.918 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.290 (now 0.3056774437427521). Wait count has been increased from 0. +Metric validation/losses/global_contrastive_loss improved by 0.038 >= min_delta = 0.0. New best score: 3.556 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 15:22:12.311512] Computing e^(tensor([12295.1126], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 15:22:12.317327] Ending with PPL=467.57336890461283 (duration: 0:05:58.283538) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 6.034 +Metric validation/losses/mim_loss improved by 0.011 >= min_delta = 0.0. New best score: 1.051 +Monitored metric validation/losses/mmm_image_loss did not improve in the last 4 records. Best score: 1.037 (now 1.0409910678863525). Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.176 >= min_delta = 0.0. New best score: 6.742 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.290 (now 0.3344399631023407). Wait count has been increased from 1. +Metric validation/losses/global_contrastive_loss improved by 0.065 >= min_delta = 0.0. New best score: 3.491 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 16:37:28.004283] Computing e^(tensor([11549.0405], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 16:37:28.009977] Ending with PPL=321.989924726889 (duration: 0:04:16.001013) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 5.782 +Metric validation/losses/mim_loss improved by 0.004 >= min_delta = 0.0. New best score: 1.047 +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 1.037 (now 1.0409910678863525). Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.200 >= min_delta = 0.0. New best score: 6.542 +Metric validation/losses/itm_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.289 +Metric validation/losses/global_contrastive_loss improved by 0.022 >= min_delta = 0.0. New best score: 3.469 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 17:45:48.193176] Computing e^(tensor([10757.2444], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 17:45:48.198751] Ending with PPL=216.72346568117086 (duration: 0:04:17.732116) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 5.459 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.047 (now 1.0608021020889282). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 1.037 (now 1.0409910678863525). Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.149 >= min_delta = 0.0. New best score: 6.393 +Metric validation/losses/itm_loss improved by 0.023 >= min_delta = 0.0. New best score: 0.267 +Metric validation/losses/global_contrastive_loss improved by 0.035 >= min_delta = 0.0. New best score: 3.434 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 18:58:51.310586] Computing e^(tensor([10271.3771], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 18:58:51.316238] Ending with PPL=169.9813165289669 (duration: 0:04:16.632562) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 5.228 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 1.042 +Monitored metric validation/losses/mmm_image_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Metric validation/losses/mmm_text_loss improved by 0.104 >= min_delta = 0.0. New best score: 6.289 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.267 (now 0.351532518863678). Wait count has been increased from 0. +Metric validation/losses/global_contrastive_loss improved by 0.016 >= min_delta = 0.0. New best score: 3.418 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 20:09:12.575388] Computing e^(tensor([9868.9246], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 20:09:12.581298] Ending with PPL=138.99838079665818 (duration: 0:05:04.394650) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 5.046 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.042 (now 1.0647696256637573). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.037 (now 1.040387511253357). Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.188 >= min_delta = 0.0. New best score: 6.101 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.267 (now 0.33304688334465027). Wait count has been increased from 1. +Metric validation/losses/global_contrastive_loss improved by 0.038 >= min_delta = 0.0. New best score: 3.381 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 21:23:47.622447] Computing e^(tensor([9501.7269], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 21:23:47.628122] Ending with PPL=115.68412879294742 (duration: 0:04:24.160570) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.898 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.042 (now 1.0709302425384521). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.037 (now 1.0785207748413086). Wait count has been increased from 1. +Metric validation/losses/mmm_text_loss improved by 0.051 >= min_delta = 0.0. New best score: 6.051 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.267 (now 0.3030518591403961). Wait count has been increased from 2. +Metric validation/losses/global_contrastive_loss improved by 0.033 >= min_delta = 0.0. New best score: 3.348 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 22:32:12.378406] Computing e^(tensor([9197.9192], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 22:32:12.384102] Ending with PPL=99.38086655112095 (duration: 0:04:25.800305) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.780 +Metric validation/losses/mim_loss improved by 0.009 >= min_delta = 0.0. New best score: 1.033 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 1.037 (now 1.044959545135498). Wait count has been increased from 2. +Metric validation/losses/mmm_text_loss improved by 0.089 >= min_delta = 0.0. New best score: 5.961 +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.267 (now 0.2990414798259735). Signaling model to ignore task. +Metric validation/losses/global_contrastive_loss improved by 0.004 >= min_delta = 0.0. New best score: 3.344 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 23:44:15.864785] Computing e^(tensor([8870.5070], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 23:44:15.870376] Ending with PPL=84.37351149990985 (duration: 0:04:21.160515) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.662 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.033 (now 1.0574204921722412). Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 1.035 +Metric validation/losses/mmm_text_loss improved by 0.237 >= min_delta = 0.0. New best score: 5.724 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.267 (now 0.2990414798259735). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 3.344 (now 3.445406675338745). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 00:49:00.597063] Computing e^(tensor([8569.5062], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 00:49:00.602686] Ending with PPL=72.58462330261187 (duration: 0:04:20.418288) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.551 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.033 (now 1.079451084136963). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.035 (now 1.0352758169174194). Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.073 >= min_delta = 0.0. New best score: 5.651 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.267 (now 0.2990414798259735). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 3.344 (now 3.3796987533569336). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 02:01:29.879542] Computing e^(tensor([8285.3527], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 02:01:29.885144] Ending with PPL=62.97112782644102 (duration: 0:04:14.516308) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.441 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 1.033 (now 1.0387765169143677). Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.020 >= min_delta = 0.0. New best score: 1.015 +Metric validation/losses/mmm_text_loss improved by 0.022 >= min_delta = 0.0. New best score: 5.629 +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Metric validation/losses/global_contrastive_loss improved by 0.038 >= min_delta = 0.0. New best score: 3.307 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 03:05:09.633039] Computing e^(tensor([8031.1662], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 03:05:09.638601] Ending with PPL=55.455621808625175 (duration: 0:04:16.792975) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.350 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 1.033 (now 1.051963210105896). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.015 (now 1.0209319591522217). Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.055 >= min_delta = 0.0. New best score: 5.575 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.267 (now 0.306975781917572). Wait count has been increased from 0. +Metric validation/losses/global_contrastive_loss improved by 0.076 >= min_delta = 0.0. New best score: 3.231 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 04:16:39.755237] Computing e^(tensor([7813.7780], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 04:16:39.760882] Ending with PPL=49.74395673832092 (duration: 0:04:14.558347) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.264 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 1.033 (now 1.0559604167938232). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.015 (now 1.0206886529922485). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 5.575 (now 5.583824634552002). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.267 (now 0.3446628451347351). Wait count has been increased from 1. +Metric validation/losses/global_contrastive_loss improved by 0.017 >= min_delta = 0.0. New best score: 3.214 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 05:25:15.214578] Computing e^(tensor([7625.1484], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 05:25:15.220196] Ending with PPL=45.26681435811498 (duration: 0:04:17.718897) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.189 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 1.033 (now 1.0402506589889526). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 1.015 (now 1.0307976007461548). Wait count has been increased from 2. +Metric validation/losses/mmm_text_loss improved by 0.133 >= min_delta = 0.0. New best score: 5.441 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.267 (now 0.34778207540512085). Wait count has been increased from 2. +Metric validation/losses/global_contrastive_loss improved by 0.048 >= min_delta = 0.0. New best score: 3.166 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 06:36:02.108132] Computing e^(tensor([7452.5784], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 06:36:02.115527] Ending with PPL=41.5247309227338 (duration: 0:04:14.330622) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.117 +Monitored metric validation/losses/mim_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 4 records. Best score: 1.015 (now 1.019075870513916). Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.070 >= min_delta = 0.0. New best score: 5.371 +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.267 (now 0.31121826171875). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 3.166 (now 3.183544874191284). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 07:40:28.995632] Computing e^(tensor([7285.6365], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 07:40:29.001322] Ending with PPL=38.199340500296735 (duration: 0:04:28.243671) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 4.059 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.033 (now 1.0384624004364014). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 1.015 (now 1.019075870513916). Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.157 >= min_delta = 0.0. New best score: 5.214 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.267 (now 0.31121826171875). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 3.166 (now 3.287428379058838). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 08:54:31.439766] Computing e^(tensor([7142.5051], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 08:54:31.445650] Ending with PPL=35.56110683239751 (duration: 0:05:53.585680) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.997 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.033 (now 1.0394536256790161). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 1.015 (now 1.019075870513916). Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.044 >= min_delta = 0.0. New best score: 5.170 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.267 (now 0.31121826171875). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 3.166 (now 3.3540890216827393). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 10:03:11.170170] Computing e^(tensor([6999.3717], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 10:03:11.174396] Ending with PPL=33.105049874952634 (duration: 0:04:24.306881) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.939 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 1.033 (now 1.0697907209396362). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Metric validation/losses/mmm_text_loss improved by 0.022 >= min_delta = 0.0. New best score: 5.148 +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.166 (now 3.273789167404175). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 11:18:23.103489] Computing e^(tensor([6873.2048], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 11:18:23.109258] Ending with PPL=31.081177673464783 (duration: 0:04:28.261926) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.892 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 1.033 (now 1.0556119680404663). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.015 (now 1.0246593952178955). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 5.148 (now 5.178205966949463). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.267 (now 0.31553247570991516). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.166 (now 3.273789167404175). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 12:29:35.285218] Computing e^(tensor([6753.1366], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 12:29:35.290987] Ending with PPL=29.27015238032404 (duration: 0:04:20.017789) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.839 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 1.033 (now 1.0489063262939453). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.015 (now 1.0230382680892944). Wait count has been increased from 1. +Metric validation/losses/mmm_text_loss improved by 0.135 >= min_delta = 0.0. New best score: 5.013 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.267 (now 0.32786425948143005). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 3.166 (now 3.273789167404175). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 13:43:54.430032] Computing e^(tensor([6664.9518], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 13:43:54.435810] Ending with PPL=28.007600142629624 (duration: 0:04:41.285078) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.802 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 1.033 (now 1.0412660837173462). Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 1.011 +Metric validation/losses/mmm_text_loss improved by 0.086 >= min_delta = 0.0. New best score: 4.927 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.267 (now 0.31982743740081787). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 14:59:13.257338] Computing e^(tensor([6556.6134], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 14:59:13.262980] Ending with PPL=26.5308095347675 (duration: 0:04:25.959836) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.765 +Monitored metric validation/losses/mim_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.011 (now 1.0761393308639526). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 4.927 (now 4.943163871765137). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.267 (now 0.291167676448822). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 3.166 (now 3.440516710281372). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 16:07:09.568554] Computing e^(tensor([6470.8783], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 16:07:09.574148] Ending with PPL=25.41753165220582 (duration: 0:04:18.168324) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.729 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.033 (now 1.0533719062805176). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.011 (now 1.0194200277328491). Wait count has been increased from 1. +Metric validation/losses/mmm_text_loss improved by 0.194 >= min_delta = 0.0. New best score: 4.732 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.267 (now 0.291167676448822). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 3.166 (now 3.3435049057006836). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 17:20:48.816530] Computing e^(tensor([6378.7277], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 17:20:48.822113] Ending with PPL=24.27298168960853 (duration: 0:04:15.521357) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.680 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.033 (now 1.0358279943466187). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 1.011 (now 1.0157787799835205). Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 4.732 (now 4.763762950897217). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.267 (now 0.291167676448822). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 3.166 (now 3.3121466636657715). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 18:31:36.845483] Computing e^(tensor([6289.8611], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 18:31:36.851308] Ending with PPL=23.218062373964386 (duration: 0:04:13.874810) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.645 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 1.033 (now 1.0441288948059082). Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.006 >= min_delta = 0.0. New best score: 1.004 +Metric validation/losses/mmm_text_loss improved by 0.111 >= min_delta = 0.0. New best score: 4.621 +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.166 (now 3.246953010559082). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 19:41:40.239585] Computing e^(tensor([6220.2529], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 19:41:40.245352] Ending with PPL=22.42388006653849 (duration: 0:04:31.501236) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.611 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 1.033 (now 1.0525074005126953). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.004 (now 1.0276223421096802). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 4.621 (now 4.764657020568848). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.267 (now 0.33064940571784973). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.166 (now 3.246953010559082). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 20:53:59.575662] Computing e^(tensor([6146.3704], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 20:53:59.581334] Ending with PPL=21.610627557066184 (duration: 0:04:19.763229) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.577 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 1.033 (now 1.0441699028015137). Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 1.000 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 4.621 (now 4.717462062835693). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.267 (now 0.33559003472328186). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 3.166 (now 3.246953010559082). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 22:03:06.444762] Computing e^(tensor([6067.6222], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 22:03:06.450550] Ending with PPL=20.776261965483595 (duration: 0:05:24.581987) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.542 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 1.033 (now 1.0536766052246094). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.000 (now 1.050683856010437). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 4.621 (now 4.664440155029297). Wait count has been increased from 2. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.267 (now 0.3336995542049408). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 23:16:15.303644] Computing e^(tensor([6008.6487], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 23:16:15.309288] Ending with PPL=20.172581630511658 (duration: 0:04:14.747442) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.512 +Monitored metric validation/losses/mim_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.000 (now 1.0494062900543213). Wait count has been increased from 1. +Metric validation/losses/mmm_text_loss improved by 0.071 >= min_delta = 0.0. New best score: 4.551 +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.267 (now 0.3197144567966461). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 3.166 (now 3.3369481563568115). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 00:19:35.042921] Computing e^(tensor([5944.2539], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 00:19:35.048572] Ending with PPL=19.533421615925572 (duration: 0:04:14.612714) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.482 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.033 (now 1.056900143623352). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 1.000 (now 1.033707857131958). Wait count has been increased from 2. +Metric validation/losses/mmm_text_loss improved by 0.096 >= min_delta = 0.0. New best score: 4.455 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.267 (now 0.3197144567966461). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 3.166 (now 3.3988962173461914). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 01:31:39.035414] Computing e^(tensor([5898.1341], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 01:31:39.041003] Ending with PPL=19.088137296762113 (duration: 0:04:23.357427) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.469 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.033 (now 1.0414454936981201). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 4 records. Best score: 1.000 (now 1.0096007585525513). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 4.455 (now 4.457240104675293). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.267 (now 0.3197144567966461). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 3.166 (now 3.3588674068450928). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 02:39:13.563669] Computing e^(tensor([5834.4212], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 02:39:13.569415] Ending with PPL=18.48964026272365 (duration: 0:04:15.269150) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.435 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 1.033 (now 1.0689091682434082). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 1.000 (now 1.0096007585525513). Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.025 >= min_delta = 0.0. New best score: 4.430 +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.166 (now 3.3058359622955322). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 03:50:00.244975] Computing e^(tensor([5784.8364], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 03:50:00.250695] Ending with PPL=18.036873942905054 (duration: 0:06:20.477674) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.412 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 1.033 (now 1.041237711906433). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 1.000 (now 1.0096007585525513). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 4.430 (now 4.484805583953857). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.267 (now 0.3118264675140381). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.166 (now 3.3058359622955322). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 05:02:57.932699] Computing e^(tensor([5740.2450], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 05:02:57.938333] Ending with PPL=17.639179021433716 (duration: 0:04:12.573453) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.383 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 1.033 (now 1.042583703994751). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 4.430 (now 4.504971504211426). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.267 (now 0.31323403120040894). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 3.166 (now 3.3058359622955322). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 06:09:54.216976] Computing e^(tensor([5683.4951], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 06:09:54.222632] Ending with PPL=17.145702335568206 (duration: 0:05:21.650430) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.365 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 1.033 (now 1.0549644231796265). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.000 (now 1.0198922157287598). Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.061 >= min_delta = 0.0. New best score: 4.369 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.267 (now 0.34513989090919495). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 07:20:24.503330] Computing e^(tensor([5651.0914], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 07:20:24.507867] Ending with PPL=16.870148608656088 (duration: 0:04:18.186047) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.353 +Monitored metric validation/losses/mim_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.000 (now 1.0453827381134033). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 4.369 (now 4.389795780181885). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.267 (now 0.3182629942893982). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 3.166 (now 3.33669376373291). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 08:23:09.737090] Computing e^(tensor([5611.1392], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 08:23:09.742753] Ending with PPL=16.536492580660013 (duration: 0:04:26.277067) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.326 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.033 (now 1.0604246854782104). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 1.000 (now 1.0426244735717773). Wait count has been increased from 2. +Metric validation/losses/mmm_text_loss improved by 0.078 >= min_delta = 0.0. New best score: 4.290 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.267 (now 0.3182629942893982). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 3.166 (now 3.3612489700317383). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 09:36:33.409540] Computing e^(tensor([5552.7271], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 09:36:33.415186] Ending with PPL=16.06051172539646 (duration: 0:04:12.737715) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.311 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.033 (now 1.0477632284164429). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 4 records. Best score: 1.000 (now 1.0305843353271484). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 4.290 (now 4.294010639190674). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.267 (now 0.3182629942893982). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 3.166 (now 3.3338236808776855). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 10:51:00.619010] Computing e^(tensor([5526.9858], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 10:51:00.624726] Ending with PPL=15.855126420547004 (duration: 0:07:28.527355) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.292 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 1.033 (now 1.0418097972869873). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 1.000 (now 1.0305843353271484). Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.084 >= min_delta = 0.0. New best score: 4.206 +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.166 (now 3.272841691970825). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 12:03:44.592635] Computing e^(tensor([5480.9356], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 12:03:44.598362] Ending with PPL=15.494231816168156 (duration: 0:04:13.616950) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.269 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 1.033 (now 1.0443882942199707). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 1.000 (now 1.0305843353271484). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 4.206 (now 4.381190299987793). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.267 (now 0.32644224166870117). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.166 (now 3.272841691970825). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 13:12:16.853138] Computing e^(tensor([5440.5318], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 13:12:16.859023] Ending with PPL=15.18435897774002 (duration: 0:05:13.360242) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.254 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 1.033 (now 1.0467736721038818). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 4.206 (now 4.353311061859131). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.267 (now 0.32196044921875). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 3.166 (now 3.272841691970825). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 14:27:28.022242] Computing e^(tensor([5400.9674], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 14:27:28.027815] Ending with PPL=14.886930774256403 (duration: 0:05:42.817203) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.240 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 1.033 (now 1.0371781587600708). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.000 (now 1.0185751914978027). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 4.206 (now 4.256248474121094). Wait count has been increased from 2. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.267 (now 0.34080639481544495). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 15:36:08.239027] Computing e^(tensor([5377.9054], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 15:36:08.244848] Ending with PPL=14.716255723572052 (duration: 0:04:12.139477) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.227 +Monitored metric validation/losses/mim_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.000 (now 1.0850657224655151). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 4.206 (now 4.260131359100342). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.267 (now 0.330444872379303). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 3.166 (now 3.3642139434814453). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 16:47:28.242979] Computing e^(tensor([5374.4786], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 16:47:28.248662] Ending with PPL=14.691062299041269 (duration: 0:04:17.414848) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.215 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.033 (now 1.0724581480026245). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 1.000 (now 1.0305688381195068). Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 4.206 (now 4.260131359100342). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.267 (now 0.330444872379303). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 3.166 (now 3.38611102104187). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 17:44:18.141497] Computing e^(tensor([5313.6482], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 17:44:18.147196] Ending with PPL=14.250957590913101 (duration: 0:04:24.665519) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.197 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.033 (now 1.0635082721710205). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 4 records. Best score: 1.000 (now 1.042533040046692). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 4.206 (now 4.260131359100342). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.267 (now 0.330444872379303). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 3.166 (now 3.36480975151062). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 18:40:13.543066] Computing e^(tensor([5274.2015], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 18:40:13.548893] Ending with PPL=13.972634464465237 (duration: 0:04:19.214676) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.179 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 1.033 (now 1.0658868551254272). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 1.000 (now 1.042533040046692). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.166 (now 3.3526792526245117). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 19:38:18.573702] Computing e^(tensor([5251.3252], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 19:38:18.579292] Ending with PPL=13.813724443972951 (duration: 0:04:29.631306) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.171 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 1.033 (now 1.0578086376190186). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 1.000 (now 1.042533040046692). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 4.206 (now 4.777081489562988). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.267 (now 0.3381176292896271). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.166 (now 3.3526792526245117). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 20:37:32.127559] Computing e^(tensor([5235.2230], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 20:37:32.133230] Ending with PPL=13.702955165070787 (duration: 0:08:00.192213) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.159 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 1.033 (now 1.0337005853652954). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 4.206 (now 4.654601573944092). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.267 (now 0.38650673627853394). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 3.166 (now 3.3526792526245117). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 21:38:19.916085] Computing e^(tensor([5208.4229], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 21:38:19.921738] Ending with PPL=13.520559683833207 (duration: 0:06:26.285299) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.155 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 1.033 (now 1.04306161403656). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.000 (now 1.012402057647705). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 4.206 (now 4.396160125732422). Wait count has been increased from 2. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.267 (now 0.3108450770378113). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 22:33:29.633586] Computing e^(tensor([5203.9085], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 22:33:29.639664] Ending with PPL=13.490075236399878 (duration: 0:05:02.197544) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.141 +Monitored metric validation/losses/mim_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.000 (now 1.0969654321670532). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 4.206 (now 4.267131805419922). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.267 (now 0.3095831573009491). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 3.166 (now 3.5606565475463867). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 23:38:59.552900] Computing e^(tensor([5177.1713], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 23:38:59.558534] Ending with PPL=13.310932021220642 (duration: 0:05:17.406861) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.129 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.033 (now 1.047646164894104). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 1.000 (now 1.044053316116333). Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 4.206 (now 4.267131805419922). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.267 (now 0.3095831573009491). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 3.166 (now 3.3750619888305664). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 00:42:15.412068] Computing e^(tensor([5142.6075], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 00:42:15.417743] Ending with PPL=13.082869949118336 (duration: 0:04:49.812191) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.114 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.033 (now 1.0575883388519287). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 4 records. Best score: 1.000 (now 1.029788851737976). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 4.206 (now 4.267131805419922). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.267 (now 0.3095831573009491). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 3.166 (now 3.305142641067505). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 01:44:55.896785] Computing e^(tensor([5132.5103], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 01:44:55.902591] Ending with PPL=13.016986190078821 (duration: 0:04:22.886360) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.108 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 1.033 (now 1.0665327310562134). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 1.000 (now 1.029788851737976). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.166 (now 3.268280506134033). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 02:41:40.129547] Computing e^(tensor([5106.8157], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 02:41:40.135190] Ending with PPL=12.850822734627203 (duration: 0:04:30.860245) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.099 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 1.033 (now 1.044053554534912). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 1.000 (now 1.029788851737976). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 4.206 (now 4.497568607330322). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.267 (now 0.312926709651947). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.166 (now 3.268280506134033). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 03:40:40.159020] Computing e^(tensor([5083.9749], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 03:40:40.164616] Ending with PPL=12.704896322953639 (duration: 0:06:21.206475) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.089 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 1.033 (now 1.047652244567871). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 4.206 (now 4.324644565582275). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.267 (now 0.3367432653903961). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 3.166 (now 3.268280506134033). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 04:45:38.997408] Computing e^(tensor([5056.2397], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 04:45:39.003014] Ending with PPL=12.529925927281793 (duration: 0:08:42.306930) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.077 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 1.033 (now 1.0554888248443604). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.000 (now 1.0463348627090454). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 4.206 (now 4.20642614364624). Wait count has been increased from 2. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.267 (now 0.31004199385643005). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 05:52:27.795378] Computing e^(tensor([5038.2973], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 05:52:27.801016] Ending with PPL=12.41801977024582 (duration: 0:04:33.131997) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.071 +Monitored metric validation/losses/mim_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.000 (now 1.0629059076309204). Wait count has been increased from 1. +Metric validation/losses/mmm_text_loss improved by 0.077 >= min_delta = 0.0. New best score: 4.129 +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.267 (now 0.28960806131362915). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 3.166 (now 3.406836748123169). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 06:54:59.854024] Computing e^(tensor([5034.1216], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 06:54:59.859865] Ending with PPL=12.392120370799933 (duration: 0:04:21.989744) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.058 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.033 (now 1.0513461828231812). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 1.000 (now 1.0293574333190918). Wait count has been increased from 2. +Metric validation/losses/mmm_text_loss improved by 0.136 >= min_delta = 0.0. New best score: 3.993 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.267 (now 0.28960806131362915). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 3.166 (now 3.4866995811462402). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 07:56:58.074034] Computing e^(tensor([4999.0695], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 07:56:58.079758] Ending with PPL=12.176827209804914 (duration: 0:04:24.794183) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.056 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.033 (now 1.0729509592056274). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 4 records. Best score: 1.000 (now 1.0317161083221436). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.993 (now 4.013975620269775). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.267 (now 0.28960806131362915). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 3.166 (now 3.316505193710327). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 09:04:31.060004] Computing e^(tensor([4995.2259], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 09:04:31.065678] Ending with PPL=12.15344857469213 (duration: 0:08:10.508423) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.045 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 1.033 (now 1.0625369548797607). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 1.000 (now 1.0317161083221436). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.993 (now 4.008838176727295). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.166 (now 3.2881224155426025). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 10:19:30.615615] Computing e^(tensor([4969.5748], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 10:19:30.621230] Ending with PPL=11.998568998062655 (duration: 0:05:23.401831) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.042 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 1.033 (now 1.046910047531128). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 1.000 (now 1.0317161083221436). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.993 (now 4.014272212982178). Wait count has been increased from 2. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.267 (now 0.3086215853691101). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.166 (now 3.2881224155426025). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 11:28:00.454850] Computing e^(tensor([4953.3556], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 11:28:00.460662] Ending with PPL=11.901659042660626 (duration: 0:04:41.727149) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.028 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 1.033 (now 1.0431599617004395). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 3.993 (now 3.9933981895446777). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.267 (now 0.3211193382740021). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 3.166 (now 3.2881224155426025). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 12:52:22.087418] Computing e^(tensor([4937.5623], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 12:52:22.093315] Ending with PPL=11.80804570587283 (duration: 0:04:28.389537) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.020 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 1.033 (now 1.069451928138733). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.000 (now 1.0556997060775757). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 3.993 (now 3.9933981895446777). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.267 (now 0.348022997379303). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 14:22:24.734671] Computing e^(tensor([4946.6365], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 14:22:24.740480] Ending with PPL=11.861741597127185 (duration: 0:04:23.323346) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.016 +Monitored metric validation/losses/mim_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.000 (now 1.053816318511963). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 3.993 (now 3.9933981895446777). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.267 (now 0.3771801292896271). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 3.166 (now 3.4383864402770996). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 16:01:31.335110] Computing e^(tensor([4928.7183], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 16:01:31.340790] Ending with PPL=11.755945711957752 (duration: 0:04:29.944680) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.013 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.033 (now 1.0451571941375732). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 1.000 (now 1.0270962715148926). Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.267 (now 0.3771801292896271). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 3.166 (now 3.325754165649414). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 17:43:27.164237] Computing e^(tensor([4908.8891], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 17:43:27.169873] Ending with PPL=11.639966201558474 (duration: 0:04:12.797153) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.004 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.033 (now 1.0763781070709229). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 4 records. Best score: 1.000 (now 1.0279066562652588). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.993 (now 4.186192512512207). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.267 (now 0.3771801292896271). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 3.166 (now 3.4083445072174072). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 19:26:50.905114] Computing e^(tensor([4890.8337], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 19:26:50.910724] Ending with PPL=11.535356919122163 (duration: 0:04:08.400601) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.996 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 1.033 (now 1.0587084293365479). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 1.000 (now 1.0279066562652588). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.993 (now 4.078798294067383). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.166 (now 3.29351806640625). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 21:10:13.439209] Computing e^(tensor([4877.9979], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 21:10:13.444915] Ending with PPL=11.461561509910965 (duration: 0:04:11.733344) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.991 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 1.033 (now 1.0581685304641724). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 1.000 (now 1.0279066562652588). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.993 (now 4.085831642150879). Wait count has been increased from 2. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.267 (now 0.33290448784828186). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.166 (now 3.29351806640625). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 23:06:21.005280] Computing e^(tensor([4863.3450], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 23:06:21.010974] Ending with PPL=11.377895635625347 (duration: 0:04:11.351259) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.987 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 1.033 (now 1.0531504154205322). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Metric validation/losses/mmm_text_loss improved by 0.010 >= min_delta = 0.0. New best score: 3.984 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.267 (now 0.34288960695266724). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 3.166 (now 3.29351806640625). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 01:06:03.115140] Computing e^(tensor([4860.4626], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 01:06:03.120766] Ending with PPL=11.361509632446818 (duration: 0:04:14.163965) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.983 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 1.033 (now 1.0415589809417725). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.000 (now 1.0220857858657837). Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.003 >= min_delta = 0.0. New best score: 3.981 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.267 (now 0.3371207118034363). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 02:57:02.902907] Computing e^(tensor([4836.3765], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 02:57:02.908562] Ending with PPL=11.225502883691307 (duration: 0:04:13.391641) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.979 +Monitored metric validation/losses/mim_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 1.000 (now 1.0538898706436157). Wait count has been increased from 1. +Metric validation/losses/mmm_text_loss improved by 0.001 >= min_delta = 0.0. New best score: 3.979 +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.267 (now 0.31680673360824585). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 3.166 (now 3.334282636642456). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 04:44:27.131608] Computing e^(tensor([4843.0752], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 04:44:27.137284] Ending with PPL=11.26316430530028 (duration: 0:04:11.315514) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.972 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.033 (now 1.0701984167099). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 1.000 (now 1.049017310142517). Wait count has been increased from 2. +Metric validation/losses/mmm_text_loss improved by 0.049 >= min_delta = 0.0. New best score: 3.930 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.267 (now 0.31680673360824585). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 3.166 (now 3.3707666397094727). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 06:31:08.408868] Computing e^(tensor([4823.0455], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 06:31:08.414604] Ending with PPL=11.150928508046617 (duration: 0:04:11.124773) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.962 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.033 (now 1.0692191123962402). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 4 records. Best score: 1.000 (now 1.0193980932235718). Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.045 >= min_delta = 0.0. New best score: 3.885 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.267 (now 0.31680673360824585). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 3.166 (now 3.29193115234375). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 08:27:37.784964] Computing e^(tensor([4806.2322], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 08:27:37.790720] Ending with PPL=11.057579236112607 (duration: 0:04:52.090842) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.957 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 1.033 (now 1.0633490085601807). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 1.000 (now 1.0193980932235718). Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.065 >= min_delta = 0.0. New best score: 3.820 +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.166 (now 3.259669780731201). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 10:14:19.332678] Computing e^(tensor([4793.1872], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 10:14:19.338412] Ending with PPL=10.985690948768191 (duration: 0:04:24.714405) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.951 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 1.033 (now 1.0587337017059326). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 1.000 (now 1.0193980932235718). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.820 (now 4.002289772033691). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.267 (now 0.3401382267475128). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.166 (now 3.259669780731201). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 12:05:35.005826] Computing e^(tensor([4778.4487], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 12:05:35.011473] Ending with PPL=10.90503193978446 (duration: 0:04:10.653710) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.946 +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 1.033 (now 1.0415818691253662). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.820 (now 3.8875861167907715). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.267 (now 0.2856729030609131). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 3.166 (now 3.259669780731201). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 13:51:09.196403] Computing e^(tensor([4764.6827], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 13:51:09.202183] Ending with PPL=10.830230478914684 (duration: 0:04:18.620052) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.942 +Monitored metric validation/losses/mim_loss did not improve in the last 6 records. Best score: 1.033 (now 1.0411232709884644). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 1.000 (now 1.013824224472046). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.820 (now 3.883531093597412). Wait count has been increased from 2. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.267 (now 0.3643595278263092). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 15:20:37.775477] Computing e^(tensor([4776.1191], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 15:20:37.781128] Ending with PPL=10.892337484113625 (duration: 0:06:17.585862) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 3.810 +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.267 (now 0.3152085542678833). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 3.166 (now 3.288705587387085). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 16:55:36.349270] Computing e^(tensor([4769.1347], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 16:55:36.354930] Ending with PPL=10.85436527573728 (duration: 0:06:11.465549) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.937 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 1.033 (now 1.0543533563613892). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 1.000 (now 1.0592750310897827). Wait count has been increased from 2. +Metric validation/losses/mmm_text_loss improved by 0.191 >= min_delta = 0.0. New best score: 3.619 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.267 (now 0.3152085542678833). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 3.166 (now 3.2944486141204834). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 18:28:09.879038] Computing e^(tensor([4752.1450], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 18:28:09.884712] Ending with PPL=10.762550097895401 (duration: 0:04:41.430844) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.932 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 1.033 (now 1.0422934293746948). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 4 records. Best score: 1.000 (now 1.0188679695129395). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 3.619 (now 3.7140743732452393). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.267 (now 0.3152085542678833). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 3.166 (now 3.236755847930908). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 19:56:43.546114] Computing e^(tensor([4742.5895], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 19:56:43.551923] Ending with PPL=10.711251713993216 (duration: 0:04:46.614387) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.924 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 1.033 (now 1.039553165435791). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 1.000 (now 1.0188679695129395). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 3.619 (now 3.686723470687866). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 3.166 (now 3.2300519943237305). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 21:29:10.766950] Computing e^(tensor([4725.4365], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 21:29:10.772630] Ending with PPL=10.61977922499319 (duration: 0:05:21.612180) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.920 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 1.033 (now 1.0468541383743286). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 1.000 (now 1.0188679695129395). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 3.619 (now 3.832977533340454). Wait count has been increased from 2. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.267 (now 0.32239624857902527). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 3.166 (now 3.2300519943237305). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 22:59:21.877188] Computing e^(tensor([4731.1959], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 22:59:21.882811] Ending with PPL=10.650405567211847 (duration: 0:04:44.643724) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00 + main() + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 212, in main + trainer.validate(model, datamodule=datamodule) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 628, in validate + return call._call_and_handle_interrupt( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 42, in _call_and_handle_interrupt + return trainer_fn(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 671, in _validate_impl + results = self._run(model, ckpt_path=ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 934, in _run + call._call_setup_hook(self) # allow user to setup lightning_module in accelerator environment + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 83, in _call_setup_hook + _call_lightning_datamodule_hook(trainer, "setup", stage=fn) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 164, in _call_lightning_datamodule_hook + return fn(*args, **kwargs) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/multidata.py", line 179, in setup + datamodule.setup(stage) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/datamodules.py", line 185, in setup + super().setup(stage) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/datamodules.py", line 43, in setup + self.train_dataset: Dataset = build_datasets_from_info(self.train_dataset_infos, split="train") + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/utils.py", line 27, in build_datasets_from_info + current_dataset = load_dataset( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/datasets/load.py", line 1785, in load_dataset + builder_instance = load_dataset_builder( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/datasets/load.py", line 1514, in load_dataset_builder + dataset_module = dataset_module_factory( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/datasets/load.py", line 1231, in dataset_module_factory + raise e1 from None + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/datasets/load.py", line 1198, in dataset_module_factory + raise e + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/datasets/load.py", line 1172, in dataset_module_factory + dataset_info = hf_api.dataset_info( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/utils/_validators.py", line 118, in _inner_fn + return fn(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/hf_api.py", line 1740, in dataset_info + r = get_session().get(path, headers=headers, timeout=timeout, params=params) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/sessions.py", line 602, in get + return self.request("GET", url, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/sessions.py", line 589, in request + resp = self.send(prep, **send_kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/sessions.py", line 703, in send + r = adapter.send(request, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/utils/_http.py", line 63, in send + return super().send(request, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/adapters.py", line 532, in send + raise ReadTimeout(e, request=request) +requests.exceptions.ReadTimeout: (ReadTimeoutError("HTTPSConnectionPool(host='huggingface.co', port=443): Read timed out. (read timeout=100.0)"), '(Request ID: 53d197df-a607-47ec-98a7-945ae46a8dd2)') +Traceback (most recent call last): + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 196, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/cluster/apps/nss/gcc-8.2.0/python/3.10.4/x86_64/lib64/python3.10/runpy.py", line 86, in _run_code + exec(code, run_globals) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 220, in + main() + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/train.py", line 212, in main + trainer.validate(model, datamodule=datamodule) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 628, in validate + return call._call_and_handle_interrupt( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 42, in _call_and_handle_interrupt + return trainer_fn(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 671, in _validate_impl + results = self._run(model, ckpt_path=ckpt_path) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 934, in _run + call._call_setup_hook(self) # allow user to setup lightning_module in accelerator environment + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 83, in _call_setup_hook + _call_lightning_datamodule_hook(trainer, "setup", stage=fn) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 164, in _call_lightning_datamodule_hook + return fn(*args, **kwargs) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/multidata.py", line 179, in setup + datamodule.setup(stage) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/datamodules.py", line 185, in setup + super().setup(stage) + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/datamodules.py", line 43, in setup + self.train_dataset: Dataset = build_datasets_from_info(self.train_dataset_infos, split="train") + File "/cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/data/utils.py", line 27, in build_datasets_from_info + current_dataset = load_dataset( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/datasets/load.py", line 1785, in load_dataset + builder_instance = load_dataset_builder( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/datasets/load.py", line 1514, in load_dataset_builder + dataset_module = dataset_module_factory( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/datasets/load.py", line 1231, in dataset_module_factory + raise e1 from None + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/datasets/load.py", line 1198, in dataset_module_factory + raise e + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/datasets/load.py", line 1172, in dataset_module_factory + dataset_info = hf_api.dataset_info( + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/utils/_validators.py", line 118, in _inner_fn + return fn(*args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/hf_api.py", line 1740, in dataset_info + r = get_session().get(path, headers=headers, timeout=timeout, params=params) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/sessions.py", line 602, in get + return self.request("GET", url, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/sessions.py", line 589, in request + resp = self.send(prep, **send_kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/sessions.py", line 703, in send + r = adapter.send(request, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/huggingface_hub/utils/_http.py", line 63, in send + return super().send(request, *args, **kwargs) + File "/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/requests/adapters.py", line 532, in send + raise ReadTimeout(e, request=request) +requests.exceptions.ReadTimeout: (ReadTimeoutError("HTTPSConnectionPool(host='huggingface.co', port=443): Read timed out. (read timeout=100.0)"), '(Request ID: 53d197df-a607-47ec-98a7-945ae46a8dd2)') +wandb: Waiting for W&B process to finish... (failed 1). Press Control-C to abort syncing. +wandb: - 0.000 MB of 0.000 MB uploaded (0.000 MB deduped) wandb: \ 0.000 MB of 0.076 MB uploaded (0.000 MB deduped) wandb: | 0.076 MB of 0.076 MB uploaded (0.000 MB deduped) wandb: +wandb: Run history: +wandb: ImageDataModule ▁▁▁▁▁▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▄▄▅▅▅▅▅▆▆▆▆▇▇▇▇█████ +wandb: MLMDataModule ▁▁▁▁▁▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▅▆▆▆▆▇▇▇▇█████ +wandb: VLDataModule ▁▁▁▁▁▂▂▂▂▃▃▃▃▃▄▄▄▄▅▅▅▅▅▅▅▅▅▆▆▆▆▇▇▇▇█████ +wandb: epoch ▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: evaluation/blimp/anaphor_agreement ▁▃▆▇█ +wandb: evaluation/blimp/argument_structure █▁▆▂▆ +wandb: evaluation/blimp/binding █▄▁▃█ +wandb: evaluation/blimp/control_raising ▁▃█▇▄ +wandb: evaluation/blimp/determiner_noun_agreement ▇▁▄█▆ +wandb: evaluation/blimp/ellipsis ▆▁▄██ +wandb: evaluation/blimp/filler_gap ▃▁█▁▅ +wandb: evaluation/blimp/irregular_forms ▅▁▃▄█ +wandb: evaluation/blimp/island_effects ▅▁▃█▅ +wandb: evaluation/blimp/npi_licensing ▂▄█▆▁ +wandb: evaluation/blimp/quantifiers ▁▆▂▁█ +wandb: evaluation/blimp/subject_verb_agreement ▆▁▃█▆ +wandb: evaluation/blimp_average ▄▁▄▆█ +wandb: evaluation/imagenet_zeroshot/top1 ▁▆█▅█ +wandb: evaluation/imagenet_zeroshot/top5 ▁▅▆▆█ +wandb: evaluation/pseudo_perplexity ▄█▂▁▁ +wandb: lr-AdamW/pg1 █████████▇▇▇▇▇▇▇▇▇▆▆▆▆▅▅▅▅▅▄▄▄▄▂▂▂▂▂▁▁▁▁ +wandb: lr-AdamW/pg2 █████████▇▇▇▇▇▇▇▇▇▆▆▆▆▅▅▅▅▅▄▄▄▄▂▂▂▂▂▁▁▁▁ +wandb: train/losses/global_contrastive_loss █▅▂▅▅▃▁▁▂ +wandb: train/losses/itm_loss ▂▁█▅▄▂▅▄▄ +wandb: train/losses/mim_loss ▃▃▇▁▃▇▄▄█ +wandb: train/losses/mlm_loss ▃▅█▅█▆▁▂▅ +wandb: train/losses/mmm_image_loss ██▁▄▄▅▃▄▃ +wandb: train/losses/mmm_text_loss █▆▅▅▅▃▂▃▁ +wandb: trainer/global_step ▁▁▁▁▂▂▂▂▂▃▃▃▃▃▃▃▃▃▄▄▄▄▅▅▅▅▅▆▆▆▆▇▇▇▇█████ +wandb: validation/losses/global_contrastive_loss █▃▂▁▁ +wandb: validation/losses/itm_loss ▇▆▁▅█ +wandb: validation/losses/mim_loss █▄▃▂▁ +wandb: validation/losses/mlm_loss ▄█▃▂▁ +wandb: validation/losses/mmm_image_loss █▄▂▂▁ +wandb: validation/losses/mmm_text_loss █▅▃▂▁ +wandb: validation/monitor/global_contrastive █▄▂▁▁ +wandb: validation/monitor/itm █▅▃▂▁ +wandb: validation/monitor/mim █▅▁▁▁ +wandb: validation/monitor/mlm █▅▁▁▁ +wandb: validation/monitor/mmm_image █▅▃▂▁ +wandb: validation/monitor/mmm_text █▅▃▂▁ +wandb: +wandb: Run summary: +wandb: ImageDataModule 3293 +wandb: MLMDataModule 33503 +wandb: VLDataModule 3178 +wandb: epoch 0 +wandb: evaluation/blimp/anaphor_agreement 96.11 +wandb: evaluation/blimp/argument_structure 68.88 +wandb: evaluation/blimp/binding 70.35 +wandb: evaluation/blimp/control_raising 65.36 +wandb: evaluation/blimp/determiner_noun_agreement 90.6 +wandb: evaluation/blimp/ellipsis 77.25 +wandb: evaluation/blimp/filler_gap 62.37 +wandb: evaluation/blimp/irregular_forms 85.7 +wandb: evaluation/blimp/island_effects 47.76 +wandb: evaluation/blimp/npi_licensing 54.13 +wandb: evaluation/blimp/quantifiers 62.73 +wandb: evaluation/blimp/subject_verb_agreement 67.79 +wandb: evaluation/blimp_average 70.7525 +wandb: evaluation/imagenet_zeroshot/top1 0.00218 +wandb: evaluation/imagenet_zeroshot/top5 0.01226 +wandb: evaluation/pseudo_perplexity 10.38773 +wandb: lr-AdamW/pg1 0.001 +wandb: lr-AdamW/pg2 0.00075 +wandb: train/losses/global_contrastive_loss 3.2813 +wandb: train/losses/itm_loss 0.33477 +wandb: train/losses/mim_loss 1.09948 +wandb: train/losses/mlm_loss 2.74652 +wandb: train/losses/mmm_image_loss 1.06691 +wandb: train/losses/mmm_text_loss 6.20304 +wandb: trainer/global_step 10527 +wandb: validation/losses/global_contrastive_loss 3.26723 +wandb: validation/losses/itm_loss 0.33642 +wandb: validation/losses/mim_loss 1.02534 +wandb: validation/losses/mlm_loss 2.8873 +wandb: validation/losses/mmm_image_loss 1.02 +wandb: validation/losses/mmm_text_loss 6.61781 +wandb: validation/monitor/global_contrastive 0.00625 +wandb: validation/monitor/itm 0.0 +wandb: validation/monitor/mim 0.0 +wandb: validation/monitor/mlm 0.0 +wandb: validation/monitor/mmm_image 0.0 +wandb: validation/monitor/mmm_text 0.0 +wandb: +wandb: 🚀 View run silver-sweep-8 at: https://wandb.ai/rycolab/alkmi-wit/runs/vwzrsjzb +wandb: Synced 3 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20230820_102209-vwzrsjzb/logs +[ImageNet Zero-Shot Evaluation 2023-08-20 18:27:13.389944] Ending with top5=0.012261961913906225 (duration: 0:02:07.247882) +Monitored metric validation/losses/mlm_loss did not improve in the last 5 records. Best score: 2.836. Signaling model to ignore task. +Monitored metric validation/losses/mim_loss did not improve in the last 5 records. Best score: 0.887. Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.228. Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 3.017. Wait count has been increased from 3. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 0.902. Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 2.885. Signaling model to ignore task. +Starting validation +2023-08-20 18:31:36.327796 Preparing data for datamodule: VLDataModule +2023-08-20 18:31:36.328036 Preparing data for datamodule: ImageDataModule +2023-08-20 18:31:36.328192 Preparing data for datamodule: MLMDataModule +2023-08-20 18:31:36.328542 Setting up datamodule: VLDataModule diff --git a/flava-wit/text100-vision10/half_bs4025_seed5501650_bf16-mixed/resume_singlenode_2023-08-21-17:42:49.log b/flava-wit/text100-vision10/half_bs4025_seed5501650_bf16-mixed/resume_singlenode_2023-08-21-17:42:49.log new file mode 100644 index 0000000000000000000000000000000000000000..25d92097e0e8f3f9f8d45c4a57fe7c44300a98ce --- /dev/null +++ b/flava-wit/text100-vision10/half_bs4025_seed5501650_bf16-mixed/resume_singlenode_2023-08-21-17:42:49.log @@ -0,0 +1,3771 @@ +wandb: Currently logged in as: tamariucai (rycolab). Use `wandb login --relogin` to force relogin +wandb: - Waiting for wandb.init()... wandb: \ Waiting for wandb.init()... wandb: wandb version 0.15.8 is available! To upgrade, please run: +wandb: $ pip install wandb --upgrade +wandb: Tracking run with wandb version 0.15.4 +wandb: Run data is saved locally in ./wandb/run-20230821_174450-vwzrsjzb +wandb: Run `wandb offline` to turn off syncing. +wandb: Resuming run silver-sweep-8 +wandb: ⭐️ View project at https://wandb.ai/rycolab/alkmi-wit +wandb: 🧹 View sweep at https://wandb.ai/rycolab/alkmi-wit/sweeps/ff69ui4h +wandb: 🚀 View run at https://wandb.ai/rycolab/alkmi-wit/runs/vwzrsjzb +[rank: 0] Global seed set to 5501650 +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-test +The current host is eu-a65-06 +The current cuDNN version is 8801 +Parameter 'text_perc' already had the value 100. +Parameter 'vision_perc' already had the value 10. +Parameter 'seed' already had the value 5501650. +Total GPU memory: 84.99 GB +Detected 80 GB of RAM, doubling batch size to 64. +Detected 80 GB of RAM, halving gradient accumulation to 64 +Detected half-sized run, (almost) doubling batch size to 96. +Detected half-sized run, (almost) halving gradient accumulation to 42 +[update_ckeckpoint_dir] Detected hyperparameter run! +[update_ckeckpoint_dir] Detected half-size run! +[update_ckeckpoint_dir] Found and updated model name flava in the checkpoint dirpath. +[update_ckeckpoint_dir] Setting checkpoint dirpath to /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-21)_time(17:44:52)/seed(5501650)-magic({'enable': True})-text_perc(100)-vision_perc(10/ +Text is the predominant modality (100 v.s. 10 vision), will sample proportionally for optimal BLiMP performance. +Registering basic callbacks +[PPL Evaluation] Loading dataset 'theodor1289/wit' with split 'test' +[PPL Evaluation] Length of the dataset is 1295591 +Building model 'flava' +Enabling TensorFloat32 tensor cores for float32 matrix multiplication +Compiling FLAVA's text model... +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-aa925072b83a8ef4_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[100%] +Loading cached processed dataset at theodor1289wit-test +Found cached dataset imagenet-1k (/cluster/scratch/tamariucai/HuggingfaceDatasets/imagenet-1k/default/1.0.0/a1e9bfc56c3a7350165007d1176b15e9128fcaf9ab972147840529aed3ae52bc) +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/lightning_fabric/plugins/environments/slurm.py:165: PossibleUserWarning: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /cluster/work/cotterell/tamariucai/acquiring-linguis ... + rank_zero_warn( +Using bfloat16 Automatic Mixed Precision (AMP) +GPU available: True (cuda), used: True +TPU available: False, using: 0 TPU cores +IPU available: False, using: 0 IPUs +HPU available: False, using: 0 HPUs +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[10%] +Loading cached processed dataset at theodor1289wit-test +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-aa925072b83a8ef4_*_of_00016.arrow +Loading cached processed dataset at /cluster/work/cotterell/tamariucai/acquiring-linguistic-knowledge/alkmi/cache-e4fe8dceb6f9ed7e_*_of_00016.arrow +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Found cached dataset parquet (/cluster/scratch/tamariucai/HuggingfaceDatasets/theodor1289___parquet/theodor1289--wit-bf532b06ea2d7bd2/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7) +Loading cached processed dataset at theodor1289wit-train[100%] +Loading cached processed dataset at theodor1289wit-test +Restoring states from the checkpoint path at /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text100-vision10/half_bs4025_seed5501650_bf16-mixed/flava-epoch=00-step=10446.ckpt +/cluster/home/tamariucai/.cache/pypoetry/virtualenvs/alkmi-CJbY6op3-py3.10/lib64/python3.10/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:337: UserWarning: The dirpath has changed from "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-20)_time(10:22:16)/seed(5501650)-magic({'enable': True})-text_perc(100)-vision_perc(10" to "/cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava_half-wit/date(2023-08-21)_time(17:44:52)/seed(5501650)-magic({'enable': True})-text_perc(100)-vision_perc(10", therefore `best_model_score`, `kth_best_model_path`, `kth_value`, `last_model_path` and `best_k_models` won't be reloaded. Only `best_model_path` will be reloaded. + warnings.warn( +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0] + + | Name | Type | Params +---------------------------------------------- +0 | model | FlavaForPreTraining | 189 M +---------------------------------------------- +136 M Trainable params +53.8 M Non-trainable params +189 M Total params +759.666 Total estimated model params size (MB) +Restored all states from the checkpoint at /cluster/work/cotterell/tamariucai/HuggingfaceCheckpoints/flava-wit/text100-vision10/half_bs4025_seed5501650_bf16-mixed/flava-epoch=00-step=10446.ckpt +Compiling FLAVA's image model... +Compiling FLAVA's multimodal model... +Compiling FLAVA's image codebook... +FLAVA will use a different learning rate for its text submodel (0.00075) compared to its other submodels (0.001) +Initializing datamodule +VLDataModule: batch_size is 96, num_workers is 4 +ImageDataModule: batch_size is 96, num_workers is 4 +MLMDataModule: batch_size is 96, num_workers is 4 +Dataset sizes (un-normalized): [10, 10, 100] +Dataset sizes (normalized): [0.08333333333333333, 0.08333333333333333, 0.8333333333333334] +Sampling weights after temperature (1.0): [0.08333333333333333, 0.08333333333333333, 0.8333333333333334] +2023-08-21 17:46:09.055777 Setting up datamodule: VLDataModule +2023-08-21 17:48:53.780393 Setting up datamodule: ImageDataModule +2023-08-21 17:49:05.149042 Setting up datamodule: MLMDataModule +Registering multimodal overfitting monitors +Adding ImageNet zeroshot callback +Callbacks registered: ['LearningRateMonitor', 'PseudoPerplexityCallback', 'LMEvalHarnessCallback', 'ModelCheckpoint', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'MultimodalOverfittingMonitor', 'ImageNetZeroshotCallback'] +Trainer successfully initialized (strategy=SingleDeviceStrategy) +Starting training +2023-08-21 18:03:21.641815 Preparing data for datamodule: VLDataModule +2023-08-21 18:03:21.641934 Preparing data for datamodule: ImageDataModule +2023-08-21 18:03:21.642021 Preparing data for datamodule: MLMDataModule +2023-08-21 18:03:21.642264 Setting up datamodule: VLDataModule +2023-08-21 18:06:19.652282 Setting up datamodule: ImageDataModule +2023-08-21 18:06:31.005555 Setting up datamodule: MLMDataModule +Resetting wait count for `mlm` since training is resuming and they are special tasks. +Not loading previous best score for `mlm`. +Not loading previous best score for `mim`. +Not loading previous best score for `itm`. +Not loading previous best score for `global_contrastive`. +Not loading previous best score for `mmm_image`. +Resetting wait count for `mmm_text` since training is resuming and they are special tasks. +Not loading previous best score for `mmm_text`. +2023-08-21 18:21:18.274380 Building train dataloader for datamodule: VLDataModule +2023-08-21 18:21:18.275078 Building train dataloader for datamodule: ImageDataModule +2023-08-21 18:21:18.275354 Building train dataloader for datamodule: MLMDataModule +2023-08-21 18:21:19.940787 Building val dataloader for datamodule: VLDataModule +2023-08-21 18:21:19.941685 Building val dataloader for datamodule: ImageDataModule +2023-08-21 18:21:19.942096 Building val dataloader for datamodule: MLMDataModule +Found cached dataset json (/cluster/scratch/tamariucai/HuggingfaceDatasets/json/default-7795097c7b181601/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96) +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-21 21:04:02.835558] Computing e^(tensor([4803.3361], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-21 21:04:02.841064] Ending with PPL=11.0415789067984 (duration: 0:03:26.391891) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.864 +Metric validation/losses/mim_loss improved by 0.012 >= min_delta = 0.0. New best score: 1.040 +Metric validation/losses/itm_loss improved by 0.022 >= min_delta = 0.0. New best score: 0.326 +Metric validation/losses/global_contrastive_loss improved by 0.140 >= min_delta = 0.0. New best score: 3.674 +Metric validation/losses/mmm_image_loss improved by 0.012 >= min_delta = 0.0. New best score: 1.042 +Metric validation/losses/mmm_text_loss improved by 0.675 >= min_delta = 0.0. New best score: 5.598 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 03:34:39.490576] Computing e^(tensor([4723.3229], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 03:34:39.494073] Ending with PPL=10.608562257087424 (duration: 0:03:22.857695) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.862 +Metric validation/losses/mim_loss improved by 0.011 >= min_delta = 0.0. New best score: 1.029 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.326 (now 0.32904335856437683). Wait count has been increased from 0. +Metric validation/losses/global_contrastive_loss improved by 0.154 >= min_delta = 0.0. New best score: 3.520 +Metric validation/losses/mmm_image_loss improved by 0.007 >= min_delta = 0.0. New best score: 1.034 +Metric validation/losses/mmm_text_loss improved by 0.310 >= min_delta = 0.0. New best score: 5.288 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 06:50:30.173687] Computing e^(tensor([4710.4828], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 06:50:30.177053] Ending with PPL=10.540673124690128 (duration: 0:05:13.868093) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.850 +Metric validation/losses/mim_loss improved by 0.020 >= min_delta = 0.0. New best score: 1.010 +Metric validation/losses/itm_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.320 +Metric validation/losses/global_contrastive_loss improved by 0.071 >= min_delta = 0.0. New best score: 3.449 +Metric validation/losses/mmm_image_loss improved by 0.020 >= min_delta = 0.0. New best score: 1.014 +Metric validation/losses/mmm_text_loss improved by 0.379 >= min_delta = 0.0. New best score: 4.909 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 10:27:24.626570] Computing e^(tensor([4703.4034], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 10:27:24.630044] Ending with PPL=10.5034280082112 (duration: 0:05:26.729397) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.838 +Metric validation/losses/mim_loss improved by 0.010 >= min_delta = 0.0. New best score: 1.000 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.320 (now 0.3364366292953491). Wait count has been increased from 0. +Metric validation/losses/global_contrastive_loss improved by 0.183 >= min_delta = 0.0. New best score: 3.266 +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 1.009 +Metric validation/losses/mmm_text_loss improved by 0.552 >= min_delta = 0.0. New best score: 4.357 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 13:59:12.267918] Computing e^(tensor([4686.2405], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 13:59:12.271441] Ending with PPL=10.413679263596759 (duration: 0:03:25.279478) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.831 +Metric validation/losses/mim_loss improved by 0.020 >= min_delta = 0.0. New best score: 0.980 +Metric validation/losses/itm_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.319 +Metric validation/losses/global_contrastive_loss improved by 0.071 >= min_delta = 0.0. New best score: 3.195 +Metric validation/losses/mmm_image_loss improved by 0.020 >= min_delta = 0.0. New best score: 0.989 +Metric validation/losses/mmm_text_loss improved by 0.631 >= min_delta = 0.0. New best score: 3.726 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 17:03:47.167358] Computing e^(tensor([4671.5849], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 17:03:47.170712] Ending with PPL=10.337648742969437 (duration: 0:03:23.952370) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.825 +Metric validation/losses/mim_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.973 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.319 (now 0.3356458842754364). Wait count has been increased from 0. +Metric validation/losses/global_contrastive_loss improved by 0.098 >= min_delta = 0.0. New best score: 3.097 +Metric validation/losses/mmm_image_loss improved by 0.017 >= min_delta = 0.0. New best score: 0.972 +Metric validation/losses/mmm_text_loss improved by 0.327 >= min_delta = 0.0. New best score: 3.398 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 19:57:45.601599] Computing e^(tensor([4659.3073], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 19:57:45.604988] Ending with PPL=10.274382153841929 (duration: 0:03:20.941912) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.817 +Metric validation/losses/mim_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.969 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.319 (now 0.32458773255348206). Wait count has been increased from 1. +Metric validation/losses/global_contrastive_loss improved by 0.033 >= min_delta = 0.0. New best score: 3.064 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.972 (now 0.978489339351654). Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.111 >= min_delta = 0.0. New best score: 3.288 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-22 23:07:43.001219] Computing e^(tensor([4617.4326], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-22 23:07:43.004635] Ending with PPL=10.061500166524892 (duration: 0:03:18.598498) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.808 +Metric validation/losses/mim_loss improved by 0.010 >= min_delta = 0.0. New best score: 0.959 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.319 (now 0.32343995571136475). Wait count has been increased from 2. +Metric validation/losses/global_contrastive_loss improved by 0.076 >= min_delta = 0.0. New best score: 2.988 +Metric validation/losses/mmm_image_loss improved by 0.016 >= min_delta = 0.0. New best score: 0.957 +Metric validation/losses/mmm_text_loss improved by 0.068 >= min_delta = 0.0. New best score: 3.220 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 01:40:49.507794] Computing e^(tensor([4625.6587], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 01:40:49.511146] Ending with PPL=10.102969043030136 (duration: 0:03:23.536648) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.804 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.959 (now 0.9601945877075195). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 4 records. Best score: 0.319 (now 0.33182385563850403). Wait count has been increased from 3. +Metric validation/losses/global_contrastive_loss improved by 0.026 >= min_delta = 0.0. New best score: 2.962 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.957 (now 0.9597057104110718). Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.046 >= min_delta = 0.0. New best score: 3.174 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 04:13:43.501769] Computing e^(tensor([4610.5455], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 04:13:43.505705] Ending with PPL=10.026912496179955 (duration: 0:03:19.804700) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.799 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 0.959 (now 0.9605151414871216). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.319 (now 0.3219744563102722). Signaling model to ignore task. +Metric validation/losses/global_contrastive_loss improved by 0.068 >= min_delta = 0.0. New best score: 2.894 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.957 (now 0.9667628407478333). Wait count has been increased from 1. +Metric validation/losses/mmm_text_loss improved by 0.060 >= min_delta = 0.0. New best score: 3.113 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 06:36:39.758931] Computing e^(tensor([4578.2073], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 06:36:39.762264] Ending with PPL=9.866090217919128 (duration: 0:03:21.527505) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.786 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.957 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.319 (now 0.3219744563102722). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.894 (now 3.223212480545044). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.957 (now 0.9580250382423401). Wait count has been increased from 2. +Metric validation/losses/mmm_text_loss improved by 0.079 >= min_delta = 0.0. New best score: 3.035 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 09:05:55.253206] Computing e^(tensor([4523.1241], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 09:05:55.256615] Ending with PPL=9.598070346795788 (duration: 0:03:24.000644) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.764 +Metric validation/losses/mim_loss improved by 0.041 >= min_delta = 0.0. New best score: 0.917 +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.319 (now 0.3219744563102722). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.894 (now 3.0981626510620117). Wait count has been increased from 1. +Metric validation/losses/mmm_image_loss improved by 0.038 >= min_delta = 0.0. New best score: 0.919 +Metric validation/losses/mmm_text_loss improved by 0.049 >= min_delta = 0.0. New best score: 2.985 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 11:37:40.981103] Computing e^(tensor([4499.3417], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 11:37:40.984450] Ending with PPL=9.484613628334014 (duration: 0:03:24.948183) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.755 +Metric validation/losses/mim_loss improved by 0.017 >= min_delta = 0.0. New best score: 0.899 +Monitored metric validation/losses/itm_loss did not improve in the last 8 records. Best score: 0.319 (now 0.3219744563102722). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.894 (now 3.048272132873535). Wait count has been increased from 2. +Metric validation/losses/mmm_image_loss improved by 0.019 >= min_delta = 0.0. New best score: 0.899 +Metric validation/losses/mmm_text_loss improved by 0.011 >= min_delta = 0.0. New best score: 2.974 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 13:56:13.256531] Computing e^(tensor([4474.9921], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 13:56:13.259893] Ending with PPL=9.369840104482902 (duration: 0:03:19.241276) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.744 +Metric validation/losses/mim_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.892 +Monitored metric validation/losses/itm_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 2.894 (now 3.0347588062286377). Wait count has been increased from 3. +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.895 +Metric validation/losses/mmm_text_loss improved by 0.021 >= min_delta = 0.0. New best score: 2.953 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 16:14:07.949337] Computing e^(tensor([4482.2433], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 16:14:07.952800] Ending with PPL=9.403873422514557 (duration: 0:03:18.539452) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.891 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.319 (now 0.3290667235851288). Wait count has been increased from 0. +Metric validation/losses/global_contrastive_loss improved by 0.244 >= min_delta = 0.0. New best score: 2.650 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.893 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.953 (now 3.0075035095214844). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 18:37:24.295445] Computing e^(tensor([4452.5733], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 18:37:24.298881] Ending with PPL=9.265396654417087 (duration: 0:03:20.039133) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.729 +Metric validation/losses/mim_loss improved by 0.019 >= min_delta = 0.0. New best score: 0.873 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.319 (now 0.3264337182044983). Wait count has been increased from 1. +Metric validation/losses/global_contrastive_loss improved by 0.011 >= min_delta = 0.0. New best score: 2.639 +Metric validation/losses/mmm_image_loss improved by 0.016 >= min_delta = 0.0. New best score: 0.876 +Metric validation/losses/mmm_text_loss improved by 0.007 >= min_delta = 0.0. New best score: 2.945 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 20:42:15.533726] Computing e^(tensor([4444.7073], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 20:42:15.537168] Ending with PPL=9.229027443122686 (duration: 0:03:17.718464) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.728 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.873 (now 0.8769270181655884). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.319 (now 0.32175183296203613). Wait count has been increased from 2. +Metric validation/losses/global_contrastive_loss improved by 0.000 >= min_delta = 0.0. New best score: 2.639 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.876 (now 0.8820454478263855). Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.015 >= min_delta = 0.0. New best score: 2.930 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-23 23:00:31.146192] Computing e^(tensor([4450.7045], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-23 23:00:31.149785] Ending with PPL=9.256742880221742 (duration: 0:03:18.357647) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.725 +Metric validation/losses/mim_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.863 +Monitored metric validation/losses/itm_loss did not improve since the last 4 records. Best score: 0.319 (now 0.3263523578643799). Wait count has been increased from 3. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.639 (now 2.646822929382324). Wait count has been increased from 0. +Metric validation/losses/mmm_image_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.867 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.930 (now 2.9424664974212646). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 01:11:49.779594] Computing e^(tensor([4428.5261], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 01:11:49.783059] Ending with PPL=9.154660320363428 (duration: 0:03:17.261098) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.717 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.863 (now 0.8695924282073975). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.319 (now 0.3229973018169403). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.639 (now 2.641955614089966). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.867 (now 0.8684881925582886). Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.036 >= min_delta = 0.0. New best score: 2.894 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 03:32:19.770792] Computing e^(tensor([4427.3554], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 03:32:19.774138] Ending with PPL=9.149302887020752 (duration: 0:03:20.959864) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.715 +Metric validation/losses/mim_loss improved by 0.003 >= min_delta = 0.0. New best score: 0.860 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.319 (now 0.3229973018169403). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.639 (now 3.011054515838623). Wait count has been increased from 2. +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.863 +Metric validation/losses/mmm_text_loss improved by 0.045 >= min_delta = 0.0. New best score: 2.849 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 05:41:27.555651] Computing e^(tensor([4416.7910], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 05:41:27.558990] Ending with PPL=9.101102177298783 (duration: 0:03:20.578869) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.708 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.860 (now 0.8641675114631653). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.319 (now 0.3229973018169403). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 2.639 (now 3.0480127334594727). Wait count has been increased from 3. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.863 (now 0.8684226870536804). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.849 (now 2.8577113151550293). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 07:50:16.323328] Computing e^(tensor([4405.7212], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 07:50:16.326856] Ending with PPL=9.050867451766164 (duration: 0:03:19.459723) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.705 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 0.860 (now 0.8631117343902588). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve in the last 8 records. Best score: 0.319 (now 0.3229973018169403). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.639 (now 3.066007375717163). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.863 (now 0.8715595006942749). Wait count has been increased from 1. +Metric validation/losses/mmm_text_loss improved by 0.030 >= min_delta = 0.0. New best score: 2.819 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 09:56:11.205502] Computing e^(tensor([4403.2880], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 09:56:11.208863] Ending with PPL=9.039862816542119 (duration: 0:03:18.101684) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.698 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 0.860 (now 0.8743023872375488). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.639 (now 3.066007375717163). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.863 (now 0.8722139000892639). Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.819 (now 2.8311002254486084). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 11:58:01.477984] Computing e^(tensor([4388.9736], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 11:58:01.481353] Ending with PPL=8.975393797868156 (duration: 0:03:12.990996) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.844 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.319 (now 0.3230973184108734). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 2.639 (now 3.066007375717163). Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.014 >= min_delta = 0.0. New best score: 0.848 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 2.819 (now 2.8485617637634277). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 13:51:46.387847] Computing e^(tensor([4387.3311], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 13:51:46.391207] Ending with PPL=8.968025648989544 (duration: 0:03:19.305968) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.693 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.844 (now 0.8448245525360107). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.319 (now 0.3217920958995819). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 8 records. Best score: 2.639 (now 3.066007375717163). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.848 (now 0.8513255715370178). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 2.819 (now 2.8598060607910156). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 15:51:03.746784] Computing e^(tensor([4393.0862], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 15:51:03.750130] Ending with PPL=8.993868938161356 (duration: 0:03:15.849003) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.842 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.319 (now 0.33002880215644836). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.848 (now 0.8509542942047119). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 4 records. Best score: 2.819 (now 2.8625664710998535). Wait count has been increased from 3. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 17:55:25.575799] Computing e^(tensor([4393.4594], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 17:55:25.579208] Ending with PPL=8.995547047313735 (duration: 0:03:13.199498) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.689 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.842 (now 0.8490172028541565). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 4 records. Best score: 0.319 (now 0.3241935670375824). Wait count has been increased from 3. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.639 (now 2.778031349182129). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.848 (now 0.8590490221977234). Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 2.819 (now 2.8758037090301514). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 20:15:56.622291] Computing e^(tensor([4361.2196], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 20:15:56.625698] Ending with PPL=8.851702358752442 (duration: 0:03:19.467725) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.689 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 0.842 (now 0.8453348875045776). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.319 (now 0.3254668712615967). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.639 (now 2.691251754760742). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 4 records. Best score: 0.848 (now 0.8559586405754089). Wait count has been increased from 3. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 2.819 (now 2.8758037090301514). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-24 22:36:39.538315] Computing e^(tensor([4350.1336], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-24 22:36:39.541694] Ending with PPL=8.802772984120397 (duration: 0:03:17.002741) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.680 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 0.842 (now 0.8456007838249207). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.319 (now 0.3254668712615967). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.639 (now 3.0499460697174072). Wait count has been increased from 2. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 0.848 (now 0.8586506843566895). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 7 records. Best score: 2.819 (now 2.8758037090301514). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 01:02:22.393310] Computing e^(tensor([4361.8611], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 01:02:22.396762] Ending with PPL=8.854542199621552 (duration: 0:03:17.809279) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.832 +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.319 (now 0.3254668712615967). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 2.639 (now 3.0505857467651367). Wait count has been increased from 3. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 6 records. Best score: 0.848 (now 0.8586506843566895). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 8 records. Best score: 2.819 (now 2.8758037090301514). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 03:22:15.980092] Computing e^(tensor([4360.7724], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 03:22:15.983502] Ending with PPL=8.84972344636333 (duration: 0:03:17.762650) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.675 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.832 (now 0.834488570690155). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 8 records. Best score: 0.319 (now 0.3254668712615967). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.639 (now 3.0760276317596436). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 7 records. Best score: 0.848 (now 0.8586506843566895). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 05:52:20.138517] Computing e^(tensor([4347.0971], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 05:52:20.142000] Ending with PPL=8.789418529873322 (duration: 0:05:24.505762) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.674 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 0.832 (now 0.8382134437561035). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 2.639 (now 3.0760276317596436). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 8 records. Best score: 0.848 (now 0.8586506843566895). Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.002 >= min_delta = 0.0. New best score: 2.817 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 08:31:09.443258] Computing e^(tensor([4328.3549], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 08:31:09.446717] Ending with PPL=8.70743687310602 (duration: 0:05:09.753155) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.672 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 0.832 (now 0.8532204627990723). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.319 (now 0.3305252492427826). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 2.639 (now 3.0760276317596436). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.817 (now 2.8572139739990234). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 10:50:48.146701] Computing e^(tensor([4325.0614], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 10:50:48.151331] Ending with PPL=8.693109298201064 (duration: 0:03:17.029672) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.670 +Metric validation/losses/mim_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.825 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.319 (now 0.33080586791038513). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 8 records. Best score: 2.639 (now 3.0760276317596436). Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.015 >= min_delta = 0.0. New best score: 0.833 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 2.817 (now 2.8581881523132324). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 13:18:21.993971] Computing e^(tensor([4316.7245], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 13:18:21.997535] Ending with PPL=8.656948057662607 (duration: 0:03:20.059146) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.668 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.825 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.319 (now 0.33412906527519226). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.828 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 2.817 (now 2.8626203536987305). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 15:47:15.803860] Computing e^(tensor([4320.3171], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 15:47:15.807237] Ending with PPL=8.672512431249924 (duration: 0:03:19.600867) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.657 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.825 (now 0.8262379765510559). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 4 records. Best score: 0.319 (now 0.32196030020713806). Wait count has been increased from 3. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.639 (now 2.811681032180786). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.828 (now 0.8353679776191711). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 4 records. Best score: 2.817 (now 2.8373749256134033). Wait count has been increased from 3. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 18:18:02.696253] Computing e^(tensor([4302.6049], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 18:18:02.699937] Ending with PPL=8.596047088084937 (duration: 0:03:20.201536) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.657 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.820 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.319 (now 0.3227922022342682). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.639 (now 2.7190604209899902). Wait count has been increased from 1. +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.823 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 2.817 (now 2.861611843109131). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 20:49:08.403864] Computing e^(tensor([4315.7884], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 20:49:08.407256] Ending with PPL=8.652897242664057 (duration: 0:03:19.346113) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.818 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.319 (now 0.3227922022342682). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.639 (now 3.0421547889709473). Wait count has been increased from 2. +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.821 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 2.817 (now 2.861611843109131). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-25 23:20:26.288975] Computing e^(tensor([4320.4755], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-25 23:20:26.292490] Ending with PPL=8.673199453216421 (duration: 0:03:19.127371) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.653 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.818 (now 0.8268820643424988). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in the last 7 records. Best score: 0.319 (now 0.3227922022342682). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 4 records. Best score: 2.639 (now 3.0464653968811035). Wait count has been increased from 3. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.821 (now 0.8282365798950195). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 7 records. Best score: 2.817 (now 2.861611843109131). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 01:53:18.267817] Computing e^(tensor([4307.0248], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 01:53:18.271200] Ending with PPL=8.615064963571482 (duration: 0:03:17.535339) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.648 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.816 +Monitored metric validation/losses/itm_loss did not improve in the last 8 records. Best score: 0.319 (now 0.3227922022342682). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 2.639 (now 3.0438857078552246). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.821 (now 0.8215692639350891). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 8 records. Best score: 2.817 (now 2.861611843109131). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 04:34:15.846053] Computing e^(tensor([4296.4472], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 04:34:15.849417] Ending with PPL=8.569621666966004 (duration: 0:03:19.765471) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.821 +Monitored metric validation/losses/mmm_text_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 07:08:16.477806] Computing e^(tensor([4295.6977], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 07:08:16.481285] Ending with PPL=8.566410895493677 (duration: 0:03:21.198833) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.646 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 0.816 (now 0.8169199824333191). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.319 (now 0.33029720187187195). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 7 records. Best score: 2.639 (now 3.0438857078552246). Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.816 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.817 (now 2.9041428565979004). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 09:43:20.624528] Computing e^(tensor([4284.7845], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 09:43:20.627931] Ending with PPL=8.51979484582422 (duration: 0:03:18.581013) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.644 +Metric validation/losses/mim_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.816 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.319 (now 0.31962987780570984). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 8 records. Best score: 2.639 (now 3.0438857078552246). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.816 (now 0.820391058921814). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 2.817 (now 2.8569629192352295). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 12:11:46.598479] Computing e^(tensor([4286.5979], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 12:11:46.601952] Ending with PPL=8.52752318280702 (duration: 0:03:19.091792) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.636 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.815 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.319 (now 0.3284609317779541). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve in 8 records. This exceeds the restart patience 8, so we will restart the task with renewed patience. +Metric validation/losses/mmm_image_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.815 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 2.817 (now 2.853163242340088). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-26 14:46:26.588646] Computing e^(tensor([4266.7365], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-26 14:46:26.591991] Ending with PPL=8.4432577858891 (duration: 0:03:24.244540) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.910 +Metric validation/losses/mmm_image_loss improved by 0.032 >= min_delta = 0.0. New best score: 0.926 +Metric validation/losses/mmm_text_loss improved by 0.164 >= min_delta = 0.0. New best score: 3.233 +Metric validation/losses/itm_loss improved by 0.025 >= min_delta = 0.0. New best score: 0.329 +Metric validation/losses/global_contrastive_loss improved by 0.678 >= min_delta = 0.0. New best score: 2.523 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 03:16:58.380077] Computing e^(tensor([4389.0338], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 03:16:58.385770] Ending with PPL=8.975664034774294 (duration: 0:04:12.842197) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.880 +Metric validation/losses/mmm_image_loss improved by 0.048 >= min_delta = 0.0. New best score: 0.879 +Metric validation/losses/mmm_text_loss improved by 0.101 >= min_delta = 0.0. New best score: 3.132 +Metric validation/losses/itm_loss improved by 0.024 >= min_delta = 0.0. New best score: 0.305 +Metric validation/losses/global_contrastive_loss improved by 0.070 >= min_delta = 0.0. New best score: 2.453 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 04:50:54.863125] Computing e^(tensor([4351.3417], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 04:50:54.868703] Ending with PPL=8.808092256555328 (duration: 0:04:17.433167) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.848 +Metric validation/losses/mmm_image_loss improved by 0.035 >= min_delta = 0.0. New best score: 0.843 +Metric validation/losses/mmm_text_loss improved by 0.038 >= min_delta = 0.0. New best score: 3.094 +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.305 (now 0.32179245352745056). Wait count has been increased from 0. +Metric validation/losses/global_contrastive_loss improved by 0.152 >= min_delta = 0.0. New best score: 2.302 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 06:32:47.595731] Computing e^(tensor([4351.5924], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 06:32:47.601469] Ending with PPL=8.809196368764296 (duration: 0:04:15.459489) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.721 +Metric validation/losses/mim_loss improved by 0.008 >= min_delta = 0.0. New best score: 0.839 +Metric validation/losses/mmm_image_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.839 +Metric validation/losses/mmm_text_loss improved by 0.027 >= min_delta = 0.0. New best score: 3.068 +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.305 (now 0.33483508229255676). Wait count has been increased from 1. +Metric validation/losses/global_contrastive_loss improved by 0.050 >= min_delta = 0.0. New best score: 2.251 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 08:09:03.621469] Computing e^(tensor([4346.3718], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 08:09:03.632304] Ending with PPL=8.786231542865464 (duration: 0:05:28.253563) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.718 +Metric validation/losses/mim_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.835 +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.834 +Metric validation/losses/mmm_text_loss improved by 0.039 >= min_delta = 0.0. New best score: 3.029 +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.305 (now 0.3264191746711731). Wait count has been increased from 2. +Metric validation/losses/global_contrastive_loss improved by 0.041 >= min_delta = 0.0. New best score: 2.210 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 09:53:10.889712] Computing e^(tensor([4341.4547], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 09:53:10.895384] Ending with PPL=8.764656783933088 (duration: 0:04:23.280055) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.828 +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.830 +Metric validation/losses/mmm_text_loss improved by 0.027 >= min_delta = 0.0. New best score: 3.002 +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.305 (now 0.3181944191455841). Signaling model to ignore task. +Metric validation/losses/global_contrastive_loss improved by 0.049 >= min_delta = 0.0. New best score: 2.161 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 11:29:25.779025] Computing e^(tensor([4317.3986], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 11:29:25.784776] Ending with PPL=8.659866513821644 (duration: 0:04:10.979272) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.712 +Metric validation/losses/mim_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.825 +Metric validation/losses/mmm_image_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.820 +Metric validation/losses/mmm_text_loss improved by 0.059 >= min_delta = 0.0. New best score: 2.943 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.305 (now 0.3181944191455841). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 2.161 (now 2.4498236179351807). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 13:07:04.755490] Computing e^(tensor([4296.5303], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 13:07:04.761189] Ending with PPL=8.569977831732757 (duration: 0:05:22.980485) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.697 +Metric validation/losses/mim_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.819 +Metric validation/losses/mmm_image_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.818 +Metric validation/losses/mmm_text_loss improved by 0.039 >= min_delta = 0.0. New best score: 2.904 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.305 (now 0.3181944191455841). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 2.161 (now 2.4485559463500977). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 14:50:47.173330] Computing e^(tensor([4267.8618], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 14:50:47.178965] Ending with PPL=8.448010073208074 (duration: 0:04:13.448510) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.691 +Metric validation/losses/mim_loss improved by 0.009 >= min_delta = 0.0. New best score: 0.810 +Metric validation/losses/mmm_image_loss improved by 0.010 >= min_delta = 0.0. New best score: 0.808 +Metric validation/losses/mmm_text_loss improved by 0.047 >= min_delta = 0.0. New best score: 2.857 +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 2.161 (now 2.3809611797332764). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 16:26:31.985514] Computing e^(tensor([4253.7311], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 16:26:31.991386] Ending with PPL=8.388532281849761 (duration: 0:04:14.614170) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.687 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.810 (now 0.8167135715484619). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.808 (now 0.8204881548881531). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.857 (now 2.925734043121338). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.305 (now 0.32809263467788696). Wait count has been increased from 0. +Metric validation/losses/global_contrastive_loss improved by 0.150 >= min_delta = 0.0. New best score: 2.011 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 18:06:41.184946] Computing e^(tensor([4251.1597], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 18:06:41.190704] Ending with PPL=8.377753736931435 (duration: 0:05:41.291322) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.683 +Metric validation/losses/mim_loss improved by 0.007 >= min_delta = 0.0. New best score: 0.803 +Metric validation/losses/mmm_image_loss improved by 0.004 >= min_delta = 0.0. New best score: 0.804 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 2.857 (now 2.9047727584838867). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.305 (now 0.3193071484565735). Wait count has been increased from 1. +Metric validation/losses/global_contrastive_loss improved by 0.010 >= min_delta = 0.0. New best score: 2.000 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 19:51:56.836092] Computing e^(tensor([4256.9455], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 19:51:56.841922] Ending with PPL=8.402024940919869 (duration: 0:04:28.351834) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.675 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.803 (now 0.8082790970802307). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.804 (now 0.8143921494483948). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 2.857 (now 2.8994200229644775). Wait count has been increased from 2. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.305 (now 0.31620657444000244). Wait count has been increased from 2. +Metric validation/losses/global_contrastive_loss improved by 0.020 >= min_delta = 0.0. New best score: 1.981 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 21:33:54.723247] Computing e^(tensor([4251.8365], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 21:33:54.728920] Ending with PPL=8.380589542332649 (duration: 0:04:16.501468) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 1.980 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-27 23:12:39.653533] Computing e^(tensor([4252.9666], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-27 23:12:39.659206] Ending with PPL=8.385326175280426 (duration: 0:04:14.521040) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.674 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 0.803 (now 0.8030930161476135). Signaling model to ignore task. +Metric validation/losses/mmm_image_loss improved by 0.006 >= min_delta = 0.0. New best score: 0.798 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 2.857 (now 2.8782997131347656). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.305 (now 0.33938735723495483). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 1.980 (now 2.3293046951293945). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 02:27:55.144714] Computing e^(tensor([4235.4909], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 02:27:55.150618] Ending with PPL=8.312375718116773 (duration: 0:04:29.684088) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.670 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.801 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.798 (now 0.8013208508491516). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 1.980 (now 2.284533977508545). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 04:06:36.127728] Computing e^(tensor([4233.3241], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 04:06:36.133390] Ending with PPL=8.30337472256278 (duration: 0:04:16.249642) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.789 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.857 (now 2.8748655319213867). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.305 (now 0.3168448507785797). Wait count has been increased from 0. +Metric validation/losses/global_contrastive_loss improved by 0.063 >= min_delta = 0.0. New best score: 1.917 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 05:42:27.954793] Computing e^(tensor([4226.0100], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 05:42:27.960575] Ending with PPL=8.273064446265417 (duration: 0:04:41.865064) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.665 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 0.801 (now 0.8035577535629272). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.789 (now 0.8002036809921265). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 2.857 (now 2.887254476547241). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve since the last 2 records. Best score: 0.305 (now 0.3238949477672577). Wait count has been increased from 1. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 1.917 (now 1.9193072319030762). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 07:22:24.829675] Computing e^(tensor([4231.5897], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 07:22:24.835482] Ending with PPL=8.29617738715471 (duration: 0:04:27.861912) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.799 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.789 (now 0.7968927621841431). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 3 records. Best score: 2.857 (now 2.861351251602173). Wait count has been increased from 2. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.305 (now 0.3304320275783539). Wait count has been increased from 2. +Metric validation/losses/global_contrastive_loss improved by 0.014 >= min_delta = 0.0. New best score: 1.904 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 09:03:00.139652] Computing e^(tensor([4210.3541], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 09:03:00.145440] Ending with PPL=8.208556372802548 (duration: 0:04:16.837740) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.796 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.789 (now 0.7976795434951782). Wait count has been increased from 2. +Metric validation/losses/mmm_text_loss improved by 0.009 >= min_delta = 0.0. New best score: 2.847 +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.305 (now 0.3363676369190216). Signaling model to ignore task. +Metric validation/losses/global_contrastive_loss improved by 0.016 >= min_delta = 0.0. New best score: 1.888 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 10:37:20.095269] Computing e^(tensor([4210.2647], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 10:37:20.100986] Ending with PPL=8.208189245780224 (duration: 0:04:15.483225) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.656 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.796 (now 0.7993056774139404). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 4 records. Best score: 0.789 (now 0.7931231260299683). Signaling model to ignore task. +Metric validation/losses/mmm_text_loss improved by 0.053 >= min_delta = 0.0. New best score: 2.794 +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.305 (now 0.3363676369190216). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 1 records. Best score: 1.888 (now 2.255509853363037). Wait count has been increased from 0. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 12:15:28.716936] Computing e^(tensor([4220.8622], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 12:15:28.722652] Ending with PPL=8.251797954972853 (duration: 0:04:13.278036) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.757 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.305 (now 0.3363676369190216). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 1.888 (now 2.2805228233337402). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 13:54:35.269685] Computing e^(tensor([4211.7227], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 13:54:35.275501] Ending with PPL=8.214175410099044 (duration: 0:04:16.815311) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 1.878 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 17:19:10.605734] Computing e^(tensor([4197.1202], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 17:19:10.611488] Ending with PPL=8.154420001126157 (duration: 0:04:11.476769) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.796 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.789 (now 0.7924204468727112). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 4 records. Best score: 2.757 (now 2.8085074424743652). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 3 records. Best score: 0.305 (now 0.32318466901779175). Wait count has been increased from 2. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 1.878 (now 1.8824124336242676). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 20:30:00.470304] Computing e^(tensor([4207.8549], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 20:30:00.475967] Ending with PPL=8.1983051779965 (duration: 0:04:11.937832) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.794 +Metric validation/losses/mmm_image_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.788 +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 2.757 (now 2.8085074424743652). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 4 records. Best score: 0.305 (now 0.31653642654418945). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 1.878 (now 1.8793147802352905). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-28 22:03:49.790998] Computing e^(tensor([4212.8023], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-28 22:03:49.796669] Ending with PPL=8.218610372446639 (duration: 0:04:14.184716) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.654 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 0.794 (now 0.7968294620513916). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 2 records. Best score: 0.788 (now 0.7934253215789795). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.305 (now 0.31653642654418945). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 1.878 (now 2.2602438926696777). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 01:26:30.789697] Computing e^(tensor([4203.6417], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 01:26:30.795455] Ending with PPL=8.181052988805108 (duration: 0:04:25.656086) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.788 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 2.757 (now 2.886892557144165). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.305 (now 0.3158150017261505). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 04:25:52.036709] Computing e^(tensor([4254.5252], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 04:25:52.042375] Ending with PPL=8.391863564062552 (duration: 0:04:10.527442) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.654 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.794 (now 0.7989629507064819). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 4 records. Best score: 0.788 (now 0.7991533875465393). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 2.757 (now 2.850931167602539). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.305 (now 0.33052584528923035). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 1.878 (now 2.305140972137451). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 10:49:16.757720] Computing e^(tensor([4201.0411], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 10:49:16.763411] Ending with PPL=8.170421848387724 (duration: 0:04:19.236904) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.654 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 0.794 (now 0.8024429082870483). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 0.788 (now 0.7991533875465393). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.305 (now 0.33052584528923035). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 1.878 (now 2.305140972137451). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 12:28:31.361671] Computing e^(tensor([4210.4072], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 12:28:31.367396] Ending with PPL=8.208774204255187 (duration: 0:04:38.272410) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.650 +Monitored metric validation/losses/mim_loss did not improve since the last 1 records. Best score: 0.794 (now 0.8015297055244446). Wait count has been increased from 0. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 4 records. Best score: 0.788 (now 0.7961271405220032). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 6 records. Best score: 2.757 (now 2.8335485458374023). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve in the last 5 records. Best score: 0.305 (now 0.33089837431907654). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 1.878 (now 2.2949960231781006). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 20:43:49.244638] Computing e^(tensor([4183.0488], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 20:43:49.250421] Ending with PPL=8.097249154578972 (duration: 0:04:07.476883) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.648 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 0.794 (now 0.7972970604896545). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 0.788 (now 0.7961271405220032). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.305 (now 0.33089837431907654). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 1.878 (now 2.2949960231781006). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-29 21:56:12.364002] Computing e^(tensor([4188.2464], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-29 21:56:12.369854] Ending with PPL=8.118319799933168 (duration: 0:04:04.742617) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.642 +Monitored metric validation/losses/mim_loss did not improve in the last 4 records. Best score: 0.794 (now 0.7976493239402771). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 2.757 (now 2.874453067779541). Wait count has been increased from 1. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.305 (now 0.3226836621761322). Wait count has been increased from 0. +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 00:34:12.003143] Computing e^(tensor([4226.0390], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 00:34:12.008810] Ending with PPL=8.273184450546305 (duration: 0:04:09.972715) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.291 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 1.878 (now 1.9264472723007202). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 04:35:47.080646] Computing e^(tensor([4193.1726], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 04:35:47.086380] Ending with PPL=8.138340561159536 (duration: 0:04:13.362299) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.278 +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 1.878 (now 1.900190830230713). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 06:07:29.923597] Computing e^(tensor([4175.3787], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 06:07:29.929246] Ending with PPL=8.066255377897523 (duration: 0:04:11.223763) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.641 +Monitored metric validation/losses/mim_loss did not improve since the last 2 records. Best score: 0.794 (now 0.79951012134552). Wait count has been increased from 1. +Monitored metric validation/losses/mmm_image_loss did not improve in the last 5 records. Best score: 0.788 (now 0.7992428541183472). Signaling model to ignore task. +Monitored metric validation/losses/mmm_text_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Metric validation/losses/itm_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.274 +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 5 records. Best score: 1.878 (now 1.900190830230713). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 07:38:57.566820] Computing e^(tensor([4179.5346], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 07:38:57.572495] Ending with PPL=8.083034106683849 (duration: 0:04:13.418223) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.261 +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 6 records. Best score: 1.878 (now 1.900190830230713). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 09:06:27.733044] Computing e^(tensor([4177.6026], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 09:06:27.738900] Ending with PPL=8.075229508289626 (duration: 0:04:14.059850) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.252 +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 10:47:32.912816] Computing e^(tensor([4218.9956], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 10:47:32.918563] Ending with PPL=8.244099926111089 (duration: 0:04:12.324135) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.241 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 1.878 (now 1.9865057468414307). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 14:15:16.610886] Computing e^(tensor([4184.3414], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 14:15:16.616635] Ending with PPL=8.10248435312026 (duration: 0:04:10.938890) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.230 +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 1.878 (now 1.8814805746078491). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-30 18:06:34.004769] Computing e^(tensor([4160.5060], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-30 18:06:34.010542] Ending with PPL=8.006494206479928 (duration: 0:04:14.716777) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.641 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.792 +Monitored metric validation/losses/mmm_image_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/mmm_text_loss did not improve since the last 2 records. Best score: 2.757 (now 2.854175567626953). Wait count has been increased from 1. +Metric validation/losses/itm_loss improved by 0.000 >= min_delta = 0.0. New best score: 0.230 +Monitored metric validation/losses/global_contrastive_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 00:20:53.246449] Computing e^(tensor([4224.4411], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 00:20:53.252347] Ending with PPL=8.266577185588458 (duration: 0:04:11.720495) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 0.218 +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 2 records. Best score: 1.878 (now 1.9495586156845093). Wait count has been increased from 1. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 04:35:15.960224] Computing e^(tensor([4164.2669], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 04:35:15.965900] Ending with PPL=8.021564106102637 (duration: 0:05:22.323150) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.634 +Monitored metric validation/losses/mim_loss did not improve in the last 3 records. Best score: 0.792 (now 0.8043619990348816). Signaling model to ignore task. +Monitored metric validation/losses/mmm_image_loss did not improve since the last 3 records. Best score: 0.788 (now 0.8009209632873535). Wait count has been increased from 2. +Monitored metric validation/losses/mmm_text_loss did not improve in the last 5 records. Best score: 2.757 (now 2.79518723487854). Signaling model to ignore task. +Monitored metric validation/losses/itm_loss did not improve since the last 1 records. Best score: 0.218 (now 0.21856604516506195). Wait count has been increased from 0. +Metric validation/losses/global_contrastive_loss improved by 0.006 >= min_delta = 0.0. New best score: 1.872 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 06:38:33.161191] Computing e^(tensor([4182.0698], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 06:38:33.166875] Ending with PPL=8.093286557799228 (duration: 0:04:11.044979) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 1.871 +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 10:42:31.536473] Computing e^(tensor([4180.7744], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 10:42:31.542237] Ending with PPL=8.088046331612269 (duration: 0:04:11.033849) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.631 +Metric validation/losses/mim_loss improved by 0.002 >= min_delta = 0.0. New best score: 0.790 +Monitored metric validation/losses/mmm_image_loss did not improve since the last 1 records. Best score: 0.788 (now 0.7904579639434814). Wait count has been increased from 0. +Metric validation/losses/mmm_text_loss improved by 0.007 >= min_delta = 0.0. New best score: 2.751 +Monitored metric validation/losses/itm_loss did not improve in the last 6 records. Best score: 0.218 (now 0.22304768860340118). Signaling model to ignore task. +Monitored metric validation/losses/global_contrastive_loss did not improve since the last 3 records. Best score: 1.871 (now 2.2108206748962402). Wait count has been increased from 2. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 16:32:17.275896] Computing e^(tensor([4153.6989], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 16:32:17.281631] Ending with PPL=7.979290024616428 (duration: 0:04:50.374774) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00= min_delta = 0.0. New best score: 2.630 +Metric validation/losses/mim_loss improved by 0.001 >= min_delta = 0.0. New best score: 0.789 +Metric validation/losses/mmm_image_loss improved by 0.005 >= min_delta = 0.0. New best score: 0.783 +Monitored metric validation/losses/mmm_text_loss did not improve since the last 1 records. Best score: 2.751 (now 2.7772316932678223). Wait count has been increased from 0. +Monitored metric validation/losses/itm_loss did not improve in 6 records. This exceeds the revival patience 6, so we will restart the task with renewed wait count. +Monitored metric validation/losses/global_contrastive_loss did not improve in the last 4 records. Best score: 1.871 (now 2.206935167312622). Signaling model to ignore task. +[PPL Evaluation] Starting from index 0 to index 2000. +[PPL Evaluation 2023-08-31 18:26:41.515311] Computing e^(tensor([4154.1657], dtype=torch.float64) / 2000) +[PPL Evaluation 2023-08-31 18:26:41.521031] Ending with PPL=7.981152873332645 (duration: 0:04:08.006729) +Starting LM Evaluation Harness +Running on blimp... + 0%| | 0/1 [00:00